//////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// /// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. /// You may obtain a copy of the License at /// /// http://www.apache.org/licenses/LICENSE-2.0 /// /// Unless required by applicable law or agreed to in writing, software /// distributed under the License is distributed on an "AS IS" BASIS, /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. /// See the License for the specific language governing permissions and /// limitations under the License. /// /// Copyright holder is ArangoDB GmbH, Cologne, Germany /// /// @author Kaveh Vahedipour //////////////////////////////////////////////////////////////////////////////// #include "Store.h" #include "StoreCallback.h" #include "Agency/Agent.h" #include "Basics/ConditionLocker.h" #include "Basics/VelocyPackHelper.h" #include #include #include #include #include #include #include using namespace arangodb::consensus; inline static bool endpointPathFromUrl ( std::string const& url, std::string& endpoint, std::string& path) { std::stringstream ep; path = "/"; size_t pos = 7; if (url.find("http://")==0) { ep << "tcp://"; } else if (url.find("https://")==0) { ep << "ssl://"; ++pos; } else { return false; } size_t slash_p = url.find("/",pos); if (slash_p==std::string::npos) { ep << url.substr(pos); } else { ep << url.substr(pos,slash_p-pos); path = url.substr(slash_p); } if (ep.str().find(':')==std::string::npos) { ep << ":8529"; } endpoint = ep.str(); return true; } struct NotEmpty { bool operator()(const std::string& s) { return !s.empty(); } }; struct Empty { bool operator()(const std::string& s) { return s.empty(); } }; /// @brief Split strings by separator std::vector split(const std::string& value, char separator) { std::vector result; std::string::size_type p = (value.find(separator) == 0) ? 1:0; std::string::size_type q; while ((q = value.find(separator, p)) != std::string::npos) { result.emplace_back(value, p, q - p); p = q + 1; } result.emplace_back(value, p); result.erase(std::find_if(result.rbegin(), result.rend(), NotEmpty()).base(), result.end()); return result; } // Construct with node name Node::Node (std::string const& name) : _node_name(name), _parent(nullptr) { _value.clear(); } // Construct with node name in tree structure Node::Node (std::string const& name, Node* parent) : _node_name(name), _parent(parent) { _value.clear(); } // Default dtor Node::~Node() {} Slice Node::slice() const { return (_value.size()==0) ? arangodb::basics::VelocyPackHelper::EmptyObjectValue() : Slice(_value.data()); } // Get name of this node std::string const& Node::name() const { return _node_name; } // Get full path of this node std::string Node::uri() const { Node *par = _parent; std::stringstream path; std::deque names; names.push_front(name()); while (par != 0) { names.push_front(par->name()); par = par->_parent; } for (size_t i = 1; i < names.size(); ++i) { path << "/" << names.at(i); } return path.str(); } // Assignment of rhs slice Node& Node::operator= (VPackSlice const& slice) { // 1. remove any existing time to live entry // 2. clear children map // 3. copy from rhs to buffer pointer // 4. inform all observers here and above // Must not copy _parent, _ttl, _observers removeTimeToLive(); _children.clear(); _value.reset(); _value.append(reinterpret_cast(slice.begin()), slice.byteSize()); return *this; } // Assignment of rhs node Node& Node::operator= (Node const& rhs) { // 1. remove any existing time to live entry // 2. clear children map // 3. copy from rhs to buffer pointer // 4. inform all observers here and above // Must not copy rhs's _parent, _ttl, _observers removeTimeToLive(); _node_name = rhs._node_name; _value = rhs._value; _children = rhs._children; return *this; } // Comparison with slice bool Node::operator== (VPackSlice const& rhs) const { return rhs.equals(slice()); } // Remove this node from store bool Node::remove () { Node& parent = *_parent; return parent.removeChild(_node_name); } // Remove child by name bool Node::removeChild (std::string const& key) { auto found = _children.find(key); if (found == _children.end()) { return false; } found->second->removeTimeToLive(); _children.erase(found); return true; } // Node type NodeType Node::type() const { return _children.size() ? NODE : LEAF; } // lh-value at path vector Node& Node::operator ()(std::vector const& pv) { if (pv.size()) { std::string const& key = pv.at(0); if (_children.find(key) == _children.end()) { _children[key] = std::make_shared(key, this); } auto pvc(pv); pvc.erase(pvc.begin()); return (*_children[key])(pvc); } else { return *this; } } // rh-value at path vector Node const& Node::operator ()(std::vector const& pv) const { if (pv.size()) { std::string const& key = pv.at(0); if (_children.find(key) == _children.end()) { throw StoreException( std::string("Node ") + key + std::string(" not found")); } const Node& child = *_children.at(key); auto pvc(pv); pvc.erase(pvc.begin()); return child(pvc); } else { return *this; } } // lh-value at path Node& Node::operator ()(std::string const& path) { return this->operator()(split(path,'/')); } // rh-value at path Node const& Node::operator ()(std::string const& path) const { return this->operator()(split(path,'/')); } // lh-store Node const& Node::root() const { Node *par = _parent, *tmp = 0; while (par != 0) { tmp = par; par = par->_parent; } return *tmp; } // rh-store Node& Node::root() { Node *par = _parent, *tmp = 0; while (par != 0) { tmp = par; par = par->_parent; } return *tmp; } // velocypack value type of this node ValueType Node::valueType() const { return slice().type(); } // file time to live entry for this node to now + millis bool Node::addTimeToLive (long millis) { auto tkey = std::chrono::system_clock::now() + std::chrono::milliseconds(millis); root()._time_table.insert( std::pair>( tkey, _parent->_children[_node_name])); _ttl = tkey; return true; } // remove time to live entry for this node bool Node::removeTimeToLive () { if (_ttl != std::chrono::system_clock::time_point()) { auto ret = root()._time_table.equal_range(_ttl); for (auto it = ret.first; it!=ret.second;) { if (it->second == _parent->_children[_node_name]) { root()._time_table.erase(it); break; } ++it; } } return true; } inline bool Node::observedBy (std::string const& url) const { auto ret = root()._observer_table.equal_range(url); for (auto it = ret.first; it!=ret.second; ++it) { if (it->second == uri()) { return true; } } return false; } namespace arangodb { namespace consensus { template<> bool Node::handle (VPackSlice const& slice) { Slice val = slice.get("new"); if (val.isObject()) { if (val.hasKey("op")) { // No longer a keyword but a regular key "op" if (_children.find("op") == _children.end()) { _children["op"] = std::make_shared("op", this); } *(_children["op"]) = val.get("op"); } else { // Deeper down this->applies(val); } } else { *this = val; } if (slice.hasKey("ttl")) { VPackSlice ttl_v = slice.get("ttl"); if (ttl_v.isNumber()) { long ttl = 1000l * ( (ttl_v.isDouble()) ? static_cast(slice.get("ttl").getDouble()): slice.get("ttl").getInt()); addTimeToLive (ttl); } else { LOG_TOPIC(WARN, Logger::AGENCY) << "Non-number value assigned to ttl: " << ttl_v.toJson(); } } return true; } template<> bool Node::handle (VPackSlice const& slice) { Builder tmp; tmp.openObject(); try { tmp.add("tmp", Value(this->slice().getInt()+1)); } catch (std::exception const&) { tmp.add("tmp",Value(1)); } tmp.close(); *this = tmp.slice().get("tmp"); return true; } template<> bool Node::handle (VPackSlice const& slice) { Builder tmp; tmp.openObject(); try { tmp.add("tmp", Value(this->slice().getInt()-1)); } catch (std::exception const&) { tmp.add("tmp",Value(-1)); } tmp.close(); *this = tmp.slice().get("tmp"); return true; } template<> bool Node::handle (VPackSlice const& slice) { if (!slice.hasKey("new")) { LOG_TOPIC(WARN, Logger::AGENCY) << "Operator push without new value: " << slice.toJson(); return false; } Builder tmp; tmp.openArray(); if (this->slice().isArray()) { for (auto const& old : VPackArrayIterator(this->slice())) tmp.add(old); } tmp.add(slice.get("new")); tmp.close(); *this = tmp.slice(); return true; } template<> bool Node::handle (VPackSlice const& slice) { Builder tmp; tmp.openArray(); if (this->slice().isArray()) { VPackArrayIterator it(this->slice()); if (it.size()>1) { size_t j = it.size()-1; for (auto old : it) { tmp.add(old); if (--j==0) break; } } } tmp.close(); *this = tmp.slice(); return true; } template<> bool Node::handle (VPackSlice const& slice) { if (!slice.hasKey("new")) { LOG_TOPIC(WARN, Logger::AGENCY) << "Operator prepend without new value: " << slice.toJson(); return false; } Builder tmp; tmp.openArray(); tmp.add(slice.get("new")); if (this->slice().isArray()) { for (auto const& old : VPackArrayIterator(this->slice())) tmp.add(old); } tmp.close(); *this = tmp.slice(); return true; } template<> bool Node::handle (VPackSlice const& slice) { Builder tmp; tmp.openArray(); if (this->slice().isArray()) { // If a VPackArrayIterator it(this->slice()); bool first = true; for (auto const& old : it) { if (first) { first = false; } else { tmp.add(old); } } } tmp.close(); *this = tmp.slice(); return true; } /// Add observer for this node template<> bool Node::handle (VPackSlice const& slice) { if (!slice.hasKey("url")) return false; if (!slice.get("url").isString()) return false; std::string url (slice.get("url").copyString()), uri (this->uri()); // check if such entry exists if (!observedBy(url)) { root()._observer_table.emplace(std::pair(url,uri)); root()._observed_table.emplace(std::pair(uri,url)); return true; } return false; } template<> bool Node::handle (VPackSlice const& slice) { if (!slice.hasKey("url")) return false; if (!slice.get("url").isString()) return false; std::string url (slice.get("url").copyString()), uri (this->uri()); // delete in both cases a single entry (ensured above) // breaking the iterators is fine then auto ret = root()._observer_table.equal_range(url); for (auto it = ret.first; it!=ret.second; ++it) { if (it->second == uri) { root()._observer_table.erase(it); break; } } ret = root()._observed_table.equal_range(uri); for (auto it = ret.first; it!=ret.second; ++it) { if (it->second == url) { root()._observed_table.erase(it); return true; } } return false; } }} bool Node::applieOp (VPackSlice const& slice) { std::string oper = slice.get("op").copyString(); if (oper == "delete") { return _parent->removeChild(_node_name); } else if (oper == "set") { // "op":"set" return handle(slice); } else if (oper == "increment") { // "op":"increment" return handle(slice); } else if (oper == "decrement") { // "op":"decrement" return handle(slice); } else if (oper == "push") { // "op":"push" return handle(slice); } else if (oper == "pop") { // "op":"pop" return handle(slice); } else if (oper == "prepend") { // "op":"prepend" return handle(slice); } else if (oper == "shift") { // "op":"shift" return handle(slice); } else if (oper == "observe") { // "op":"observe" return handle(slice); } else if (oper == "unobserve") { // "op":"unobserve" return handle(slice); } else { // "op" might not be a key word after all LOG_TOPIC(INFO, Logger::AGENCY) << "Keyword 'op' without known operation. Handling as regular key."; } return false; } // Apply slice to this node bool Node::applies (VPackSlice const& slice) { if (slice.isObject()) { // Object is an operation? if (slice.hasKey("op")) { if (applieOp(slice)) { return true; } } // Object is special case json for (auto const& i : VPackObjectIterator(slice)) { std::string key = i.key.copyString(); if (key.find('/')!=std::string::npos) { (*this)(key).applies(i.value); } else { auto found = _children.find(key); if (found == _children.end()) { _children[key] = std::make_shared(key, this); } _children[key]->applies(i.value); } } } else { *this = slice; } return true; } void Node::toBuilder (Builder& builder) const { try { if (type()==NODE) { VPackObjectBuilder guard(&builder); for (auto const& child : _children) { builder.add(VPackValue(child.first)); child.second->toBuilder(builder); } } else { if (!slice().isNone()) { builder.add(slice()); } } } catch (std::exception const& e) { LOG_TOPIC(ERR, Logger::AGENCY) << e.what(); } } // Print internals to ostream std::ostream& Node::print (std::ostream& o) const { Node const* par = _parent; while (par != 0) { par = par->_parent; o << " "; } o << _node_name << " : "; if (type() == NODE) { o << std::endl; for (auto const& i : _children) o << *(i.second); } else { o << ((slice().isNone()) ? "NONE" : slice().toJson()); if (_ttl != std::chrono::system_clock::time_point()) { o << " ttl! "; } o << std::endl; } if (!_time_table.empty()) { for (auto const& i : _time_table) { o << i.second.get() << std::endl; } } return o; } // Create with name Store::Store (std::string const& name) : Node(name), Thread(name) {} // Default ctor Store::~Store () {} // Apply queries multiple queries to store std::vector Store::apply (query_t const& query) { std::vector applied; MUTEX_LOCKER(storeLocker, _storeLock); for (auto const& i : VPackArrayIterator(query->slice())) { switch (i.length()) { case 1: applied.push_back(applies(i[0])); break; // no precond case 2: if (check(i[1])) { applied.push_back(applies(i[0])); // precondition } else { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Precondition failed!"; applied.push_back(false); } break; default: // wrong LOG_TOPIC(ERR, Logger::AGENCY) << "We can only handle log entry with or without precondition!"; applied.push_back(false); break; } } _cv.signal(); return applied; } //template std::multimap std::ostream& operator<< ( std::ostream& os, std::multimap const& m) { for (auto const& i : m) { os << i.first << ": " << i.second << std::endl; } return os; } // Apply external struct notify_t { std::string key; std::string modified; std::string oper; notify_t (std::string const& k, std::string const& m, std::string const& o) : key(k), modified(m), oper(o) {} }; std::vector Store::apply ( std::vector const& queries, bool inform) { std::vector applied; { MUTEX_LOCKER(storeLocker, _storeLock); for (auto const& i : queries) { applied.push_back(applies(i)); // no precond } } std::multimap> in; for (auto const& i : queries) { for (auto const& j : VPackObjectIterator(i)) { if (j.value.isObject() && j.value.hasKey("op")) { std::string oper = j.value.get("op").copyString(); if (!(oper == "observe" || oper == "unobserve")) { std::string uri = j.key.copyString(); while (true) { auto ret = _observed_table.equal_range(uri); for (auto it = ret.first; it!=ret.second; ++it) { in.emplace ( it->second, std::make_shared( it->first, j.key.copyString(), oper)); } size_t pos = uri.find_last_of('/'); if (pos == std::string::npos || pos == 0) { break; } else { uri = uri.substr(0,pos); } } } } } } std::vector urls; for (auto it = in.begin(), end = in.end(); it != end; it = in.upper_bound(it->first)) { urls.push_back(it->first); } for (auto const& url : urls) { Builder body; // host body.openObject(); body.add("term",VPackValue(0)); body.add("index",VPackValue(0)); auto ret = in.equal_range(url); for (auto it = ret.first; it!=ret.second; ++it) { body.add(it->second->key,VPackValue(VPackValueType::Object)); body.add(it->second->modified,VPackValue(VPackValueType::Object)); body.add("op",VPackValue(it->second->oper)); body.close(); body.close(); } body.close(); std::string endpoint, path; if (endpointPathFromUrl (url,endpoint,path)) { std::unique_ptr> headerFields = std::make_unique >(); ClusterCommResult res = arangodb::ClusterComm::instance()->asyncRequest( "1", 1, endpoint, GeneralRequest::RequestType::POST, path, std::make_shared(body.toString()), headerFields, std::make_shared(), 0.0, true); } else { LOG_TOPIC(WARN, Logger::AGENCY) << "Malformed URL " << url; } } return applied; } // Check precondition bool Store::check (VPackSlice const& slice) const { if (!slice.isObject()) { LOG_TOPIC(WARN, Logger::AGENCY) << "Cannot check precondition: " << slice.toJson(); return false; } for (auto const& precond : VPackObjectIterator(slice)) { std::string path = precond.key.copyString(); bool found = false; Node node ("precond"); try { node = (*this)(path); found = true; } catch (StoreException const&) {} if (precond.value.isObject()) { for (auto const& op : VPackObjectIterator(precond.value)) { std::string const& oper = op.key.copyString(); if (oper == "old") { // old return (node == op.value); } else if (oper == "isArray") { // isArray if (!op.value.isBoolean()) { LOG (FATAL) << "Non boolsh expression for 'isArray' precondition"; return false; } bool isArray = (node.type() == LEAF && node.slice().isArray()); return op.value.getBool() ? isArray : !isArray; } else if (oper == "oldEmpty") { // isEmpty if (!op.value.isBoolean()) { LOG (FATAL) << "Non boolsh expression for 'oldEmpty' precondition"; return false; } return op.value.getBool() ? !found : found; } } } else { return node == precond.value; } } return true; } // Read queries into result std::vector Store::read (query_t const& queries, query_t& result) const { std::vector success; MUTEX_LOCKER(storeLocker, _storeLock); if (queries->slice().isArray()) { result->add(VPackValue(VPackValueType::Array)); // top node array for (auto const& query : VPackArrayIterator(queries->slice())) { success.push_back(read (query, *result)); } result->close(); } else { LOG_TOPIC(ERR, Logger::AGENCY) << "Read queries to stores must be arrays"; } return success; } // read single query into ret bool Store::read (VPackSlice const& query, Builder& ret) const { bool success = true; // Collect all paths std::list query_strs; if (query.isArray()) { for (auto const& sub_query : VPackArrayIterator(query)) { query_strs.push_back(sub_query.copyString()); } } else { return false; } // Remove double ranges (inclusion / identity) query_strs.sort(); // sort paths for (auto i = query_strs.begin(), j = i; i != query_strs.end(); ++i) { if (i!=j && i->compare(0,j->size(),*j)==0) { *i=""; } else { j = i; } } auto cut = std::remove_if(query_strs.begin(), query_strs.end(), Empty()); query_strs.erase (cut,query_strs.end()); // Create response tree Node copy("copy"); for (auto const path : query_strs) { try { copy(path) = (*this)(path); } catch (StoreException const&) { std::vector pv = split(path,'/'); while (!pv.empty()) { std::string end = pv.back(); pv.pop_back(); copy(pv).removeChild(end); try { (*this)(pv); break; } catch(...) {} } if (copy(pv).type() == LEAF && copy(pv).slice().isNone()) { copy(pv) = arangodb::basics::VelocyPackHelper::EmptyObjectValue(); } } } // Into result builder copy.toBuilder(ret); return success; } // Shutdown void Store::beginShutdown() { Thread::beginShutdown(); CONDITION_LOCKER(guard, _cv); guard.broadcast(); } // TTL clear values from store query_t Store::clearExpired () const { query_t tmp = std::make_shared(); tmp->openArray(); { MUTEX_LOCKER(storeLocker, _storeLock); for (auto it = _time_table.cbegin(); it != _time_table.cend(); ++it) { if (it->first < std::chrono::system_clock::now()) { tmp->openArray(); tmp->openObject(); tmp->add(it->second->uri(), VPackValue(VPackValueType::Object)); tmp->add("op",VPackValue("delete")); tmp->close(); tmp->close(); tmp->close(); } else { break; } } } tmp->close(); return tmp; } // Dump internal data to builder void Store::dumpToBuilder (Builder& builder) const { MUTEX_LOCKER(storeLocker, _storeLock); toBuilder(builder); { VPackObjectBuilder guard(&builder); for (auto const& i : _time_table) { auto in_time_t = std::chrono::system_clock::to_time_t(i.first); std::string ts = ctime(&in_time_t); ts.resize(ts.size()-1); builder.add(ts, VPackValue((size_t)i.second.get())); } } { VPackArrayBuilder garray(&builder); for (auto const& i : _observer_table) { VPackObjectBuilder guard(&builder); builder.add(i.first, VPackValue(i.second)); } } { VPackArrayBuilder garray(&builder); for (auto const& i : _observed_table) { VPackObjectBuilder guard(&builder); builder.add(i.first, VPackValue(i.second)); } } } size_t Store::matchPath (std::vector const& pv) const { // Node* cur(this); /* for (size_t i = 0; i < pv.size(); ++i) { if (cur.find(pv.at(i))) { } }*/ return 0; } // Start thread bool Store::start () { Thread::start(); return true; } // Start thread with agent bool Store::start (Agent* agent) { _agent = agent; return start(); } // Work ttls and callbacks void Store::run() { CONDITION_LOCKER(guard, _cv); while (!this->isStopping()) { // Check timetable and remove overage entries std::chrono::microseconds t{0}; query_t toClear; { // any entries in time table? MUTEX_LOCKER(storeLocker, _storeLock); if (!_time_table.empty()) { t = std::chrono::duration_cast( _time_table.begin()->first - std::chrono::system_clock::now()); } } if (t != std::chrono::microseconds{0}) { _cv.wait(t.count()); } else { _cv.wait(); } toClear = clearExpired(); _agent->write(toClear); } }