From 3a1a9c898c40c76d22b7f17fec41e9bf5b073dee Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 5 Dec 2016 15:44:53 +0100 Subject: [PATCH 01/60] correct handling of distributeShardsLike in FailedFollower --- arangod/Agency/FailedFollower.cpp | 35 ++++++++++++++++++++++++------- arangod/Agency/FailedServer.cpp | 19 ++++++++++++----- arangod/Agency/Inception.cpp | 4 ++-- arangod/Agency/Job.cpp | 20 ++++++++++++++++++ arangod/Agency/Job.h | 4 ++++ arangod/Agency/Node.cpp | 12 +++++++++++ arangod/Agency/Node.h | 6 ++++-- 7 files changed, 83 insertions(+), 17 deletions(-) diff --git a/arangod/Agency/FailedFollower.cpp b/arangod/Agency/FailedFollower.cpp index 1ee296537e..4208d03c3c 100644 --- a/arangod/Agency/FailedFollower.cpp +++ b/arangod/Agency/FailedFollower.cpp @@ -66,7 +66,32 @@ bool FailedFollower::create() { << "Todo: failed Follower for " + _shard + " from " + _from + " to " + _to; std::string path = _agencyPrefix + toDoPrefix + _jobId; + std::string planPath = + planColPrefix + _database + "/" + _collection + "/shards"; + + auto const& myClones = clones(_snapshot, _database, _collection); + if (!myClones.empty()) { + size_t sub = 0; + auto myshards = _snapshot( + planColPrefix + _database + "/" + _collection + "/shards").children(); + auto mpos = std::distance(myshards.begin(), myshards.find(_shard)); + + // Deal with my clones + for (auto const& collection : myClones) { + auto othershards = + _snapshot(planColPrefix + _database + "/" + collection + "/shards") + .children(); + auto opos = othershards.begin(); + std::advance(opos, mpos); + auto const& shard = opos->first; + + FailedFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++), + _jobId, _agencyPrefix, _database, collection, shard, + _from, _to); + } + } + _jb = std::make_shared(); _jb->openArray(); _jb->openObject(); @@ -182,16 +207,10 @@ bool FailedFollower::start() { pending.close(); - // Precondition - // --- Check that Current servers are as we expect - + // Preconditions pending.openObject(); - /* pending.add(_agencyPrefix + curPath, VPackValue(VPackValueType::Object)); - pending.add("old", current.slice()); - pending.close(); - */ - // --- Check if shard is not blocked + // --- Check if shard is not blocked by other job pending.add(_agencyPrefix + blockedShardsPrefix + _shard, VPackValue(VPackValueType::Object)); pending.add("oldEmpty", VPackValue(true)); diff --git a/arangod/Agency/FailedServer.cpp b/arangod/Agency/FailedServer.cpp index d96f4ba697..6f750a5e2b 100644 --- a/arangod/Agency/FailedServer.cpp +++ b/arangod/Agency/FailedServer.cpp @@ -130,14 +130,23 @@ bool FailedServer::start() { auto cdatabase = current.at(database.first)->children(); for (auto const& collptr : database.second->children()) { - Node const& collection = *(collptr.second); + auto const& collection = *(collptr.second); if (!cdatabase.find(collptr.first)->second->children().empty()) { - Node const& collection = *(collptr.second); - Node const& replicationFactor = collection("replicationFactor"); + + auto const& collection = *(collptr.second); + auto const& replicationFactor = collection("replicationFactor"); + if (replicationFactor.slice().getUInt() > 1) { - auto available = availableServers(); + + bool isClone = false; + try { // Clone + if(!collection("distributeShardsLike").slice().copyString().empty()) { + isClone = true; + } + } catch (...) {} // Not clone + auto available = availableServers(); for (auto const& shard : collection("shards").children()) { @@ -167,7 +176,7 @@ bool FailedServer::start() { ++pos; } - if (found && available.size() > 0) { + if (found && !available.empty() > 0 && !isClone) { auto randIt = available.begin(); std::advance(randIt, std::rand() % available.size()); FailedFollower( diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index e72acecf9b..9fadccd0ef 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -368,7 +368,7 @@ bool Inception::estimateRAFTInterval() { auto config = _agent->config(); auto myid = _agent->id(); - double to = 0.25; + auto to = std::chrono::duration(1.0); // for (size_t i = 0; i < nrep; ++i) { for (auto const& peer : config.pool()) { @@ -383,7 +383,7 @@ bool Inception::estimateRAFTInterval() { 2.0, true); } } - std::this_thread::sleep_for(std::chrono::duration(to)); + std::this_thread::sleep_for(to); to *= 1.01; } diff --git a/arangod/Agency/Job.cpp b/arangod/Agency/Job.cpp index b13a4fe549..3270b45a91 100644 --- a/arangod/Agency/Job.cpp +++ b/arangod/Agency/Job.cpp @@ -177,3 +177,23 @@ std::vector Job::availableServers() const { return ret; } + +std::vector Job::clones(Node const& snapshot, + std::string const& database, + std::string const& collection) { + + std::vector ret; + std::string databasePath = planColPrefix + database; + try { + for (const auto& colptr : snapshot(databasePath).children()) { // databases + try { + auto const col = *colptr.second; + if (col("distributeShardsLike").slice().copyString() == collection) { + ret.push_back(colptr.first); + } + } catch(...) {} + } + } catch (...) {} + return ret; + +} diff --git a/arangod/Agency/Job.h b/arangod/Agency/Job.h index 087321f43f..5c9fef1ecf 100644 --- a/arangod/Agency/Job.h +++ b/arangod/Agency/Job.h @@ -111,6 +111,10 @@ struct Job { virtual std::vector availableServers() const; + static std::vector clones( + Node const& snapshot, std::string const& database, + std::string const& collection); + Node const _snapshot; Agent* _agent; std::string _jobId; diff --git a/arangod/Agency/Node.cpp b/arangod/Agency/Node.cpp index 8813949a66..b2f7363c7a 100644 --- a/arangod/Agency/Node.cpp +++ b/arangod/Agency/Node.cpp @@ -781,3 +781,15 @@ std::string Node::getString() const { } return slice().copyString(); } + +Slice Node::getArray() const { + if (type() == NODE) { + throw StoreException("Must not convert NODE type to array"); + } + if (!_isArray) { + throw StoreException("Not an array type"); + } + rebuildVecBuf(); + return Slice(_vecBuf.data()); +} + diff --git a/arangod/Agency/Node.h b/arangod/Agency/Node.h index e2df04f1ae..4cf67d5e31 100644 --- a/arangod/Agency/Node.h +++ b/arangod/Agency/Node.h @@ -217,6 +217,9 @@ class Node { /// @brief Get string value (throws if type NODE or if conversion fails) std::string getString() const; + /// @brief Get array value + Slice getArray() const; + protected: /// @brief Add time to live entry virtual bool addTimeToLive(long millis); @@ -231,8 +234,7 @@ class Node { Store* _store; ///< @brief Store Children _children; ///< @brief child nodes TimePoint _ttl; ///< @brief my expiry - // Buffer _value; ///< @brief my value - std::vector> _value; ///< @brief my value + std::vector> _value; ///< @brief my value mutable Buffer _vecBuf; mutable bool _vecBufDirty; bool _isArray; From 11bd9381d515ed04c21eb5e6ea64e07da435427d Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 6 Dec 2016 16:40:50 +0100 Subject: [PATCH 02/60] Add satellite collections --- .../Administration/Replication/README.mdpp | 8 + .../Synchronous/Satellites/README.mdpp | 32 ++++ .../Collections/DatabaseMethods.mdpp | 8 +- Documentation/Books/Manual/SUMMARY.md | 1 + arangod/Aql/ClusterNodes.h | 21 ++- arangod/Aql/Collection.h | 7 +- arangod/Aql/ExecutionEngine.cpp | 160 ++++++++++++------ arangod/Aql/ExecutionNode.cpp | 7 +- arangod/Aql/IndexNode.cpp | 1 + arangod/Aql/Optimizer.cpp | 6 + arangod/Aql/Optimizer.h | 5 + arangod/Aql/OptimizerRules.h | 3 + arangod/Aql/RestAqlHandler.cpp | 1 - arangod/Cluster/ClusterInfo.cpp | 10 +- arangod/Cluster/ClusterInfo.h | 1 + arangod/Cluster/ClusterMethods.cpp | 5 + .../RestHandler/RestReplicationHandler.cpp | 2 +- arangod/Utils/AqlTransaction.cpp | 2 +- arangod/V8Server/v8-collection.cpp | 14 +- arangod/V8Server/v8-vocindex.cpp | 2 +- arangod/VocBase/LogicalCollection.cpp | 79 +++++---- arangod/VocBase/LogicalCollection.h | 3 +- js/client/modules/@arangodb/testing.js | 2 + js/common/modules/@arangodb/aql/explainer.js | 4 +- js/server/modules/@arangodb/cluster.js | 1 + 25 files changed, 292 insertions(+), 93 deletions(-) create mode 100644 Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp diff --git a/Documentation/Books/Manual/Administration/Replication/README.mdpp b/Documentation/Books/Manual/Administration/Replication/README.mdpp index ffc46da746..e9581a0b18 100644 --- a/Documentation/Books/Manual/Administration/Replication/README.mdpp +++ b/Documentation/Books/Manual/Administration/Replication/README.mdpp @@ -13,6 +13,14 @@ Synchronous replication only works in in a cluster and is typically used for mis Synchronous replication is organized in a way that every shard has a leader and r-1 followers. The number of followers can be controlled using the `replicationFactor` whenever you create a collection, the `replicationFactor` is the total number of copies being kept, that is, it is one plus the number of followers. +### Satellite collections + +Satellite collections are synchronously replicated collections having a dynamic replicationFactor. +They will replicate all data to all database servers allowing the database servers to join data +locally instead of doing heavy network operations. + +Satellite collections are an enterprise only feature. + ### Asynchronous replication In ArangoDB any write operation will be logged to the write-ahead log. When using Asynchronous replication slaves will connect to a master and apply all the events from the log in the same order locally. After that, they will have the same state of data as the master database. diff --git a/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp b/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp new file mode 100644 index 0000000000..2f44734675 --- /dev/null +++ b/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp @@ -0,0 +1,32 @@ +Satellite Collections +===================== + +Satellite Collections are an *Enterprise* only feature. When doing Joins in an +ArangoDB cluster data has to exchanged between different servers. + +Joins will be executed on a coordinator. It will prepare an execution plan +and execute it. When executing the coordinator will contact all shards of the +starting point of the join and ask for their data. The database servers carrying +out this operation will load all their local data and then ask the cluster for +the other part of the join. This again will be distributed to all involved shards +of this join part. + +In sum this results in much network traffic and slow results depending of the +amount of data that has to be sent throughout the cluster. + +Satellite collections are collections that are intended to address this issue. + +They will facilitate the synchronous replication and replicate all its data +to all database servers that are part of the cluster. + +This enables the database servers to execute that part of any Join locally. + +This greatly improves performance for such joins at the costs of increased +storage requirements and poorer write performance on this data. + +To create a satellite collection set the *replicationFactor* of this collection +to "satellite". + +Using arangosh: + + arangosh> db._create("satellite", {"replicationFactor": "satellite"}); diff --git a/Documentation/Books/Manual/DataModeling/Collections/DatabaseMethods.mdpp b/Documentation/Books/Manual/DataModeling/Collections/DatabaseMethods.mdpp index 6ffc593a85..9151c3712e 100644 --- a/Documentation/Books/Manual/DataModeling/Collections/DatabaseMethods.mdpp +++ b/Documentation/Books/Manual/DataModeling/Collections/DatabaseMethods.mdpp @@ -138,7 +138,13 @@ to the [naming conventions](../NamingConventions/README.md). If a server fails, this is detected automatically and one of the servers holding copies take over, usually without an error being - reported. + reported. + + When using the *Enterprise* version of ArangoDB the replicationFactor + may be set to "satellite" making the collection locally joinable + on every database server. This reduces the number of network hops + dramatically when using joins in AQL at the costs of reduced write + performance on these collections. `db._create(collection-name, properties, type)` diff --git a/Documentation/Books/Manual/SUMMARY.md b/Documentation/Books/Manual/SUMMARY.md index b75c815779..0557ba0fff 100644 --- a/Documentation/Books/Manual/SUMMARY.md +++ b/Documentation/Books/Manual/SUMMARY.md @@ -162,6 +162,7 @@ * [Synchronous Replication](Administration/Replication/Synchronous/README.md) * [Implementation](Administration/Replication/Synchronous/Implementation.md) * [Configuration](Administration/Replication/Synchronous/Configuration.md) + * [Satellite Collections](Administration/Replication/Synchronous/Satellites/README.md) * [Sharding](Administration/Sharding/README.md) # * [Authentication](Administration/Sharding/Authentication.md) # * [Firewall setup](Administration/Sharding/FirewallSetup.md) diff --git a/arangod/Aql/ClusterNodes.h b/arangod/Aql/ClusterNodes.h index c2cbd47172..8c12e96104 100644 --- a/arangod/Aql/ClusterNodes.h +++ b/arangod/Aql/ClusterNodes.h @@ -183,6 +183,9 @@ class ScatterNode : public ExecutionNode { /// @brief return the collection Collection const* collection() const { return _collection; } + /// @brief set collection + void setCollection(Collection const* collection) { _collection = collection; } + private: /// @brief the underlying database TRI_vocbase_t* _vocbase; @@ -302,7 +305,8 @@ class GatherNode : public ExecutionNode { public: GatherNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, Collection const* collection) - : ExecutionNode(plan, id), _vocbase(vocbase), _collection(collection) {} + : ExecutionNode(plan, id), _vocbase(vocbase), _collection(collection), + _auxiliaryCollections() {} GatherNode(ExecutionPlan*, arangodb::velocypack::Slice const& base, SortElementVector const& elements); @@ -357,6 +361,18 @@ class GatherNode : public ExecutionNode { /// @brief return the collection Collection const* collection() const { return _collection; } + void setCollection(Collection const* collection) { _collection = collection; } + + std::unordered_set auxiliaryCollections() const { + return _auxiliaryCollections; + } + + void addAuxiliaryCollection(Collection const* auxiliaryCollection) { + _auxiliaryCollections.emplace(auxiliaryCollection); + } + + bool hasAuxiliaryCollections() const { return !_auxiliaryCollections.empty(); } + private: /// @brief pairs, consisting of variable and sort direction /// (true = ascending | false = descending) @@ -367,6 +383,9 @@ class GatherNode : public ExecutionNode { /// @brief the underlying collection Collection const* _collection; + + /// @brief (optional) auxiliary collections (satellites) + std::unordered_set _auxiliaryCollections; }; } // namespace arangodb::aql diff --git a/arangod/Aql/Collection.h b/arangod/Aql/Collection.h index ab06a70304..88d88b5ca6 100644 --- a/arangod/Aql/Collection.h +++ b/arangod/Aql/Collection.h @@ -86,12 +86,17 @@ struct Collection { /// @brief either use the set collection or get one from ClusterInfo: std::shared_ptr getCollection() const; - + /// @brief check smartness of the underlying collection bool isSmart() const { return getCollection()->isSmart(); } + /// @brief check if collection is a satellite collection + bool isSatellite() const { + return getCollection()->isSatellite(); + } + private: arangodb::LogicalCollection* collection; diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index e84fe3840c..5a8c22f378 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -22,6 +22,7 @@ //////////////////////////////////////////////////////////////////////////////// #include "ExecutionEngine.h" + #include "Aql/BasicBlocks.h" #include "Aql/CalculationBlock.h" #include "Aql/ClusterBlocks.h" @@ -343,38 +344,78 @@ struct CoordinatorInstanciator : public WalkerWorker { id(id), nodes(), part(p), - idOfRemoteNode(idOfRemoteNode) {} - - Collection* getCollection() const { - Collection* collection = nullptr; + idOfRemoteNode(idOfRemoteNode), + collection(nullptr), + auxiliaryCollections(), + populated(false) { + } + void populate() { + // mop: compiler should inline that I suppose :S + auto collectionFn = [&](Collection* col) -> void { + if (col->isSatellite()) { + auxiliaryCollections.emplace(col); + } else { + collection = col; + } + }; + Collection* localCollection = nullptr; for (auto en = nodes.rbegin(); en != nodes.rend(); ++en) { // find the collection to be used if ((*en)->getType() == ExecutionNode::ENUMERATE_COLLECTION) { - collection = const_cast( + localCollection = const_cast( static_cast((*en))->collection()); + collectionFn(localCollection); } else if ((*en)->getType() == ExecutionNode::INDEX) { - collection = const_cast( + localCollection = const_cast( static_cast((*en))->collection()); + collectionFn(localCollection); } else if ((*en)->getType() == ExecutionNode::INSERT || (*en)->getType() == ExecutionNode::UPDATE || (*en)->getType() == ExecutionNode::REPLACE || (*en)->getType() == ExecutionNode::REMOVE || (*en)->getType() == ExecutionNode::UPSERT) { - collection = const_cast( + localCollection = const_cast( static_cast((*en))->collection()); + collectionFn(localCollection); } } + // mop: no non satellite collection found + if (collection == nullptr) { + // mop: just take the last satellite then + collection = localCollection; + } + // mop: ok we are actually only working with a satellite... + // so remove its shardId from the auxiliaryShards again + if (collection != nullptr && collection->isSatellite()) { + auxiliaryCollections.erase(collection); + } + populated = true; + } + Collection* getCollection() { + if (!populated) { + populate(); + } TRI_ASSERT(collection != nullptr); return collection; } + std::unordered_set getAuxiliaryCollections() { + if (!populated) { + populate(); + } + return auxiliaryCollections; + } + EngineLocation const location; size_t const id; std::vector nodes; arangodb::aql::QueryPart part; // only relevant for DBserver parts size_t idOfRemoteNode; // id of the remote node + Collection* collection; + std::unordered_set auxiliaryCollections; + bool populated; // in the original plan that needs this engine }; @@ -392,6 +433,8 @@ struct CoordinatorInstanciator : public WalkerWorker { // query or a dependent one. std::unordered_map queryIds; + + std::unordered_set auxiliaryCollections; // this map allows to find the queries which are the parts of the big // query. There are two cases, the first is for the remote queries on // the DBservers, for these, the key is: @@ -435,7 +478,7 @@ struct CoordinatorInstanciator : public WalkerWorker { /// @brief generatePlanForOneShard void generatePlanForOneShard(VPackBuilder& builder, size_t nr, - EngineInfo const& info, QueryId& connectedId, + EngineInfo* info, QueryId& connectedId, std::string const& shardId, bool verbose) { // copy the relevant fragment of the plan for each shard // Note that in these parts of the query there are no SubqueryNodes, @@ -443,7 +486,7 @@ struct CoordinatorInstanciator : public WalkerWorker { ExecutionPlan plan(query->ast()); ExecutionNode* previous = nullptr; - for (ExecutionNode const* current : info.nodes) { + for (ExecutionNode const* current : info->nodes) { auto clone = current->clone(&plan, false, false); // UNNECESSARY, because clone does it: plan.registerNode(clone); @@ -473,34 +516,42 @@ struct CoordinatorInstanciator : public WalkerWorker { /// @brief distributePlanToShard, send a single plan to one shard void distributePlanToShard(arangodb::CoordTransactionID& coordTransactionID, - EngineInfo const& info, Collection* collection, + EngineInfo* info, QueryId& connectedId, std::string const& shardId, VPackSlice const& planSlice) { - // inject the current shard id into the collection - collection->setCurrentShard(shardId); - + Collection* collection = info->getCollection(); // create a JSON representation of the plan VPackBuilder result; result.openObject(); result.add("plan", VPackValue(VPackValueType::Object)); - + VPackBuilder tmp; query->ast()->variables()->toVelocyPack(tmp); result.add("variables", tmp.slice()); result.add("collections", VPackValue(VPackValueType::Array)); - // add the collection result.openObject(); - result.add("name", VPackValue(collection->getName())); + result.add("name", VPackValue(shardId)); result.add("type", VPackValue(TRI_TransactionTypeGetStr(collection->accessType))); result.close(); + + // mop: this is currently only working for satellites and hardcoded to their structure + for (auto auxiliaryCollection: info->getAuxiliaryCollections()) { + TRI_ASSERT(auxiliaryCollection->isSatellite()); + + // add the collection + result.openObject(); + result.add("name", VPackValue((*auxiliaryCollection->shardIds())[0])); + result.add("type", VPackValue(TRI_TransactionTypeGetStr(collection->accessType))); + result.close(); + } result.close(); // collections - + result.add(VPackObjectIterator(planSlice)); result.close(); // plan - if (info.part == arangodb::aql::PART_MAIN) { + if (info->part == arangodb::aql::PART_MAIN) { result.add("part", VPackValue("main")); } else { result.add("part", VPackValue("dependent")); @@ -522,13 +573,13 @@ struct CoordinatorInstanciator : public WalkerWorker { auto body = std::make_shared(result.slice().toJson()); - // std::cout << "GENERATED A PLAN FOR THE REMOTE SERVERS: " << *(body.get()) + //LOG(ERR) << "GENERATED A PLAN FOR THE REMOTE SERVERS: " << *(body.get()); // << "\n"; auto cc = arangodb::ClusterComm::instance(); - std::string const url("/_db/" + arangodb::basics::StringUtils::urlEncode( - collection->vocbase->name()) + + std::string const url("/_db/" + + arangodb::basics::StringUtils::urlEncode(collection->vocbase->name()) + "/_api/aql/instantiate"); auto headers = std::make_unique>(); @@ -539,7 +590,7 @@ struct CoordinatorInstanciator : public WalkerWorker { } /// @brief aggregateQueryIds, get answers for all shards in a Scatter/Gather - void aggregateQueryIds(EngineInfo const& info, arangodb::ClusterComm*& cc, + void aggregateQueryIds(EngineInfo* info, arangodb::ClusterComm*& cc, arangodb::CoordTransactionID& coordTransactionID, Collection* collection) { // pick up the remote query ids @@ -569,13 +620,13 @@ struct CoordinatorInstanciator : public WalkerWorker { // res.answer->body() << ", REMOTENODEID: " << info.idOfRemoteNode << // " SHARDID:" << res.shardID << ", QUERYID: " << queryId << "\n"; std::string theID = - arangodb::basics::StringUtils::itoa(info.idOfRemoteNode) + ":" + + arangodb::basics::StringUtils::itoa(info->idOfRemoteNode) + ":" + res.shardID; - if (info.part == arangodb::aql::PART_MAIN) { - queryIds.emplace(theID, queryId + "*"); - } else { - queryIds.emplace(theID, queryId); + + if (info->part == arangodb::aql::PART_MAIN) { + queryId += "*"; } + queryIds.emplace(theID, queryId); } else { error += "DB SERVER ANSWERED WITH ERROR: "; error += res.answer->payload().toJson(); @@ -589,7 +640,7 @@ struct CoordinatorInstanciator : public WalkerWorker { } } - // std::cout << "GOT ALL RESPONSES FROM DB SERVERS: " << nrok << "\n"; + //LOG(ERR) << "GOT ALL RESPONSES FROM DB SERVERS: " << nrok << "\n"; if (nrok != (int)shardIds->size()) { if (errorCode == TRI_ERROR_NO_ERROR) { @@ -600,9 +651,16 @@ struct CoordinatorInstanciator : public WalkerWorker { } /// @brief distributePlansToShards, for a single Scatter/Gather block - void distributePlansToShards(EngineInfo const& info, QueryId connectedId) { - // std::cout << "distributePlansToShards: " << info.id << std::endl; - Collection* collection = info.getCollection(); + void distributePlansToShards(EngineInfo* info, QueryId connectedId) { + //LOG(ERR) << "distributePlansToShards: " << info.id; + Collection* collection = info->getCollection(); + + auto auxiliaryCollections = info->getAuxiliaryCollections(); + for (auto const& auxiliaryCollection: auxiliaryCollections) { + TRI_ASSERT(auxiliaryCollection->shardIds()->size() == 1); + auxiliaryCollection->setCurrentShard((*auxiliaryCollection->shardIds())[0]); + } + // now send the plan to the remote servers arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer(); auto cc = arangodb::ClusterComm::instance(); @@ -613,23 +671,27 @@ struct CoordinatorInstanciator : public WalkerWorker { auto shardIds = collection->shardIds(); for (auto const& shardId : *shardIds) { // inject the current shard id into the collection - collection->setCurrentShard(shardId); VPackBuilder b; + collection->setCurrentShard(shardId); generatePlanForOneShard(b, nr++, info, connectedId, shardId, true); - distributePlanToShard(coordTransactionID, info, collection, connectedId, - shardId, b.slice()); + distributePlanToShard(coordTransactionID, info, + connectedId, shardId, + b.slice()); + } + collection->resetCurrentShard(); + for (auto const& auxiliaryCollection: auxiliaryCollections) { + TRI_ASSERT(auxiliaryCollection->shardIds()->size() == 1); + auxiliaryCollection->resetCurrentShard(); } - // fix collection - collection->resetCurrentShard(); aggregateQueryIds(info, cc, coordTransactionID, collection); } /// @brief buildEngineCoordinator, for a single piece - ExecutionEngine* buildEngineCoordinator(EngineInfo& info) { + ExecutionEngine* buildEngineCoordinator(EngineInfo* info) { Query* localQuery = query; - bool needToClone = info.id > 0; // use the original for the main part + bool needToClone = info->id > 0; // use the original for the main part if (needToClone) { // need a new query instance on the coordinator localQuery = query->clone(PART_DEPENDENT, false); @@ -646,7 +708,7 @@ struct CoordinatorInstanciator : public WalkerWorker { std::unordered_map cache; RemoteNode* remoteNode = nullptr; - for (auto en = info.nodes.begin(); en != info.nodes.end(); ++en) { + for (auto en = info->nodes.begin(); en != info->nodes.end(); ++en) { auto const nodeType = (*en)->getType(); if (nodeType == ExecutionNode::REMOTE) { @@ -688,15 +750,15 @@ struct CoordinatorInstanciator : public WalkerWorker { // now we'll create a remote node for each shard and add it to the // gather node - Collection const* collection = - static_cast((*en))->collection(); + auto gatherNode = static_cast(*en); + Collection const* collection = gatherNode->collection(); auto shardIds = collection->shardIds(); - for (auto const& shardId : *shardIds) { std::string theId = arangodb::basics::StringUtils::itoa(remoteNode->id()) + ":" + shardId; + auto it = queryIds.find(theId); if (it == queryIds.end()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, @@ -962,11 +1024,12 @@ struct CoordinatorInstanciator : public WalkerWorker { QueryId id = 0; for (auto it = engines.rbegin(); it != engines.rend(); ++it) { - // std::cout << "Doing engine: " << it->id << " location:" - // << it->location << std::endl; - if ((*it).location == COORDINATOR) { + EngineInfo* info = &(*it); + //LOG(ERR) << "Doing engine: " << it->id << " location:" + // << it->location; + if (info->location == COORDINATOR) { // create a coordinator-based engine - engine = buildEngineCoordinator(*it); + engine = buildEngineCoordinator(info); TRI_ASSERT(engine != nullptr); if ((*it).id > 0) { @@ -997,12 +1060,11 @@ struct CoordinatorInstanciator : public WalkerWorker { } else { // create an engine on a remote DB server // hand in the previous engine's id - distributePlansToShards((*it), id); + distributePlansToShards(info, id); } } TRI_ASSERT(engine != nullptr); - // return the last created coordinator-based engine // this is the local engine that we'll use to run the query return engine; @@ -1104,6 +1166,7 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan( for (auto& q : inst.get()->queryIds) { std::string theId = q.first; std::string queryId = q.second; + // std::cout << "queryIds: " << theId << " : " << queryId << // std::endl; auto pos = theId.find(':'); @@ -1156,6 +1219,7 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan( std::string const& shardId = p.first; std::string const& queryId = p.second.first; bool isTraverserEngine = p.second.second; + // Lock shard on DBserver: arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer(); auto cc = arangodb::ClusterComm::instance(); diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 6d0e2340ea..72ead910ce 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -1153,7 +1153,11 @@ EnumerateCollectionNode::EnumerateCollectionNode( _collection(plan->getAst()->query()->collections()->get( base.get("collection").copyString())), _outVariable(varFromVPack(plan->getAst(), base, "outVariable")), - _random(base.get("random").getBoolean()) {} + _random(base.get("random").getBoolean()) { + TRI_ASSERT(_vocbase != nullptr); + TRI_ASSERT(_collection != nullptr); + TRI_ASSERT(_outVariable != nullptr); +} /// @brief toVelocyPack, for EnumerateCollectionNode void EnumerateCollectionNode::toVelocyPackHelper(VPackBuilder& nodes, @@ -1167,6 +1171,7 @@ void EnumerateCollectionNode::toVelocyPackHelper(VPackBuilder& nodes, nodes.add(VPackValue("outVariable")); _outVariable->toVelocyPack(nodes); nodes.add("random", VPackValue(_random)); + nodes.add("satellite", VPackValue(_collection->isSatellite())); // And close it: nodes.close(); diff --git a/arangod/Aql/IndexNode.cpp b/arangod/Aql/IndexNode.cpp index 0770acfcc3..365550aa62 100644 --- a/arangod/Aql/IndexNode.cpp +++ b/arangod/Aql/IndexNode.cpp @@ -65,6 +65,7 @@ void IndexNode::toVelocyPackHelper(VPackBuilder& nodes, bool verbose) const { // Now put info about vocbase and cid in there nodes.add("database", VPackValue(_vocbase->name())); nodes.add("collection", VPackValue(_collection->getName())); + nodes.add("satellite", VPackValue(_collection->isSatellite())); nodes.add(VPackValue("outVariable")); _outVariable->toVelocyPack(nodes); diff --git a/arangod/Aql/Optimizer.cpp b/arangod/Aql/Optimizer.cpp index c61731fc01..428f98a498 100644 --- a/arangod/Aql/Optimizer.cpp +++ b/arangod/Aql/Optimizer.cpp @@ -512,5 +512,11 @@ void Optimizer::setupRules() { registerRule("undistribute-remove-after-enum-coll", undistributeRemoveAfterEnumCollRule, undistributeRemoveAfterEnumCollRule_pass10, DoesNotCreateAdditionalPlans, true); + +#ifdef USE_ENTERPRISE + registerRule("remove-satellite-joins", + removeSatelliteJoinsRule, + removeSatelliteJoinsRule_pass10, DoesNotCreateAdditionalPlans, true); +#endif } } diff --git a/arangod/Aql/Optimizer.h b/arangod/Aql/Optimizer.h index b47755135b..9a98ce0f1b 100644 --- a/arangod/Aql/Optimizer.h +++ b/arangod/Aql/Optimizer.h @@ -191,6 +191,11 @@ class Optimizer { // only a SingletonNode and possibly some CalculationNodes as dependencies removeUnnecessaryRemoteScatterRule_pass10 = 1040, + // remove any superflous satellite collection joins... + // put it after Scatter rule because we would do + // the work twice otherwise + removeSatelliteJoinsRule_pass10 = 1045, + // recognize that a RemoveNode can be moved to the shards undistributeRemoveAfterEnumCollRule_pass10 = 1050, diff --git a/arangod/Aql/OptimizerRules.h b/arangod/Aql/OptimizerRules.h index e0659d0547..cedcab94fa 100644 --- a/arangod/Aql/OptimizerRules.h +++ b/arangod/Aql/OptimizerRules.h @@ -125,6 +125,9 @@ void distributeInClusterRule(Optimizer*, ExecutionPlan*, #ifdef USE_ENTERPRISE void distributeInClusterRuleSmartEdgeCollection(Optimizer*, ExecutionPlan*, Optimizer::Rule const*); + +/// @brief remove scatter/gather and remote nodes for satellite collections +void removeSatelliteJoinsRule(Optimizer*, ExecutionPlan*, Optimizer::Rule const*); #endif void distributeFilternCalcToClusterRule(Optimizer*, ExecutionPlan*, diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 3388dde21a..f40c2b13b5 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -564,7 +564,6 @@ RestStatus RestAqlHandler::execute() { // GeneralRequest::translateMethod(_request->requestType()) << ", // " << arangodb::ServerState::instance()->getId() << ": " << // _request->fullUrl() << ": " << _request->body() << "\n\n"; - std::vector const& suffixes = _request->suffixes(); // extract the sub-request type diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index 55db3dfff8..93043febdb 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -1029,6 +1029,7 @@ int ClusterInfo::dropDatabaseCoordinator(std::string const& name, int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, std::string const& collectionID, uint64_t numberOfShards, + uint64_t replicationFactor, VPackSlice const& json, std::string& errorMsg, double timeout) { @@ -1074,6 +1075,7 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, std::shared_ptr dbServerResult = std::make_shared(-1); std::shared_ptr errMsg = std::make_shared(); + auto dbServers = getCurrentDBServers(); std::function dbServerChanged = [=](VPackSlice const& result) { if (result.isObject() && result.length() == (size_t)numberOfShards) { @@ -1081,6 +1083,13 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, bool tmpHaveError = false; for (auto const& p : VPackObjectIterator(result)) { + if (replicationFactor == 0) { + VPackSlice servers = p.value.get("servers"); + if (!servers.isArray() || servers.length() < dbServers.size()) { + return true; + } + } + if (arangodb::basics::VelocyPackHelper::getBooleanValue( p.value, "error", false)) { tmpHaveError = true; @@ -1149,7 +1158,6 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName, // Update our cache: loadPlan(); - if (numberOfShards == 0) { loadCurrent(); events::CreateCollection(name, TRI_ERROR_NO_ERROR); diff --git a/arangod/Cluster/ClusterInfo.h b/arangod/Cluster/ClusterInfo.h index 1babe2ac9d..20af9e8344 100644 --- a/arangod/Cluster/ClusterInfo.h +++ b/arangod/Cluster/ClusterInfo.h @@ -349,6 +349,7 @@ class ClusterInfo { int createCollectionCoordinator(std::string const& databaseName, std::string const& collectionID, uint64_t numberOfShards, + uint64_t replicationFactor, arangodb::velocypack::Slice const& json, std::string& errorMsg, double timeout); diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 91b888bcf3..732192e420 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -2059,6 +2059,11 @@ std::unordered_map> distributeShards( random_shuffle(dbServers.begin(), dbServers.end()); } + // mop: distribute satellite collections on all servers + if (replicationFactor == 0) { + replicationFactor = dbServers.size(); + } + // fetch a unique id for each shard to create uint64_t const id = ci->uniqid(numberOfShards); diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index cf9f345633..4b0127f29c 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -1761,7 +1761,7 @@ int RestReplicationHandler::processRestoreCollectionCoordinator( VPackCollection::merge(parameters, sliceToMerge, false); VPackSlice const merged = mergedBuilder.slice(); - int res = ci->createCollectionCoordinator(dbName, newId, numberOfShards, + int res = ci->createCollectionCoordinator(dbName, newId, numberOfShards, replicationFactor, merged, errorMsg, 0.0); if (res != TRI_ERROR_NO_ERROR) { errorMsg = diff --git a/arangod/Utils/AqlTransaction.cpp b/arangod/Utils/AqlTransaction.cpp index 8ce0bf5cb1..50171ac820 100644 --- a/arangod/Utils/AqlTransaction.cpp +++ b/arangod/Utils/AqlTransaction.cpp @@ -23,6 +23,7 @@ #include "AqlTransaction.h" #include "CollectionNameResolver.h" +#include "Logger/Logger.h" #include "VocBase/LogicalCollection.h" using namespace arangodb; @@ -89,7 +90,6 @@ LogicalCollection* AqlTransaction::documentCollection(TRI_voc_cid_t cid) { int AqlTransaction::lockCollections() { auto trx = getInternals(); - for (auto& trxCollection : trx->_collections) { int res = TRI_LockCollectionTransaction(trxCollection, trxCollection->_accessType, 0); diff --git a/arangod/V8Server/v8-collection.cpp b/arangod/V8Server/v8-collection.cpp index b7eff74af4..7cccd9d7dd 100644 --- a/arangod/V8Server/v8-collection.cpp +++ b/arangod/V8Server/v8-collection.cpp @@ -1176,7 +1176,7 @@ static void JS_PropertiesVocbaseCol( TRI_V8_THROW_EXCEPTION_PARAMETER( "indexBuckets must be a two-power between 1 and 1024"); } - + int res = info->update(slice, false); if (res != TRI_ERROR_NO_ERROR) { @@ -1217,9 +1217,15 @@ static void JS_PropertiesVocbaseCol( TRI_GET_GLOBAL_STRING(KeyOptionsKey); result->Set(KeyOptionsKey, TRI_VPackToV8(isolate, keyOpts)->ToObject()); } - result->Set( - TRI_V8_ASCII_STRING("replicationFactor"), - v8::Number::New(isolate, static_cast(c->replicationFactor()))); + if (c->isSatellite()) { + result->Set( + TRI_V8_ASCII_STRING("replicationFactor"), + TRI_V8_STD_STRING(std::string("satellite"))); + } else { + result->Set( + TRI_V8_ASCII_STRING("replicationFactor"), + v8::Number::New(isolate, static_cast(c->replicationFactor()))); + } std::string shardsLike = c->distributeShardsLike(); if (!shardsLike.empty()) { CollectionNameResolver resolver(c->vocbase()); diff --git a/arangod/V8Server/v8-vocindex.cpp b/arangod/V8Server/v8-vocindex.cpp index c3ee93c32f..f81bf9eeb4 100644 --- a/arangod/V8Server/v8-vocindex.cpp +++ b/arangod/V8Server/v8-vocindex.cpp @@ -705,7 +705,7 @@ std::unique_ptr CreateCollectionCoordinator(LogicalCollection std::string errorMsg; int myerrno = ci->createCollectionCoordinator( parameters->dbName(), parameters->cid_as_string(), - parameters->numberOfShards(), velocy.slice(), errorMsg, 240.0); + parameters->numberOfShards(), parameters->replicationFactor(), velocy.slice(), errorMsg, 240.0); if (myerrno != TRI_ERROR_NO_ERROR) { if (errorMsg.empty()) { diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index f114a6887c..d5fba18143 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -381,7 +381,7 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, _version(ReadNumericValue(info, "version", currentVersion())), _indexBuckets(ReadNumericValue( info, "indexBuckets", DatabaseFeature::defaultIndexBuckets())), - _replicationFactor(ReadNumericValue(info, "replicationFactor", 1)), + _replicationFactor(1), _numberOfShards(ReadNumericValue(info, "numberOfShards", 1)), _allowUserKeys(ReadBooleanValue(info, "allowUserKeys", true)), _shardIds(new ShardMap()), @@ -412,7 +412,8 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, "with the --database.auto-upgrade option."); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_FAILED, errorMsg); - } + } + if (_isVolatile && _waitForSync) { // Illegal collection configuration @@ -429,8 +430,49 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, VPackSlice shardKeysSlice = info.get("shardKeys"); bool const isCluster = ServerState::instance()->isRunningInCluster(); + // Cluster only tests + if (ServerState::instance()->isCoordinator()) { + if ( (_numberOfShards == 0 && !_isSmart) || _numberOfShards > 1000) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "invalid number of shards"); + } - if (shardKeysSlice.isNone()) { + VPackSlice keyGenSlice = info.get("keyOptions"); + if (keyGenSlice.isObject()) { + keyGenSlice = keyGenSlice.get("type"); + if (keyGenSlice.isString()) { + StringRef tmp(keyGenSlice); + if (!tmp.empty() && tmp != "traditional") { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_UNSUPPORTED, + "non-traditional key generators are " + "not supported for sharded " + "collections"); + } + } + } + auto replicationFactorSlice = info.get("replicationFactor"); + if (!replicationFactorSlice.isNone()) { + bool isError = true; + if (replicationFactorSlice.isString() && replicationFactorSlice.copyString() == "satellite") { + _replicationFactor = 0; + _numberOfShards = 1; + _distributeShardsLike = ""; + isError = false; + } else if (replicationFactorSlice.isNumber()) { + _replicationFactor = replicationFactorSlice.getNumber(); + // mop: only allow satellite collections to be created explicitly + if (_replicationFactor > 0 || _replicationFactor <= 10) { + isError = false; + } + } + if (isError) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "invalid replicationFactor"); + } + } + } + + if (shardKeysSlice.isNone() || isSatellite()) { // Use default. _shardKeys.emplace_back(StaticStrings::KeyString); } else { @@ -470,32 +512,6 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, } - // Cluster only tests - if (ServerState::instance()->isCoordinator()) { - if ( (_numberOfShards == 0 && !_isSmart) || _numberOfShards > 1000) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "invalid number of shards"); - } - - VPackSlice keyGenSlice = info.get("keyOptions"); - if (keyGenSlice.isObject()) { - keyGenSlice = keyGenSlice.get("type"); - if (keyGenSlice.isString()) { - StringRef tmp(keyGenSlice); - if (!tmp.empty() && tmp != "traditional") { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_UNSUPPORTED, - "non-traditional key generators are " - "not supported for sharded " - "collections"); - } - } - } - - if (_replicationFactor == 0 || _replicationFactor > 10) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "invalid replicationFactor"); - } - } _keyGenerator.reset(KeyGenerator::factory(info.get("keyOptions"))); @@ -3579,3 +3595,8 @@ bool LogicalCollection::skipForAqlWrite(arangodb::velocypack::Slice document, return false; } #endif + +bool LogicalCollection::isSatellite() const { + return _replicationFactor == 0; +} + diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index acaa9a398e..95a880b735 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -240,6 +240,7 @@ class LogicalCollection { // SECTION: Replication int replicationFactor() const; + bool isSatellite() const; // SECTION: Sharding @@ -540,7 +541,7 @@ class LogicalCollection { std::vector> _indexes; // SECTION: Replication - size_t const _replicationFactor; + size_t _replicationFactor; // SECTION: Sharding size_t _numberOfShards; diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index d412af3634..17c77e7b7f 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -1466,6 +1466,8 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir) instanceArgs['server.endpoint'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.my-address'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.supervision-grace-period'] = '5'; + instanceArgs['agency.election-timeout-min'] = '0.5'; + instanceArgs['agency.election-timeout-max'] = '4.0'; if (i === N - 1) { diff --git a/js/common/modules/@arangodb/aql/explainer.js b/js/common/modules/@arangodb/aql/explainer.js index 666f37dfcb..5e6c46823a 100644 --- a/js/common/modules/@arangodb/aql/explainer.js +++ b/js/common/modules/@arangodb/aql/explainer.js @@ -825,7 +825,7 @@ function processQuery (query, explain) { return keyword('EMPTY') + ' ' + annotation('/* empty result set */'); case 'EnumerateCollectionNode': collectionVariables[node.outVariable.id] = node.collection; - return keyword('FOR') + ' ' + variableName(node.outVariable) + ' ' + keyword('IN') + ' ' + collection(node.collection) + ' ' + annotation('/* full collection scan' + (node.random ? ', random order' : '') + ' */'); + return keyword('FOR') + ' ' + variableName(node.outVariable) + ' ' + keyword('IN') + ' ' + collection(node.collection) + ' ' + annotation('/* full collection scan' + (node.random ? ', random order' : '') + (node.satellite ? ', satellite' : '') + ' */'); case 'EnumerateListNode': return keyword('FOR') + ' ' + variableName(node.outVariable) + ' ' + keyword('IN') + ' ' + variableName(node.inVariable) + ' ' + annotation('/* list iteration */'); case 'IndexNode': @@ -845,7 +845,7 @@ function processQuery (query, explain) { } indexes.push(idx); }); - return keyword('FOR') + ' ' + variableName(node.outVariable) + ' ' + keyword('IN') + ' ' + collection(node.collection) + ' ' + annotation('/* ' + types.join(', ') + ' */'); + return keyword('FOR') + ' ' + variableName(node.outVariable) + ' ' + keyword('IN') + ' ' + collection(node.collection) + ' ' + annotation('/* ' + types.join(', ') + (node.satellite ? ', satellite' : '') + ' */'); case 'IndexRangeNode': collectionVariables[node.outVariable.id] = node.collection; var index = node.index; diff --git a/js/server/modules/@arangodb/cluster.js b/js/server/modules/@arangodb/cluster.js index 083d6ecad7..7529e716e8 100644 --- a/js/server/modules/@arangodb/cluster.js +++ b/js/server/modules/@arangodb/cluster.js @@ -472,6 +472,7 @@ function createLocalCollections (plannedCollections, planVersion, var payload = { error: error.error, errorNum: error.errorNum, errorMessage: error.errorMessage, + satellite: collInfo.replicationFactor === 0, indexes: collInfo.indexes, servers: [ ourselves ], planVersion: planVersion }; From 8add7f40af947142eedf9400dafd1d89f37a69b6 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 6 Dec 2016 17:07:54 +0100 Subject: [PATCH 03/60] Disable creation of satellites on community edition --- arangod/VocBase/LogicalCollection.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index d5fba18143..1cb119445a 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -453,18 +453,21 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, auto replicationFactorSlice = info.get("replicationFactor"); if (!replicationFactorSlice.isNone()) { bool isError = true; - if (replicationFactorSlice.isString() && replicationFactorSlice.copyString() == "satellite") { - _replicationFactor = 0; - _numberOfShards = 1; - _distributeShardsLike = ""; - isError = false; - } else if (replicationFactorSlice.isNumber()) { + if (replicationFactorSlice.isNumber()) { _replicationFactor = replicationFactorSlice.getNumber(); // mop: only allow satellite collections to be created explicitly if (_replicationFactor > 0 || _replicationFactor <= 10) { isError = false; } } +#ifdef USE_ENTERPRISE + else if (replicationFactorSlice.isString() && replicationFactorSlice.copyString() == "satellite") { + _replicationFactor = 0; + _numberOfShards = 1; + _distributeShardsLike = ""; + isError = false; + } +#endif if (isError) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "invalid replicationFactor"); From 51b279346b7ecc5f674b309cf5afe4732e3322eb Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Tue, 6 Dec 2016 17:10:15 +0100 Subject: [PATCH 04/60] redirects to myelf should be hinstory --- arangod/Agency/Inception.cpp | 8 ++++---- arangod/Agency/Supervision.cpp | 25 ++++++++++++++++++++++++- arangod/Agency/Supervision.h | 3 +++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 9fadccd0ef..9be987de34 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -55,7 +55,7 @@ void Inception::gossip() { auto s = std::chrono::system_clock::now(); std::chrono::seconds timeout(3600); size_t j = 0; - bool complete = false; + //bool complete = false; long waitInterval = 250000; CONDITION_LOCKER(guard, _cv); @@ -123,14 +123,14 @@ void Inception::gossip() { // We're done if (config.poolComplete()) { - if (complete) { +// if (complete) { LOG_TOPIC(INFO, Logger::AGENCY) << "Agent pool completed. Stopping " "active gossipping. Starting RAFT process."; _agent->startConstituent(); break; } - complete = true; - } +// complete = true; +// } // Timed out? :( if ((std::chrono::system_clock::now() - s) > timeout) { diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index bb3b64f809..55f851d758 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -550,6 +550,7 @@ bool Supervision::handleJobs() { // Do supervision shrinkCluster(); workJobs(); + enforceReplication(); return true; } @@ -610,6 +611,28 @@ void Supervision::workJobs() { } } +void Supervision::enforceReplication() { + + auto const& plannedDBs = _snapshot(planColPrefix).children(); + + for (const auto& db_ : plannedDBs) { // Planned databases + auto const& db = *(db_.second); + for (const auto& col_ : db.children()) { // Planned collections + auto const& col = *(col_.second); + auto const& replicationFactor = col("replicationFactor").slice().getUInt(); + for (auto const& shard_ : col("shards").children()) { // Pl shards + auto const& shard = *(shard_.second); + if (replicationFactor != shard.slice().length()) { + LOG(WARN) << shard.slice().type() + << " target repl(" << replicationFactor + << ") actual repl(" << shard.slice().length() << ")"; + } + } + } + } + +} + // Shrink cluster if applicable, guarded by caller void Supervision::shrinkCluster() { // Get servers from plan @@ -705,7 +728,7 @@ void Supervision::shrinkCluster() { **/ // Find greatest replication factor among all collections uint64_t maxReplFact = 1; - Node::Children const& databases = _snapshot("/Plan/Collections").children(); + Node::Children const& databases = _snapshot(planColPrefix).children(); for (auto const& database : databases) { for (auto const& collptr : database.second->children()) { uint64_t replFact{0}; diff --git a/arangod/Agency/Supervision.h b/arangod/Agency/Supervision.h index 7449ca4f96..235dc331c3 100644 --- a/arangod/Agency/Supervision.h +++ b/arangod/Agency/Supervision.h @@ -117,6 +117,9 @@ class Supervision : public arangodb::Thread { private: + /// @brief Check for inconsistencies in replication factor vs dbs entries + void enforceReplication(); + /// @brief Update agency prefix from agency itself bool updateAgencyPrefix(size_t nTries = 10, int intervalSec = 1); From 0d3020434a43d06488ff2ca368324b7b3f447beb Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Tue, 6 Dec 2016 17:18:00 +0100 Subject: [PATCH 05/60] redirects to myelf should be hinstory --- arangod/Agency/Agent.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp index 214dccd03e..72ed9cfd21 100644 --- a/arangod/Agency/Agent.cpp +++ b/arangod/Agency/Agent.cpp @@ -575,8 +575,9 @@ query_t Agent::lastAckedAgo() const { trans_ret_t Agent::transact(query_t const& queries) { arangodb::consensus::index_t maxind = 0; // maximum write index - if (!_constituent.leading()) { - return trans_ret_t(false, _constituent.leaderID()); + auto leader = _constituent.leaderID(); + if (leader != id()) { + return trans_ret_t(false, leader); } // Apply to spearhead and get indices for log entries @@ -635,8 +636,9 @@ write_ret_t Agent::write(query_t const& query) { std::vector applied; std::vector indices; - if (!_constituent.leading()) { - return write_ret_t(false, _constituent.leaderID()); + auto leader = _constituent.leaderID(); + if (leader != id()) { + return write_ret_t(false, leader); } // Apply to spearhead and get indices for log entries @@ -668,8 +670,10 @@ write_ret_t Agent::write(query_t const& query) { /// Read from store read_ret_t Agent::read(query_t const& query) { - if (!_constituent.leading()) { - return read_ret_t(false, _constituent.leaderID()); + + auto leader = _constituent.leaderID(); + if (leader != id()) { + return read_ret_t(false, leader); } MUTEX_LOCKER(mutexLocker, _ioLock); From 0c97df252753003510094583c9fae4b1ab52a232 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 6 Dec 2016 17:20:28 +0100 Subject: [PATCH 06/60] Fix compilation --- arangod/Agency/Supervision.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 55f851d758..4ce39a087d 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -623,7 +623,7 @@ void Supervision::enforceReplication() { for (auto const& shard_ : col("shards").children()) { // Pl shards auto const& shard = *(shard_.second); if (replicationFactor != shard.slice().length()) { - LOG(WARN) << shard.slice().type() + LOG(WARN) << shard.slice().typeName() << " target repl(" << replicationFactor << ") actual repl(" << shard.slice().length() << ")"; } From 7701922ef25f427fbaebdf4e16828850e57256f7 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 6 Dec 2016 17:26:20 +0100 Subject: [PATCH 07/60] Do not hardcode timeouts --- js/client/modules/@arangodb/testing.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 17c77e7b7f..a3a39694ba 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -1466,8 +1466,8 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir) instanceArgs['server.endpoint'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.my-address'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.supervision-grace-period'] = '5'; - instanceArgs['agency.election-timeout-min'] = '0.5'; - instanceArgs['agency.election-timeout-max'] = '4.0'; + //instanceArgs['agency.election-timeout-min'] = '0.5'; + //instanceArgs['agency.election-timeout-max'] = '4.0'; if (i === N - 1) { From 654edc893cf8f260e43190fc50dc2528f789e618 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 6 Dec 2016 17:35:58 +0100 Subject: [PATCH 08/60] fixed documentation --- Documentation/Books/AQL/Invocation/WithArangosh.mdpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Books/AQL/Invocation/WithArangosh.mdpp b/Documentation/Books/AQL/Invocation/WithArangosh.mdpp index 4b886bb433..5e8742df47 100644 --- a/Documentation/Books/AQL/Invocation/WithArangosh.mdpp +++ b/Documentation/Books/AQL/Invocation/WithArangosh.mdpp @@ -103,7 +103,7 @@ allowed to use: @EXAMPLE_ARANGOSH_OUTPUT{02_workWithAQL_memoryLimit} |db._query( | 'FOR i IN 1..100000 SORT i RETURN i', {}, { - | options: { memoryLimit: 100000 } + | memoryLimit: 100000 }).toArray(); @END_EXAMPLE_ARANGOSH_OUTPUT @endDocuBlock 02_workWithAQL_memoryLimit From 30a0243e3da23001d598d9ffd58a58ae0715bdf2 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Tue, 6 Dec 2016 18:05:05 +0100 Subject: [PATCH 09/60] redirects to myelf should be hinstory --- arangod/Agency/RestAgencyHandler.cpp | 1 + js/client/modules/@arangodb/testing.js | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arangod/Agency/RestAgencyHandler.cpp b/arangod/Agency/RestAgencyHandler.cpp index ffc410c9db..e7bd618582 100644 --- a/arangod/Agency/RestAgencyHandler.cpp +++ b/arangod/Agency/RestAgencyHandler.cpp @@ -406,6 +406,7 @@ inline RestStatus RestAgencyHandler::handleRead() { return RestStatus::DONE; } else { + TRI_ASSERT(ret.redirect != _agent->id()); redirectRequest(ret.redirect); } return RestStatus::DONE; diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 17c77e7b7f..396e528dd2 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -1,3 +1,4 @@ + /* jshint strict: false, sub: true */ /* global print, arango */ 'use strict'; @@ -1466,8 +1467,8 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir) instanceArgs['server.endpoint'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.my-address'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.supervision-grace-period'] = '5'; - instanceArgs['agency.election-timeout-min'] = '0.5'; - instanceArgs['agency.election-timeout-max'] = '4.0'; + //instanceArgs['agency.election-timeout-min'] = '0.5'; + //instanceArgs['agency.election-timeout-max'] = '4.0'; if (i === N - 1) { From 70fb21ce2d42c6c47204bc612779fd6f81829636 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Tue, 6 Dec 2016 18:29:29 +0100 Subject: [PATCH 10/60] update scriptStandAlone with new inception --- scripts/startStandAloneAgency.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/startStandAloneAgency.sh b/scripts/startStandAloneAgency.sh index e76c85528f..74e5150670 100755 --- a/scripts/startStandAloneAgency.sh +++ b/scripts/startStandAloneAgency.sh @@ -170,10 +170,22 @@ count=1 for aid in "${aaid[@]}"; do port=$(( $BASE + $aid )) - if [ "$GOSSIP_MODE" = 2 ]; then + + if [ "$GOSSIP_MODE" = "2" ]; then nport=$(( $BASE + $(( $(( $aid + 1 )) % 3 )))) GOSSIP_PEERS=" --agency.endpoint $TRANSPORT://localhost:$nport" fi + + if [ "$GOSSIP_MODE" = "3" ]; then + GOSSIP_PEERS="" + for id in "${aaid[@]}"; do + if [ ! "$id" = "$aid" ]; then + nport=$(( $BASE + $(( $id )) )) + GOSSIP_PEERS+=" --agency.endpoint $TRANSPORT://localhost:$nport" + fi + done + fi + printf " starting agent %s " "$aid" build/bin/arangod \ -c none \ From 0700cfa86a7f9e75e2c1bd099253dd609365e78b Mon Sep 17 00:00:00 2001 From: Alan Plum Date: Tue, 6 Dec 2016 18:16:31 +0100 Subject: [PATCH 11/60] Implement OAuth 1.0a --- js/common/modules/@arangodb/request.js | 36 ++-- js/server/modules/@arangodb/foxx/oauth1.js | 220 +++++++++++++++++++++ js/server/modules/@arangodb/foxx/oauth2.js | 17 +- 3 files changed, 249 insertions(+), 24 deletions(-) create mode 100644 js/server/modules/@arangodb/foxx/oauth1.js diff --git a/js/common/modules/@arangodb/request.js b/js/common/modules/@arangodb/request.js index 8cfacd480c..cc144f623e 100644 --- a/js/common/modules/@arangodb/request.js +++ b/js/common/modules/@arangodb/request.js @@ -1,5 +1,4 @@ /* jshint sub: true */ -/* global exports: true */ 'use strict'; // ////////////////////////////////////////////////////////////////////////////// @@ -31,7 +30,6 @@ const internal = require('internal'); const Buffer = require('buffer').Buffer; -const extend = require('lodash').extend; const httperr = require('http-errors'); const is = require('@arangodb/is'); const querystring = require('querystring'); @@ -92,7 +90,7 @@ function request (req) { let pathObj = typeof path === 'string' ? url.parse(path) : path; if (pathObj.auth) { let auth = pathObj.auth.split(':'); - req = extend({ + req = Object.assign({ auth: { username: decodeURIComponent(auth[0]), password: decodeURIComponent(auth[1]) @@ -143,7 +141,7 @@ function request (req) { } if (req.auth) { - headers['authorization'] = ( // eslint-disable-line dot-notation + headers.authorization = ( req.auth.bearer ? 'Bearer ' + req.auth.bearer : 'Basic ' + new Buffer( @@ -175,21 +173,17 @@ function request (req) { return new Response(result, req.encoding, req.json); } -exports = request; -exports.request = request; -exports.Response = Response; +module.exports = request; +request.request = request; +request.Response = Response; -['delete', 'get', 'head', 'patch', 'post', 'put'] - .forEach(function (method) { - exports[method.toLowerCase()] = function (url, options) { - if (typeof url === 'object') { - options = url; - url = undefined; - } else if (typeof url === 'string') { - options = extend({}, options, {url: url}); - } - return request(extend({method: method.toUpperCase()}, options)); - }; - }); - -module.exports = exports; +for (const method of ['DELETE', 'GET', 'HEAD', 'PATCH', 'POST', 'PUT']) { + request[method.toLowerCase()] = function (url, options) { + if (typeof url === 'object') { + options = url; + } else if (typeof url === 'string') { + options = Object.assign({}, options, {url}); + } + return request(Object.assign({method}, options)); + }; +} diff --git a/js/server/modules/@arangodb/foxx/oauth1.js b/js/server/modules/@arangodb/foxx/oauth1.js new file mode 100644 index 0000000000..59fdbe9b5d --- /dev/null +++ b/js/server/modules/@arangodb/foxx/oauth1.js @@ -0,0 +1,220 @@ +/* eslint camelcase: 0 */ +'use strict'; + +// ////////////////////////////////////////////////////////////////////////////// +// / DISCLAIMER +// / +// / Copyright 2016 ArangoDB GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License") +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is ArangoDB GmbH, Cologne, Germany +// / +// / @author Alan Plum +// ////////////////////////////////////////////////////////////////////////////// + +const assert = require('assert'); +const url = require('url'); +const parseUrl = url.parse; +const formatUrl = url.format; +const parseQuery = require('querystring').parse; +const request = require('@arangodb/request'); +const crypto = require('@arangodb/crypto'); + +function nonceAndTime () { + const oauth_timestamp = Math.floor(Date.now() / 1000); + const oauth_nonce = crypto.genRandomAlphaNumbers(32); + return {oauth_timestamp, oauth_nonce}; +} + +function normalizeUrl (requestUrl, parameters) { + const urlParts = url.parse(requestUrl); + const urlTokens = [urlParts.protocol, '//', urlParts.hostname]; + if (urlParts.port && ( + (urlParts.protocol === 'http:' && urlParts.port !== '80') || + (urlParts.protocol === 'https:' && urlParts.port !== '443') + )) { + urlTokens.push(':', urlParts.port); + } + urlTokens.push(urlParts.pathname); + const params = []; + if (urlParts.query) { + const qs = parseQuery(urlParts.query); + parameters = Object.assign(qs, parameters); + } + for (const key of Object.keys(parameters)) { + const value = parameters[key] === undefined ? '' : String(parameters[key]); + params.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`); + } + return { + url: urlTokens.join(''), + params: params.sort().join('&') + }; +} + +function createSignatureBaseString (method, normalizedUrl, normalizedParams) { + return [ + method.toUpperCase(), + encodeURIComponent(normalizedUrl), + encodeURIComponent(normalizedParams) + ].join('&'); +} + +function parse (str) { + try { + return JSON.parse(str); + } catch (e) { + if (e instanceof SyntaxError) { + return parseQuery(str); + } + throw e; + } +} + +module.exports = function oauth1 (cfg) { + if (!cfg) { + cfg = {}; + } + assert(cfg.requestTokenEndpoint, 'No Request Token URL specified'); + assert(cfg.authEndpoint, 'No User Authorization URL specified'); + assert(cfg.accessTokenEndpoint, 'No Access Token URL specified'); + assert(cfg.clientId, 'No Client ID specified'); + assert(cfg.clientSecret, 'No Client Secret specified'); + const oauth_consumer_key = cfg.clientId; + const oauth_signature_method = cfg.signatureMethod || 'HMAC-SHA1'; + assert(( + oauth_signature_method === 'HMAC-SHA1' || + oauth_signature_method === 'PLAINTEXT' + ), `Unsupported signature method: "${oauth_signature_method}"`); + + function createSignature (signatureBaseString, token_secret) { + const key = `${cfg.clientSecret || ''}&${token_secret || ''}`; + if (oauth_signature_method === 'PLAINTEXT') { + return key; + } + const hex = crypto.hmac(key, signatureBaseString, 'sha1'); + return new Buffer(hex, 'hex').toString('base64'); + } + + function createRequest (method, url, opts, token_secret) { + const normalized = normalizeUrl( + url, + Object.assign( + nonceAndTime(), + { + oauth_consumer_key, + oauth_signature_method, + oauth_version: '1.0' + }, + opts + ) + ); + const signatureBaseString = createSignatureBaseString( + 'POST', + normalized.url, + normalized.params + ); + const signature = createSignature(signatureBaseString, token_secret); + normalized.params += `&oauth_signature=${encodeURIComponent(signature)}`; + const queryParams = []; + const authParams = []; + if (cfg.realm) { + authParams.push(`realm="${encodeURIComponent(cfg.realm)}"`); + } + for (const param of normalized.params.split('&')) { + if (param.startsWith('oauth_')) { + const [key, value] = param.split('='); + authParams.push(`${key}="${value}"`); + } else { + queryParams.push(param); + } + } + return { + url: normalized.url, + qs: queryParams.join('&'), + headers: { + accept: 'application/json', + authorization: `OAuth ${authParams.join(', ')}` + } + }; + } + + return { + fetchRequestToken (oauth_callback, opts) { + assert(oauth_callback, 'No Callback URL specified'); + const req = createRequest( + 'POST', + cfg.requestTokenEndpoint, + Object.assign({oauth_callback}, opts) + ); + const res = request.post(req); + if (!res.body) { + throw new Error(`OAuth provider returned empty response with HTTP status ${res.status}`); + } + return parse(res.body); + }, + getAuthUrl (oauth_token, opts) { + assert(oauth_token, 'No Request Token specified'); + const endpoint = parseUrl(cfg.authEndpoint); + delete endpoint.search; + endpoint.query = Object.assign( + parseQuery(endpoint.query), + opts, + {oauth_token} + ); + return formatUrl(endpoint); + }, + exchangeRequestToken (oauth_token, oauth_verifier, opts) { + assert(oauth_token, 'No Request Token specified'); + assert(oauth_verifier, 'No Verification Code specified'); + const req = createRequest( + 'POST', + cfg.accessTokenEndpoint, + Object.assign({oauth_token, oauth_verifier}, opts) + ); + const res = request.post(req); + if (!res.body) { + throw new Error(`OAuth provider returned empty response with HTTP status ${res.status}`); + } + return parse(res.body); + }, + fetchActiveUser (oauth_token, oauth_token_secret, opts) { + assert(oauth_token, 'No Access Token specified'); + assert(oauth_token_secret, 'No Token Secret specified'); + if (!cfg.activeUserEndpoint) { + return null; + } + const req = createRequest( + 'GET', + cfg.activeUserEndpoint, + Object.assign({oauth_token}, opts), + oauth_token_secret + ); + const res = request.get(req); + if (!res.body) { + throw new Error(`OAuth provider returned empty response with HTTP status ${res.status}`); + } + return parse(res.body); + }, + createSignedRequest (method, url, query, oauth_token, oauth_token_secret) { + assert(oauth_token, 'No Access Token specified'); + assert(oauth_token_secret, 'No Token Secret specified'); + return createRequest( + method, + url, + Object.assign({oauth_token}, query), + oauth_token_secret + ); + } + }; +}; diff --git a/js/server/modules/@arangodb/foxx/oauth2.js b/js/server/modules/@arangodb/foxx/oauth2.js index 0980501b88..c84009e845 100644 --- a/js/server/modules/@arangodb/foxx/oauth2.js +++ b/js/server/modules/@arangodb/foxx/oauth2.js @@ -23,6 +23,7 @@ // / @author Alan Plum // ////////////////////////////////////////////////////////////////////////////// +const assert = require('assert'); const url = require('url'); const parseUrl = url.parse; const formatUrl = url.format; @@ -41,6 +42,14 @@ function parse (str) { } module.exports = function oauth2 (cfg) { + if (!cfg) { + cfg = {}; + } + assert(cfg.tokenEndpoint, 'No Token Request URL specified'); + assert(cfg.authEndpoint, 'No User Authorization URL specified'); + assert(cfg.clientId, 'No Client ID specified'); + assert(cfg.clientSecret, 'No Client Secret specified'); + function getTokenRequest (code, redirect_uri) { const endpoint = parseUrl(cfg.tokenEndpoint); const body = Object.assign( @@ -70,7 +79,7 @@ module.exports = function oauth2 (cfg) { } return { - getAuthUrl(redirect_uri, opts) { + getAuthUrl (redirect_uri, opts) { if (typeof redirect_uri !== 'string') { opts = redirect_uri; redirect_uri = undefined; @@ -88,7 +97,8 @@ module.exports = function oauth2 (cfg) { } return formatUrl(endpoint); }, - exchangeGrantToken(code, redirect_uri) { + exchangeGrantToken (code, redirect_uri) { + assert(code, 'No Grant Token specified'); const req = getTokenRequest(code, redirect_uri); const res = request.post(req.url, { headers: {accept: 'application/json'}, @@ -99,7 +109,8 @@ module.exports = function oauth2 (cfg) { } return parse(res.body); }, - fetchActiveUser(access_token) { + fetchActiveUser (access_token) { + assert(access_token, 'No Access Token specified'); if (!cfg.activeUserEndpoint) { return null; } From aecb1891501d3127419d47382b7ff9ad571a8788 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 7 Dec 2016 11:18:55 +0100 Subject: [PATCH 12/60] pull once more; rename cx->xc --- Installation/Jenkins/build.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 449c55d384..f8e73ea977 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -323,14 +323,14 @@ while [ $# -gt 0 ]; do CLEAN_IT=1 shift ;; - --cxArmV8) + --xcArmV8) ARMV8=1 - CXGCC=1 + XCGCC=1 shift ;; - --cxArmV7) + --xcArmV7) ARMV7=1 - CXGCC=1 + XCGCC=1 shift ;; @@ -388,7 +388,7 @@ elif [ "$CLANG36" == 1 ]; then CC=/usr/bin/clang-3.6 CXX=/usr/bin/clang++-3.6 CXXFLAGS="${CXXFLAGS} -std=c++11" -elif [ "${CXGCC}" = 1 ]; then +elif [ "${XCGCC}" = 1 ]; then USE_JEMALLOC=0 if [ "${ARMV8}" = 1 ]; then export TOOL_PREFIX=aarch64-linux-gnu @@ -397,7 +397,7 @@ elif [ "${CXGCC}" = 1 ]; then export TOOL_PREFIX=aarch64-linux-gnu BUILD_DIR="${BUILD_DIR}-ARMV7" else - echo "Unknown CX-Compiler!" + echo "Unknown XC-Compiler!" exit 1; fi @@ -527,7 +527,7 @@ if test -n "${ENTERPRISE_GIT_URL}" ; then if test ! -d enterprise; then git clone ${ENTERPRISE_GIT_URL} enterprise fi - (cd enterprise; git checkout master; git fetch --tags; git pull --all; git checkout ${GITARGS} ) + (cd enterprise; git checkout master; git fetch --tags; git pull --all; git checkout ${GITARGS}; git pull ) fi From 0ac353e1094aa8f9074b262a7a178c7f9e3409c6 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Wed, 7 Dec 2016 11:44:04 +0100 Subject: [PATCH 13/60] fix solaris compile error --- lib/Basics/operating-system.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Basics/operating-system.h b/lib/Basics/operating-system.h index c8e8130a84..396ad6c1d5 100644 --- a/lib/Basics/operating-system.h +++ b/lib/Basics/operating-system.h @@ -143,7 +143,7 @@ #define TRI_DIR_SEPARATOR_STR "/" #define TRI_O_CLOEXEC O_CLOEXEC -#define TRI_NOATIME O_NOATIME +#define TRI_NOATIME 0 #define TRI_CHDIR ::chdir #define TRI_CLOSE ::close From 47463a2f1c0fd31be070aac5615e4672eb8a8cff Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 11:56:41 +0100 Subject: [PATCH 14/60] Agency startup redone after revisit of design document --- arangod/Agency/Agent.cpp | 15 ++ arangod/Agency/AgentConfiguration.cpp | 14 +- arangod/Agency/Constituent.cpp | 5 + arangod/Agency/Inception.cpp | 283 +++++++++++--------------- scripts/startStandAloneAgency.sh | 2 +- 5 files changed, 147 insertions(+), 172 deletions(-) diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp index 72ed9cfd21..c69d9cad1d 100644 --- a/arangod/Agency/Agent.cpp +++ b/arangod/Agency/Agent.cpp @@ -704,6 +704,13 @@ void Agent::run() { // Leader working only if (leading()) { + + // Really leading? + if (challengeLeadership()) { + _constituent.candidate(); + } + + // Don't panic _appendCV.wait(1000); // Append entries to followers @@ -914,6 +921,8 @@ void Agent::notifyInactive() const { out.add("id", VPackValue(id())); out.add("active", _config.activeToBuilder()->slice()); out.add("pool", _config.poolToBuilder()->slice()); + out.add("min ping", VPackValue(_config.minPing())); + out.add("max ping", VPackValue(_config.maxPing())); out.close(); for (auto const& p : pool) { @@ -958,6 +967,12 @@ void Agent::notify(query_t const& message) { if (!slice.hasKey("pool") || !slice.get("pool").isObject()) { THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL); } + if (!slice.hasKey("min ping") || !slice.get("min ping").isNumber()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL); + } + if (!slice.hasKey("max ping") || !slice.get("max ping").isNumber()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL); + } _config.update(message); _state.persistActiveAgents(_config.activeToBuilder(), diff --git a/arangod/Agency/AgentConfiguration.cpp b/arangod/Agency/AgentConfiguration.cpp index 4c555bf8f6..dd31f575c1 100644 --- a/arangod/Agency/AgentConfiguration.cpp +++ b/arangod/Agency/AgentConfiguration.cpp @@ -345,13 +345,15 @@ void config_t::update(query_t const& message) { VPackSlice slice = message->slice(); std::map pool; bool changed = false; - for (auto const& p : VPackObjectIterator(slice.get("pool"))) { + for (auto const& p : VPackObjectIterator(slice.get(poolStr))) { pool[p.key.copyString()] = p.value.copyString(); } std::vector active; - for (auto const& a : VPackArrayIterator(slice.get("active"))) { + for (auto const& a : VPackArrayIterator(slice.get(activeStr))) { active.push_back(a.copyString()); } + double minPing = slice.get(minPingStr).getDouble(); + double maxPing = slice.get(maxPingStr).getDouble(); WRITE_LOCKER(writeLocker, _lock); if (pool != _pool) { _pool = pool; @@ -361,6 +363,14 @@ void config_t::update(query_t const& message) { _active = active; changed=true; } + if (minPing != _minPing) { + _minPing = minPing; + changed=true; + } + if (maxPing != _maxPing) { + _maxPing = maxPing; + changed=true; + } if (changed) { ++_version; } diff --git a/arangod/Agency/Constituent.cpp b/arangod/Agency/Constituent.cpp index 751238fa5a..ac5e08acb0 100644 --- a/arangod/Agency/Constituent.cpp +++ b/arangod/Agency/Constituent.cpp @@ -303,6 +303,11 @@ bool Constituent::checkLeader(term_t term, std::string id, index_t prevLogIndex, /// @brief Vote bool Constituent::vote(term_t termOfPeer, std::string id, index_t prevLogIndex, term_t prevLogTerm) { + + if (!_agent->ready()) { + return false; + } + TRI_ASSERT(_vocbase != nullptr); LOG_TOPIC(TRACE, Logger::AGENCY) diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 9be987de34..3e122de942 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -55,7 +55,6 @@ void Inception::gossip() { auto s = std::chrono::system_clock::now(); std::chrono::seconds timeout(3600); size_t j = 0; - //bool complete = false; long waitInterval = 250000; CONDITION_LOCKER(guard, _cv); @@ -101,6 +100,7 @@ void Inception::gossip() { } // pool entries + bool complete = true; for (auto const& pair : config.pool()) { if (pair.second != config.endpoint()) { { @@ -109,6 +109,7 @@ void Inception::gossip() { continue; } } + complete = false; std::string clientid = config.id() + std::to_string(j++); auto hf = std::make_unique>(); @@ -123,14 +124,13 @@ void Inception::gossip() { // We're done if (config.poolComplete()) { -// if (complete) { + if (complete) { LOG_TOPIC(INFO, Logger::AGENCY) << "Agent pool completed. Stopping " "active gossipping. Starting RAFT process."; _agent->startConstituent(); break; } -// complete = true; -// } + } // Timed out? :( if ((std::chrono::system_clock::now() - s) > timeout) { @@ -145,176 +145,125 @@ void Inception::gossip() { // don't panic just yet _cv.wait(waitInterval); - waitInterval *= 2; - - } - -} - - -// @brief Active agency from persisted database -bool Inception::activeAgencyFromPersistence() { - - LOG_TOPIC(INFO, Logger::AGENCY) << "Found persisted agent pool ..."; - - auto myConfig = _agent->config(); - std::string const path = pubApiPrefix + "config"; - - // Can only be done responcibly, if we are complete - if (myConfig.poolComplete()) { - - // Contact hosts on pool in hopes of finding a leader Id - for (auto const& pair : myConfig.pool()) { - - if (pair.first != myConfig.id()) { - - auto comres = arangodb::ClusterComm::instance()->syncRequest( - myConfig.id(), 1, pair.second, rest::RequestType::GET, path, - std::string(), std::unordered_map(), 1.0); - - if (comres->status == CL_COMM_SENT) { - - auto body = comres->result->getBodyVelocyPack(); - auto theirConfig = body->slice(); - - std::string leaderId; - - // LeaderId in configuration? - try { - leaderId = theirConfig.get("leaderId").copyString(); - } catch (std::exception const& e) { - LOG_TOPIC(DEBUG, Logger::AGENCY) - << "Failed to get leaderId from" << pair.second << ": " - << e.what(); - } - - if (leaderId != "") { // Got leaderId. Let's get do it. - - try { - LOG_TOPIC(DEBUG, Logger::AGENCY) - << "Found active agency with leader " << leaderId - << " at endpoint " - << theirConfig.get("configuration").get( - "pool").get(leaderId).copyString(); - } catch (std::exception const& e) { - LOG_TOPIC(DEBUG, Logger::AGENCY) - << "Failed to get leaderId from" << pair.second << ": " - << e.what(); - } - - auto agency = std::make_shared(); - agency->openObject(); - agency->add("term", theirConfig.get("term")); - agency->add("id", VPackValue(leaderId)); - agency->add("active", theirConfig.get("configuration").get("active")); - agency->add("pool", theirConfig.get("configuration").get("pool")); - agency->close(); - _agent->notify(agency); - - return true; - - } else { // No leaderId. Move on. - - LOG_TOPIC(DEBUG, Logger::AGENCY) - << "Failed to get leaderId from" << pair.second; - - } - - } - - } - + if (waitInterval < 2500000) { // 2.5s + waitInterval *= 2; } - } - return false; + } } bool Inception::restartingActiveAgent() { - auto myConfig = _agent->config(); - std::string const path = pubApiPrefix + "config"; + auto const path = pubApiPrefix + "config"; - auto s = std::chrono::system_clock::now(); - std::chrono::seconds timeout(60); + auto myConfig = _agent->config(); + auto startTime = std::chrono::system_clock::now(); + auto pool = myConfig.pool(); + auto active = myConfig.active(); + auto activeOrig = active; + auto clientId = myConfig.id(); + auto majority = (myConfig.size()+1)/2; - // Can only be done responcibly, if we are complete - if (myConfig.poolComplete()) { + + std::chrono::seconds timeout(3600); + + CONDITION_LOCKER(guard, _cv); + + long waitInterval(500000); + + while (!this->isStopping() && !_agent->isStopping()) { - auto pool = myConfig.pool(); - auto active = myConfig.active(); + active.erase( + std::remove(active.begin(), active.end(), myConfig.id()), active.end()); + active.erase( + std::remove(active.begin(), active.end(), ""), active.end()); - CONDITION_LOCKER(guard, _cv); - - long waitInterval(500000); - - while (!this->isStopping() && !_agent->isStopping()) { - - active.erase( - std::remove(active.begin(), active.end(), myConfig.id()), active.end()); - active.erase( - std::remove(active.begin(), active.end(), ""), active.end()); - - if (active.empty()) { - return true; - } + if (active.size() < majority) { + LOG_TOPIC(INFO, Logger::AGENCY) + << "Found majority of agents in agreement over active pool" + " finishing startup sequence."; + return true; + } + + for (auto& p : pool) { - for (auto& i : active) { + if (p.first != myConfig.id() && p.first != "") { - if (i != myConfig.id() && i != "") { - - auto clientId = myConfig.id(); - auto comres = arangodb::ClusterComm::instance()->syncRequest( - clientId, 1, pool.at(i), rest::RequestType::GET, path, std::string(), - std::unordered_map(), 2.0); - - if (comres->status == CL_COMM_SENT) { + auto comres = arangodb::ClusterComm::instance()->syncRequest( + clientId, 1, p.second, rest::RequestType::GET, path, std::string(), + std::unordered_map(), 2.0); + + if (comres->status == CL_COMM_SENT) { + try { - try { - - auto theirActive = comres->result->getBodyVelocyPack()-> - slice().get("configuration").get("active").toJson(); - auto myActive = myConfig.activeToBuilder()->toJson(); - + auto const theirConfigVP = comres->result->getBodyVelocyPack(); + auto const& theirConfig = theirConfigVP->slice(); + auto const& theirLeaderId = theirConfig.get("leaderId").copyString(); + auto const& tcc = theirConfig.get("configuration"); + + // Known leader. We are done. + if (!theirLeaderId.empty()) { + LOG_TOPIC(INFO, Logger::AGENCY) << + "Found active RAFTing agency lead by " << theirLeaderId; + auto agency = std::make_shared(); + agency->openObject(); + agency->add("term", theirConfig.get("term")); + agency->add("id", VPackValue(theirLeaderId)); + agency->add("active", tcc.get("active")); + agency->add("pool", tcc.get("pool")); + agency->add("min ping", tcc.get("min ping")); + agency->add("max ping", tcc.get("max ping")); + agency->close(); + _agent->notify(agency); + return true; + } + + auto theirActive = tcc.get("active").toJson(); + auto myActive = myConfig.activeToBuilder()->toJson(); + auto i = std::find(active.begin(),active.end(),p.first); + + if (i != active.end()) { if (theirActive != myActive) { LOG_TOPIC(FATAL, Logger::AGENCY) - << "Assumed active RAFT peer and I disagree on active membership." - << "Administrative intervention needed."; + << "Assumed active RAFT peer and I disagree on active " + "membership. Administrative intervention needed."; FATAL_ERROR_EXIT(); return false; } else { - i = ""; + *i = ""; } - - } catch (std::exception const& e) { - LOG_TOPIC(FATAL, Logger::AGENCY) - << "Assumed active RAFT peer has no active agency list: " << e.what() - << "Administrative intervention needed."; - FATAL_ERROR_EXIT(); - return false; } - } - } - + + } catch (std::exception const& e) { + LOG_TOPIC(FATAL, Logger::AGENCY) + << "Assumed active RAFT peer has no active agency list: " + << e.what() << "Administrative intervention needed."; + FATAL_ERROR_EXIT(); + return false; + } + } } - // Timed out? :( - if ((std::chrono::system_clock::now() - s) > timeout) { - if (myConfig.poolComplete()) { - LOG_TOPIC(DEBUG, Logger::AGENCY) << "Joined complete pool!"; - } else { - LOG_TOPIC(ERR, Logger::AGENCY) - << "Failed to find complete pool of agents. Giving up!"; - } - break; - } - - _cv.wait(waitInterval); - waitInterval *= 2; - } + + // Timed out? :( + if ((std::chrono::system_clock::now() - startTime) > timeout) { + if (myConfig.poolComplete()) { + LOG_TOPIC(DEBUG, Logger::AGENCY) << "Joined complete pool!"; + } else { + LOG_TOPIC(ERR, Logger::AGENCY) + << "Failed to find complete pool of agents. Giving up!"; + } + break; + } + + _cv.wait(waitInterval); + if (waitInterval < 2500000) { // 2.5s + waitInterval *= 2; + } + } return false; @@ -512,11 +461,6 @@ bool Inception::estimateRAFTInterval() { } -// @brief Active agency from persisted database -bool Inception::activeAgencyFromCommandLine() { - return false; -} - // @brief Thread main void Inception::run() { while (arangodb::rest::RestHandlerFactory::isMaintenance() && @@ -528,23 +472,23 @@ void Inception::run() { } config_t config = _agent->config(); - // 1. If active agency, do as you're told - if (config.startup() == "persistence" && activeAgencyFromPersistence()) { - _agent->ready(true); + + // Are we starting from persisted pool? + if (config.startup() == "persistence") { + if (restartingActiveAgent()) { + LOG_TOPIC(INFO, Logger::AGENCY) << "Activating agent."; + _agent->ready(true); + } else { + // FATAL ERROR + } + return; } - // 2. If we think that we used to be active agent - if (!_agent->ready() && restartingActiveAgent()) { - _agent->ready(true); - } - - // 3. Else gossip + // Gossip config = _agent->config(); - if (!_agent->ready() && !config.poolComplete()) { - gossip(); - } + gossip(); - // 4. If still incomplete bail out :( + // No complete pool after gossip? config = _agent->config(); if (!_agent->ready() && !config.poolComplete()) { LOG_TOPIC(FATAL, Logger::AGENCY) @@ -552,12 +496,13 @@ void Inception::run() { FATAL_ERROR_EXIT(); } - // 5. If command line RAFT timings have not been set explicitly - // Try good estimate of RAFT time limits + // If command line RAFT timings have not been set explicitly + // Try good estimate of RAFT time limits if (!config.cmdLineTimings()) { estimateRAFTInterval(); } + LOG_TOPIC(INFO, Logger::AGENCY) << "Activating agent."; _agent->ready(true); } diff --git a/scripts/startStandAloneAgency.sh b/scripts/startStandAloneAgency.sh index 74e5150670..2b7c090ba2 100755 --- a/scripts/startStandAloneAgency.sh +++ b/scripts/startStandAloneAgency.sh @@ -158,7 +158,7 @@ if [ "$GOSSIP_MODE" = "0" ]; then GOSSIP_PEERS=" --agency.endpoint $TRANSPORT://localhost:$BASE" fi -rm -rf agency +#rm -rf agency mkdir -p agency PIDS="" From 3b4266962a94fc042e037e3a8b70cbd037e529ee Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 12:06:38 +0100 Subject: [PATCH 15/60] Fatal error exit missing --- arangod/Agency/Inception.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 3e122de942..54b5fd164c 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -479,6 +479,9 @@ void Inception::run() { LOG_TOPIC(INFO, Logger::AGENCY) << "Activating agent."; _agent->ready(true); } else { + LOG_TOPIC(FATAL, Logger::AGENCY) + << "Unable to restart with persisted pool. Fatal exit."; + FATAL_ERROR_EXIT(); // FATAL ERROR } return; From 9bf7d3fb5b19031d4b664b073ca477d1c4995d69 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 12:12:49 +0100 Subject: [PATCH 16/60] Useful logging in failure mode of inception's persistence --- arangod/Agency/Inception.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 54b5fd164c..0e3f7c5894 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -228,7 +228,11 @@ bool Inception::restartingActiveAgent() { if (theirActive != myActive) { LOG_TOPIC(FATAL, Logger::AGENCY) << "Assumed active RAFT peer and I disagree on active " - "membership. Administrative intervention needed."; + "membership:"; + LOG_TOPIC(FATAL, Logger::AGENCY) + << "Their active list is " << theirActive; + LOG_TOPIC(FATAL, Logger::AGENCY) + << "My active list is " << myActive; FATAL_ERROR_EXIT(); return false; } else { From b8c72dece8389fa1f0046ffe85ea8a09d6be0187 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Wed, 7 Dec 2016 12:18:18 +0100 Subject: [PATCH 17/60] fixed issue #2211 --- CHANGELOG | 9 ++++++ UnitTests/HttpInterface/api-cursor-spec.rb | 28 +++++++++++++++++++ ...api-export-spec-timecritical-noncluster.rb | 27 ++++++++++++++++++ arangod/RestHandler/RestCursorHandler.cpp | 4 +-- arangod/RestHandler/RestExportHandler.cpp | 4 +-- arangod/Utils/Cursor.h | 15 ++++++++-- arangod/Utils/CursorRepository.cpp | 14 ++++++++-- arangod/Utils/CursorRepository.h | 4 +-- arangod/V8Server/v8-voccursor.cpp | 2 +- 9 files changed, 96 insertions(+), 11 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index caf6b8e703..09bce453c5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -26,6 +26,15 @@ edge attribute `label`. * process.stdout.isTTY now returns `true` in arangosh and when running arangod with the `--console` flag + +v3.1.4 (XXXX-XX-XX) +------------------- + +* fixed issue #2211 + +* fixed issue #2204 + + v3.1.3 (xxxx-xx-xx) ------------------- diff --git a/UnitTests/HttpInterface/api-cursor-spec.rb b/UnitTests/HttpInterface/api-cursor-spec.rb index 38ab000011..d47eb633ca 100644 --- a/UnitTests/HttpInterface/api-cursor-spec.rb +++ b/UnitTests/HttpInterface/api-cursor-spec.rb @@ -598,6 +598,34 @@ describe ArangoDB do doc.parsed_response['cached'].should eq(false) end + it "calls wrong export API" do + cmd = api + body = "{ \"query\" : \"FOR u IN #{@cn} LIMIT 5 RETURN u.n\", \"count\" : true, \"batchSize\" : 2 }" + doc = ArangoDB.log_post("#{prefix}-create-wrong-api", cmd, :body => body) + + doc.code.should eq(201) + doc.headers['content-type'].should eq("application/json; charset=utf-8") + doc.parsed_response['error'].should eq(false) + doc.parsed_response['code'].should eq(201) + doc.parsed_response['id'].should be_kind_of(String) + doc.parsed_response['id'].should match(@reId) + doc.parsed_response['hasMore'].should eq(true) + doc.parsed_response['count'].should eq(5) + doc.parsed_response['result'].length.should eq(2) + doc.parsed_response['cached'].should eq(false) + + id = doc.parsed_response['id'] + + cmd = "/_api/export/#{id}" + doc = ArangoDB.log_put("#{prefix}-create-wrong-api", cmd) + + doc.code.should eq(404) + doc.headers['content-type'].should eq("application/json; charset=utf-8") + doc.parsed_response['error'].should eq(true) + doc.parsed_response['code'].should eq(404) + doc.parsed_response['errorNum'].should eq(1600) + end + it "creates a query that survives memory limit constraints" do cmd = api body = "{ \"query\" : \"FOR i IN 1..10000 SORT i RETURN i\", \"memoryLimit\" : 10000000, \"batchSize\": 10 }" diff --git a/UnitTests/HttpInterface/api-export-spec-timecritical-noncluster.rb b/UnitTests/HttpInterface/api-export-spec-timecritical-noncluster.rb index 7cda0fe6d0..48ac173a02 100644 --- a/UnitTests/HttpInterface/api-export-spec-timecritical-noncluster.rb +++ b/UnitTests/HttpInterface/api-export-spec-timecritical-noncluster.rb @@ -678,7 +678,34 @@ describe ArangoDB do doc.parsed_response['count'].should eq(2000) doc.parsed_response['result'].length.should eq(2000) end + + it "calls wrong cursor API" do + cmd = api + "?collection=#{@cn}" + body = "{ \"count\" : true, \"batchSize\" : 100, \"flush\" : true }" + doc = ArangoDB.log_post("#{prefix}-limit-return", cmd, :body => body) + + doc.code.should eq(201) + doc.headers['content-type'].should eq("application/json; charset=utf-8") + doc.parsed_response['error'].should eq(false) + doc.parsed_response['code'].should eq(201) + doc.parsed_response['id'].should be_kind_of(String) + doc.parsed_response['id'].should match(@reId) + doc.parsed_response['hasMore'].should eq(true) + doc.parsed_response['count'].should eq(2000) + doc.parsed_response['result'].length.should eq(100) + id = doc.parsed_response['id'] + + # intentionally wrong + cmd = "/_api/cursor/#{id}" + doc = ArangoDB.log_put("#{prefix}-return-cont", cmd) + + doc.code.should eq(404) + doc.headers['content-type'].should eq("application/json; charset=utf-8") + doc.parsed_response['error'].should eq(true) + doc.parsed_response['code'].should eq(404) + doc.parsed_response['errorNum'].should eq(1600) + end end ################################################################################ diff --git a/arangod/RestHandler/RestCursorHandler.cpp b/arangod/RestHandler/RestCursorHandler.cpp index 5e71fb876c..686c270ff3 100644 --- a/arangod/RestHandler/RestCursorHandler.cpp +++ b/arangod/RestHandler/RestCursorHandler.cpp @@ -447,7 +447,7 @@ void RestCursorHandler::modifyCursor() { auto cursorId = static_cast( arangodb::basics::StringUtils::uint64(id)); bool busy; - auto cursor = cursors->find(cursorId, busy); + auto cursor = cursors->find(cursorId, Cursor::CURSOR_VPACK, busy); if (cursor == nullptr) { if (busy) { @@ -499,7 +499,7 @@ void RestCursorHandler::deleteCursor() { auto cursorId = static_cast( arangodb::basics::StringUtils::uint64(id)); - bool found = cursors->remove(cursorId); + bool found = cursors->remove(cursorId, Cursor::CURSOR_VPACK); if (!found) { generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_CURSOR_NOT_FOUND); diff --git a/arangod/RestHandler/RestExportHandler.cpp b/arangod/RestHandler/RestExportHandler.cpp index 79fa91858f..669b87dc65 100644 --- a/arangod/RestHandler/RestExportHandler.cpp +++ b/arangod/RestHandler/RestExportHandler.cpp @@ -306,7 +306,7 @@ void RestExportHandler::modifyCursor() { auto cursorId = static_cast( arangodb::basics::StringUtils::uint64(id)); bool busy; - auto cursor = cursors->find(cursorId, busy); + auto cursor = cursors->find(cursorId, Cursor::CURSOR_EXPORT, busy); if (cursor == nullptr) { if (busy) { @@ -356,7 +356,7 @@ void RestExportHandler::deleteCursor() { auto cursorId = static_cast( arangodb::basics::StringUtils::uint64(id)); - bool found = cursors->remove(cursorId); + bool found = cursors->remove(cursorId, Cursor::CURSOR_EXPORT); if (!found) { generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_CURSOR_NOT_FOUND); diff --git a/arangod/Utils/Cursor.h b/arangod/Utils/Cursor.h index 362e1f1df4..481b1de02c 100644 --- a/arangod/Utils/Cursor.h +++ b/arangod/Utils/Cursor.h @@ -44,6 +44,11 @@ typedef TRI_voc_tick_t CursorId; class Cursor { public: + enum CursorType { + CURSOR_VPACK, + CURSOR_EXPORT + }; + Cursor(Cursor const&) = delete; Cursor& operator=(Cursor const&) = delete; @@ -90,6 +95,8 @@ class Cursor { _isUsed = false; } + virtual CursorType type() const = 0; + virtual bool hasNext() = 0; virtual arangodb::velocypack::Slice next() = 0; @@ -110,7 +117,7 @@ class Cursor { bool _isUsed; }; -class VelocyPackCursor : public Cursor { +class VelocyPackCursor final : public Cursor { public: VelocyPackCursor(TRI_vocbase_t*, CursorId, aql::QueryResult&&, size_t, std::shared_ptr, double, @@ -120,6 +127,8 @@ class VelocyPackCursor : public Cursor { public: aql::QueryResult const* result() const { return &_result; } + + CursorType type() const override final { return CURSOR_VPACK; } bool hasNext() override final; @@ -136,7 +145,7 @@ class VelocyPackCursor : public Cursor { bool _cached; }; -class ExportCursor : public Cursor { +class ExportCursor final : public Cursor { public: ExportCursor(TRI_vocbase_t*, CursorId, arangodb::CollectionExport*, size_t, double, bool); @@ -144,6 +153,8 @@ class ExportCursor : public Cursor { ~ExportCursor(); public: + CursorType type() const override final { return CURSOR_EXPORT; } + bool hasNext() override final; arangodb::velocypack::Slice next() override final; diff --git a/arangod/Utils/CursorRepository.cpp b/arangod/Utils/CursorRepository.cpp index 44e3b0e401..41c3140f05 100644 --- a/arangod/Utils/CursorRepository.cpp +++ b/arangod/Utils/CursorRepository.cpp @@ -140,7 +140,7 @@ ExportCursor* CursorRepository::createFromExport(arangodb::CollectionExport* ex, /// @brief remove a cursor by id //////////////////////////////////////////////////////////////////////////////// -bool CursorRepository::remove(CursorId id) { +bool CursorRepository::remove(CursorId id, Cursor::CursorType type) { arangodb::Cursor* cursor = nullptr; { @@ -159,6 +159,11 @@ bool CursorRepository::remove(CursorId id) { return false; } + if (cursor->type() != type) { + // wrong type + return false; + } + if (cursor->isUsed()) { // cursor is in use by someone else. now mark as deleted cursor->deleted(); @@ -181,7 +186,7 @@ bool CursorRepository::remove(CursorId id) { /// it must be returned later using release() //////////////////////////////////////////////////////////////////////////////// -Cursor* CursorRepository::find(CursorId id, bool& busy) { +Cursor* CursorRepository::find(CursorId id, Cursor::CursorType type, bool& busy) { arangodb::Cursor* cursor = nullptr; busy = false; @@ -201,6 +206,11 @@ Cursor* CursorRepository::find(CursorId id, bool& busy) { return nullptr; } + if (cursor->type() != type) { + // wrong cursor type + return nullptr; + } + if (cursor->isUsed()) { busy = true; return nullptr; diff --git a/arangod/Utils/CursorRepository.h b/arangod/Utils/CursorRepository.h index 1d2ae7439b..dcc21c790c 100644 --- a/arangod/Utils/CursorRepository.h +++ b/arangod/Utils/CursorRepository.h @@ -79,7 +79,7 @@ class CursorRepository { /// @brief remove a cursor by id ////////////////////////////////////////////////////////////////////////////// - bool remove(CursorId); + bool remove(CursorId, Cursor::CursorType); ////////////////////////////////////////////////////////////////////////////// /// @brief find an existing cursor by id @@ -87,7 +87,7 @@ class CursorRepository { /// it must be returned later using release() ////////////////////////////////////////////////////////////////////////////// - Cursor* find(CursorId, bool&); + Cursor* find(CursorId, Cursor::CursorType, bool&); ////////////////////////////////////////////////////////////////////////////// /// @brief return a cursor diff --git a/arangod/V8Server/v8-voccursor.cpp b/arangod/V8Server/v8-voccursor.cpp index c4727be5ad..8463434d83 100644 --- a/arangod/V8Server/v8-voccursor.cpp +++ b/arangod/V8Server/v8-voccursor.cpp @@ -137,7 +137,7 @@ static void JS_JsonCursor(v8::FunctionCallbackInfo const& args) { TRI_ASSERT(cursors != nullptr); bool busy; - auto cursor = cursors->find(cursorId, busy); + auto cursor = cursors->find(cursorId, Cursor::CURSOR_VPACK, busy); if (cursor == nullptr) { if (busy) { From d2742f2ef3d296533f8651de0786f2427becf458 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Wed, 7 Dec 2016 12:18:34 +0100 Subject: [PATCH 18/60] fixed issue #2210 --- arangod/Aql/Ast.cpp | 92 ++++++++++++++++++++++++++++++++++++-- arangod/Aql/Ast.h | 5 ++- arangod/Aql/AstNode.cpp | 62 +++---------------------- arangod/Aql/Condition.cpp | 10 ++++- arangod/Aql/IndexBlock.cpp | 30 +++++++++++-- 5 files changed, 133 insertions(+), 66 deletions(-) diff --git a/arangod/Aql/Ast.cpp b/arangod/Aql/Ast.cpp index 19ab95009a..0a04312fe0 100644 --- a/arangod/Aql/Ast.cpp +++ b/arangod/Aql/Ast.cpp @@ -1634,6 +1634,7 @@ void Ast::validateAndOptimize() { struct TraversalContext { std::unordered_set writeCollectionsSeen; std::unordered_map collectionsFirstSeen; + std::unordered_map variableDefinitions; int64_t stopOptimizationRequests = 0; int64_t nestingLevel = 0; bool isInFilter = false; @@ -1769,7 +1770,7 @@ void Ast::validateAndOptimize() { // attribute access if (node->type == NODE_TYPE_ATTRIBUTE_ACCESS) { - return this->optimizeAttributeAccess(node); + return this->optimizeAttributeAccess(node, static_cast(data)->variableDefinitions); } // passthru node @@ -1813,6 +1814,22 @@ void Ast::validateAndOptimize() { // LET if (node->type == NODE_TYPE_LET) { + // remember variable assignments + TRI_ASSERT(node->numMembers() == 2); + auto context = static_cast(data); + Variable const* variable = static_cast(node->getMember(0)->getData()); + AstNode const* definition = node->getMember(1); + // recursively process assignments so we can track LET a = b LET c = b + + while (definition->type == NODE_TYPE_REFERENCE) { + auto it = context->variableDefinitions.find(static_cast(definition->getData())); + if (it == context->variableDefinitions.end()) { + break; + } + definition = (*it).second; + } + + context->variableDefinitions.emplace(variable, definition); return this->optimizeLet(node); } @@ -2669,12 +2686,21 @@ AstNode* Ast::optimizeTernaryOperator(AstNode* node) { } /// @brief optimizes an attribute access -AstNode* Ast::optimizeAttributeAccess(AstNode* node) { +AstNode* Ast::optimizeAttributeAccess(AstNode* node, std::unordered_map const& variableDefinitions) { TRI_ASSERT(node != nullptr); TRI_ASSERT(node->type == NODE_TYPE_ATTRIBUTE_ACCESS); TRI_ASSERT(node->numMembers() == 1); - AstNode* what = node->getMember(0); + AstNode const* what = node->getMember(0); + + if (what->type == NODE_TYPE_REFERENCE) { + // check if the access value is a variable and if it is an alias + auto it = variableDefinitions.find(static_cast(what->getData())); + + if (it != variableDefinitions.end()) { + what = (*it).second; + } + } if (!what->isConstant()) { return node; @@ -2689,6 +2715,7 @@ AstNode* Ast::optimizeAttributeAccess(AstNode* node) { for (size_t i = 0; i < n; ++i) { AstNode const* member = what->getMember(0); + if (member->type == NODE_TYPE_OBJECT_ELEMENT && member->getStringLength() == length && memcmp(name, member->getStringValue(), length) == 0) { @@ -2942,6 +2969,8 @@ AstNode* Ast::nodeFromVPack(VPackSlice const& slice, bool copyStringValues) { for (auto const& it : VPackArrayIterator(slice)) { node->addMember(nodeFromVPack(it, copyStringValues)); } + + node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT); return node; } @@ -2963,6 +2992,8 @@ AstNode* Ast::nodeFromVPack(VPackSlice const& slice, bool copyStringValues) { node->addMember(createNodeObjectElement( attributeName, static_cast(nameLength), nodeFromVPack(it.value, copyStringValues))); } + + node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT); return node; } @@ -2970,6 +3001,61 @@ AstNode* Ast::nodeFromVPack(VPackSlice const& slice, bool copyStringValues) { return createNodeValueNull(); } +/// @brief resolve an attribute access +AstNode const* Ast::resolveConstAttributeAccess(AstNode const* node) { + TRI_ASSERT(node != nullptr); + TRI_ASSERT(node->type == NODE_TYPE_ATTRIBUTE_ACCESS); + + std::vector attributeNames; + + while (node->type == NODE_TYPE_ATTRIBUTE_ACCESS) { + attributeNames.emplace_back(node->getString()); + node = node->getMember(0); + } + + size_t which = attributeNames.size(); + TRI_ASSERT(which > 0); + + while (which > 0) { + TRI_ASSERT(node->type == NODE_TYPE_VALUE || node->type == NODE_TYPE_ARRAY || + node->type == NODE_TYPE_OBJECT); + + bool found = false; + + if (node->type == NODE_TYPE_OBJECT) { + TRI_ASSERT(which > 0); + std::string const& attributeName = attributeNames[which - 1]; + --which; + + size_t const n = node->numMembers(); + for (size_t i = 0; i < n; ++i) { + auto member = node->getMember(i); + + if (member->type == NODE_TYPE_OBJECT_ELEMENT && + member->getString() == attributeName) { + // found the attribute + node = member->getMember(0); + if (which == 0) { + // we found what we looked for + return node; + } + // we found the correct attribute but there is now an attribute + // access on the result + found = true; + break; + } + } + } + + if (!found) { + break; + } + } + + // attribute not found or non-array + return createNodeValueNull(); +} + /// @brief traverse the AST, using pre- and post-order visitors AstNode* Ast::traverseAndModify( AstNode* node, std::function preVisitor, diff --git a/arangod/Aql/Ast.h b/arangod/Aql/Ast.h index 267e3fe5f7..7595ef303f 100644 --- a/arangod/Aql/Ast.h +++ b/arangod/Aql/Ast.h @@ -412,6 +412,9 @@ class Ast { /// @brief create an AST node from vpack AstNode* nodeFromVPack(arangodb::velocypack::Slice const&, bool); + + /// @brief resolve an attribute access + static AstNode const* resolveConstAttributeAccess(AstNode const*); /// @brief traverse the AST using a depth-first visitor static AstNode* traverseAndModify(AstNode*, @@ -457,7 +460,7 @@ class Ast { AstNode* optimizeTernaryOperator(AstNode*); /// @brief optimizes an attribute access - AstNode* optimizeAttributeAccess(AstNode*); + AstNode* optimizeAttributeAccess(AstNode*, std::unordered_map const&); /// @brief optimizes a call to a built-in function AstNode* optimizeFunctionCall(AstNode*); diff --git a/arangod/Aql/AstNode.cpp b/arangod/Aql/AstNode.cpp index 96e20b8154..c785aabe8c 100644 --- a/arangod/Aql/AstNode.cpp +++ b/arangod/Aql/AstNode.cpp @@ -187,61 +187,6 @@ std::unordered_map const AstNode::ValueTypeNames{ namespace { -/// @brief resolve an attribute access -static AstNode const* ResolveAttribute(AstNode const* node) { - TRI_ASSERT(node != nullptr); - TRI_ASSERT(node->type == NODE_TYPE_ATTRIBUTE_ACCESS); - - std::vector attributeNames; - - while (node->type == NODE_TYPE_ATTRIBUTE_ACCESS) { - attributeNames.emplace_back(node->getString()); - node = node->getMember(0); - } - - size_t which = attributeNames.size(); - TRI_ASSERT(which > 0); - - while (which > 0) { - TRI_ASSERT(node->type == NODE_TYPE_VALUE || node->type == NODE_TYPE_ARRAY || - node->type == NODE_TYPE_OBJECT); - - bool found = false; - - if (node->type == NODE_TYPE_OBJECT) { - TRI_ASSERT(which > 0); - std::string const& attributeName = attributeNames[which - 1]; - --which; - - size_t const n = node->numMembers(); - for (size_t i = 0; i < n; ++i) { - auto member = node->getMember(i); - - if (member->type == NODE_TYPE_OBJECT_ELEMENT && - member->getString() == attributeName) { - // found the attribute - node = member->getMember(0); - if (which == 0) { - // we found what we looked for - return node; - } - // we found the correct attribute but there is now an attribute - // access on the result - found = true; - break; - } - } - } - - if (!found) { - break; - } - } - - // attribute not found or non-array - return Ast::createNodeValueNull(); -} - /// @brief get the node type for inter-node comparisons static VPackValueType GetNodeCompareType(AstNode const* node) { TRI_ASSERT(node != nullptr); @@ -276,10 +221,10 @@ int arangodb::aql::CompareAstNodes(AstNode const* lhs, AstNode const* rhs, TRI_ASSERT(rhs != nullptr); if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) { - lhs = ResolveAttribute(lhs); + lhs = Ast::resolveConstAttributeAccess(lhs); } if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) { - rhs = ResolveAttribute(rhs); + rhs = Ast::resolveConstAttributeAccess(rhs); } auto lType = GetNodeCompareType(lhs); @@ -419,6 +364,7 @@ AstNode::AstNode(bool v, AstNodeValueType valueType) value.value._bool = v; TRI_ASSERT(flags == 0); TRI_ASSERT(computedValue == nullptr); + setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT); } /// @brief create an int node, with defining a value @@ -428,6 +374,7 @@ AstNode::AstNode(int64_t v, AstNodeValueType valueType) value.value._int = v; TRI_ASSERT(flags == 0); TRI_ASSERT(computedValue == nullptr); + setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT); } /// @brief create a string node, with defining a value @@ -437,6 +384,7 @@ AstNode::AstNode(char const* v, size_t length, AstNodeValueType valueType) setStringValue(v, length); TRI_ASSERT(flags == 0); TRI_ASSERT(computedValue == nullptr); + setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT); } /// @brief create the node from VPack diff --git a/arangod/Aql/Condition.cpp b/arangod/Aql/Condition.cpp index 2a186a6919..7af12546d6 100644 --- a/arangod/Aql/Condition.cpp +++ b/arangod/Aql/Condition.cpp @@ -722,14 +722,20 @@ void Condition::optimize(ExecutionPlan* plan) { auto operand = andNode->getMemberUnchecked(j); if (operand->isComparisonOperator()) { - auto lhs = operand->getMember(0); - auto rhs = operand->getMember(1); + AstNode const* lhs = operand->getMember(0); + AstNode const* rhs = operand->getMember(1); if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) { + if (lhs->isConstant()) { + lhs = Ast::resolveConstAttributeAccess(lhs); + } storeAttributeAccess(varAccess, variableUsage, lhs, j, ATTRIBUTE_LEFT); } if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS || rhs->type == NODE_TYPE_EXPANSION) { + if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS && rhs->isConstant()) { + rhs = Ast::resolveConstAttributeAccess(rhs); + } storeAttributeAccess(varAccess, variableUsage, rhs, j, ATTRIBUTE_RIGHT); } } diff --git a/arangod/Aql/IndexBlock.cpp b/arangod/Aql/IndexBlock.cpp index 7a54317ba3..5dddd72cc9 100644 --- a/arangod/Aql/IndexBlock.cpp +++ b/arangod/Aql/IndexBlock.cpp @@ -53,6 +53,30 @@ IndexBlock::IndexBlock(ExecutionEngine* engine, IndexNode const* en) _hasV8Expression(false) { _mmdr.reset(new ManagedDocumentResult(transaction())); + + if (_condition != nullptr) { + // fix const attribute accesses, e.g. { "a": 1 }.a + for (size_t i = 0; i < _condition->numMembers(); ++i) { + auto andCond = _condition->getMemberUnchecked(i); + for (size_t j = 0; j < andCond->numMembers(); ++j) { + auto leaf = andCond->getMemberUnchecked(j); + + // We only support binary conditions + TRI_ASSERT(leaf->numMembers() == 2); + AstNode* lhs = leaf->getMember(0); + AstNode* rhs = leaf->getMember(1); + + if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS && lhs->isConstant()) { + lhs = const_cast(Ast::resolveConstAttributeAccess(lhs)); + leaf->changeMember(0, lhs); + } + if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS && rhs->isConstant()) { + rhs = const_cast(Ast::resolveConstAttributeAccess(rhs)); + leaf->changeMember(1, rhs); + } + } + } + } } IndexBlock::~IndexBlock() { @@ -126,7 +150,7 @@ int IndexBlock::initialize() { auto en = static_cast(getPlanNode()); auto ast = en->_plan->getAst(); - + // instantiate expressions: auto instantiateExpression = [&](size_t i, size_t j, size_t k, AstNode* a) -> void { @@ -176,8 +200,8 @@ int IndexBlock::initialize() { // We only support binary conditions TRI_ASSERT(leaf->numMembers() == 2); - auto lhs = leaf->getMember(0); - auto rhs = leaf->getMember(1); + AstNode* lhs = leaf->getMember(0); + AstNode* rhs = leaf->getMember(1); if (lhs->isAttributeAccessForVariable(outVariable, false)) { // Index is responsible for the left side, check if right side has to be From 5911f14bbaacabb40819c0fd98fdb1bcb03c7c74 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Wed, 7 Dec 2016 13:12:02 +0100 Subject: [PATCH 19/60] updated CHANGELOG --- CHANGELOG | 23 ++++++++++++++++---- js/server/tests/aql/aql-optimizer-indexes.js | 23 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 09bce453c5..67a7054bab 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -35,7 +35,7 @@ v3.1.4 (XXXX-XX-XX) * fixed issue #2204 -v3.1.3 (xxxx-xx-xx) +v3.1.3 (2016-12-02) ------------------- * fix a traversal bug when using skiplist indexes: @@ -48,7 +48,8 @@ v3.1.3 (xxxx-xx-xx) * fix endless loop when trying to create a collection with replicationFactor: -1 -v3.1.2 (20XX-XX-XX) + +v3.1.2 (2016-11-24) ------------------- * added support for descriptions field in Foxx dependencies @@ -58,7 +59,7 @@ Now they state correctly how many documents were fetched from the index and how have been filtered. -v3.1.1 (XXXX-XX-XX) +v3.1.1 (2016-11-15) ------------------- * fixed issue #2176 @@ -352,13 +353,27 @@ v3.1.0 (2016-10-29) * fixed issue #2156 -v3.0.12 (XXXX-XX-XX) +v3.0.13 (XXXX-XX-XX) -------------------- +* fixed issue #2210 + + +v3.0.12 (2016-11-23) +-------------------- + +* fixed issue #2176 + +* fixed issue #2168 + * fixed issues #2149, #2159 * fixed error reporting for issue #2158 +* fixed assembly linkage bug in CRC4 module + +* added support for descriptions field in Foxx dependencies + v3.0.11 (2016-11-08) -------------------- diff --git a/js/server/tests/aql/aql-optimizer-indexes.js b/js/server/tests/aql/aql-optimizer-indexes.js index a793ac8796..344cd23beb 100644 --- a/js/server/tests/aql/aql-optimizer-indexes.js +++ b/js/server/tests/aql/aql-optimizer-indexes.js @@ -54,6 +54,29 @@ function optimizerIndexesTestSuite () { db._drop("UnitTestsCollection"); }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test same results for const access queries +//////////////////////////////////////////////////////////////////////////////// + + testSameResultsConstAccess : function () { + var bind = { doc : { key: "test1" } }; + var q1 = `RETURN (FOR item IN UnitTestsCollection FILTER (@doc.key == item._key) LIMIT 1 RETURN item)[0]`; + var q2 = `LET doc = @doc RETURN (FOR item IN UnitTestsCollection FILTER (doc.key == item._key) LIMIT 1 RETURN item)[0]`; + var q3 = `LET doc = { key: "test1" } RETURN (FOR item IN UnitTestsCollection FILTER (doc.key == item._key) LIMIT 1 RETURN item)[0]`; + + var results = AQL_EXECUTE(q1, bind); + assertEqual(1, results.json.length); + assertEqual("test1", results.json[0]._key); + + results = AQL_EXECUTE(q2, bind); + assertEqual(1, results.json.length); + assertEqual("test1", results.json[0]._key); + + results = AQL_EXECUTE(q3); + assertEqual(1, results.json.length); + assertEqual("test1", results.json[0]._key); + }, + //////////////////////////////////////////////////////////////////////////////// /// @brief test _id //////////////////////////////////////////////////////////////////////////////// From b20de61ae8a22f6bcb6da738c043cd87f325dea7 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 14:29:48 +0100 Subject: [PATCH 20/60] Useful logging in failure mode of inception's persistence --- arangod/Agency/FailedFollower.cpp | 7 +--- arangod/Agency/Inception.cpp | 62 ++++++++++++++++--------------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/arangod/Agency/FailedFollower.cpp b/arangod/Agency/FailedFollower.cpp index 4208d03c3c..67fe1f7788 100644 --- a/arangod/Agency/FailedFollower.cpp +++ b/arangod/Agency/FailedFollower.cpp @@ -73,15 +73,12 @@ bool FailedFollower::create() { if (!myClones.empty()) { size_t sub = 0; - auto myshards = _snapshot( - planColPrefix + _database + "/" + _collection + "/shards").children(); + auto myshards = _snapshot(planPath).children(); auto mpos = std::distance(myshards.begin(), myshards.find(_shard)); // Deal with my clones for (auto const& collection : myClones) { - auto othershards = - _snapshot(planColPrefix + _database + "/" + collection + "/shards") - .children(); + auto othershards = _snapshot(planPath).children(); auto opos = othershards.begin(); std::advance(opos, mpos); auto const& shard = opos->first; diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 0e3f7c5894..85c854109f 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -51,9 +51,10 @@ Inception::~Inception() { shutdown(); } void Inception::gossip() { LOG_TOPIC(INFO, Logger::AGENCY) << "Entering gossip phase ..."; + using namespace std::chrono; - auto s = std::chrono::system_clock::now(); - std::chrono::seconds timeout(3600); + auto startTime = system_clock::now(); + seconds timeout(3600); size_t j = 0; long waitInterval = 250000; @@ -61,11 +62,11 @@ void Inception::gossip() { while (!this->isStopping() && !_agent->isStopping()) { - config_t config = _agent->config(); // get a copy of conf - size_t version = config.version(); + auto const config = _agent->config(); // get a copy of conf + auto const version = config.version(); // Build gossip message - query_t out = std::make_shared(); + auto out = std::make_shared(); out->openObject(); out->add("endpoint", VPackValue(config.endpoint())); out->add("id", VPackValue(config.id())); @@ -76,7 +77,7 @@ void Inception::gossip() { out->close(); out->close(); - std::string path = privApiPrefix + "gossip"; + auto const path = privApiPrefix + "gossip"; // gossip peers for (auto const& p : config.gossipPeers()) { @@ -110,7 +111,7 @@ void Inception::gossip() { } } complete = false; - std::string clientid = config.id() + std::to_string(j++); + auto const clientid = config.id() + std::to_string(j++); auto hf = std::make_unique>(); LOG_TOPIC(DEBUG, Logger::AGENCY) << "Sending gossip message: " @@ -133,7 +134,7 @@ void Inception::gossip() { } // Timed out? :( - if ((std::chrono::system_clock::now() - s) > timeout) { + if ((system_clock::now() - startTime) > timeout) { if (config.poolComplete()) { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Stopping active gossipping!"; } else { @@ -156,34 +157,36 @@ void Inception::gossip() { bool Inception::restartingActiveAgent() { - auto const path = pubApiPrefix + "config"; + LOG_TOPIC(INFO, Logger::AGENCY) << "Restarting agent from persistence ..."; - auto myConfig = _agent->config(); - auto startTime = std::chrono::system_clock::now(); - auto pool = myConfig.pool(); - auto active = myConfig.active(); - auto activeOrig = active; - auto clientId = myConfig.id(); - auto majority = (myConfig.size()+1)/2; + using namespace std::chrono; + auto const path = pubApiPrefix + "config"; + auto const myConfig = _agent->config(); + auto const startTime = system_clock::now(); + auto pool = myConfig.pool(); + auto active = myConfig.active(); + auto const& clientId = myConfig.id(); + auto const majority = (myConfig.size()+1)/2; - std::chrono::seconds timeout(3600); + seconds const timeout(3600); CONDITION_LOCKER(guard, _cv); long waitInterval(500000); + active.erase( + std::remove(active.begin(), active.end(), myConfig.id()), active.end()); + while (!this->isStopping() && !_agent->isStopping()) { - active.erase( - std::remove(active.begin(), active.end(), myConfig.id()), active.end()); active.erase( std::remove(active.begin(), active.end(), ""), active.end()); if (active.size() < majority) { LOG_TOPIC(INFO, Logger::AGENCY) - << "Found majority of agents in agreement over active pool" - " finishing startup sequence."; + << "Found majority of agents in agreement over active pool. " + "Finishing startup sequence."; return true; } @@ -198,7 +201,7 @@ bool Inception::restartingActiveAgent() { if (comres->status == CL_COMM_SENT) { try { - auto const theirConfigVP = comres->result->getBodyVelocyPack(); + auto const theirConfigVP = comres->result->getBodyVelocyPack(); auto const& theirConfig = theirConfigVP->slice(); auto const& theirLeaderId = theirConfig.get("leaderId").copyString(); auto const& tcc = theirConfig.get("configuration"); @@ -206,7 +209,8 @@ bool Inception::restartingActiveAgent() { // Known leader. We are done. if (!theirLeaderId.empty()) { LOG_TOPIC(INFO, Logger::AGENCY) << - "Found active RAFTing agency lead by " << theirLeaderId; + "Found active RAFTing agency lead by " << theirLeaderId << + "Finishing startup sequence."; auto agency = std::make_shared(); agency->openObject(); agency->add("term", theirConfig.get("term")); @@ -220,15 +224,14 @@ bool Inception::restartingActiveAgent() { return true; } - auto theirActive = tcc.get("active").toJson(); - auto myActive = myConfig.activeToBuilder()->toJson(); + auto const theirActive = tcc.get("active").toJson(); + auto const myActive = myConfig.activeToBuilder()->toJson(); auto i = std::find(active.begin(),active.end(),p.first); if (i != active.end()) { if (theirActive != myActive) { LOG_TOPIC(FATAL, Logger::AGENCY) - << "Assumed active RAFT peer and I disagree on active " - "membership:"; + << "Assumed active RAFT peer and I disagree on active membership:"; LOG_TOPIC(FATAL, Logger::AGENCY) << "Their active list is " << theirActive; LOG_TOPIC(FATAL, Logger::AGENCY) @@ -253,7 +256,7 @@ bool Inception::restartingActiveAgent() { } // Timed out? :( - if ((std::chrono::system_clock::now() - startTime) > timeout) { + if ((system_clock::now() - startTime) > timeout) { if (myConfig.poolComplete()) { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Joined complete pool!"; } else { @@ -321,7 +324,7 @@ bool Inception::estimateRAFTInterval() { auto config = _agent->config(); auto myid = _agent->id(); - auto to = std::chrono::duration(1.0); // + auto to = duration(1.0); // for (size_t i = 0; i < nrep; ++i) { for (auto const& peer : config.pool()) { @@ -492,7 +495,6 @@ void Inception::run() { } // Gossip - config = _agent->config(); gossip(); // No complete pool after gossip? From 1b7a0eec23e029f5a403ea2d918863d3c4348e4c Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 7 Dec 2016 14:52:45 +0100 Subject: [PATCH 21/60] finalize ARM crosscompiling --- 3rdParty/CMakeLists.txt | 7 ++++--- Installation/Jenkins/build.sh | 38 +++++++++++++++-------------------- scripts/build-xc-deb.sh | 29 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 25 deletions(-) create mode 100755 scripts/build-xc-deb.sh diff --git a/3rdParty/CMakeLists.txt b/3rdParty/CMakeLists.txt index 52ca321e97..64ca1ae997 100644 --- a/3rdParty/CMakeLists.txt +++ b/3rdParty/CMakeLists.txt @@ -99,9 +99,10 @@ set(CMAKE_USE_OPENSSL ON CACHE type BOOL) # mop: super important...if this is off curl will not handle request timeouts < 1000ms set(ENABLE_THREADED_RESOLVER ON CACHE type BOOL) # bugfix for HAVE_POSIX_STRERROR_R define on cross compiling. -if (CROSS_COMPILING) - add_definitions("-DHAVE_POSIX_STRERROR_R=1") -endif() +#if (CROSS_COMPILING) +# add_definitions("-DHAVE_POSIX_STRERROR_R=1") +# add_definitions("-DHAVE_POSIX_STRERROR_R__TRYRUN_OUTPUT=TRUE") +#endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/curl/curl-7.50.3) ################################################################################ diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index f8e73ea977..4cfaa7310a 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -323,13 +323,9 @@ while [ $# -gt 0 ]; do CLEAN_IT=1 shift ;; - --xcArmV8) - ARMV8=1 - XCGCC=1 + --xcArm) shift - ;; - --xcArmV7) - ARMV7=1 + TOOL_PREFIX=$1 XCGCC=1 shift ;; @@ -390,18 +386,18 @@ elif [ "$CLANG36" == 1 ]; then CXXFLAGS="${CXXFLAGS} -std=c++11" elif [ "${XCGCC}" = 1 ]; then USE_JEMALLOC=0 - if [ "${ARMV8}" = 1 ]; then - export TOOL_PREFIX=aarch64-linux-gnu - BUILD_DIR="${BUILD_DIR}-ARMV8" - elif [ "${ARMV7}" = 1 ]; then - export TOOL_PREFIX=aarch64-linux-gnu - BUILD_DIR="${BUILD_DIR}-ARMV7" - else - echo "Unknown XC-Compiler!" - exit 1; - fi + BUILD_DIR="${BUILD_DIR}-${TOOL_PREFIX}" - CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true" # -DCMAKE_LIBRARY_ARCHITECTURE=${TOOL_PREFIX} " + # tell cmake we're cross compiling: + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true -DCMAKE_SYSTEM_NAME=Linux" + # -DCMAKE_LIBRARY_ARCHITECTURE=${TOOL_PREFIX} " + # these options would be evaluated using TRY_RUN(), which obviously doesn't work: + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DHAVE_POLL_FINE_EXITCODE=0" + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DHAVE_GLIBC_STRERROR_R=0" + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DHAVE_GLIBC_STRERROR_R__TRYRUN_OUTPUT=TRUE" + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DHAVE_POSIX_STRERROR_R=1" + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DHAVE_POSIX_STRERROR_R__TRYRUN_OUTPUT=FALSE" + export CXX=$TOOL_PREFIX-g++ export AR=$TOOL_PREFIX-ar export RANLIB=$TOOL_PREFIX-ranlib @@ -409,13 +405,9 @@ elif [ "${XCGCC}" = 1 ]; then export LD=$TOOL_PREFIX-g++ export LINK=$TOOL_PREFIX-g++ export STRIP=$TOOL_PREFIX-strip - # we need ARM LD: GOLD=0; - # tell cmake we're cross compiling: - CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true" - # V8's mksnapshot won't work - ignore it: MAKE_PARAMS="${MAKE_PARAMS} -i" fi @@ -470,9 +462,11 @@ if [ -z "${MSVC}" ]; then if [ ! -f ${STRIP} ] ; then STRIP=`which strip` fi - CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCMAKE_STRIP=${STRIP}" export STRIP fi + if test -n "${STRIP}"; then + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCMAKE_STRIP=${STRIP}" + fi fi CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} ${MAINTAINER_MODE}" diff --git a/scripts/build-xc-deb.sh b/scripts/build-xc-deb.sh new file mode 100755 index 0000000000..6274317aa7 --- /dev/null +++ b/scripts/build-xc-deb.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd ${DIR}/.. + +EP="" +for i in $@; do + if test "$i" == "--enterprise"; then + EP="EP" + fi +done + + +./Installation/Jenkins/build.sh \ + standard \ + --rpath \ + --parallel 25 \ + --package DEB \ + $SNAP \ + --xcArm arm-linux-gnueabihf \ + --buildDir build-${EP}deb \ + --targetDir /var/tmp/ \ + --noopt \ + $@ + +cd ${DIR}/.. From 886ed6492d01744b83b11eb2eb27c7badaa3c006 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 7 Dec 2016 14:53:06 +0100 Subject: [PATCH 22/60] finalize ARM crosscompiling --- 3rdParty/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/3rdParty/CMakeLists.txt b/3rdParty/CMakeLists.txt index 64ca1ae997..d0c4682d26 100644 --- a/3rdParty/CMakeLists.txt +++ b/3rdParty/CMakeLists.txt @@ -98,11 +98,6 @@ set(CMAKE_USE_LIBSSH2 OFF CACHE type BOOL) set(CMAKE_USE_OPENSSL ON CACHE type BOOL) # mop: super important...if this is off curl will not handle request timeouts < 1000ms set(ENABLE_THREADED_RESOLVER ON CACHE type BOOL) -# bugfix for HAVE_POSIX_STRERROR_R define on cross compiling. -#if (CROSS_COMPILING) -# add_definitions("-DHAVE_POSIX_STRERROR_R=1") -# add_definitions("-DHAVE_POSIX_STRERROR_R__TRYRUN_OUTPUT=TRUE") -#endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/curl/curl-7.50.3) ################################################################################ From 3f6eff9baf80d6ac3a60c6b28d71cb28e961a7fb Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 7 Dec 2016 15:16:21 +0100 Subject: [PATCH 23/60] allow empty commits and make shure the enterprise directory is managed properly. --- Installation/release.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Installation/release.sh b/Installation/release.sh index 1eb313efd9..ef8ab04237 100755 --- a/Installation/release.sh +++ b/Installation/release.sh @@ -3,7 +3,7 @@ set -ex SCRIPT_DIR=`dirname $0` SRC_DIR="${SCRIPT_DIR}/../" -ENTERPRISE_SRC_DIR=${SRC_DIR}/enterprise +ENTERPRISE_SRC_DIR=${SRC_DIR}enterprise FORCE_TAG=0 TAG=1 @@ -113,7 +113,7 @@ else fi echo "I'm on Branch: ${GITARGS}" fi -(cd enterprise; git checkout master; git fetch --tags; git pull --all; git checkout ${GITARGS} ) +(cd enterprise; git checkout master; git fetch --tags; git pull --all; git checkout ${GITARGS}; git pull ) @@ -220,7 +220,7 @@ if [ "$TAG" == "1" ]; then fi cd ${ENTERPRISE_SRC_DIR} - git commit -m "release version $VERSION enterprise" -a + git commit --allow-empty -m "release version $VERSION enterprise" -a git push if test "${FORCE_TAG}" == 0; then From 75474ad665b2fbd357544bf15d0e59f372167000 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 7 Dec 2016 15:28:46 +0100 Subject: [PATCH 24/60] fixed docu PUT /_api/simple/by-example --- .../Rest/Simple Queries/JSA_put_api_simple_by_example.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md b/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md index fe27be1bf3..b2f37ab09e 100644 --- a/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md +++ b/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md @@ -17,6 +17,12 @@ The number of documents to skip in the query (optional). The maximal amount of documents to return. The *skip* is applied before the *limit* restriction. (optional) +@RESTBODYPARAM{batchSize,integer,optional,int64} +maximum number of result documents to be transferred from +the server to the client in one roundtrip. If this attribute is +not set, a server-controlled default value will be used. A *batchSize* value of +*0* is disallowed. + @RESTDESCRIPTION This will find all documents matching a given example. From b930b23fc249cf7514c96d68c351882768fd9c47 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 16:20:47 +0100 Subject: [PATCH 25/60] AddFollower jobs for newly arrived db server to satisfy replication factors --- arangod/Agency/AddFollower.cpp | 24 ++++++++++++++++++++++++ arangod/Agency/CleanOutServer.cpp | 2 +- arangod/Agency/FailedFollower.cpp | 3 ++- arangod/Agency/FailedServer.cpp | 2 +- arangod/Agency/Job.cpp | 8 ++++---- arangod/Agency/Job.h | 3 ++- arangod/Agency/RemoveServer.cpp | 2 +- arangod/Agency/Supervision.cpp | 31 +++++++++++++++++++++++++------ 8 files changed, 60 insertions(+), 15 deletions(-) diff --git a/arangod/Agency/AddFollower.cpp b/arangod/Agency/AddFollower.cpp index 21e1d4c9a6..c7fa57165e 100644 --- a/arangod/Agency/AddFollower.cpp +++ b/arangod/Agency/AddFollower.cpp @@ -73,6 +73,30 @@ bool AddFollower::create() { TRI_ASSERT(current[0].isString()); #endif + std::string planPath = + planColPrefix + _database + "/" + _collection + "/shards"; + + auto const& myClones = clones(_snapshot, _database, _collection); + if (!myClones.empty()) { + + size_t sub = 0; + auto myshards = _snapshot(planPath).children(); + auto mpos = std::distance(myshards.begin(), myshards.find(_shard)); + + // Deal with my clones + for (auto const& collection : myClones) { + auto othershards = _snapshot( + planColPrefix + _database + "/" + collection + "/shards").children(); + auto opos = othershards.begin(); + std::advance(opos, mpos); + auto const& shard = opos->first; + + AddFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++), + _jobId, _agencyPrefix, _database, collection, shard, + _newFollower); + } + } + _jb = std::make_shared(); _jb->openArray(); _jb->openObject(); diff --git a/arangod/Agency/CleanOutServer.cpp b/arangod/Agency/CleanOutServer.cpp index bce1fb58c1..c8043511b2 100644 --- a/arangod/Agency/CleanOutServer.cpp +++ b/arangod/Agency/CleanOutServer.cpp @@ -243,7 +243,7 @@ bool CleanOutServer::start() { bool CleanOutServer::scheduleMoveShards() { - std::vector servers = availableServers(); + std::vector servers = availableServers(_snapshot); // Minimum 1 DB server must remain if (servers.size() == 1) { diff --git a/arangod/Agency/FailedFollower.cpp b/arangod/Agency/FailedFollower.cpp index 67fe1f7788..92c0de7629 100644 --- a/arangod/Agency/FailedFollower.cpp +++ b/arangod/Agency/FailedFollower.cpp @@ -78,7 +78,8 @@ bool FailedFollower::create() { // Deal with my clones for (auto const& collection : myClones) { - auto othershards = _snapshot(planPath).children(); + auto othershards = _snapshot( + planColPrefix + _database + "/" + collection + "/shards").children(); auto opos = othershards.begin(); std::advance(opos, mpos); auto const& shard = opos->first; diff --git a/arangod/Agency/FailedServer.cpp b/arangod/Agency/FailedServer.cpp index 6f750a5e2b..2324c0333c 100644 --- a/arangod/Agency/FailedServer.cpp +++ b/arangod/Agency/FailedServer.cpp @@ -146,7 +146,7 @@ bool FailedServer::start() { } } catch (...) {} // Not clone - auto available = availableServers(); + auto available = availableServers(_snapshot); for (auto const& shard : collection("shards").children()) { diff --git a/arangod/Agency/Job.cpp b/arangod/Agency/Job.cpp index 3270b45a91..09de7dac85 100644 --- a/arangod/Agency/Job.cpp +++ b/arangod/Agency/Job.cpp @@ -143,12 +143,12 @@ bool Job::finish(std::string const& type, bool success, } -std::vector Job::availableServers() const { +std::vector Job::availableServers(Node const& snapshot) { std::vector ret; // Get servers from plan - Node::Children const& dbservers = _snapshot(plannedServers).children(); + Node::Children const& dbservers = snapshot(plannedServers).children(); for (auto const& srv : dbservers) { ret.push_back(srv.first); } @@ -156,7 +156,7 @@ std::vector Job::availableServers() const { // Remove cleaned servers from ist try { for (auto const& srv : - VPackArrayIterator(_snapshot(cleanedPrefix).slice())) { + VPackArrayIterator(snapshot(cleanedPrefix).slice())) { ret.erase( std::remove(ret.begin(), ret.end(), srv.copyString()), ret.end()); @@ -167,7 +167,7 @@ std::vector Job::availableServers() const { // Remove failed servers from list try { for (auto const& srv : - VPackArrayIterator(_snapshot(failedServersPrefix).slice())) { + VPackArrayIterator(snapshot(failedServersPrefix).slice())) { ret.erase( std::remove(ret.begin(), ret.end(), srv.copyString()), ret.end()); diff --git a/arangod/Agency/Job.h b/arangod/Agency/Job.h index 5c9fef1ecf..e1406414e3 100644 --- a/arangod/Agency/Job.h +++ b/arangod/Agency/Job.h @@ -109,7 +109,8 @@ struct Job { virtual bool start() = 0; - virtual std::vector availableServers() const; + static std::vector availableServers( + const arangodb::consensus::Node&); static std::vector clones( Node const& snapshot, std::string const& database, diff --git a/arangod/Agency/RemoveServer.cpp b/arangod/Agency/RemoveServer.cpp index 5744ad1e3c..c3d241a80b 100644 --- a/arangod/Agency/RemoveServer.cpp +++ b/arangod/Agency/RemoveServer.cpp @@ -286,7 +286,7 @@ bool RemoveServer::start() { bool RemoveServer::scheduleAddFollowers() { - std::vector servers = availableServers(); + std::vector servers = availableServers(_snapshot); // Minimum 1 DB server must remain if (servers.size() == 1) { diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 4ce39a087d..a3d5187939 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -548,6 +548,7 @@ bool Supervision::handleJobs() { } // Do supervision + shrinkCluster(); workJobs(); enforceReplication(); @@ -614,18 +615,36 @@ void Supervision::workJobs() { void Supervision::enforceReplication() { auto const& plannedDBs = _snapshot(planColPrefix).children(); + auto available = Job::availableServers(_snapshot); for (const auto& db_ : plannedDBs) { // Planned databases auto const& db = *(db_.second); for (const auto& col_ : db.children()) { // Planned collections auto const& col = *(col_.second); auto const& replicationFactor = col("replicationFactor").slice().getUInt(); - for (auto const& shard_ : col("shards").children()) { // Pl shards - auto const& shard = *(shard_.second); - if (replicationFactor != shard.slice().length()) { - LOG(WARN) << shard.slice().typeName() - << " target repl(" << replicationFactor - << ") actual repl(" << shard.slice().length() << ")"; + + bool clone = false; + try { + clone = !col("distributeShardsLike").slice().copyString().empty(); + } catch (...) {} + + if (!clone) { + for (auto const& shard_ : col("shards").children()) { // Pl shards + auto const& shard = *(shard_.second); + + // Enough DBServer to + if (replicationFactor > shard.slice().length() && + available.size() >= replicationFactor) { + for (auto const& i : VPackArrayIterator(shard.slice())) { + available.erase( + std::remove( + available.begin(), available.end(), i.copyString()), + available.end()); + } + AddFollower( + _snapshot, _agent, std::to_string(_jobId++), "supervision", + _agencyPrefix, db_.first, col_.first, shard_.first, available.back()); + } } } } From 9746f6c7add8609847be1df56f476c5556f007b8 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 7 Dec 2016 16:23:54 +0100 Subject: [PATCH 26/60] Finally fix condition for coordinator bootstrap. --- arangod/Cluster/ClusterFeature.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arangod/Cluster/ClusterFeature.cpp b/arangod/Cluster/ClusterFeature.cpp index 944bfe1b59..ee987e50c5 100644 --- a/arangod/Cluster/ClusterFeature.cpp +++ b/arangod/Cluster/ClusterFeature.cpp @@ -308,7 +308,8 @@ void ClusterFeature::prepare() { LOG(INFO) << "Waiting for DBservers to show up..."; ci->loadCurrentDBServers(); std::vector DBServers = ci->getCurrentDBServers(); - if (DBServers.size() > 1 || TRI_microtime() - start > 30.0) { + if (DBServers.size() >= 1 && + (DBServers.size() > 1 || TRI_microtime() - start > 15.0)) { LOG(INFO) << "Found " << DBServers.size() << " DBservers."; break; } From e1b1eb2048e39ce263dde540cc560128d4546a35 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Wed, 7 Dec 2016 16:33:20 +0100 Subject: [PATCH 27/60] finalize xc compile --- Installation/Jenkins/build.sh | 3 ++- cmake/packages/client/deb.txt | 10 +++++++++- cmake/packages/deb.cmake | 2 +- scripts/build-xc-deb.sh | 3 +-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 4cfaa7310a..d0d8f6855e 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -386,7 +386,8 @@ elif [ "$CLANG36" == 1 ]; then CXXFLAGS="${CXXFLAGS} -std=c++11" elif [ "${XCGCC}" = 1 ]; then USE_JEMALLOC=0 - BUILD_DIR="${BUILD_DIR}-${TOOL_PREFIX}" + + BUILD_DIR="${BUILD_DIR}-`basename ${TOOL_PREFIX}`" # tell cmake we're cross compiling: CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true -DCMAKE_SYSTEM_NAME=Linux" diff --git a/cmake/packages/client/deb.txt b/cmake/packages/client/deb.txt index b72fda75c8..4b08be07d4 100644 --- a/cmake/packages/client/deb.txt +++ b/cmake/packages/client/deb.txt @@ -10,6 +10,8 @@ cmake_minimum_required(VERSION 2.8) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "@PROJECT_BINARY_DIR@/bin/") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_X ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CMAKE_INSTALL_DO_STRIP 1) +set(CMAKE_STRIP @CMAKE_STRIP@) +set(CROSS_COMPILING @CROSS_COMPILING@) set(CMAKE_INSTALL_BINDIR @CMAKE_INSTALL_BINDIR@) set(CMAKE_INSTALL_FULL_BINDIR @CMAKE_INSTALL_FULL_BINDIR@) @@ -63,7 +65,13 @@ set(CPACK_DEBIAN_PACKAGE_SECTION "shell") set(CPACK_PACKAGE_VENDOR ${ARANGODB_PACKAGE_VENDOR}) set(CPACK_PACKAGE_CONTACT ${ARANGODB_PACKAGE_CONTACT}) set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${ARANGODB_URL_INFO_ABOUT}) -set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) +# build of dependecies don't work on cross compiling (yet) +if (CROSS_COMPILING) + set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS OFF) + set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.14), libgcc1 (>= 1:3.4), libssl1.0.0 (>= 1.0.1), libstdc++6 (>= 5.2)") +else() + set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) +endif() set(CPACK_DEBIAN_PACKAGE_CONFLICTS "arangodb, ${CPACKG_PACKAGE_CONFLICTS}, ${CPACKG_PACKAGE_CONFLICTS}-client, ${CPACK_PACKAGE_NAME}") set(CPACK_DEBIAN_COMPRESSION_TYPE "xz") set(CPACK_COMPONENTS_ALL debian-extras) diff --git a/cmake/packages/deb.cmake b/cmake/packages/deb.cmake index 7e31cb783a..fdf7fe1d4f 100644 --- a/cmake/packages/deb.cmake +++ b/cmake/packages/deb.cmake @@ -10,7 +10,7 @@ set(PACKAGING_HANDLE_CONFIG_FILES true) FILE(READ "${PROJECT_SOURCE_DIR}/Installation/debian/packagedesc.txt" CPACK_DEBIAN_PACKAGE_DESCRIPTION) set(CPACK_DEBIAN_PACKAGE_SECTION "database") set(CPACK_DEBIAN_PACKAGE_CONFLICTS "arangodb, ${CPACKG_PACKAGE_CONFLICTS}, ${CPACKG_PACKAGE_CONFLICTS}-client, ${CPACK_PACKAGE_NAME}-client") -# build of dependecies (yet) don't works on cross compiling +# build of dependecies don't work on cross compiling (yet) if (CROSS_COMPILING) set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS OFF) set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.14), libgcc1 (>= 1:3.4), libssl1.0.0 (>= 1.0.1), libstdc++6 (>= 5.2)") diff --git a/scripts/build-xc-deb.sh b/scripts/build-xc-deb.sh index 6274317aa7..ed6d757710 100755 --- a/scripts/build-xc-deb.sh +++ b/scripts/build-xc-deb.sh @@ -16,11 +16,10 @@ done ./Installation/Jenkins/build.sh \ standard \ - --rpath \ --parallel 25 \ --package DEB \ $SNAP \ - --xcArm arm-linux-gnueabihf \ + --xcArm /usr/bin/arm-linux-gnueabihf \ --buildDir build-${EP}deb \ --targetDir /var/tmp/ \ --noopt \ From 2fadea8332d4930c963c0e74918cd670426d9d6f Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 16:42:35 +0100 Subject: [PATCH 28/60] AddFollower tests --- arangod/Agency/Supervision.cpp | 4 +++- scripts/startLocalCluster.sh | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index a3d5187939..8c570e5c30 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -641,9 +641,11 @@ void Supervision::enforceReplication() { available.begin(), available.end(), i.copyString()), available.end()); } + auto randIt = available.begin(); + std::advance(randIt, std::rand() % available.size()); AddFollower( _snapshot, _agent, std::to_string(_jobId++), "supervision", - _agencyPrefix, db_.first, col_.first, shard_.first, available.back()); + _agencyPrefix, db_.first, col_.first, shard_.first, *randIt); } } } diff --git a/scripts/startLocalCluster.sh b/scripts/startLocalCluster.sh index 81888f4aa9..f07bc5c8cd 100755 --- a/scripts/startLocalCluster.sh +++ b/scripts/startLocalCluster.sh @@ -223,7 +223,7 @@ start() { --database.directory cluster/data$PORT \ --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \ --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \ - --server.endpoint $TRANSPORT://127.0.0.1:$PORT \ + --server.endpoint $TRANSPORT://0.0.0.0:$PORT \ --cluster.my-role $ROLE \ --log.file cluster/$PORT.log \ --log.level info \ @@ -254,7 +254,7 @@ startTerminal() { --database.directory cluster/data$PORT \ --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \ --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \ - --server.endpoint $TRANSPORT://127.0.0.1:$PORT \ + --server.endpoint $TRANSPORT://0.0.0.0:$PORT \ --cluster.my-role $ROLE \ --log.file cluster/$PORT.log \ --log.level info \ @@ -283,7 +283,7 @@ startDebugger() { --database.directory cluster/data$PORT \ --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \ --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \ - --server.endpoint $TRANSPORT://127.0.0.1:$PORT \ + --server.endpoint $TRANSPORT://0.0.0.0:$PORT \ --cluster.my-role $ROLE \ --log.file cluster/$PORT.log \ --log.level info \ @@ -312,7 +312,7 @@ startRR() { --database.directory cluster/data$PORT \ --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \ --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \ - --server.endpoint $TRANSPORT://127.0.0.1:$PORT \ + --server.endpoint $TRANSPORT://0.0.0.0:$PORT \ --cluster.my-role $ROLE \ --log.file cluster/$PORT.log \ --log.level info \ @@ -397,7 +397,7 @@ if [ "$SECONDARIES" == "1" ] ; then --database.directory cluster/data$PORT \ --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \ --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \ - --server.endpoint $TRANSPORT://127.0.0.1:$PORT \ + --server.endpoint $TRANSPORT://0.0.0.0:$PORT \ --cluster.my-id $CLUSTER_ID \ --log.file cluster/$PORT.log \ --server.statistics true \ From be3b23469c00728084b8d36ac83dbdaecda280f3 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Wed, 7 Dec 2016 16:56:15 +0100 Subject: [PATCH 29/60] more diagnostics --- arangod/VocBase/replication-dump.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/VocBase/replication-dump.cpp b/arangod/VocBase/replication-dump.cpp index 297be4e063..4ed12aef0b 100644 --- a/arangod/VocBase/replication-dump.cpp +++ b/arangod/VocBase/replication-dump.cpp @@ -398,7 +398,6 @@ static int SliceifyMarker(TRI_replication_dump_t* dump, builder.add("rev", slice.get(StaticStrings::RevString)); } // convert 2300 markers to 2301 markers for edges - Append(dump, ",\"type\":"); if (type == TRI_DF_MARKER_VPACK_DOCUMENT && isEdgeCollection) { builder.add("type", VPackValue(2301)); } else { @@ -553,6 +552,7 @@ static int DumpCollection(TRI_replication_dump_t* dump, } if (res != TRI_ERROR_NO_ERROR) { + LOG(ERR) << "got error during dump dump of collection '" << collection->name() << "': " << TRI_errno_string(res); THROW_ARANGO_EXCEPTION(res); } From c10f600ac7a3e3bc933928e3bf53e750f6f328e9 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 7 Dec 2016 13:37:24 +0100 Subject: [PATCH 30/60] Fix potential race --- arangod/Aql/ExecutionEngine.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index 5a8c22f378..f510c9bd8d 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -542,7 +542,8 @@ struct CoordinatorInstanciator : public WalkerWorker { // add the collection result.openObject(); - result.add("name", VPackValue((*auxiliaryCollection->shardIds())[0])); + auto auxiliaryShards = auxiliaryCollection->shardIds(); + result.add("name", VPackValue((*auxiliaryShards)[0])); result.add("type", VPackValue(TRI_TransactionTypeGetStr(collection->accessType))); result.close(); } From 42e5e4318d7f8e82d3c3698b46bfdecd4e2e4407 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 7 Dec 2016 15:21:34 +0100 Subject: [PATCH 31/60] Fix communicator loglevel --- lib/SimpleHttpClient/Communicator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/SimpleHttpClient/Communicator.cpp b/lib/SimpleHttpClient/Communicator.cpp index 50378a10e2..ee9e4fc93d 100644 --- a/lib/SimpleHttpClient/Communicator.cpp +++ b/lib/SimpleHttpClient/Communicator.cpp @@ -379,7 +379,7 @@ void Communicator::handleResult(CURL* handle, CURLcode rc) { } std::string prefix("Communicator(" + std::to_string(rip->_ticketId) + ") // "); - LOG_TOPIC(TRACE, Logger::REQUESTS) + LOG_TOPIC(TRACE, Logger::COMMUNICATION) << prefix << "Curl rc is : " << rc << " after " << Logger::FIXED(TRI_microtime() - rip->_startTime) << " s"; if (strlen(rip->_errorBuffer) != 0) { From 10f7d756514c2bc446a8454a75cb788df3be541b Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Fri, 25 Nov 2016 15:14:24 +0100 Subject: [PATCH 32/60] Fix uniform shard distribution when creating collections --- CHANGELOG | 2 +- arangod/Cluster/ClusterMethods.cpp | 46 +++++++------ .../resilience/shard-distribution-spec.js | 68 +++++++++++++++++++ 3 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 js/server/tests/resilience/shard-distribution-spec.js diff --git a/CHANGELOG b/CHANGELOG index 67a7054bab..2766633a50 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,7 +5,6 @@ devel db._query("FOR i IN 1..100000 SORT i RETURN i", {}, { options: { memoryLimit: 100000 } }); - * added convenience function to create vertex-centric indexes. Usage: `db.collection.ensureVertexCentricIndex("label", {type: "hash", direction: "outbound"})` That will create an index that can be used on OUTBOUND with filtering on the @@ -58,6 +57,7 @@ v3.1.2 (2016-11-24) Now they state correctly how many documents were fetched from the index and how many have been filtered. +* Prevent uniform shard distribution when replicationFactor == numServers v3.1.1 (2016-11-15) ------------------- diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 732192e420..c55ca19900 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -2066,33 +2066,35 @@ std::unordered_map> distributeShards( // fetch a unique id for each shard to create uint64_t const id = ci->uniqid(numberOfShards); - - // now create the shards - size_t count = 0; + + size_t leaderIndex = 0; + size_t followerIndex = 0; for (uint64_t i = 0; i < numberOfShards; ++i) { // determine responsible server(s) std::vector serverIds; for (uint64_t j = 0; j < replicationFactor; ++j) { - std::string candidate; - size_t count2 = 0; - bool found = true; - do { - candidate = dbServers[count++]; - if (count >= dbServers.size()) { - count = 0; - } - if (++count2 == dbServers.size() + 1) { - LOG_TOPIC(WARN, Logger::CLUSTER) - << "createCollectionCoordinator: replicationFactor is " - << "too large for the number of DBservers"; - found = false; - break; - } - } while (std::find(serverIds.begin(), serverIds.end(), candidate) != - serverIds.end()); - if (found) { - serverIds.push_back(candidate); + if (j >= dbServers.size()) { + LOG_TOPIC(WARN, Logger::CLUSTER) + << "createCollectionCoordinator: replicationFactor is " + << "too large for the number of DBservers"; + break; } + std::string candidate; + // mop: leader + if (serverIds.size() == 0) { + candidate = dbServers[leaderIndex++]; + if (leaderIndex >= dbServers.size()) { + leaderIndex = 0; + } + } else { + do { + candidate = dbServers[followerIndex++]; + if (followerIndex >= dbServers.size()) { + followerIndex = 0; + } + } while (candidate == serverIds[0]); // mop: ignore leader + } + serverIds.push_back(candidate); } // determine shard id diff --git a/js/server/tests/resilience/shard-distribution-spec.js b/js/server/tests/resilience/shard-distribution-spec.js new file mode 100644 index 0000000000..628ca592be --- /dev/null +++ b/js/server/tests/resilience/shard-distribution-spec.js @@ -0,0 +1,68 @@ +/* global describe, beforeEach, afterEach, it*/ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2016-2016 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Andreas Streichardt +/// @author Copyright 2016, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// +'use strict'; + +const expect = require('chai').expect; +const internal = require('internal'); +const download = require('internal').download; + +let coordinator = instanceInfo.arangods.filter(arangod => { + return arangod.role === 'coordinator'; +})[0]; + +let dbServerCount = instanceInfo.arangods.filter(arangod => { + return arangod.role === 'dbserver'; +}).length; + +describe('Shard distribution', function () { + afterEach(function() { + internal.db._drop('distributionTest'); + }); + it('should properly distribute a collection', function() { + internal.db._create('distributionTest', {replicationFactor: 2, numberOfShards: 16}); + let distribution = JSON.parse(download(coordinator.url + '/_admin/cluster/shardDistribution').body).results; + + let leaders = Object.keys(distribution.distributionTest.Current).reduce((current, shardKey) => { + let shard = distribution.distributionTest.Current[shardKey]; + if (current.indexOf(shard.leader) === -1) { + current.push(shard.leader); + } + return current; + }, []); + expect(leaders).to.have.length.of.at.least(2); + }); + it('should properly distribute a collection with full replication factor', function() { + internal.db._create('distributionTest', {replicationFactor: dbServerCount, numberOfShards: 16}); + let distribution = JSON.parse(download(coordinator.url + '/_admin/cluster/shardDistribution').body).results; + + let leaders = Object.keys(distribution.distributionTest.Current).reduce((current, shardKey) => { + let shard = distribution.distributionTest.Current[shardKey]; + if (current.indexOf(shard.leader) === -1) { + current.push(shard.leader); + } + return current; + }, []); + expect(leaders).to.have.length.of.at.least(2); + }); +}); From 82682f8d2501dd4438b22f4e9b4b95b3af089866 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 7 Dec 2016 18:37:50 +0100 Subject: [PATCH 33/60] Wait for synchronous replication to settle --- arangod/Aql/EnumerateCollectionBlock.cpp | 29 ++ arangod/Aql/ExecutionNode.cpp | 1 - arangod/CMakeLists.txt | 1 + arangod/Cluster/ClusterInfo.cpp | 223 --------------- arangod/Cluster/ClusterInfo.h | 47 ---- arangod/Cluster/FollowerInfo.cpp | 253 ++++++++++++++++++ arangod/Cluster/FollowerInfo.h | 81 ++++++ .../RestHandler/RestReplicationHandler.cpp | 1 + arangod/Utils/Transaction.cpp | 1 + arangod/V8Server/v8-collection.cpp | 1 + arangod/VocBase/LogicalCollection.cpp | 42 +-- js/common/bootstrap/errors.js | 1 + lib/Basics/errors.dat | 1 + lib/Basics/voc-errors.cpp | 1 + lib/Basics/voc-errors.h | 14 + 15 files changed, 406 insertions(+), 291 deletions(-) create mode 100644 arangod/Cluster/FollowerInfo.cpp create mode 100644 arangod/Cluster/FollowerInfo.h diff --git a/arangod/Aql/EnumerateCollectionBlock.cpp b/arangod/Aql/EnumerateCollectionBlock.cpp index 5c06761f9b..a2dd551dce 100644 --- a/arangod/Aql/EnumerateCollectionBlock.cpp +++ b/arangod/Aql/EnumerateCollectionBlock.cpp @@ -22,11 +22,13 @@ //////////////////////////////////////////////////////////////////////////////// #include "EnumerateCollectionBlock.h" + #include "Aql/AqlItemBlock.h" #include "Aql/Collection.h" #include "Aql/CollectionScanner.h" #include "Aql/ExecutionEngine.h" #include "Basics/Exceptions.h" +#include "Cluster/FollowerInfo.h" #include "VocBase/ManagedDocumentResult.h" #include "VocBase/vocbase.h" @@ -115,6 +117,33 @@ int EnumerateCollectionBlock::initialize() { auto ep = static_cast(_exeNode); _mustStoreResult = ep->isVarUsedLater(ep->_outVariable); + if (_collection->isSatellite()) { + auto logicalCollection = _collection->getCollection(); + auto cid = logicalCollection->planId(); + auto dbName = logicalCollection->dbName(); + auto collectionInfoCurrent = ClusterInfo::instance()->getCollectionCurrent(dbName, std::to_string(cid)); + + bool inSync = false; + unsigned long waitInterval = 10000; + double startTime = TRI_microtime(); + double endTime = startTime + 60.0; + + while (!inSync) { + auto followers = collectionInfoCurrent->servers(_collection->getName()); + inSync = std::find(followers.begin(), followers.end(), ServerState::instance()->getId()) != followers.end(); + if (!inSync) { + usleep(waitInterval); + } + if (TRI_microtime() > endTime) { + break; + } + } + + if (!inSync) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_AQL_COLLECTION_OUT_OF_SYNC, "collection " + _collection->name); + } + } + return ExecutionBlock::initialize(); // cppcheck-suppress style diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 72ead910ce..663d8ee7fc 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -1171,7 +1171,6 @@ void EnumerateCollectionNode::toVelocyPackHelper(VPackBuilder& nodes, nodes.add(VPackValue("outVariable")); _outVariable->toVelocyPack(nodes); nodes.add("random", VPackValue(_random)); - nodes.add("satellite", VPackValue(_collection->isSatellite())); // And close it: nodes.close(); diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index a28f6969f1..beff9553ff 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -182,6 +182,7 @@ SET(ARANGOD_SOURCES Cluster/ClusterInfo.cpp Cluster/ClusterMethods.cpp Cluster/ClusterTraverser.cpp + Cluster/FollowerInfo.cpp Cluster/DBServerAgencySync.cpp Cluster/HeartbeatThread.cpp Cluster/RestAgencyCallbacksHandler.cpp diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index 93043febdb..d0f6ec300e 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -2561,226 +2561,3 @@ std::shared_ptr ClusterInfo::getCurrent() { READ_LOCKER(readLocker, _currentProt.lock); return _current; } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief get information about current followers of a shard. -//////////////////////////////////////////////////////////////////////////////// - -std::shared_ptr const> FollowerInfo::get() { - MUTEX_LOCKER(locker, _mutex); - return _followers; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief change JSON under -/// Current/Collection/// -/// to add or remove a serverID, if add flag is true, the entry is added -/// (if it is not yet there), otherwise the entry is removed (if it was -/// there). -//////////////////////////////////////////////////////////////////////////////// - -static VPackBuilder newShardEntry(VPackSlice oldValue, ServerID const& sid, - bool add) { - VPackBuilder newValue; - VPackSlice servers; - { - VPackObjectBuilder b(&newValue); - // Now need to find the `servers` attribute, which is a list: - for (auto const& it : VPackObjectIterator(oldValue)) { - if (it.key.isEqualString("servers")) { - servers = it.value; - } else { - newValue.add(it.key); - newValue.add(it.value); - } - } - newValue.add(VPackValue("servers")); - if (servers.isArray() && servers.length() > 0) { - VPackArrayBuilder bb(&newValue); - newValue.add(servers[0]); - VPackArrayIterator it(servers); - bool done = false; - for (++it; it.valid(); ++it) { - if ((*it).isEqualString(sid)) { - if (add) { - newValue.add(*it); - done = true; - } - } else { - newValue.add(*it); - } - } - if (add && !done) { - newValue.add(VPackValue(sid)); - } - } else { - VPackArrayBuilder bb(&newValue); - newValue.add(VPackValue(ServerState::instance()->getId())); - if (add) { - newValue.add(VPackValue(sid)); - } - } - } - return newValue; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief add a follower to a shard, this is only done by the server side -/// of the "get-in-sync" capabilities. This reports to the agency under -/// `/Current` but in asynchronous "fire-and-forget" way. -//////////////////////////////////////////////////////////////////////////////// - -void FollowerInfo::add(ServerID const& sid) { - MUTEX_LOCKER(locker, _mutex); - - // Fully copy the vector: - auto v = std::make_shared>(*_followers); - v->push_back(sid); // add a single entry - _followers = v; // will cast to std::vector const - // Now tell the agency, path is - // Current/Collections/// - std::string path = "Current/Collections/"; - path += _docColl->vocbase()->name(); - path += "/"; - path += std::to_string(_docColl->planId()); - path += "/"; - path += _docColl->name(); - AgencyComm ac; - double startTime = TRI_microtime(); - bool success = false; - do { - AgencyCommResult res = ac.getValues(path); - - if (res.successful()) { - velocypack::Slice currentEntry = - res.slice()[0].get(std::vector( - {AgencyCommManager::path(), "Current", "Collections", - _docColl->vocbase()->name(), std::to_string(_docColl->planId()), - _docColl->name()})); - - if (!currentEntry.isObject()) { - LOG_TOPIC(ERR, Logger::CLUSTER) - << "FollowerInfo::add, did not find object in " << path; - if (!currentEntry.isNone()) { - LOG_TOPIC(ERR, Logger::CLUSTER) << "Found: " << currentEntry.toJson(); - } - } else { - auto newValue = newShardEntry(currentEntry, sid, true); - std::string key = "Current/Collections/" + _docColl->vocbase()->name() + - "/" + std::to_string(_docColl->planId()) + "/" + - _docColl->name(); - AgencyWriteTransaction trx; - trx.preconditions.push_back(AgencyPrecondition( - key, AgencyPrecondition::Type::VALUE, currentEntry)); - trx.operations.push_back(AgencyOperation( - key, AgencyValueOperationType::SET, newValue.slice())); - trx.operations.push_back(AgencyOperation( - "Current/Version", AgencySimpleOperationType::INCREMENT_OP)); - AgencyCommResult res2 = ac.sendTransactionWithFailover(trx); - if (res2.successful()) { - success = true; - break; // - } else { - LOG_TOPIC(WARN, Logger::CLUSTER) - << "FollowerInfo::add, could not cas key " << path; - } - } - } else { - LOG_TOPIC(ERR, Logger::CLUSTER) << "FollowerInfo::add, could not read " - << path << " in agency."; - } - usleep(500000); - } while (TRI_microtime() < startTime + 30); - if (!success) { - LOG_TOPIC(ERR, Logger::CLUSTER) - << "FollowerInfo::add, timeout in agency operation for key " << path; - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief remove a follower from a shard, this is only done by the -/// server if a synchronous replication request fails. This reports to -/// the agency under `/Current` but in asynchronous "fire-and-forget" -/// way. The method fails silently, if the follower information has -/// since been dropped (see `dropFollowerInfo` below). -//////////////////////////////////////////////////////////////////////////////// - -void FollowerInfo::remove(ServerID const& sid) { - MUTEX_LOCKER(locker, _mutex); - - auto v = std::make_shared>(); - v->reserve(_followers->size() - 1); - for (auto const& i : *_followers) { - if (i != sid) { - v->push_back(i); - } - } - _followers = v; // will cast to std::vector const - // Now tell the agency, path is - // Current/Collections/// - std::string path = "Current/Collections/"; - path += _docColl->vocbase()->name(); - path += "/"; - path += std::to_string(_docColl->planId()); - path += "/"; - path += _docColl->name(); - AgencyComm ac; - double startTime = TRI_microtime(); - bool success = false; - do { - AgencyCommResult res = ac.getValues(path); - if (res.successful()) { - velocypack::Slice currentEntry = - res.slice()[0].get(std::vector( - {AgencyCommManager::path(), "Current", "Collections", - _docColl->vocbase()->name(), std::to_string(_docColl->planId()), - _docColl->name()})); - - if (!currentEntry.isObject()) { - LOG_TOPIC(ERR, Logger::CLUSTER) - << "FollowerInfo::remove, did not find object in " << path; - if (!currentEntry.isNone()) { - LOG_TOPIC(ERR, Logger::CLUSTER) << "Found: " << currentEntry.toJson(); - } - } else { - auto newValue = newShardEntry(currentEntry, sid, false); - std::string key = "Current/Collections/" + _docColl->vocbase()->name() + - "/" + std::to_string(_docColl->planId()) + "/" + - _docColl->name(); - AgencyWriteTransaction trx; - trx.preconditions.push_back(AgencyPrecondition( - key, AgencyPrecondition::Type::VALUE, currentEntry)); - trx.operations.push_back(AgencyOperation( - key, AgencyValueOperationType::SET, newValue.slice())); - trx.operations.push_back(AgencyOperation( - "Current/Version", AgencySimpleOperationType::INCREMENT_OP)); - AgencyCommResult res2 = ac.sendTransactionWithFailover(trx); - if (res2.successful()) { - success = true; - break; // - } else { - LOG_TOPIC(WARN, Logger::CLUSTER) - << "FollowerInfo::remove, could not cas key " << path; - } - } - } else { - LOG_TOPIC(ERR, Logger::CLUSTER) << "FollowerInfo::remove, could not read " - << path << " in agency."; - } - usleep(500000); - } while (TRI_microtime() < startTime + 30); - if (!success) { - LOG_TOPIC(ERR, Logger::CLUSTER) - << "FollowerInfo::remove, timeout in agency operation for key " << path; - } -} - -////////////////////////////////////////////////////////////////////////////// -/// @brief clear follower list, no changes in agency necesary -////////////////////////////////////////////////////////////////////////////// - -void FollowerInfo::clear() { - MUTEX_LOCKER(locker, _mutex); - auto v = std::make_shared>(); - _followers = v; // will cast to std::vector const -} diff --git a/arangod/Cluster/ClusterInfo.h b/arangod/Cluster/ClusterInfo.h index 20af9e8344..340cadbe58 100644 --- a/arangod/Cluster/ClusterInfo.h +++ b/arangod/Cluster/ClusterInfo.h @@ -650,53 +650,6 @@ class ClusterInfo { std::vector _failedServers; }; -//////////////////////////////////////////////////////////////////////////////// -/// @brief a class to track followers that are in sync for a shard -//////////////////////////////////////////////////////////////////////////////// - -class FollowerInfo { - std::shared_ptr const> _followers; - Mutex _mutex; - arangodb::LogicalCollection* _docColl; - - public: - - explicit FollowerInfo(arangodb::LogicalCollection* d) - : _followers(new std::vector()), _docColl(d) { } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief get information about current followers of a shard. - ////////////////////////////////////////////////////////////////////////////// - - std::shared_ptr const> get(); - - ////////////////////////////////////////////////////////////////////////////// - /// @brief add a follower to a shard, this is only done by the server side - /// of the "get-in-sync" capabilities. This reports to the agency under - /// `/Current` but in asynchronous "fire-and-forget" way. The method - /// fails silently, if the follower information has since been dropped - /// (see `dropFollowerInfo` below). - ////////////////////////////////////////////////////////////////////////////// - - void add(ServerID const& s); - - ////////////////////////////////////////////////////////////////////////////// - /// @brief remove a follower from a shard, this is only done by the - /// server if a synchronous replication request fails. This reports to - /// the agency under `/Current` but in an asynchronous "fire-and-forget" - /// way. - ////////////////////////////////////////////////////////////////////////////// - - void remove(ServerID const& s); - - ////////////////////////////////////////////////////////////////////////////// - /// @brief clear follower list, no changes in agency necesary - ////////////////////////////////////////////////////////////////////////////// - - void clear(); - -}; - } // end namespace arangodb #endif diff --git a/arangod/Cluster/FollowerInfo.cpp b/arangod/Cluster/FollowerInfo.cpp new file mode 100644 index 0000000000..1f39e9e845 --- /dev/null +++ b/arangod/Cluster/FollowerInfo.cpp @@ -0,0 +1,253 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Andreas Streichardt +//////////////////////////////////////////////////////////////////////////////// + +#include "FollowerInfo.h" + +#include "Cluster/ServerState.h" +#include "VocBase/LogicalCollection.h" + +using namespace arangodb; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief get information about current followers of a shard. +//////////////////////////////////////////////////////////////////////////////// + +std::shared_ptr const> FollowerInfo::get() { + MUTEX_LOCKER(locker, _mutex); + return _followers; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief change JSON under +/// Current/Collection/// +/// to add or remove a serverID, if add flag is true, the entry is added +/// (if it is not yet there), otherwise the entry is removed (if it was +/// there). +//////////////////////////////////////////////////////////////////////////////// + +static VPackBuilder newShardEntry(VPackSlice oldValue, ServerID const& sid, + bool add) { + VPackBuilder newValue; + VPackSlice servers; + { + VPackObjectBuilder b(&newValue); + // Now need to find the `servers` attribute, which is a list: + for (auto const& it : VPackObjectIterator(oldValue)) { + if (it.key.isEqualString("servers")) { + servers = it.value; + } else { + newValue.add(it.key); + newValue.add(it.value); + } + } + newValue.add(VPackValue("servers")); + if (servers.isArray() && servers.length() > 0) { + VPackArrayBuilder bb(&newValue); + newValue.add(servers[0]); + VPackArrayIterator it(servers); + bool done = false; + for (++it; it.valid(); ++it) { + if ((*it).isEqualString(sid)) { + if (add) { + newValue.add(*it); + done = true; + } + } else { + newValue.add(*it); + } + } + if (add && !done) { + newValue.add(VPackValue(sid)); + } + } else { + VPackArrayBuilder bb(&newValue); + newValue.add(VPackValue(ServerState::instance()->getId())); + if (add) { + newValue.add(VPackValue(sid)); + } + } + } + return newValue; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief add a follower to a shard, this is only done by the server side +/// of the "get-in-sync" capabilities. This reports to the agency under +/// `/Current` but in asynchronous "fire-and-forget" way. +//////////////////////////////////////////////////////////////////////////////// + +void FollowerInfo::add(ServerID const& sid) { + MUTEX_LOCKER(locker, _mutex); + + // Fully copy the vector: + auto v = std::make_shared>(*_followers); + v->push_back(sid); // add a single entry + _followers = v; // will cast to std::vector const + // Now tell the agency, path is + // Current/Collections/// + std::string path = "Current/Collections/"; + path += _docColl->vocbase()->name(); + path += "/"; + path += std::to_string(_docColl->planId()); + path += "/"; + path += _docColl->name(); + AgencyComm ac; + double startTime = TRI_microtime(); + bool success = false; + do { + AgencyCommResult res = ac.getValues(path); + + if (res.successful()) { + velocypack::Slice currentEntry = + res.slice()[0].get(std::vector( + {AgencyCommManager::path(), "Current", "Collections", + _docColl->vocbase()->name(), std::to_string(_docColl->planId()), + _docColl->name()})); + + if (!currentEntry.isObject()) { + LOG_TOPIC(ERR, Logger::CLUSTER) + << "FollowerInfo::add, did not find object in " << path; + if (!currentEntry.isNone()) { + LOG_TOPIC(ERR, Logger::CLUSTER) << "Found: " << currentEntry.toJson(); + } + } else { + auto newValue = newShardEntry(currentEntry, sid, true); + std::string key = "Current/Collections/" + _docColl->vocbase()->name() + + "/" + std::to_string(_docColl->planId()) + "/" + + _docColl->name(); + AgencyWriteTransaction trx; + trx.preconditions.push_back(AgencyPrecondition( + key, AgencyPrecondition::Type::VALUE, currentEntry)); + trx.operations.push_back(AgencyOperation( + key, AgencyValueOperationType::SET, newValue.slice())); + trx.operations.push_back(AgencyOperation( + "Current/Version", AgencySimpleOperationType::INCREMENT_OP)); + AgencyCommResult res2 = ac.sendTransactionWithFailover(trx); + if (res2.successful()) { + success = true; + break; // + } else { + LOG_TOPIC(WARN, Logger::CLUSTER) + << "FollowerInfo::add, could not cas key " << path; + } + } + } else { + LOG_TOPIC(ERR, Logger::CLUSTER) << "FollowerInfo::add, could not read " + << path << " in agency."; + } + usleep(500000); + } while (TRI_microtime() < startTime + 30); + if (!success) { + LOG_TOPIC(ERR, Logger::CLUSTER) + << "FollowerInfo::add, timeout in agency operation for key " << path; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief remove a follower from a shard, this is only done by the +/// server if a synchronous replication request fails. This reports to +/// the agency under `/Current` but in asynchronous "fire-and-forget" +/// way. The method fails silently, if the follower information has +/// since been dropped (see `dropFollowerInfo` below). +//////////////////////////////////////////////////////////////////////////////// + +void FollowerInfo::remove(ServerID const& sid) { + MUTEX_LOCKER(locker, _mutex); + + auto v = std::make_shared>(); + v->reserve(_followers->size() - 1); + for (auto const& i : *_followers) { + if (i != sid) { + v->push_back(i); + } + } + _followers = v; // will cast to std::vector const + // Now tell the agency, path is + // Current/Collections/// + std::string path = "Current/Collections/"; + path += _docColl->vocbase()->name(); + path += "/"; + path += std::to_string(_docColl->planId()); + path += "/"; + path += _docColl->name(); + AgencyComm ac; + double startTime = TRI_microtime(); + bool success = false; + do { + AgencyCommResult res = ac.getValues(path); + if (res.successful()) { + velocypack::Slice currentEntry = + res.slice()[0].get(std::vector( + {AgencyCommManager::path(), "Current", "Collections", + _docColl->vocbase()->name(), std::to_string(_docColl->planId()), + _docColl->name()})); + + if (!currentEntry.isObject()) { + LOG_TOPIC(ERR, Logger::CLUSTER) + << "FollowerInfo::remove, did not find object in " << path; + if (!currentEntry.isNone()) { + LOG_TOPIC(ERR, Logger::CLUSTER) << "Found: " << currentEntry.toJson(); + } + } else { + auto newValue = newShardEntry(currentEntry, sid, false); + std::string key = "Current/Collections/" + _docColl->vocbase()->name() + + "/" + std::to_string(_docColl->planId()) + "/" + + _docColl->name(); + AgencyWriteTransaction trx; + trx.preconditions.push_back(AgencyPrecondition( + key, AgencyPrecondition::Type::VALUE, currentEntry)); + trx.operations.push_back(AgencyOperation( + key, AgencyValueOperationType::SET, newValue.slice())); + trx.operations.push_back(AgencyOperation( + "Current/Version", AgencySimpleOperationType::INCREMENT_OP)); + AgencyCommResult res2 = ac.sendTransactionWithFailover(trx); + if (res2.successful()) { + success = true; + break; // + } else { + LOG_TOPIC(WARN, Logger::CLUSTER) + << "FollowerInfo::remove, could not cas key " << path; + } + } + } else { + LOG_TOPIC(ERR, Logger::CLUSTER) << "FollowerInfo::remove, could not read " + << path << " in agency."; + } + usleep(500000); + } while (TRI_microtime() < startTime + 30); + if (!success) { + LOG_TOPIC(ERR, Logger::CLUSTER) + << "FollowerInfo::remove, timeout in agency operation for key " << path; + } +} + +////////////////////////////////////////////////////////////////////////////// +/// @brief clear follower list, no changes in agency necesary +////////////////////////////////////////////////////////////////////////////// + +void FollowerInfo::clear() { + MUTEX_LOCKER(locker, _mutex); + auto v = std::make_shared>(); + _followers = v; // will cast to std::vector const +} diff --git a/arangod/Cluster/FollowerInfo.h b/arangod/Cluster/FollowerInfo.h new file mode 100644 index 0000000000..fa12def019 --- /dev/null +++ b/arangod/Cluster/FollowerInfo.h @@ -0,0 +1,81 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Andreas Streichardt +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_CLUSTER_FOLLOWER_INFO_H +#define ARANGOD_CLUSTER_FOLLOWER_INFO_H 1 + +#include "ClusterInfo.h" + +namespace arangodb { + + //////////////////////////////////////////////////////////////////////////////// +/// @brief a class to track followers that are in sync for a shard +//////////////////////////////////////////////////////////////////////////////// + +class FollowerInfo { + std::shared_ptr const> _followers; + Mutex _mutex; + arangodb::LogicalCollection* _docColl; + + public: + + explicit FollowerInfo(arangodb::LogicalCollection* d) + : _followers(new std::vector()), _docColl(d) { } + + ////////////////////////////////////////////////////////////////////////////// + /// @brief get information about current followers of a shard. + ////////////////////////////////////////////////////////////////////////////// + + std::shared_ptr const> get(); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief add a follower to a shard, this is only done by the server side + /// of the "get-in-sync" capabilities. This reports to the agency under + /// `/Current` but in asynchronous "fire-and-forget" way. The method + /// fails silently, if the follower information has since been dropped + /// (see `dropFollowerInfo` below). + ////////////////////////////////////////////////////////////////////////////// + + void add(ServerID const& s); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief remove a follower from a shard, this is only done by the + /// server if a synchronous replication request fails. This reports to + /// the agency under `/Current` but in an asynchronous "fire-and-forget" + /// way. + ////////////////////////////////////////////////////////////////////////////// + + void remove(ServerID const& s); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief clear follower list, no changes in agency necesary + ////////////////////////////////////////////////////////////////////////////// + + void clear(); + +}; +} // end namespace arangodb + +#endif +#define ARANGOD_CLUSTER_CLUSTER_INFO_H 1 diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 4b0127f29c..c6597eaf3b 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -29,6 +29,7 @@ #include "Basics/conversions.h" #include "Basics/files.h" #include "Cluster/ClusterComm.h" +#include "Cluster/FollowerInfo.h" #include "Cluster/ClusterMethods.h" #include "GeneralServer/GeneralServer.h" #include "Indexes/EdgeIndex.h" diff --git a/arangod/Utils/Transaction.cpp b/arangod/Utils/Transaction.cpp index ffe4686816..59d2106d2d 100644 --- a/arangod/Utils/Transaction.cpp +++ b/arangod/Utils/Transaction.cpp @@ -34,6 +34,7 @@ #include "Basics/VelocyPackHelper.h" #include "Cluster/ClusterComm.h" #include "Cluster/ClusterMethods.h" +#include "Cluster/FollowerInfo.h" #include "Cluster/ServerState.h" #include "Indexes/EdgeIndex.h" #include "Indexes/HashIndex.h" diff --git a/arangod/V8Server/v8-collection.cpp b/arangod/V8Server/v8-collection.cpp index 7cccd9d7dd..92798c14f4 100644 --- a/arangod/V8Server/v8-collection.cpp +++ b/arangod/V8Server/v8-collection.cpp @@ -34,6 +34,7 @@ #include "Basics/VelocyPackHelper.h" #include "Basics/WriteLocker.h" #include "Cluster/ClusterInfo.h" +#include "Cluster/FollowerInfo.h" #include "Cluster/ClusterMethods.h" #include "Indexes/PrimaryIndex.h" #include "RestServer/DatabaseFeature.h" diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 1cb119445a..7153804939 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -34,6 +34,7 @@ #include "Aql/QueryCache.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ClusterMethods.h" +#include "Cluster/FollowerInfo.h" #include "Cluster/ServerState.h" #include "Indexes/EdgeIndex.h" #include "Indexes/FulltextIndex.h" @@ -428,7 +429,7 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, } VPackSlice shardKeysSlice = info.get("shardKeys"); - + bool const isCluster = ServerState::instance()->isRunningInCluster(); // Cluster only tests if (ServerState::instance()->isCoordinator()) { @@ -450,28 +451,29 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, } } } - auto replicationFactorSlice = info.get("replicationFactor"); - if (!replicationFactorSlice.isNone()) { - bool isError = true; - if (replicationFactorSlice.isNumber()) { - _replicationFactor = replicationFactorSlice.getNumber(); - // mop: only allow satellite collections to be created explicitly - if (_replicationFactor > 0 || _replicationFactor <= 10) { - isError = false; - } - } -#ifdef USE_ENTERPRISE - else if (replicationFactorSlice.isString() && replicationFactorSlice.copyString() == "satellite") { - _replicationFactor = 0; - _numberOfShards = 1; - _distributeShardsLike = ""; + } + + auto replicationFactorSlice = info.get("replicationFactor"); + if (!replicationFactorSlice.isNone()) { + bool isError = true; + if (replicationFactorSlice.isNumber()) { + _replicationFactor = replicationFactorSlice.getNumber(); + // mop: only allow satellite collections to be created explicitly + if (_replicationFactor > 0 || _replicationFactor <= 10) { isError = false; } + } +#ifdef USE_ENTERPRISE + else if (replicationFactorSlice.isString() && replicationFactorSlice.copyString() == "satellite") { + _replicationFactor = 0; + _numberOfShards = 1; + _distributeShardsLike = ""; + isError = false; + } #endif - if (isError) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "invalid replicationFactor"); - } + if (isError) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "invalid replicationFactor"); } } diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index 38894e04d8..032178c096 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -153,6 +153,7 @@ "ERROR_CLUSTER_BACKEND_UNAVAILABLE" : { "code" : 1478, "message" : "A cluster backend which was required for the operation could not be reached" }, "ERROR_CLUSTER_UNKNOWN_CALLBACK_ENDPOINT" : { "code" : 1479, "message" : "An endpoint couldn't be found" }, "ERROR_CLUSTER_AGENCY_STRUCTURE_INVALID" : { "code" : 1480, "message" : "Invalid agency structure" }, + "ERROR_CLUSTER_AQL_COLLECTION_OUT_OF_SYNC" : { "code" : 1481, "message" : "collection is out of sync" }, "ERROR_QUERY_KILLED" : { "code" : 1500, "message" : "query killed" }, "ERROR_QUERY_PARSE" : { "code" : 1501, "message" : "%s" }, "ERROR_QUERY_EMPTY" : { "code" : 1502, "message" : "query is empty" }, diff --git a/lib/Basics/errors.dat b/lib/Basics/errors.dat index a64d3a0203..2193b8e361 100755 --- a/lib/Basics/errors.dat +++ b/lib/Basics/errors.dat @@ -189,6 +189,7 @@ ERROR_CLUSTER_ONLY_ON_DBSERVER,1477,"this operation is only valid on a DBserver ERROR_CLUSTER_BACKEND_UNAVAILABLE,1478,"A cluster backend which was required for the operation could not be reached","Will be raised if a required db server can't be reached." ERROR_CLUSTER_UNKNOWN_CALLBACK_ENDPOINT,1479,"An endpoint couldn't be found","An endpoint couldn't be found" ERROR_CLUSTER_AGENCY_STRUCTURE_INVALID,1480,"Invalid agency structure","The structure in the agency is invalid" +ERROR_CLUSTER_AQL_COLLECTION_OUT_OF_SYNC,1481,"collection is out of sync","Will be raised if a collection needed during query execution is out of sync. This currently can only happen when using satellite collections" ################################################################################ ## ArangoDB query errors diff --git a/lib/Basics/voc-errors.cpp b/lib/Basics/voc-errors.cpp index ee07b7cb9b..2bf618e9fe 100644 --- a/lib/Basics/voc-errors.cpp +++ b/lib/Basics/voc-errors.cpp @@ -149,6 +149,7 @@ void TRI_InitializeErrorMessages () { REG_ERROR(ERROR_CLUSTER_BACKEND_UNAVAILABLE, "A cluster backend which was required for the operation could not be reached"); REG_ERROR(ERROR_CLUSTER_UNKNOWN_CALLBACK_ENDPOINT, "An endpoint couldn't be found"); REG_ERROR(ERROR_CLUSTER_AGENCY_STRUCTURE_INVALID, "Invalid agency structure"); + REG_ERROR(ERROR_CLUSTER_AQL_COLLECTION_OUT_OF_SYNC, "collection is out of sync"); REG_ERROR(ERROR_QUERY_KILLED, "query killed"); REG_ERROR(ERROR_QUERY_PARSE, "%s"); REG_ERROR(ERROR_QUERY_EMPTY, "query is empty"); diff --git a/lib/Basics/voc-errors.h b/lib/Basics/voc-errors.h index 700f8b6d81..fa171d08c4 100644 --- a/lib/Basics/voc-errors.h +++ b/lib/Basics/voc-errors.h @@ -365,6 +365,9 @@ /// An endpoint couldn't be found /// - 1480: @LIT{Invalid agency structure} /// The structure in the agency is invalid +/// - 1481: @LIT{collection is out of sync} +/// Will be raised if a collection needed during query execution is out of +/// sync. This currently can only happen when using satellite collections /// - 1500: @LIT{query killed} /// Will be raised when a running query is killed by an explicit admin /// command. @@ -2159,6 +2162,17 @@ void TRI_InitializeErrorMessages (); #define TRI_ERROR_CLUSTER_AGENCY_STRUCTURE_INVALID (1480) +//////////////////////////////////////////////////////////////////////////////// +/// @brief 1481: ERROR_CLUSTER_AQL_COLLECTION_OUT_OF_SYNC +/// +/// collection is out of sync +/// +/// Will be raised if a collection needed during query execution is out of +/// sync. This currently can only happen when using satellite collections +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_ERROR_CLUSTER_AQL_COLLECTION_OUT_OF_SYNC (1481) + //////////////////////////////////////////////////////////////////////////////// /// @brief 1500: ERROR_QUERY_KILLED /// From 9628420e688b2c8a6dcfedfc5d88a0d693ba57e2 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Wed, 7 Dec 2016 18:53:44 +0100 Subject: [PATCH 34/60] Fix compile error --- arangod/VocBase/replication-dump.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arangod/VocBase/replication-dump.cpp b/arangod/VocBase/replication-dump.cpp index 4ed12aef0b..71d666063d 100644 --- a/arangod/VocBase/replication-dump.cpp +++ b/arangod/VocBase/replication-dump.cpp @@ -537,7 +537,8 @@ static int DumpCollection(TRI_replication_dump_t* dump, bool bufferFull = false; auto callback = [&dump, &lastFoundTick, &databaseId, &collectionId, - &withTicks, &isEdgeCollection, &bufferFull, &useVpp]( + &withTicks, &isEdgeCollection, &bufferFull, &useVpp, + &collection]( TRI_voc_tick_t foundTick, TRI_df_marker_t const* marker) { // note the last tick we processed lastFoundTick = foundTick; From 13408007da5067074645b0b5706a88499f49ec11 Mon Sep 17 00:00:00 2001 From: Simran Brucherseifer Date: Thu, 8 Dec 2016 01:34:54 +0100 Subject: [PATCH 35/60] Docs: Gitbook 3 upgrade --- Documentation/Books/AQL/HEADER.html | 7 -- Documentation/Books/AQL/book.json | 14 ++-- Documentation/Books/AQL/styles/header.js | 76 ++++++++++++++++-- Documentation/Books/HTTP/HEADER.html | 7 -- Documentation/Books/HTTP/book.json | 11 ++- Documentation/Books/HTTP/styles/header.js | 76 ++++++++++++++++-- Documentation/Books/Makefile | 4 +- .../GettingStarted/Installing/MacOSX.mdpp | 4 +- .../Books/Manual/GettingStarted/README.mdpp | 2 +- Documentation/Books/Manual/HEADER.html | 7 -- .../Books/Manual/Indexing/IndexBasics.mdpp | 2 +- Documentation/Books/Manual/book.json | 11 ++- Documentation/Books/Manual/styles/header.js | 80 ++++++++++++++++--- 13 files changed, 233 insertions(+), 68 deletions(-) diff --git a/Documentation/Books/AQL/HEADER.html b/Documentation/Books/AQL/HEADER.html index 603484fe86..970dc9e9a6 100644 --- a/Documentation/Books/AQL/HEADER.html +++ b/Documentation/Books/AQL/HEADER.html @@ -14,13 +14,6 @@ - - - - - - -
diff --git a/Documentation/Books/AQL/book.json b/Documentation/Books/AQL/book.json index e538979260..6288b4182b 100644 --- a/Documentation/Books/AQL/book.json +++ b/Documentation/Books/AQL/book.json @@ -1,10 +1,10 @@ { - "gitbook": "^2.6.7", + "gitbook": "^3.2.2", "title": "ArangoDB VERSION_NUMBER AQL Documentation", "author": "ArangoDB GmbH", "description": "Official AQL manual for ArangoDB - the multi-model NoSQL database", "language": "en", - "plugins":["-search", "-sharing", "toggle-chapters", "addcssjs", "anchorjs", "add-header", "piwik"], + "plugins":["-search", "-lunr", "-sharing", "toggle-chapters", "addcssjs", "anchorjs", "piwik", "sitemap-general"], "pdf": { "fontSize": 12, "toc": true, @@ -23,12 +23,12 @@ "js": ["styles/header.js"], "css": ["styles/header.css"] }, - "add-header": { - "BASE_PATH": "https://docs.arangodb.com/devel" - }, "piwik": { - "URL": "www.arangodb.com/piwik/", - "siteId": 12 + "URL": "www.arangodb.com/piwik/", + "siteId": 12 + }, + "sitemap-general": { + "prefix": "https://docs.arangodb.com/devel/AQL/" } } } diff --git a/Documentation/Books/AQL/styles/header.js b/Documentation/Books/AQL/styles/header.js index d2410bf667..8d6936a35d 100644 --- a/Documentation/Books/AQL/styles/header.js +++ b/Documentation/Books/AQL/styles/header.js @@ -1,5 +1,6 @@ // Try to set the version number early, jQuery not available yet document.addEventListener("DOMContentLoaded", function(event) { + if (!gitbook.state.root) return; var bookVersion = gitbook.state.root.match(/\/(\d\.\d(\.\d)?|devel)\//); var switcher = document.getElementsByClassName("arangodb-version-switcher")[0]; if (bookVersion) { @@ -15,25 +16,81 @@ window.localStorage.removeItem(":keyword"); $(document).ready(function() { function appendHeader() { -/* + var div = document.createElement('div'); - div.innerHTML = ''; + div.innerHTML = '
\n' + + ' \n' + + '
\n' + + ' \n' + + ' \n' + + ' \n' + + '
\n' + + ' \n' + + '
\n' + + ' \n' + + '
\n' + + ' \n' + + '
\n'; $('.book').before(div.innerHTML); -*/ + }; function rerenderNavbar() { $('#header').remove(); appendHeader(); - renderGoogleSearch(); }; - function renderGoogleSearch() { - }; //render header - //rerenderNavbar(); + rerenderNavbar(); function addGoogleSrc() { var cx = '010085642145132923492:6ymjhhr677k'; var gcse = document.createElement('script'); @@ -68,7 +125,10 @@ function appendHeader() { if (urlSplit.length == 6) { urlSplit.pop(); // "" var currentBook = urlSplit.pop(); // e.g. "Manual" - urlSplit.pop() // e.g. "3.0" + var version = urlSplit.pop() // e.g. "3.0" + if (version < "2.9") { + currentBook = "Users"; + } window.location.href = urlSplit.join("/") + "/" + e.target.value + "/" + currentBook + "/"; } else { window.location.href = "https://docs.arangodb.com/" + e.target.value; diff --git a/Documentation/Books/HTTP/HEADER.html b/Documentation/Books/HTTP/HEADER.html index 96d5af7c7d..a2474a911b 100644 --- a/Documentation/Books/HTTP/HEADER.html +++ b/Documentation/Books/HTTP/HEADER.html @@ -14,13 +14,6 @@ - - - - - - -
diff --git a/Documentation/Books/HTTP/book.json b/Documentation/Books/HTTP/book.json index 9e2ee02966..ca8562eca0 100644 --- a/Documentation/Books/HTTP/book.json +++ b/Documentation/Books/HTTP/book.json @@ -1,10 +1,10 @@ { - "gitbook": "^2.6.7", + "gitbook": "^3.2.2", "title": "ArangoDB VERSION_NUMBER HTTP API Documentation", "author": "ArangoDB GmbH", "description": "Official HTTP API manual for ArangoDB - the multi-model NoSQL database", "language": "en", - "plugins":["-search", "-sharing", "toggle-chapters", "addcssjs", "anchorjs", "add-header", "piwik"], + "plugins":["-search", "-lunr", "-sharing", "toggle-chapters", "addcssjs", "anchorjs", "piwik", "sitemap-general"], "pdf": { "fontSize": 12, "toc": true, @@ -24,8 +24,11 @@ "css": ["styles/header.css"] }, "piwik": { - "URL": "www.arangodb.com/piwik/", - "siteId": 12 + "URL": "www.arangodb.com/piwik/", + "siteId": 12 + }, + "sitemap-general": { + "prefix": "https://docs.arangodb.com/devel/HTTP/" } } } diff --git a/Documentation/Books/HTTP/styles/header.js b/Documentation/Books/HTTP/styles/header.js index e343a499ad..7bab523d75 100644 --- a/Documentation/Books/HTTP/styles/header.js +++ b/Documentation/Books/HTTP/styles/header.js @@ -1,5 +1,6 @@ // Try to set the version number early, jQuery not available yet document.addEventListener("DOMContentLoaded", function(event) { + if (!gitbook.state.root) return; var bookVersion = gitbook.state.root.match(/\/(\d\.\d(\.\d)?|devel)\//); var switcher = document.getElementsByClassName("arangodb-version-switcher")[0]; if (bookVersion) { @@ -15,25 +16,81 @@ window.localStorage.removeItem(":keyword"); $(document).ready(function() { function appendHeader() { -/* + var div = document.createElement('div'); - div.innerHTML = ''; + div.innerHTML = '
\n' + + ' \n' + + '
\n' + + ' \n' + + ' \n' + + ' \n' + + '
\n' + + ' \n' + + '
\n' + + ' \n' + + '
\n' + + ' \n' + + '
\n'; $('.book').before(div.innerHTML); -*/ + }; function rerenderNavbar() { $('.arangodb-header').remove(); appendHeader(); - renderGoogleSearch(); }; - function renderGoogleSearch() { - }; //render header - //rerenderNavbar(); + rerenderNavbar(); function addGoogleSrc() { var cx = '010085642145132923492:fixi4yzeiz8'; var gcse = document.createElement('script'); @@ -68,7 +125,10 @@ function appendHeader() { if (urlSplit.length == 6) { urlSplit.pop(); // "" var currentBook = urlSplit.pop(); // e.g. "Manual" - urlSplit.pop() // e.g. "3.0" + var version = urlSplit.pop() // e.g. "3.0" + if (version < "2.9") { + currentBook = "Users"; + } window.location.href = urlSplit.join("/") + "/" + e.target.value + "/" + currentBook + "/"; } else { window.location.href = "https://docs.arangodb.com/" + e.target.value; diff --git a/Documentation/Books/Makefile b/Documentation/Books/Makefile index 18bdbbea00..afd995231b 100644 --- a/Documentation/Books/Makefile +++ b/Documentation/Books/Makefile @@ -121,9 +121,9 @@ book-check-markdown-leftovers: echo "${RESET}"; \ exit 1; \ fi - @if test "`find books/$(NAME) -name '*.html' -exec grep '\.md\"' {} \; | grep -v data-filepath | wc -l`" -gt 0; then \ + @if test "`find books/$(NAME) -name '*.html' -exec grep '\.md\"[ />]' {} \; | grep -v data-filepath | wc -l`" -gt 0; then \ echo "${ERR_COLOR}"; \ - find books/$(NAME) -name '*.html' -exec grep '\.md"' {} \; -print | grep -v data-filepath; \ + find books/$(NAME) -name '*.html' -exec grep '\.md"[ />]' {} \; -print | grep -v data-filepath; \ echo "found dangling markdown links; see the list above "; \ echo "${RESET}"; \ exit 1; \ diff --git a/Documentation/Books/Manual/GettingStarted/Installing/MacOSX.mdpp b/Documentation/Books/Manual/GettingStarted/Installing/MacOSX.mdpp index 25fb6d45b9..c4c48b2984 100644 --- a/Documentation/Books/Manual/GettingStarted/Installing/MacOSX.mdpp +++ b/Documentation/Books/Manual/GettingStarted/Installing/MacOSX.mdpp @@ -3,7 +3,7 @@ Mac OS X The preferred method for installing ArangoDB under Mac OS X is [homebrew](#homebrew). However, in case you are not using homebrew, we -provide a [command-line app](#commandline-app) or [graphical +provide a [command-line app](#command-line-app) or [graphical app](#graphical-app) which contains all the executables. Homebrew @@ -58,7 +58,7 @@ download it from [here](https://www.arangodb.com/download). Choose *Mac OS X*. Download and install the application *ArangoDB* in your application folder. -Command-Line App +Command line App ---------------- In case you are not using homebrew, we also provide a command-line app. You can download it from [here](https://www.arangodb.com/download). diff --git a/Documentation/Books/Manual/GettingStarted/README.mdpp b/Documentation/Books/Manual/GettingStarted/README.mdpp index 51f1131013..d7667f635f 100644 --- a/Documentation/Books/Manual/GettingStarted/README.mdpp +++ b/Documentation/Books/Manual/GettingStarted/README.mdpp @@ -428,7 +428,7 @@ not always have to be an array of objects: Now let's do something crazy: for every document in the users collection, iterate over all user documents again and return user pairs, e.g. John and Katie. We can use a loop inside a loop for this to get the cross product (every possible -combination of all user records, 3 * 3 = 9). We don't want pairings like *John + +combination of all user records, 3 \* 3 = 9). We don't want pairings like *John + John* however, so let's eliminate them with a filter condition: ```js diff --git a/Documentation/Books/Manual/HEADER.html b/Documentation/Books/Manual/HEADER.html index f5a86696a1..eb0bb74770 100644 --- a/Documentation/Books/Manual/HEADER.html +++ b/Documentation/Books/Manual/HEADER.html @@ -14,13 +14,6 @@ - - - - - - -
diff --git a/Documentation/Books/Manual/Indexing/IndexBasics.mdpp b/Documentation/Books/Manual/Indexing/IndexBasics.mdpp index 377b2821b3..ffd5b1d413 100644 --- a/Documentation/Books/Manual/Indexing/IndexBasics.mdpp +++ b/Documentation/Books/Manual/Indexing/IndexBasics.mdpp @@ -269,7 +269,7 @@ The geo index stores two-dimensional coordinates. It can be created on either tw separate document attributes (latitude and longitude) or a single array attribute that contains both latitude and longitude. Latitude and longitude must be numeric values. -Th geo index provides operations to find documents with coordinates nearest to a given +The geo index provides operations to find documents with coordinates nearest to a given comparison coordinate, and to find documents with coordinates that are within a specifiable radius around a comparison coordinate. diff --git a/Documentation/Books/Manual/book.json b/Documentation/Books/Manual/book.json index 02a6534e5c..2f8b76518f 100644 --- a/Documentation/Books/Manual/book.json +++ b/Documentation/Books/Manual/book.json @@ -1,10 +1,10 @@ { - "gitbook": "^2.6.7", + "gitbook": "^3.2.2", "title": "ArangoDB VERSION_NUMBER Documentation", "author": "ArangoDB GmbH", "description": "Official manual for ArangoDB - the multi-model NoSQL database", "language": "en", - "plugins":["-search", "-sharing", "toggle-chapters", "addcssjs", "anchorjs", "add-header", "piwik"], + "plugins":["-search", "-lunr", "-sharing", "toggle-chapters", "addcssjs", "anchorjs", "piwik", "sitemap-general"], "pdf": { "fontSize": 12, "toc": true, @@ -24,8 +24,11 @@ "css": ["styles/header.css"] }, "piwik": { - "URL": "www.arangodb.com/piwik/", - "siteId": 12 + "URL": "www.arangodb.com/piwik/", + "siteId": 12 + }, + "sitemap-general": { + "prefix": "https://docs.arangodb.com/devel/Manual/" } } } diff --git a/Documentation/Books/Manual/styles/header.js b/Documentation/Books/Manual/styles/header.js index 83a4d094b5..ba7845c50b 100644 --- a/Documentation/Books/Manual/styles/header.js +++ b/Documentation/Books/Manual/styles/header.js @@ -1,5 +1,6 @@ // Try to set the version number early, jQuery not available yet document.addEventListener("DOMContentLoaded", function(event) { + if (!gitbook.state.root) return; var bookVersion = gitbook.state.root.match(/\/(\d\.\d(\.\d)?|devel)\//); var switcher = document.getElementsByClassName("arangodb-version-switcher")[0]; if (bookVersion) { @@ -15,25 +16,81 @@ window.localStorage.removeItem(":keyword"); $(document).ready(function() { function appendHeader() { -/* - var div = document.createElement('div'); - div.innerHTML = ''; - $('.book').before(div.innerHTML); -*/ + var div = document.createElement('div'); + div.innerHTML = '
\n' + + ' \n' + + '
\n' + + ' \n' + + ' \n' + + ' \n' + + '
\n' + + ' \n' + + '
\n' + + ' \n' + + '
\n' + + ' \n' + + '
\n'; + + $('.book').before(div.innerHTML); + }; function rerenderNavbar() { $('.arangodb-header').remove(); appendHeader(); - renderGoogleSearch(); }; - function renderGoogleSearch() { - }; //render header - //rerenderNavbar(); + rerenderNavbar(); function addGoogleSrc() { var cx = '010085642145132923492:djexw6vlsgo'; var gcse = document.createElement('script'); @@ -68,7 +125,10 @@ function appendHeader() { if (urlSplit.length == 6) { urlSplit.pop(); // "" var currentBook = urlSplit.pop(); // e.g. "Manual" - urlSplit.pop() // e.g. "3.0" + var version = urlSplit.pop() // e.g. "3.0" + if (version < "2.9") { + currentBook = "Users"; + } window.location.href = urlSplit.join("/") + "/" + e.target.value + "/" + currentBook + "/"; } else { window.location.href = "https://docs.arangodb.com/" + e.target.value; From 9fedba9ffff7a06d184341677379d3159a5e7a21 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 10:14:38 +0100 Subject: [PATCH 36/60] Fix eslint --- js/server/tests/resilience/shard-distribution-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/server/tests/resilience/shard-distribution-spec.js b/js/server/tests/resilience/shard-distribution-spec.js index 628ca592be..c1778e5149 100644 --- a/js/server/tests/resilience/shard-distribution-spec.js +++ b/js/server/tests/resilience/shard-distribution-spec.js @@ -1,4 +1,4 @@ -/* global describe, beforeEach, afterEach, it*/ +/* global describe, beforeEach, afterEach, it, instanceInfo */ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// From 40fac5c36ae150555c50d613b4f2e3f5956ec49b Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 11:39:49 +0100 Subject: [PATCH 37/60] Fix condition circus --- arangod/VocBase/LogicalCollection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 7153804939..7ff8b6a68c 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -459,7 +459,7 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, if (replicationFactorSlice.isNumber()) { _replicationFactor = replicationFactorSlice.getNumber(); // mop: only allow satellite collections to be created explicitly - if (_replicationFactor > 0 || _replicationFactor <= 10) { + if (_replicationFactor > 0 && _replicationFactor <= 10) { isError = false; } } From 2ff0bc933150e1202cd4b7f1e0fc0ce90d71e0e3 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Thu, 8 Dec 2016 13:12:03 +0100 Subject: [PATCH 38/60] added Utf8Validator for VelocyPack --- .../include/velocypack/Utf8Helper.h | 42 +++++++++++ .../include/velocypack/velocypack-aliases.h | 7 ++ 3rdParty/velocypack/src/Utf8Helper.cpp | 69 +++++++++++++++++++ 3rdParty/velocypack/src/Validator.cpp | 15 +++- lib/CMakeLists.txt | 1 + 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 3rdParty/velocypack/include/velocypack/Utf8Helper.h create mode 100644 3rdParty/velocypack/src/Utf8Helper.cpp diff --git a/3rdParty/velocypack/include/velocypack/Utf8Helper.h b/3rdParty/velocypack/include/velocypack/Utf8Helper.h new file mode 100644 index 0000000000..0f48cc601a --- /dev/null +++ b/3rdParty/velocypack/include/velocypack/Utf8Helper.h @@ -0,0 +1,42 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief Library to build up VPack documents. +/// +/// DISCLAIMER +/// +/// Copyright 2015 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#ifndef VELOCYPACK_UTF8HELPER_H +#define VELOCYPACK_UTF8HELPER_H 1 + +#include "velocypack/velocypack-common.h" + +namespace arangodb { +namespace velocypack { + +struct Utf8Helper { + static bool isValidUtf8(uint8_t const* p, ValueLength len); +}; + +} +} + +#endif diff --git a/3rdParty/velocypack/include/velocypack/velocypack-aliases.h b/3rdParty/velocypack/include/velocypack/velocypack-aliases.h index 42c6158e1c..cb1a4edde7 100644 --- a/3rdParty/velocypack/include/velocypack/velocypack-aliases.h +++ b/3rdParty/velocypack/include/velocypack/velocypack-aliases.h @@ -151,6 +151,13 @@ using VPackSlimBuffer = arangodb::velocypack::SliceContainer; #endif #endif +#ifdef VELOCYPACK_UTF8HELPER_H +#ifndef VELOCYPACK_ALIAS_UTF8HELPER +#define VELOCYPACK_ALIAS_UTF8HELPER +using VPackUtf8Helper = arangodb::velocypack::Utf8Helper; +#endif +#endif + #ifdef VELOCYPACK_VALIDATOR_H #ifndef VELOCYPACK_ALIAS_VALIDATOR #define VELOCYPACK_ALIAS_VALIDATOR diff --git a/3rdParty/velocypack/src/Utf8Helper.cpp b/3rdParty/velocypack/src/Utf8Helper.cpp new file mode 100644 index 0000000000..1f7e7b23e9 --- /dev/null +++ b/3rdParty/velocypack/src/Utf8Helper.cpp @@ -0,0 +1,69 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief Library to build up VPack documents. +/// +/// DISCLAIMER +/// +/// Copyright 2015 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "velocypack/velocypack-common.h" +#include "velocypack/Utf8Helper.h" + +using namespace arangodb::velocypack; + +namespace { + +static constexpr uint8_t ValidChar = 0; +static constexpr uint8_t InvalidChar = 1; + +static const uint8_t states[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df + 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef + 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff + 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2 + 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4 + 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6 + 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8 +}; + +} + +bool Utf8Helper::isValidUtf8(uint8_t const* p, ValueLength len) { + uint8_t const* end = p + len; + + uint8_t state = ValidChar; + while (p < end) { + state = states[256 + state * 16 + states[*p]]; + if (state == InvalidChar) { + return false; + } + ++p; + } + + return (state == ValidChar); +} diff --git a/3rdParty/velocypack/src/Validator.cpp b/3rdParty/velocypack/src/Validator.cpp index f89c37fc7d..f9d91cad65 100644 --- a/3rdParty/velocypack/src/Validator.cpp +++ b/3rdParty/velocypack/src/Validator.cpp @@ -28,6 +28,7 @@ #include "velocypack/Validator.h" #include "velocypack/Exception.h" #include "velocypack/Slice.h" +#include "velocypack/Utf8Helper.h" #include "velocypack/ValueType.h" using namespace arangodb::velocypack; @@ -88,11 +89,23 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons } case ValueType::String: { + uint8_t const* p; + ValueLength len; if (head == 0xbfU) { // long UTF-8 string. must be at least 9 bytes long so we // can read the entire string length safely validateBufferLength(1 + 8, length, true); - } + p = ptr + 1 + 8; + len = readInteger(p, 8); + } else { + p = ptr + 1; + len = head - 0x40U; + } + validateBufferLength(length - (p - ptr), len, true); + + if (options->validateUtf8Strings && !Utf8Helper::isValidUtf8(p, len)) { + throw Exception(Exception::InvalidUtf8Sequence); + } break; } diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 4ffe7488da..a4e088f5a4 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -71,6 +71,7 @@ set(LIB_ARANGO_VPACK ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Options.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Parser.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Slice.cpp + ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Utf8Helper.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/ValueType.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Validator.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Version.cpp From ce03d08e8b29463d2bb2878d2dc02e7fd261eb02 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Thu, 8 Dec 2016 13:32:48 +0100 Subject: [PATCH 39/60] added specialized fasthash version for uint64 input --- UnitTests/Basics/hashes-test.cpp | 32 ++++++++++++++++++++++++++++++++ arangod/Indexes/EdgeIndex.cpp | 2 +- arangod/Indexes/HashIndex.h | 2 +- arangod/V8Server/v8-query.cpp | 2 +- lib/Basics/fasthash.cpp | 10 ++++++++++ lib/Basics/fasthash.h | 2 ++ 6 files changed, 47 insertions(+), 3 deletions(-) diff --git a/UnitTests/Basics/hashes-test.cpp b/UnitTests/Basics/hashes-test.cpp index a4431e4a28..ac6883f02a 100644 --- a/UnitTests/Basics/hashes-test.cpp +++ b/UnitTests/Basics/hashes-test.cpp @@ -86,6 +86,38 @@ BOOST_FIXTURE_TEST_SUITE(CHashesTest, CHashesSetup) /// @brief test fasthash64 //////////////////////////////////////////////////////////////////////////////// +BOOST_AUTO_TEST_CASE (tst_fasthash64_uint64) { + uint64_t value; + + value = 0; + BOOST_CHECK_EQUAL((uint64_t) 606939172421154273ULL, fasthash64(&value, sizeof(value), 0x12345678)); + BOOST_CHECK_EQUAL((uint64_t) 606939172421154273ULL, fasthash64_uint64(value, 0x12345678)); + + value = 1; + BOOST_CHECK_EQUAL((uint64_t) 2986466439906256014ULL, fasthash64(&value, sizeof(value), 0x12345678)); + BOOST_CHECK_EQUAL((uint64_t) 2986466439906256014ULL, fasthash64_uint64(value, 0x12345678)); + + value = 123456; + BOOST_CHECK_EQUAL((uint64_t) 10846706210321519612ULL, fasthash64(&value, sizeof(value), 0x12345678)); + BOOST_CHECK_EQUAL((uint64_t) 10846706210321519612ULL, fasthash64_uint64(value, 0x12345678)); + + value = 123456789012345ULL; + BOOST_CHECK_EQUAL((uint64_t) 11872028338155052138ULL, fasthash64(&value, sizeof(value), 0x12345678)); + BOOST_CHECK_EQUAL((uint64_t) 11872028338155052138ULL, fasthash64_uint64(value, 0x12345678)); + + value = 0xffffff000000ULL; + BOOST_CHECK_EQUAL((uint64_t) 5064027312035038651ULL, fasthash64(&value, sizeof(value), 0x12345678)); + BOOST_CHECK_EQUAL((uint64_t) 5064027312035038651ULL, fasthash64_uint64(value, 0x12345678)); + + value = 0xffffffffffffULL; + BOOST_CHECK_EQUAL((uint64_t) 12472603196990564371ULL, fasthash64(&value, sizeof(value), 0x12345678)); + BOOST_CHECK_EQUAL((uint64_t) 12472603196990564371ULL, fasthash64_uint64(value, 0x12345678)); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test fasthash64 +//////////////////////////////////////////////////////////////////////////////// + BOOST_AUTO_TEST_CASE (tst_fasthash64) { std::string buffer; diff --git a/arangod/Indexes/EdgeIndex.cpp b/arangod/Indexes/EdgeIndex.cpp index fdf8df8e45..4ce2915907 100644 --- a/arangod/Indexes/EdgeIndex.cpp +++ b/arangod/Indexes/EdgeIndex.cpp @@ -64,7 +64,7 @@ static uint64_t HashElementEdge(void*, SimpleIndexElement const& element, bool b } TRI_voc_rid_t revisionId = element.revisionId(); - return fasthash64(&revisionId, sizeof(revisionId), 0x56781234); + return fasthash64_uint64(revisionId, 0x56781234); } /// @brief checks if key and element match diff --git a/arangod/Indexes/HashIndex.h b/arangod/Indexes/HashIndex.h index 2e0eeac8bd..ae7af66607 100644 --- a/arangod/Indexes/HashIndex.h +++ b/arangod/Indexes/HashIndex.h @@ -237,7 +237,7 @@ class HashIndex final : public PathBasedIndex { } TRI_voc_rid_t revisionId = element->revisionId(); - return fasthash64(&revisionId, sizeof(revisionId), hash); + return fasthash64_uint64(revisionId, hash); } }; diff --git a/arangod/V8Server/v8-query.cpp b/arangod/V8Server/v8-query.cpp index cf2ddfe4ff..7f3d165a10 100644 --- a/arangod/V8Server/v8-query.cpp +++ b/arangod/V8Server/v8-query.cpp @@ -394,7 +394,7 @@ static void JS_ChecksumCollection( if (withData) { // with data uint64_t const n = slice.length() ^ 0xf00ba44ba5; - uint64_t seed = fasthash64(&n, sizeof(n), 0xdeadf054); + uint64_t seed = fasthash64_uint64(n, 0xdeadf054); for (auto const& it : VPackObjectIterator(slice, false)) { // loop over all attributes, but exclude _rev, _id and _key diff --git a/lib/Basics/fasthash.cpp b/lib/Basics/fasthash.cpp index 742c4954d3..ceaee8262a 100644 --- a/lib/Basics/fasthash.cpp +++ b/lib/Basics/fasthash.cpp @@ -35,6 +35,16 @@ static inline uint64_t mix(uint64_t h) { return h; } + +uint64_t fasthash64_uint64(uint64_t value, uint64_t seed) { + size_t const len = sizeof(uint64_t); + uint64_t const m = 0x880355f21e6d1965ULL; + uint64_t h = seed ^ (len * m); + h ^= mix(value); + h *= m; + + return mix(h); +} uint64_t fasthash64(const void* buf, size_t len, uint64_t seed) { #ifdef __arm__ diff --git a/lib/Basics/fasthash.h b/lib/Basics/fasthash.h index c03d1f0449..7ca29252f3 100644 --- a/lib/Basics/fasthash.h +++ b/lib/Basics/fasthash.h @@ -27,6 +27,8 @@ #define LIB_BASICS_FASTHASH_H 1 #include "Basics/Common.h" +uint64_t fasthash64_uint64(uint64_t value, uint64_t seed); + /** * fasthash32 - 32-bit implementation of fasthash * @buf: data buffer From 29bb68031efe7eae2c6a9f83d8476d736061a8dd Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 14:54:27 +0100 Subject: [PATCH 40/60] Wait for initial collection sync..(not yet working properly) --- arangod/Utils/AqlTransaction.cpp | 15 +++++++++++++-- arangod/VocBase/LogicalCollection.cpp | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/arangod/Utils/AqlTransaction.cpp b/arangod/Utils/AqlTransaction.cpp index 50171ac820..dc5566e0ae 100644 --- a/arangod/Utils/AqlTransaction.cpp +++ b/arangod/Utils/AqlTransaction.cpp @@ -56,10 +56,21 @@ int AqlTransaction::processCollectionCoordinator(aql::Collection* collection) { /// @brief add a regular collection to the transaction int AqlTransaction::processCollectionNormal(aql::Collection* collection) { - arangodb::LogicalCollection const* col = - this->resolver()->getCollectionStruct(collection->getName()); TRI_voc_cid_t cid = 0; + arangodb::LogicalCollection const* col = + this->resolver()->getCollectionStruct(collection->getName()); + if (col == nullptr) { + auto startTime = TRI_microtime(); + auto endTime = startTime + 60.0; + do { + usleep(10000); + if (TRI_microtime() > endTime) { + break; + } + col = this->resolver()->getCollectionStruct(collection->getName()); + } while (col == nullptr); + } if (col != nullptr) { cid = col->cid(); } diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 7ff8b6a68c..db92f4ee18 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -461,6 +461,10 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, // mop: only allow satellite collections to be created explicitly if (_replicationFactor > 0 && _replicationFactor <= 10) { isError = false; +#ifdef USE_ENTERPRISE + } else if (_replicationFactor == 0) { + isError = false; +#endif } } #ifdef USE_ENTERPRISE From 76c0789e3cbe6384e831fb8f7c5e5d33cc86837b Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 15:11:22 +0100 Subject: [PATCH 41/60] Distribute satellites across cluster --- arangod/Agency/Supervision.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 8c570e5c30..ae20767786 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -621,7 +621,12 @@ void Supervision::enforceReplication() { auto const& db = *(db_.second); for (const auto& col_ : db.children()) { // Planned collections auto const& col = *(col_.second); - auto const& replicationFactor = col("replicationFactor").slice().getUInt(); + auto replicationFactor = col("replicationFactor").slice().getUInt(); + + // mop: satellites => distribute to every server + if (replicationFactor == 0) { + replicationFactor = available.size(); + } bool clone = false; try { From c6ef45b64d0a85bd76f3074cad1a74f09f8e279d Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Thu, 8 Dec 2016 15:12:05 +0100 Subject: [PATCH 42/60] AddFollower to handle multiple followers at the same time --- arangod/Agency/AddFollower.cpp | 57 +++++++++++++++++++++++++++------ arangod/Agency/AddFollower.h | 30 +++++++++++------ arangod/Agency/Inception.h | 6 ---- arangod/Agency/RemoveServer.cpp | 2 +- arangod/Agency/Supervision.cpp | 19 ++++++++--- 5 files changed, 85 insertions(+), 29 deletions(-) diff --git a/arangod/Agency/AddFollower.cpp b/arangod/Agency/AddFollower.cpp index c7fa57165e..d247a6d3e2 100644 --- a/arangod/Agency/AddFollower.cpp +++ b/arangod/Agency/AddFollower.cpp @@ -32,7 +32,34 @@ AddFollower::AddFollower(Node const& snapshot, Agent* agent, std::string const& prefix, std::string const& database, std::string const& collection, std::string const& shard, - std::string const& newFollower) + std::initializer_list const& newFollower) + : Job(snapshot, agent, jobId, creator, prefix), + _database(database), + _collection(collection), + _shard(shard), + _newFollower(newFollower) { + try { + JOB_STATUS js = status(); + + if (js == TODO) { + start(); + } else if (js == NOTFOUND) { + if (create()) { + start(); + } + } + } catch (std::exception const& e) { + LOG_TOPIC(WARN, Logger::AGENCY) << e.what() << __FILE__ << __LINE__; + finish("Shards/" + _shard, false, e.what()); + } +} + +AddFollower::AddFollower(Node const& snapshot, Agent* agent, + std::string const& jobId, std::string const& creator, + std::string const& prefix, std::string const& database, + std::string const& collection, + std::string const& shard, + std::vector const& newFollower) : Job(snapshot, agent, jobId, creator, prefix), _database(database), _collection(collection), @@ -109,7 +136,13 @@ bool AddFollower::create() { _jb->add("database", VPackValue(_database)); _jb->add("collection", VPackValue(_collection)); _jb->add("shard", VPackValue(_shard)); - _jb->add("newFollower", VPackValue(_newFollower)); + _jb->add(VPackValue("newFollower")); + { + VPackArrayBuilder b(_jb.get()); + for (auto const& i : _newFollower) { + _jb->add(VPackValue(i)); + } + } _jb->add("jobId", VPackValue(_jobId)); _jb->add("timeCreated", VPackValue(now)); @@ -142,7 +175,7 @@ bool AddFollower::start() { for (auto const& srv : VPackArrayIterator(current)) { TRI_ASSERT(srv.isString()); - if (srv.copyString() == _newFollower) { + if (srv.copyString() == _newFollower.front()) { finish("Shards/" + _shard, false, "newFollower must not be already holding the shard."); return false; @@ -150,7 +183,7 @@ bool AddFollower::start() { } for (auto const& srv : VPackArrayIterator(planned)) { TRI_ASSERT(srv.isString()); - if (srv.copyString() == _newFollower) { + if (srv.copyString() == _newFollower.front()) { finish("Shards/" + _shard, false, "newFollower must not be planned for shard already."); return false; @@ -206,7 +239,9 @@ bool AddFollower::start() { for (auto const& srv : VPackArrayIterator(planned)) { pending.add(srv); } - pending.add(VPackValue(_newFollower)); + for (auto const& i : _newFollower) { + pending.add(VPackValue(i)); + } pending.close(); // --- Increment Plan/Version @@ -237,7 +272,7 @@ bool AddFollower::start() { if (res.accepted && res.indices.size() == 1 && res.indices[0]) { LOG_TOPIC(INFO, Logger::AGENCY) - << "Pending: Addfollower " + _newFollower + " to shard " + _shard; + << "Pending: Addfollower " << _newFollower << " to shard " << _shard; return true; } @@ -253,8 +288,12 @@ JOB_STATUS AddFollower::status() { try { _database = _snapshot(pos[status] + _jobId + "/database").getString(); _collection = _snapshot(pos[status] + _jobId + "/collection").getString(); - _newFollower = - _snapshot(pos[status] + _jobId + "/newFollower").getString(); + for (auto const& i : + VPackArrayIterator( + _snapshot(pos[status] + _jobId + "/newFollower").getArray())) { + _newFollower.push_back(i.copyString()); + } + _snapshot(pos[status] + _jobId + "/newFollower").getArray(); _shard = _snapshot(pos[status] + _jobId + "/shard").getString(); } catch (std::exception const& e) { std::stringstream err; @@ -271,7 +310,7 @@ JOB_STATUS AddFollower::status() { Slice current = _snapshot(curPath).slice(); for (auto const& srv : VPackArrayIterator(current)) { - if (srv.copyString() == _newFollower) { + if (srv.copyString() == _newFollower.front()) { if (finish("Shards/" + _shard)) { return FINISHED; } diff --git a/arangod/Agency/AddFollower.h b/arangod/Agency/AddFollower.h index bab6d481f6..1bda810d97 100644 --- a/arangod/Agency/AddFollower.h +++ b/arangod/Agency/AddFollower.h @@ -33,14 +33,26 @@ namespace consensus { struct AddFollower : public Job { AddFollower (Node const& snapshot, - Agent* agent, - std::string const& jobId, - std::string const& creator, - std::string const& prefix, - std::string const& database = std::string(), - std::string const& collection = std::string(), - std::string const& shard = std::string(), - std::string const& newFollower = std::string()); + Agent* agent, + std::string const& jobId, + std::string const& creator, + std::string const& prefix, + std::string const& database, + std::string const& collection, + std::string const& shard, + std::initializer_list const&); + + + AddFollower (Node const& snapshot, + Agent* agent, + std::string const& jobId, + std::string const& creator, + std::string const& prefix, + std::string const& database = std::string(), + std::string const& collection = std::string(), + std::string const& shard = std::string(), + std::vector const& newFollowers = {}); + virtual ~AddFollower (); @@ -51,7 +63,7 @@ struct AddFollower : public Job { std::string _database; std::string _collection; std::string _shard; - std::string _newFollower; + std::vector _newFollower; }; diff --git a/arangod/Agency/Inception.h b/arangod/Agency/Inception.h index 40cd81f833..e89b5609cd 100644 --- a/arangod/Agency/Inception.h +++ b/arangod/Agency/Inception.h @@ -69,15 +69,9 @@ public: private: - /// @brief Find active agency from persisted - bool activeAgencyFromPersistence(); - /// @brief We are a restarting active RAFT agent bool restartingActiveAgent(); - /// @brief Find active agency from command line - bool activeAgencyFromCommandLine(); - /// @brief Try to estimate good RAFT min/max timeouts bool estimateRAFTInterval(); diff --git a/arangod/Agency/RemoveServer.cpp b/arangod/Agency/RemoveServer.cpp index c3d241a80b..d2180a97da 100644 --- a/arangod/Agency/RemoveServer.cpp +++ b/arangod/Agency/RemoveServer.cpp @@ -355,7 +355,7 @@ bool RemoveServer::scheduleAddFollowers() { AddFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++), _jobId, _agencyPrefix, database.first, collptr.first, - shard.first, newServer); + shard.first, {newServer}); } } } diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 8c570e5c30..0ead0da084 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -634,18 +634,29 @@ void Supervision::enforceReplication() { // Enough DBServer to if (replicationFactor > shard.slice().length() && - available.size() >= replicationFactor) { + available.size() > shard.slice().length()) { for (auto const& i : VPackArrayIterator(shard.slice())) { available.erase( std::remove( available.begin(), available.end(), i.copyString()), available.end()); } - auto randIt = available.begin(); - std::advance(randIt, std::rand() % available.size()); + + size_t optimal = replicationFactor - shard.slice().length(); + std::vector newFollowers; + for (size_t i = 0; i < optimal; ++i) { + auto randIt = available.begin(); + std::advance(randIt, std::rand() % available.size()); + newFollowers.push_back(*randIt); + available.erase(randIt); + if (available.empty()) { + break; + } + } + AddFollower( _snapshot, _agent, std::to_string(_jobId++), "supervision", - _agencyPrefix, db_.first, col_.first, shard_.first, *randIt); + _agencyPrefix, db_.first, col_.first, shard_.first, newFollowers); } } } From c55afca64ddfd10218578cc1fb4ebfa2de9f20ff Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 8 Dec 2016 15:20:22 +0100 Subject: [PATCH 43/60] add new directory --- Installation/Jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index d0d8f6855e..d9e599f7da 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -576,7 +576,7 @@ if test -n "${TARGET_DIR}"; then touch arangosh/.keepme tar -c -f ${TARFILE_TMP} \ - VERSION utils scripts etc/relative UnitTests Documentation js \ + VERSION utils scripts etc/relative etc/testing UnitTests Documentation js \ lib/Basics/errors.dat \ 3rdParty/.keepme \ arangod/.keepme \ From fbcae1382fa53fd88424d713d81b6af90459a63a Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 15:21:24 +0100 Subject: [PATCH 44/60] Remove useless logging --- arangod/Agency/AgencyComm.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/arangod/Agency/AgencyComm.cpp b/arangod/Agency/AgencyComm.cpp index 7696329ce3..9e0ea51611 100644 --- a/arangod/Agency/AgencyComm.cpp +++ b/arangod/Agency/AgencyComm.cpp @@ -124,7 +124,6 @@ void AgencyPrecondition::toGeneralBuilder(VPackBuilder& builder) const { AgencyOperation::AgencyOperation(std::string const& key) : _key(AgencyCommManager::path(key)), _opType() { _opType.type = AgencyOperationType::Type::READ; - LOG(WARN) << _opType.toString(); } AgencyOperation::AgencyOperation(std::string const& key, From 03161cd15bc3cd9681d2a0999ee2a08072738072 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Thu, 8 Dec 2016 16:08:13 +0100 Subject: [PATCH 45/60] fixed string validation --- 3rdParty/velocypack/src/Validator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdParty/velocypack/src/Validator.cpp b/3rdParty/velocypack/src/Validator.cpp index f9d91cad65..9315536560 100644 --- a/3rdParty/velocypack/src/Validator.cpp +++ b/3rdParty/velocypack/src/Validator.cpp @@ -101,7 +101,7 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons p = ptr + 1; len = head - 0x40U; } - validateBufferLength(length - (p - ptr), len, true); + validateBufferLength(length - (p - ptr), length, true); if (options->validateUtf8Strings && !Utf8Helper::isValidUtf8(p, len)) { throw Exception(Exception::InvalidUtf8Sequence); From fbe8851ba2f654190069e511c2a061b36c268cdd Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 16:49:14 +0100 Subject: [PATCH 46/60] comment out waiting for now --- arangod/Utils/AqlTransaction.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arangod/Utils/AqlTransaction.cpp b/arangod/Utils/AqlTransaction.cpp index dc5566e0ae..313e306ff8 100644 --- a/arangod/Utils/AqlTransaction.cpp +++ b/arangod/Utils/AqlTransaction.cpp @@ -60,7 +60,7 @@ int AqlTransaction::processCollectionNormal(aql::Collection* collection) { arangodb::LogicalCollection const* col = this->resolver()->getCollectionStruct(collection->getName()); - if (col == nullptr) { + /*if (col == nullptr) { auto startTime = TRI_microtime(); auto endTime = startTime + 60.0; do { @@ -71,6 +71,7 @@ int AqlTransaction::processCollectionNormal(aql::Collection* collection) { col = this->resolver()->getCollectionStruct(collection->getName()); } while (col == nullptr); } + */ if (col != nullptr) { cid = col->cid(); } From 29e3c1487e40879d647f1906b7442a88ce00463b Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 16:49:33 +0100 Subject: [PATCH 47/60] Make satellite wait sync configurable --- arangod/Aql/EnumerateCollectionBlock.cpp | 10 ++++++++-- arangod/Aql/ExecutionEngine.cpp | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/arangod/Aql/EnumerateCollectionBlock.cpp b/arangod/Aql/EnumerateCollectionBlock.cpp index a2dd551dce..fdf236b701 100644 --- a/arangod/Aql/EnumerateCollectionBlock.cpp +++ b/arangod/Aql/EnumerateCollectionBlock.cpp @@ -123,18 +123,24 @@ int EnumerateCollectionBlock::initialize() { auto dbName = logicalCollection->dbName(); auto collectionInfoCurrent = ClusterInfo::instance()->getCollectionCurrent(dbName, std::to_string(cid)); + double maxWait = _engine->getQuery()->getNumericOption("satelliteSyncWait", 60.0); bool inSync = false; unsigned long waitInterval = 10000; double startTime = TRI_microtime(); - double endTime = startTime + 60.0; + double now = startTime; + double endTime = startTime + maxWait; while (!inSync) { auto followers = collectionInfoCurrent->servers(_collection->getName()); inSync = std::find(followers.begin(), followers.end(), ServerState::instance()->getId()) != followers.end(); if (!inSync) { + if (endTime - now < waitInterval) { + waitInterval = endTime - now; + } usleep(waitInterval); } - if (TRI_microtime() > endTime) { + now = TRI_microtime(); + if (now > endTime) { break; } } diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index f510c9bd8d..24e5d45f28 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -566,6 +566,8 @@ struct CoordinatorInstanciator : public WalkerWorker { result.close(); // options.optimizer double tracing = query->getNumericOption("tracing", 0); result.add("tracing", VPackValue(tracing)); + double satelliteSyncWait = query->getNumericOption("satelliteSyncWait", 60.0); + result.add("satelliteSyncWait", VPackValue(satelliteSyncWait)); result.close(); // options result.close(); From 8b486e50fa6400f428f74533366591add81a8441 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 8 Dec 2016 16:59:41 +0100 Subject: [PATCH 48/60] move windows special install targets out of nsis package code --- cmake/ArangoDBInstall.cmake | 28 ++++++++++++++++++++++++++++ cmake/packages/nsis.cmake | 22 ---------------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/cmake/ArangoDBInstall.cmake b/cmake/ArangoDBInstall.cmake index f4b9af96af..57dc6d0d9a 100644 --- a/cmake/ArangoDBInstall.cmake +++ b/cmake/ArangoDBInstall.cmake @@ -171,3 +171,31 @@ configure_file ( "${CMAKE_CURRENT_BINARY_DIR}/lib/Basics/directories.h" NEWLINE_STYLE UNIX ) + + +if (MSVC) + # so we don't need to ship dll's twice, make it one directory: + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/InstallMacros.cmake) + set(CMAKE_INSTALL_FULL_SBINDIR "${CMAKE_INSTALL_FULL_BINDIR}") + + # other platforms link the file into the binary + install(FILES ${ICU_DT} + DESTINATION "${INSTALL_ICU_DT_DEST}" + RENAME ${ICU_DT_DEST}) + + install_readme(README.windows README.windows.txt) + + # install the visual studio runtime: + set(CMAKE_INSTALL_UCRT_LIBRARIES 1) + include(InstallRequiredSystemLibraries) + INSTALL(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS} DESTINATION ${CMAKE_INSTALL_SBINDIR} COMPONENT Libraries) + INSTALL(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_COMPONENT} DESTINATION ${CMAKE_INSTALL_SBINDIR} COMPONENT Libraries) + + # install openssl + if (NOT LIB_EAY_RELEASE_DLL OR NOT SSL_EAY_RELEASE_DLL) + message(FATAL_ERROR, "BUNDLE_OPENSSL set but couldn't locate SSL DLLs. Please set LIB_EAY_RELEASE_DLL and SSL_EAY_RELEASE_DLL") + endif() + + install (FILES "${LIB_EAY_RELEASE_DLL}" DESTINATION "${CMAKE_INSTALL_BINDIR}/" COMPONENT Libraries) + install (FILES "${SSL_EAY_RELEASE_DLL}" DESTINATION "${CMAKE_INSTALL_BINDIR}/" COMPONENT Libraries) +endif() diff --git a/cmake/packages/nsis.cmake b/cmake/packages/nsis.cmake index d6c20071e3..ba50e8f75f 100644 --- a/cmake/packages/nsis.cmake +++ b/cmake/packages/nsis.cmake @@ -1,6 +1,3 @@ -# so we don't need to ship dll's twice, make it one directory: -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/InstallMacros.cmake) -set(CMAKE_INSTALL_FULL_SBINDIR "${CMAKE_INSTALL_FULL_BINDIR}") set(W_INSTALL_FILES "${PROJECT_SOURCE_DIR}/Installation/Windows/") if (${USE_ENTERPRISE}) set(CPACK_PACKAGE_NAME "ArangoDB3e") @@ -28,21 +25,6 @@ else () SET(BITS 32) endif () -install_readme(README.windows README.windows.txt) - -# install the visual studio runtime: -set(CMAKE_INSTALL_UCRT_LIBRARIES 1) -include(InstallRequiredSystemLibraries) -INSTALL(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS} DESTINATION ${CMAKE_INSTALL_SBINDIR} COMPONENT Libraries) -INSTALL(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_COMPONENT} DESTINATION ${CMAKE_INSTALL_SBINDIR} COMPONENT Libraries) - -# install openssl -if (NOT LIB_EAY_RELEASE_DLL OR NOT SSL_EAY_RELEASE_DLL) - message(FATAL_ERROR, "BUNDLE_OPENSSL set but couldn't locate SSL DLLs. Please set LIB_EAY_RELEASE_DLL and SSL_EAY_RELEASE_DLL") -endif() - -install (FILES "${LIB_EAY_RELEASE_DLL}" DESTINATION "${CMAKE_INSTALL_BINDIR}/" COMPONENT Libraries) -install (FILES "${SSL_EAY_RELEASE_DLL}" DESTINATION "${CMAKE_INSTALL_BINDIR}/" COMPONENT Libraries) # icon paths set (ICON_PATH "${W_INSTALL_FILES}/Icons/") @@ -82,10 +64,6 @@ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${AR ################################################################################ # hook to build the server package ################################################################################ -# other platforms link the file into the binary -install(FILES ${ICU_DT} - DESTINATION "${INSTALL_ICU_DT_DEST}" - RENAME ${ICU_DT_DEST}) add_custom_target(package-arongodb-server-nsis COMMAND ${CMAKE_COMMAND} . From 6920bb65f1c119cda720f485be0f553d5d2bd7e3 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Thu, 8 Dec 2016 17:31:54 +0100 Subject: [PATCH 49/60] added IndexBucket.h --- arangod/VocBase/datafile.cpp | 108 +-------------- lib/Basics/AssocMulti.h | 196 ++++++++------------------ lib/Basics/AssocUnique.h | 156 +++++++-------------- lib/Basics/IndexBucket.h | 257 +++++++++++++++++++++++++++++++++++ lib/Basics/files.cpp | 121 +++++++++++++++-- lib/Basics/files.h | 16 ++- 6 files changed, 487 insertions(+), 367 deletions(-) create mode 100644 lib/Basics/IndexBucket.h diff --git a/arangod/VocBase/datafile.cpp b/arangod/VocBase/datafile.cpp index a646f055a9..7a0cac01c0 100644 --- a/arangod/VocBase/datafile.cpp +++ b/arangod/VocBase/datafile.cpp @@ -144,104 +144,6 @@ static bool CheckCrcMarker(TRI_df_marker_t const* marker, char const* end) { } -/// @brief creates a new datafile -/// -/// returns the file descriptor or -1 if the file cannot be created -static int CreateDatafile(std::string const& filename, TRI_voc_size_t maximalSize) { - TRI_ERRORBUF; - - // open the file - int fd = TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC | TRI_NOATIME, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); - - TRI_IF_FAILURE("CreateDatafile1") { - // intentionally fail - TRI_CLOSE(fd); - fd = -1; - errno = ENOSPC; - } - - if (fd < 0) { - if (errno == ENOSPC) { - TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL); - LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_last_error(); - } else { - TRI_SYSTEM_ERROR(); - - TRI_set_errno(TRI_ERROR_SYS_ERROR); - LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF; - } - return -1; - } - - // no fallocate present, or at least pretend it's not there... - int res = TRI_ERROR_NOT_IMPLEMENTED; - -#ifdef __linux__ -#ifdef FALLOC_FL_ZERO_RANGE - // try fallocate - res = fallocate(fd, FALLOC_FL_ZERO_RANGE, 0, maximalSize); -#endif -#endif - - if (res != TRI_ERROR_NO_ERROR) { - // either fallocate failed or it is not there... - - // fill file with zeros from FileNullBuffer - size_t writeSize = TRI_GetNullBufferSizeFiles(); - size_t written = 0; - while (written < maximalSize) { - if (writeSize + written > maximalSize) { - writeSize = maximalSize - written; - } - - ssize_t writeResult = - TRI_WRITE(fd, TRI_GetNullBufferFiles(), static_cast(writeSize)); - - TRI_IF_FAILURE("CreateDatafile2") { - // intentionally fail - writeResult = -1; - errno = ENOSPC; - } - - if (writeResult < 0) { - if (errno == ENOSPC) { - TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL); - LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_last_error(); - } else { - TRI_SYSTEM_ERROR(); - TRI_set_errno(TRI_ERROR_SYS_ERROR); - LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF; - } - - TRI_CLOSE(fd); - TRI_UnlinkFile(filename.c_str()); - - return -1; - } - - written += static_cast(writeResult); - } - } - - // go back to offset 0 - TRI_lseek_t offset = TRI_LSEEK(fd, (TRI_lseek_t)0, SEEK_SET); - - if (offset == (TRI_lseek_t)-1) { - TRI_SYSTEM_ERROR(); - TRI_set_errno(TRI_ERROR_SYS_ERROR); - TRI_CLOSE(fd); - - // remove empty file - TRI_UnlinkFile(filename.c_str()); - - LOG(ERR) << "cannot seek in datafile '" << filename << "': '" << TRI_GET_ERRORBUF << "'"; - return -1; - } - - return fd; -} - //////////////////////////////////////////////////////////////////////////////// /// @brief extract the numeric part from a filename /// the filename must look like this: /.*type-abc\.ending$/, where abc is @@ -292,8 +194,8 @@ static TRI_datafile_t* CreateAnonymousDatafile(TRI_voc_fid_t fid, // memory map the data void* data; void* mmHandle; - ssize_t res = TRI_MMFile(nullptr, maximalSize, PROT_WRITE | PROT_READ, flags, - fd, &mmHandle, 0, &data); + int res = TRI_MMFile(nullptr, maximalSize, PROT_WRITE | PROT_READ, flags, + fd, &mmHandle, 0, &data); #ifdef MAP_ANONYMOUS // nothing to do @@ -327,7 +229,7 @@ static TRI_datafile_t* CreatePhysicalDatafile(std::string const& filename, TRI_voc_size_t maximalSize) { TRI_ASSERT(!filename.empty()); - int fd = CreateDatafile(filename, maximalSize); + int fd = TRI_CreateDatafile(filename, maximalSize); if (fd < 0) { // an error occurred @@ -342,8 +244,8 @@ static TRI_datafile_t* CreatePhysicalDatafile(std::string const& filename, // try populating the mapping already flags |= MAP_POPULATE; #endif - ssize_t res = TRI_MMFile(0, maximalSize, PROT_WRITE | PROT_READ, flags, fd, - &mmHandle, 0, &data); + int res = TRI_MMFile(0, maximalSize, PROT_WRITE | PROT_READ, flags, fd, + &mmHandle, 0, &data); if (res != TRI_ERROR_NO_ERROR) { TRI_set_errno(res); diff --git a/lib/Basics/AssocMulti.h b/lib/Basics/AssocMulti.h index e356b0b0d9..6741120985 100644 --- a/lib/Basics/AssocMulti.h +++ b/lib/Basics/AssocMulti.h @@ -30,7 +30,7 @@ // #define TRI_CHECK_MULTI_POINTER_HASH 1 #include "Basics/Common.h" -#include "Basics/memory-map.h" +#include "Basics/IndexBucket.h" #include "Basics/Mutex.h" #include "Basics/MutexLocker.h" #include "Basics/prime-numbers.h" @@ -144,22 +144,12 @@ class AssocMulti { typedef std::function IsEqualElementElementFuncType; typedef std::function CallbackElementFuncType; - + private: typedef Entry EntryType; - - struct Bucket { - IndexType _nrAlloc; // the size of the table - IndexType _nrUsed; // the number of used entries - IndexType _nrCollisions; // the number of entries that have - // a key that was previously in the table - EntryType* _table; // the table itself - - Bucket() : _nrAlloc(0), _nrUsed(0), _nrCollisions(0), _table(nullptr) {} - // Intentionally no destructor, the AssocMulti class takes - // care of freeing the tables! - }; - + + typedef arangodb::basics::IndexBucket Bucket; + std::vector _buckets; size_t _bucketsMask; @@ -222,44 +212,21 @@ class AssocMulti { } numberBuckets = nr; _bucketsMask = nr - 1; - - _buckets.reserve(numberBuckets); + + _buckets.resize(numberBuckets); try { for (size_t j = 0; j < numberBuckets; j++) { - _buckets.emplace_back(); - Bucket& b = _buckets.back(); - b._nrAlloc = initialSize; - b._table = nullptr; - - // may fail... - b._table = new EntryType[b._nrAlloc](); - -#ifdef __linux__ - if (b._nrAlloc > 1000000) { - uintptr_t mem = reinterpret_cast(b._table); - uintptr_t pageSize = getpagesize(); - mem = (mem / pageSize) * pageSize; - void* memptr = reinterpret_cast(mem); - TRI_MMFileAdvise(memptr, b._nrAlloc * sizeof(EntryType), - TRI_MADVISE_RANDOM); - } -#endif + _buckets[j].allocate(_initialSize); } } catch (...) { - for (auto& b : _buckets) { - delete[] b._table; - b._table = nullptr; - b._nrAlloc = 0; - } + _buckets.clear(); throw; } } ~AssocMulti() { - for (auto& b : _buckets) { - delete[] b._table; - } + _buckets.clear(); } ////////////////////////////////////////////////////////////////////////////// @@ -268,11 +235,8 @@ class AssocMulti { size_t memoryUsage() const { size_t res = 0; - // size_t count = 0; for (auto& b : _buckets) { - res += static_cast(b._nrAlloc) * sizeof(EntryType); - // std::cout << "Bucket: " << count++ << " _nrAlloc=" << b._nrAlloc - // << " _nrUsed=" << b._nrUsed << std::endl; + res += b.memoryUsage(); } return res; } @@ -534,42 +498,12 @@ class AssocMulti { } return res.load(); } - + void truncate(CallbackElementFuncType callback) { - std::vector empty; - empty.reserve(_buckets.size()); - - try { - for (size_t i = 0; i < _buckets.size(); ++i) { - auto newBucket = new EntryType[static_cast(_initialSize)](); - try { - // shouldn't fail as enough space was reserved above, but let's be paranoid - empty.emplace_back(newBucket); - } catch (...) { - delete[] newBucket; - throw; - } - } - - size_t i = 0; - for (auto& b : _buckets) { - invokeOnAllElements(callback, b); - - // now bucket is empty - delete[] b._table; - b._table = empty[i]; - b._nrAlloc = _initialSize; - b._nrUsed = 0; - - empty[i] = nullptr; // pass ownership - ++i; - } - } catch (...) { - // prevent leaks - for (auto& it : empty) { - delete[] it; - } - throw; + for (auto& b : _buckets) { + invokeOnAllElements(callback, b); + b.deallocate(); + b.allocate(_initialSize); } } @@ -1209,84 +1143,68 @@ class AssocMulti { /// @brief resize the array, internal method ////////////////////////////////////////////////////////////////////////////// - void resizeInternal(UserData* userData, Bucket& b, size_t size) { + void resizeInternal(UserData* userData, Bucket& b, size_t targetSize) { std::string const cb(_contextCallback()); - LOG(TRACE) << "resizing hash " << cb << ", target size: " << size; + LOG(TRACE) << "resizing hash " << cb << ", target size: " << targetSize; LOG_TOPIC(TRACE, Logger::PERFORMANCE) << - "hash-resize " << cb << ", target size: " << size; + "hash-resize " << cb << ", target size: " << targetSize; double start = TRI_microtime(); + + targetSize = TRI_NearPrime(targetSize); + + Bucket copy; + copy.allocate(targetSize); - EntryType* oldTable = b._table; - IndexType oldAlloc = b._nrAlloc; - - b._nrAlloc = - static_cast(TRI_NearPrime(static_cast(size))); - - try { - b._table = new EntryType[b._nrAlloc](); -#ifdef __linux__ - if (b._nrAlloc > 1000000) { - uintptr_t mem = reinterpret_cast(b._table); - uintptr_t pageSize = getpagesize(); - mem = (mem / pageSize) * pageSize; - void* memptr = reinterpret_cast(mem); - TRI_MMFileAdvise(memptr, b._nrAlloc * sizeof(EntryType), - TRI_MADVISE_RANDOM); - } -#endif - } catch (...) { - b._nrAlloc = oldAlloc; - b._table = oldTable; - throw; - } - - b._nrUsed = 0; - b._nrCollisions = 0; #ifdef TRI_INTERNAL_STATS _nrResizes++; #endif // table is already clear by allocate, copy old data - IndexType j; - for (j = 0; j < oldAlloc; j++) { - if (oldTable[j].value && oldTable[j].prev == INVALID_INDEX) { - // This is a "first" one in its doubly linked list: - uint64_t hashByKey; - if (useHashCache) { - hashByKey = oldTable[j].readHashCache(); - } else { - hashByKey = _hashElement(userData, oldTable[j].value, true); - } - IndexType insertPosition = - insertFirst(userData, b, oldTable[j].value, hashByKey); - // Now walk to the end of the list: - IndexType k = j; - while (oldTable[k].next != INVALID_INDEX) { - k = oldTable[k].next; - } - // Now insert all of them backwards, not repeating k: - while (k != j) { - uint64_t hashByElm; + if (b._nrUsed > 0) { + EntryType* oldTable = b._table; + IndexType const oldAlloc = b._nrAlloc; + TRI_ASSERT(oldAlloc > 0); + + for (IndexType j = 0; j < oldAlloc; j++) { + if (oldTable[j].value && oldTable[j].prev == INVALID_INDEX) { + // This is a "first" one in its doubly linked list: + uint64_t hashByKey; if (useHashCache) { - hashByElm = oldTable[k].readHashCache(); + hashByKey = oldTable[j].readHashCache(); } else { - hashByElm = _hashElement(userData, oldTable[k].value, false); + hashByKey = _hashElement(userData, oldTable[j].value, true); + } + IndexType insertPosition = + insertFirst(userData, copy, oldTable[j].value, hashByKey); + // Now walk to the end of the list: + IndexType k = j; + while (oldTable[k].next != INVALID_INDEX) { + k = oldTable[k].next; + } + // Now insert all of them backwards, not repeating k: + while (k != j) { + uint64_t hashByElm; + if (useHashCache) { + hashByElm = oldTable[k].readHashCache(); + } else { + hashByElm = _hashElement(userData, oldTable[k].value, false); + } + insertFurther(userData, copy, oldTable[k].value, hashByKey, hashByElm, + insertPosition); + k = oldTable[k].prev; } - insertFurther(userData, b, oldTable[k].value, hashByKey, hashByElm, - insertPosition); - k = oldTable[k].prev; } } } - - delete[] oldTable; + + b = std::move(copy); LOG(TRACE) << "resizing hash " << cb << " done"; - LOG_TOPIC(TRACE, Logger::PERFORMANCE) << "[timer] " << Logger::FIXED(TRI_microtime() - start) << " s, hash-resize, " << cb << ", target size: " << size; + LOG_TOPIC(TRACE, Logger::PERFORMANCE) << "[timer] " << Logger::FIXED(TRI_microtime() - start) << " s, hash-resize, " << cb << ", target size: " << targetSize; } #ifdef TRI_CHECK_MULTI_POINTER_HASH diff --git a/lib/Basics/AssocUnique.h b/lib/Basics/AssocUnique.h index aa0f3f71eb..88d79c3931 100644 --- a/lib/Basics/AssocUnique.h +++ b/lib/Basics/AssocUnique.h @@ -32,9 +32,9 @@ #include #include +#include "Basics/IndexBucket.h" #include "Basics/MutexLocker.h" #include "Basics/gcd.h" -#include "Basics/memory-map.h" #include "Basics/prime-numbers.h" #include "Logger/Logger.h" #include "Random/RandomGenerator.h" @@ -78,14 +78,9 @@ class AssocUnique { typedef std::function CallbackElementFuncType; + typedef arangodb::basics::IndexBucket Bucket; + private: - struct Bucket { - uint64_t _nrAlloc; // the size of the table - uint64_t _nrUsed; // the number of used entries - - Element* _table; // the table itself, aligned to a cache line boundary - }; - std::vector _buckets; size_t _bucketsMask; @@ -122,33 +117,21 @@ class AssocUnique { } numberBuckets = nr; _bucketsMask = nr - 1; - - _buckets.reserve(numberBuckets); + + _buckets.resize(numberBuckets); try { for (size_t j = 0; j < numberBuckets; j++) { - _buckets.emplace_back(); - Bucket& b = _buckets.back(); - b._nrAlloc = initialSize(); - b._table = nullptr; - - // may fail... - b._table = new Element[static_cast(b._nrAlloc)](); + _buckets[j].allocate(initialSize()); } } catch (...) { - for (auto& b : _buckets) { - delete[] b._table; - b._table = nullptr; - b._nrAlloc = 0; - } + _buckets.clear(); throw; } } ~AssocUnique() { - for (auto& b : _buckets) { - delete[] b._table; - } + _buckets.clear(); } ////////////////////////////////////////////////////////////////////////////// @@ -176,13 +159,13 @@ class AssocUnique { void resizeInternal(UserData* userData, Bucket& b, uint64_t targetSize, bool allowShrink) { - if (b._nrAlloc >= targetSize && !allowShrink) { + if (b._nrAlloc > targetSize && !allowShrink) { return; } if (allowShrink && b._nrAlloc >= targetSize && b._nrAlloc < 1.25 * targetSize) { - // no need to shrink the bucket if it's not big enough + // no need to shrink return; } @@ -200,31 +183,19 @@ class AssocUnique { "hash-resize " << cb << ", target size: " << targetSize; } - Element* oldTable = b._table; - uint64_t oldAlloc = b._nrAlloc; - TRI_ASSERT(targetSize > 0); targetSize = TRI_NearPrime(targetSize); - // This might throw, is catched outside - b._table = new Element[static_cast(targetSize)](); - - b._nrAlloc = targetSize; - -#ifdef __linux__ - if (b._nrAlloc > 1000000) { - uintptr_t mem = reinterpret_cast(b._table); - uintptr_t pageSize = getpagesize(); - mem = (mem / pageSize) * pageSize; - void* memptr = reinterpret_cast(mem); - TRI_MMFileAdvise(memptr, b._nrAlloc * sizeof(Element), - TRI_MADVISE_RANDOM); - } -#endif + Bucket copy; + copy.allocate(targetSize); if (b._nrUsed > 0) { - uint64_t const n = b._nrAlloc; + Element* oldTable = b._table; + uint64_t const oldAlloc = b._nrAlloc; + TRI_ASSERT(oldAlloc > 0); + + uint64_t const n = copy._nrAlloc; TRI_ASSERT(n > 0); for (uint64_t j = 0; j < oldAlloc; j++) { @@ -234,19 +205,20 @@ class AssocUnique { uint64_t i, k; i = k = _hashElement(userData, element) % n; - for (; i < n && b._table[i]; ++i) + for (; i < n && copy._table[i]; ++i) ; if (i == n) { - for (i = 0; i < k && b._table[i]; ++i) + for (i = 0; i < k && copy._table[i]; ++i) ; } - b._table[i] = element; + copy._table[i] = element; + ++copy._nrUsed; } } } - delete[] oldTable; + b = std::move(copy); LOG(TRACE) << "resizing hash " << cb << " done"; @@ -277,14 +249,14 @@ class AssocUnique { UserData* userData, BucketPosition& position, uint64_t const step, BucketPosition const& initial) const { Element found; - Bucket b = _buckets[position.bucketId]; + Bucket const* b = &_buckets[position.bucketId]; do { - found = b._table[position.position]; + found = b->_table[position.position]; position.position += step; - while (position.position >= b._nrAlloc) { - position.position -= b._nrAlloc; + while (position.position >= b->_nrAlloc) { + position.position -= b->_nrAlloc; position.bucketId = (position.bucketId + 1) % _buckets.size(); - b = _buckets[position.bucketId]; + b = &_buckets[position.bucketId]; } if (position == initial) { // We are done. Return the last element we have in hand @@ -330,42 +302,10 @@ class AssocUnique { public: void truncate(CallbackElementFuncType callback) { - std::vector empty; - empty.reserve(_buckets.size()); - - try { - uint64_t const nrAlloc = initialSize(); - - for (size_t i = 0; i < _buckets.size(); ++i) { - auto newBucket = new Element[static_cast(nrAlloc)](); - try { - // shouldn't fail as enough space was reserved above, but let's be paranoid - empty.emplace_back(newBucket); - } catch (...) { - delete[] newBucket; - throw; - } - } - - size_t i = 0; - for (auto& b : _buckets) { - invokeOnAllElements(callback, b); - - // now bucket is empty - delete[] b._table; - b._table = empty[i]; - b._nrAlloc = initialSize(); - b._nrUsed = 0; - - empty[i] = nullptr; // pass ownership - ++i; - } - } catch (...) { - // prevent leaks - for (auto& it : empty) { - delete[] it; - } - throw; + for (auto& b : _buckets) { + invokeOnAllElements(callback, b); + b.deallocate(); + b.allocate(initialSize()); } } @@ -391,11 +331,11 @@ class AssocUnique { ////////////////////////////////////////////////////////////////////////////// size_t memoryUsage() const { - size_t sum = 0; + size_t res = 0; for (auto& b : _buckets) { - sum += static_cast(b._nrAlloc * sizeof(Element)); + res += b.memoryUsage(); } - return sum; + return res; } ////////////////////////////////////////////////////////////////////////////// @@ -923,15 +863,17 @@ class AssocUnique { /// @brief a method to iterate over all elements in a bucket. this method /// can NOT be used for deleting elements bool invokeOnAllElements(CallbackElementFuncType const& callback, Bucket& b) { - for (size_t i = 0; i < b._nrAlloc; ++i) { - if (!b._table[i] || b._nrUsed == 0) { - continue; - } - if (!b._table[i]) { - continue; - } - if (!callback(b._table[i])) { - return false; + if (b._nrUsed > 0) { + for (size_t i = 0; i < b._nrAlloc; ++i) { + if (!b._table[i]) { + continue; + } + if (!callback(b._table[i])) { + return false; + } + if (b._nrUsed == 0) { + break; + } } } return true; @@ -1055,10 +997,10 @@ class AssocUnique { position.position = _buckets[position.bucketId]._nrAlloc - 1; } - Bucket b = _buckets[position.bucketId]; + Bucket const* b = &_buckets[position.bucketId]; Element found; do { - found = b._table[position.position]; + found = b->_table[position.position]; if (position.position == 0) { if (position.bucketId == 0) { @@ -1068,8 +1010,8 @@ class AssocUnique { } --position.bucketId; - b = _buckets[position.bucketId]; - position.position = b._nrAlloc - 1; + b = &_buckets[position.bucketId]; + position.position = b->_nrAlloc - 1; } else { --position.position; } diff --git a/lib/Basics/IndexBucket.h b/lib/Basics/IndexBucket.h new file mode 100644 index 0000000000..27883a2a48 --- /dev/null +++ b/lib/Basics/IndexBucket.h @@ -0,0 +1,257 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_BASICS_INDEX_BUCKET_H +#define ARANGODB_BASICS_INDEX_BUCKET_H 1 + +#include "Basics/Common.h" +#include "Basics/files.h" +#include "Basics/memory-map.h" +#include "Logger/Logger.h" + +namespace arangodb { +namespace basics { + +template +struct IndexBucket { + IndexType _nrAlloc; // the size of the table + IndexType _nrUsed; // the number of used entries + IndexType _nrCollisions; // the number of entries that have + // a key that was previously in the table + EntryType* _table; // the table itself + int _file; // file descriptor for memory mapped file (-1 = no file) + char* _filename; // name of memory mapped file (nullptr = no file) + void* _mmHandle; + + IndexBucket() + : _nrAlloc(0), + _nrUsed(0), + _nrCollisions(0), + _table(nullptr), + _file(-1), + _filename(nullptr), + _mmHandle(nullptr) {} + IndexBucket(IndexBucket const&) = delete; + IndexBucket& operator=(IndexBucket const&) = delete; + + // move ctor. this takes over responsibility for the resources from other + IndexBucket(IndexBucket&& other) + : _nrAlloc(other._nrAlloc), + _nrUsed(other._nrUsed), + _nrCollisions(other._nrCollisions), + _table(other._table), + _file(other._file), + _filename(other._filename), + _mmHandle(other._mmHandle) { + other._nrAlloc = 0; + other._nrUsed = 0; + other._nrCollisions = 0; + other._table = nullptr; + other._file = -1; + other._filename = nullptr; + other._mmHandle = nullptr; + } + + IndexBucket& operator=(IndexBucket&& other) { + deallocate(); // free own resources first + + _nrAlloc = other._nrAlloc; + _nrUsed = other._nrUsed; + _nrCollisions = other._nrCollisions; + _table = other._table; + _file = other._file; + _filename = other._filename; + _mmHandle = other._mmHandle; + + other._nrAlloc = 0; + other._nrUsed = 0; + other._nrCollisions = 0; + other._table = nullptr; + other._file = -1; + other._filename = nullptr; + other._mmHandle = nullptr; + + return *this; + } + + ~IndexBucket() { + deallocate(); + } + + public: + size_t memoryUsage() const { + return requiredSize(_nrAlloc); + } + + size_t requiredSize(size_t numberElements) const { + return numberElements * sizeof(EntryType); + } + + void allocate(size_t numberElements) { + TRI_ASSERT(_nrAlloc == 0); + TRI_ASSERT(_nrUsed == 0); + TRI_ASSERT(_table == nullptr); + TRI_ASSERT(_file == -1); + TRI_ASSERT(_filename == nullptr); + TRI_ASSERT(_mmHandle == nullptr); + + size_t const totalSize = requiredSize(numberElements); + + _file = allocateTempfile(_filename, totalSize); + + try { + _table = allocateMemory(numberElements); + TRI_ASSERT(_table != nullptr); + +#ifdef __linux__ + if (numberElements > 1000000) { + uintptr_t mem = reinterpret_cast(_table); + uintptr_t pageSize = getpagesize(); + mem = (mem / pageSize) * pageSize; + void* memptr = reinterpret_cast(mem); + TRI_MMFileAdvise(memptr, totalSize, TRI_MADVISE_RANDOM); + } +#endif + + _nrAlloc = numberElements; + } catch (...) { + deallocateTempfile(); + TRI_ASSERT(_file == -1); + throw; + } + } + + void deallocate() { + deallocateMemory(); + deallocateTempfile(); + + TRI_ASSERT(_file == -1); + TRI_ASSERT(_filename == nullptr); + TRI_ASSERT(_table == nullptr); + } + + private: + EntryType* allocateMemory(size_t numberElements) { + TRI_ASSERT(numberElements > 0); + + if (_file == -1) { + return new EntryType[numberElements](); + } + + size_t const totalSize = requiredSize(numberElements); + + // initialize the file + TRI_ASSERT(_file > 0); + TRI_ASSERT(_mmHandle == nullptr); + + void* data; + int res = TRI_MMFile(nullptr, totalSize, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_POPULATE, _file, &_mmHandle, 0, &data); + + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + + TRI_ASSERT(data != nullptr); + + try { + // call placement new constructor + (void) new (data) EntryType[numberElements](); + + return static_cast(data); + } catch (...) { + TRI_UNMMFile(data, totalSize, _file, &_mmHandle); + _mmHandle = nullptr; + throw; + } + } + + void deallocateMemory() { + if (_table == nullptr) { + TRI_ASSERT(_nrAlloc == 0); + TRI_ASSERT(_nrUsed == 0); + return; + } + + if (_file == -1) { + delete[] _table; + } else { + if (TRI_UNMMFile(_table, requiredSize(_nrAlloc), _file, &_mmHandle) != TRI_ERROR_NO_ERROR) { + // unmapping failed + LOG(WARN) << "munmap failed"; + } + _mmHandle = nullptr; + } + _table = nullptr; + _nrAlloc = 0; + _nrUsed = 0; + } + + int allocateTempfile(char*& filename, size_t filesize) { + TRI_ASSERT(filename == nullptr); + + if (filesize < FileBackedThreshold) { + // use new/malloc + return -1; + } + + // create a temporary file + long errorCode; + std::string errorMessage; + + if (TRI_GetTempName(nullptr, &filename, false, errorCode, errorMessage) != TRI_ERROR_NO_ERROR) { + // go on without file, but with regular new/malloc + return -1; + } + + TRI_ASSERT(filename != nullptr); + + int fd = TRI_CreateDatafile(filename, filesize); + + if (fd < 0) { + TRI_Free(TRI_CORE_MEM_ZONE, filename); + filename = nullptr; + } + + return fd; + } + + void deallocateTempfile() { + if (_file >= 0) { + // close file pointer and reset fd + TRI_CLOSE(_file); + _file = -1; + } + if (_filename != nullptr) { + TRI_UnlinkFile(_filename); + TRI_Free(TRI_CORE_MEM_ZONE, _filename); + _filename = nullptr; + } + TRI_ASSERT(_file == -1); + } +}; + +} +} + +#endif diff --git a/lib/Basics/files.cpp b/lib/Basics/files.cpp index 4d817e2d06..c6cf0609f1 100644 --- a/lib/Basics/files.cpp +++ b/lib/Basics/files.cpp @@ -2397,6 +2397,115 @@ char* TRI_GetNullBufferFiles() { return &NullBuffer[0]; } size_t TRI_GetNullBufferSizeFiles() { return sizeof(NullBuffer); } +/// @brief creates a new datafile +/// returns the file descriptor or -1 if the file cannot be created +int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) { + TRI_ERRORBUF; + + // open the file + int fd = TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC | TRI_NOATIME, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); + + TRI_IF_FAILURE("CreateDatafile1") { + // intentionally fail + TRI_CLOSE(fd); + fd = -1; + errno = ENOSPC; + } + + if (fd < 0) { + if (errno == ENOSPC) { + TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL); + LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_last_error(); + } else { + TRI_SYSTEM_ERROR(); + + TRI_set_errno(TRI_ERROR_SYS_ERROR); + LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF; + } + return -1; + } + + // no fallocate present, or at least pretend it's not there... + int res = TRI_ERROR_NOT_IMPLEMENTED; + +#ifdef __linux__ +#ifdef FALLOC_FL_ZERO_RANGE + // try fallocate + res = fallocate(fd, FALLOC_FL_ZERO_RANGE, 0, maximalSize); +#endif +#endif + + if (res != TRI_ERROR_NO_ERROR) { + // either fallocate failed or it is not there... + + // fill file with zeros from FileNullBuffer + size_t writeSize = TRI_GetNullBufferSizeFiles(); + size_t written = 0; + while (written < maximalSize) { + if (writeSize + written > maximalSize) { + writeSize = maximalSize - written; + } + + ssize_t writeResult = + TRI_WRITE(fd, TRI_GetNullBufferFiles(), static_cast(writeSize)); + + TRI_IF_FAILURE("CreateDatafile2") { + // intentionally fail + writeResult = -1; + errno = ENOSPC; + } + + if (writeResult < 0) { + if (errno == ENOSPC) { + TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL); + LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_last_error(); + } else { + TRI_SYSTEM_ERROR(); + TRI_set_errno(TRI_ERROR_SYS_ERROR); + LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF; + } + + TRI_CLOSE(fd); + TRI_UnlinkFile(filename.c_str()); + + return -1; + } + + written += static_cast(writeResult); + } + } + + // go back to offset 0 + TRI_lseek_t offset = TRI_LSEEK(fd, (TRI_lseek_t)0, SEEK_SET); + + if (offset == (TRI_lseek_t)-1) { + TRI_SYSTEM_ERROR(); + TRI_set_errno(TRI_ERROR_SYS_ERROR); + TRI_CLOSE(fd); + + // remove empty file + TRI_UnlinkFile(filename.c_str()); + + LOG(ERR) << "cannot seek in datafile '" << filename << "': '" << TRI_GET_ERRORBUF << "'"; + return -1; + } + + return fd; +} + + +#if _WIN32 +bool TRI_PathIsAbsolute(const std::string &path) { + return !PathIsRelative(path.c_str()); +} + +#else +bool TRI_PathIsAbsolute(const std::string &path) { + return path.c_str()[0] == '/'; +} +#endif + //////////////////////////////////////////////////////////////////////////////// /// @brief initialize the files subsystem //////////////////////////////////////////////////////////////////////////////// @@ -2412,15 +2521,3 @@ void TRI_InitializeFiles() { void TRI_ShutdownFiles() {} - - -#if _WIN32 -bool TRI_PathIsAbsolute(const std::string &path) { - return !PathIsRelative(path.c_str()); -} - -#else -bool TRI_PathIsAbsolute(const std::string &path) { - return path.c_str()[0] == '/'; -} -#endif diff --git a/lib/Basics/files.h b/lib/Basics/files.h index 9d71cef14c..a41ed674ed 100644 --- a/lib/Basics/files.h +++ b/lib/Basics/files.h @@ -357,6 +357,16 @@ char* TRI_GetNullBufferFiles(); size_t TRI_GetNullBufferSizeFiles(); +/// @brief creates a new datafile +/// returns the file descriptor or -1 if the file cannot be created +int TRI_CreateDatafile(std::string const& filename, size_t maximalSize); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief checks whether path is full qualified or relative +//////////////////////////////////////////////////////////////////////////////// + +bool TRI_PathIsAbsolute(const std::string &path); + //////////////////////////////////////////////////////////////////////////////// /// @brief initialize the files subsystem //////////////////////////////////////////////////////////////////////////////// @@ -369,10 +379,4 @@ void TRI_InitializeFiles(); void TRI_ShutdownFiles(); -//////////////////////////////////////////////////////////////////////////////// -/// @brief checks whether path is full qualified or relative -//////////////////////////////////////////////////////////////////////////////// - -bool TRI_PathIsAbsolute(const std::string &path); - #endif From 350da367bd1e20833c5e8f32c65967ff7d087294 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Thu, 8 Dec 2016 17:32:46 +0100 Subject: [PATCH 50/60] fixes for Visual Studio --- arangod/Agency/Supervision.cpp | 2 +- arangod/Aql/EnumerateCollectionBlock.cpp | 2 +- arangod/V8Server/v8-vocbase.cpp | 4 ++-- lib/Basics/fasthash.cpp | 10 ++++------ 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index f866f4403c..6f10af52d4 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -462,7 +462,7 @@ void Supervision::run() { } } } - _cv.wait(1000000 * _frequency); + _cv.wait(static_cast(1000000 * _frequency)); } } if (shutdown) { diff --git a/arangod/Aql/EnumerateCollectionBlock.cpp b/arangod/Aql/EnumerateCollectionBlock.cpp index fdf236b701..4e4948ba01 100644 --- a/arangod/Aql/EnumerateCollectionBlock.cpp +++ b/arangod/Aql/EnumerateCollectionBlock.cpp @@ -135,7 +135,7 @@ int EnumerateCollectionBlock::initialize() { inSync = std::find(followers.begin(), followers.end(), ServerState::instance()->getId()) != followers.end(); if (!inSync) { if (endTime - now < waitInterval) { - waitInterval = endTime - now; + waitInterval = static_cast(endTime - now); } usleep(waitInterval); } diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index 46edfa6b94..7613ae72f3 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -2733,7 +2733,7 @@ static void JS_DecodeRev(v8::FunctionCallbackInfo const& args) { #endif char buffer[32]; strftime(buffer, 32, "%Y-%m-%dT%H:%M:%S.000Z", &date); - buffer[20] = (millis / 100) + '0'; + buffer[20] = static_cast(millis / 100) + '0'; buffer[21] = ((millis / 10) % 10) + '0'; buffer[22] = (millis % 10) + '0'; buffer[24] = 0; @@ -2742,7 +2742,7 @@ static void JS_DecodeRev(v8::FunctionCallbackInfo const& args) { result->Set(TRI_V8_ASCII_STRING("date"), TRI_V8_ASCII_STRING(buffer)); result->Set(TRI_V8_ASCII_STRING("count"), - v8::Number::New(isolate, count)); + v8::Number::New(isolate, static_cast(count))); TRI_V8_RETURN(result); diff --git a/lib/Basics/fasthash.cpp b/lib/Basics/fasthash.cpp index ceaee8262a..c5ca3360ef 100644 --- a/lib/Basics/fasthash.cpp +++ b/lib/Basics/fasthash.cpp @@ -24,6 +24,8 @@ */ #include "fasthash.h" + +static constexpr uint64_t m = 0x880355f21e6d1965ULL; // Compression function for Merkle-Damgard construction. // This function is generated using the framework provided. @@ -37,9 +39,7 @@ static inline uint64_t mix(uint64_t h) { } uint64_t fasthash64_uint64(uint64_t value, uint64_t seed) { - size_t const len = sizeof(uint64_t); - uint64_t const m = 0x880355f21e6d1965ULL; - uint64_t h = seed ^ (len * m); + uint64_t h = seed ^ 4619197404915747624ULL; // this is h = seed ^ (sizeof(uint64_t) * m), but prevents VS warning C4307: integral constant overflow h ^= mix(value); h *= m; @@ -51,7 +51,6 @@ uint64_t fasthash64(const void* buf, size_t len, uint64_t seed) { // byte-wise hashing to support platforms that don't permit // unaligned accesses of uint64_t values (which is the default // memory access strategy of fasthash64) - uint64_t const m = 0x880355f21e6d1965ULL; uint8_t const* pos = (uint8_t const*)buf; uint8_t const* end = pos + len; uint64_t h = seed ^ (len * m); @@ -92,7 +91,6 @@ uint64_t fasthash64(const void* buf, size_t len, uint64_t seed) { // uint64_t-wise hashing for platforms that allow dereferencing // unaligned pointers to uint64_t memory // this is the original version of fasthash64 - uint64_t const m = 0x880355f21e6d1965ULL; uint64_t const* pos = (uint64_t const*)buf; uint64_t const* end = pos + (len / 8); const unsigned char* pos2; @@ -104,7 +102,7 @@ uint64_t fasthash64(const void* buf, size_t len, uint64_t seed) { h ^= mix(v); h *= m; } - + pos2 = (const unsigned char*)pos; v = 0; From 404e04baa4b0527833ef1cd57bd97ab28063644b Mon Sep 17 00:00:00 2001 From: jsteemann Date: Thu, 8 Dec 2016 17:36:42 +0100 Subject: [PATCH 51/60] fix art --- arangod/Agency/FailedServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Agency/FailedServer.cpp b/arangod/Agency/FailedServer.cpp index 2324c0333c..d94156d4c2 100644 --- a/arangod/Agency/FailedServer.cpp +++ b/arangod/Agency/FailedServer.cpp @@ -176,7 +176,7 @@ bool FailedServer::start() { ++pos; } - if (found && !available.empty() > 0 && !isClone) { + if (found && !available.empty() && !isClone) { auto randIt = available.begin(); std::advance(randIt, std::rand() % available.size()); FailedFollower( From 9238463cc199447f2ddc376d93669922abc1a9b0 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Thu, 8 Dec 2016 17:37:51 +0100 Subject: [PATCH 52/60] More documentation for satellites --- .../Synchronous/Satellites/README.mdpp | 102 ++++++++++++++++++ .../Rest/Cursors/JSF_post_api_cursor.md | 6 ++ 2 files changed, 108 insertions(+) diff --git a/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp b/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp index 2f44734675..32f11637f0 100644 --- a/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp +++ b/Documentation/Books/Manual/Administration/Replication/Synchronous/Satellites/README.mdpp @@ -30,3 +30,105 @@ to "satellite". Using arangosh: arangosh> db._create("satellite", {"replicationFactor": "satellite"}); + +### A full example + + arangosh> var explain = require("@arangodb/aql/explainer").explain + arangosh> db._create("satellite", {"replicationFactor": "satellite"}) + arangosh> db._create("nonsatellite", {numberOfShards: 8}) + arangosh> db._create("nonsatellite2", {numberOfShards: 8}) + +Let's analyse a normal join not involving satellite collections: + +``` +arangosh> explain("FOR doc in nonsatellite FOR doc2 in nonsatellite2 RETURN 1") + +Query string: + FOR doc in nonsatellite FOR doc2 in nonsatellite2 RETURN 1 + +Execution plan: + Id NodeType Site Est. Comment + 1 SingletonNode DBS 1 * ROOT + 4 CalculationNode DBS 1 - LET #2 = 1 /* json expression */ /* const assignment */ + 2 EnumerateCollectionNode DBS 0 - FOR doc IN nonsatellite /* full collection scan */ + 12 RemoteNode COOR 0 - REMOTE + 13 GatherNode COOR 0 - GATHER + 6 ScatterNode COOR 0 - SCATTER + 7 RemoteNode DBS 0 - REMOTE + 3 EnumerateCollectionNode DBS 0 - FOR doc2 IN nonsatellite2 /* full collection scan */ + 8 RemoteNode COOR 0 - REMOTE + 9 GatherNode COOR 0 - GATHER + 5 ReturnNode COOR 0 - RETURN #2 + +Indexes used: + none + +Optimization rules applied: + Id RuleName + 1 move-calculations-up + 2 scatter-in-cluster + 3 remove-unnecessary-remote-scatter +``` + +All shards involved querying the `nonsatellite` collection will fan out via the +coordinator to the shards of `nonsatellite`. In sum 8 shards will open 8 connections +to the coordinator asking for the results of the `nonsatellite2` join. The coordinator +will fan out to the 8 shards of `nonsatellite2`. So there will be quite some +network traffic. + +Let's now have a look at the same using satellite collections: + +``` +arangosh> db._query("FOR doc in nonsatellite FOR doc2 in satellite RETURN 1") + +Query string: + FOR doc in nonsatellite FOR doc2 in satellite RETURN 1 + +Execution plan: + Id NodeType Site Est. Comment + 1 SingletonNode DBS 1 * ROOT + 4 CalculationNode DBS 1 - LET #2 = 1 /* json expression */ /* const assignment */ + 2 EnumerateCollectionNode DBS 0 - FOR doc IN nonsatellite /* full collection scan */ + 3 EnumerateCollectionNode DBS 0 - FOR doc2 IN satellite /* full collection scan, satellite */ + 8 RemoteNode COOR 0 - REMOTE + 9 GatherNode COOR 0 - GATHER + 5 ReturnNode COOR 0 - RETURN #2 + +Indexes used: + none + +Optimization rules applied: + Id RuleName + 1 move-calculations-up + 2 scatter-in-cluster + 3 remove-unnecessary-remote-scatter + 4 remove-satellite-joins +``` + +In this scenario all shards of nonsatellite will be contacted. However +as the join is a satellite join all shards can do the join locally +as the data is replicated to all servers reducing the network overhead +dramatically. + +### Caveats + +The cluster will automatically keep all satellite collections on all servers in sync +by facilitating the synchronous replication. This means that write will be executed +on the leader only and this server will coordinate replication to the followers. +If a follower doesn't answer in time (due to network problems, temporary shutdown etc.) +it may be removed as a follower. This is being reported to the Agency. + +The follower (once back in business) will then periodically check the Agency and know +that it is out of sync. It will then automatically catch up. This may take a while +depending on how much data has to be synced. When doing a join involving the satellite +you can specify how long the DBServer is allowed to wait for sync until the query +is being aborted. + +Check [https://docs.arangodb.com/3.1/HTTP/AqlQueryCursor/AccessingCursors.html] for +details. + +During network failure there is also a minimal chance that a query was properly +distributed to the DBServers but that a previous satellite write could not be +replicated to a follower and the leader dropped the follower. The follower however +only checks every few seconds if it is really in sync so it might indeed deliver +stale results. diff --git a/Documentation/DocuBlocks/Rest/Cursors/JSF_post_api_cursor.md b/Documentation/DocuBlocks/Rest/Cursors/JSF_post_api_cursor.md index 1609cbe74a..608636ec53 100644 --- a/Documentation/DocuBlocks/Rest/Cursors/JSF_post_api_cursor.md +++ b/Documentation/DocuBlocks/Rest/Cursors/JSF_post_api_cursor.md @@ -73,6 +73,12 @@ If set to *true*, then the additional query profiling information will be return in the sub-attribute *profile* of the *extra* return attribute if the query result is not served from the query cache. +@RESTSTRUCT{satelliteSyncWait,JSF_post_api_cursor_opts,boolean,optional,} +This *enterprise* parameter allows to configure how long a DBServer will have time +to bring the satellite collections involved in the query into sync. +The default value is *60.0* (seconds). When the max time has been reached the query +will be stopped. + @RESTDESCRIPTION The query details include the query string plus optional query options and bind parameters. These values need to be passed in a JSON representation in From e7be3e5a1dfb59e0e4eac5dc76e0a43bbb2db973 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 8 Dec 2016 18:41:46 +0100 Subject: [PATCH 53/60] only git pull after doing the enterprise checkout if we're on a branch. --- Installation/Jenkins/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index d9e599f7da..4d9031dc04 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -514,6 +514,7 @@ if test -n "${ENTERPRISE_GIT_URL}" ; then GITARGS=devel fi echo "I'm on Branch: ${GITARGS}" + FINAL_PULL="git pull" fi # clean up if we're commanded to: if test -d enterprise -a ${CLEAN_IT} -eq 1; then @@ -522,7 +523,7 @@ if test -n "${ENTERPRISE_GIT_URL}" ; then if test ! -d enterprise; then git clone ${ENTERPRISE_GIT_URL} enterprise fi - (cd enterprise; git checkout master; git fetch --tags; git pull --all; git checkout ${GITARGS}; git pull ) + (cd enterprise; git checkout master; git fetch --tags; git pull --all; git checkout ${GITARGS}; ${FINAL_PULL} ) fi From 5945e575beb2e4c2f086dee1da51df89c337d6ad Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 8 Dec 2016 18:43:47 +0100 Subject: [PATCH 54/60] export it for the subshell --- Installation/Jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 4d9031dc04..0cabaa94ba 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -514,7 +514,7 @@ if test -n "${ENTERPRISE_GIT_URL}" ; then GITARGS=devel fi echo "I'm on Branch: ${GITARGS}" - FINAL_PULL="git pull" + export FINAL_PULL="git pull" fi # clean up if we're commanded to: if test -d enterprise -a ${CLEAN_IT} -eq 1; then From 7ff241251bba795dae1d82ff1c6299af8baa1a01 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 8 Dec 2016 19:47:10 +0100 Subject: [PATCH 55/60] add explanation about the differences on path-filters vs. edge/vertex filters --- Documentation/Books/AQL/Graphs/Traversals.mdpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/Books/AQL/Graphs/Traversals.mdpp b/Documentation/Books/AQL/Graphs/Traversals.mdpp index 315e79098f..a3ad32aed8 100644 --- a/Documentation/Books/AQL/Graphs/Traversals.mdpp +++ b/Documentation/Books/AQL/Graphs/Traversals.mdpp @@ -284,6 +284,18 @@ FOR v, e, p IN 1..5 OUTBOUND 'circles/A' GRAPH 'traversalGraph' It is guaranteed that at least one, but potentially more edges fulfill the condition. All of the above filters can be defined on vertices in the exact same way. +### Filtering on the path vs. filtering on vertices or edges +Filtering on the path influences the Iteration on your graph. If certain conditions +aren't met, the traversal may stop continuing along this path. + +In contrast filters on vertex or edge only express whether you're interestet in the actual value of these +documents. Thus, it influences the list of returned documents (if you return v or e) similar +as specifying a non-null `min` value. If you specify a min value of 2, the traversal over the first +two nodes of these paths has to be executed - you just won't see them in your result array. + +Similar are filters on vertices or edges - the traverser has to walk along these nodes, since +you may be interested in documents further down the path. + ### Examples From 576ccaaa72094c1b602a112bda1ff6ed801c01c4 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 8 Dec 2016 20:01:22 +0100 Subject: [PATCH 56/60] reuse tar installation process to create the CI tarball --- Installation/Jenkins/build.sh | 12 ++++++++++-- cmake/packages/packages.cmake | 10 ++++++++++ cmake/packages/snap.cmake | 8 +------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 0cabaa94ba..89f18cf3b2 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -564,18 +564,26 @@ if test -n "${TARGET_DIR}"; then ${PACKAGE_MAKE} copy_packages ${PACKAGE_MAKE} clean_packages else + # we re-use a generic cpack tarball: + ${PACKAGE_MAKE} TGZ_package + PKG_NAME=`grep CPACK_PACKAGE_FILE_NAME CPackConfig.cmake |sed -e 's;".$;;' -e 's;.*";;'` + + TARFILE=arangodb-`uname`${TAR_SUFFIX}.tar.gz TARFILE_TMP=`pwd`/arangodb.tar.$$ + trap "rm -rf ${TARFILE_TMP}" EXIT mkdir -p ${dir} - trap "rm -rf ${TARFILE_TMP}" EXIT + + cp -a ${SOURCE_DIR}/_CPack_Packages/*/TGZ/${PKG_NAME}/* ${dir} + rm -rf ${dir}/share/arangodb3/js (cd ${SOURCE_DIR} touch 3rdParty/.keepme touch arangod/.keepme touch arangosh/.keepme - + tar -c -f ${TARFILE_TMP} \ VERSION utils scripts etc/relative etc/testing UnitTests Documentation js \ lib/Basics/errors.dat \ diff --git a/cmake/packages/packages.cmake b/cmake/packages/packages.cmake index 93c586dcfa..2e6a293bcc 100644 --- a/cmake/packages/packages.cmake +++ b/cmake/packages/packages.cmake @@ -37,6 +37,16 @@ elseif (MSVC) include(packages/nsis) endif () +################################################################################ +## generic tarball +################################################################################ +set(CPACK_PACKAGE_TGZ "${CMAKE_BINARY_DIR}/${CPACK_PACKAGE_FILE_NAME}.tar.gz") +add_custom_target(TGZ_package + COMMENT "create TGZ-package" + COMMAND ${CMAKE_CPACK_COMMAND} -G TGZ -C ${CMAKE_BUILD_TYPE} + ) + + ################################################################################ ## SNAPCRAFT PACKAGE ################################################################################ diff --git a/cmake/packages/snap.cmake b/cmake/packages/snap.cmake index 5e873e2e64..ea7dcf0cd1 100644 --- a/cmake/packages/snap.cmake +++ b/cmake/packages/snap.cmake @@ -7,7 +7,6 @@ endif() if(SNAPCRAFT_FOUND) set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${ARANGODB_PACKAGE_REVISION}_${ARANGODB_PACKAGE_ARCHITECTURE}") set(SNAPCRAFT_TEMPLATE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/Installation/Ubuntu") - set(CPACK_PACKAGE_TGZ "${CMAKE_BINARY_DIR}/${CPACK_PACKAGE_FILE_NAME}.tar.gz") set(SNAPCRAFT_SOURCE_DIR "${CMAKE_BINARY_DIR}/_CPack_Packages/SNAP") message(STATUS "Create snap package") @@ -35,16 +34,11 @@ if(SNAPCRAFT_FOUND) DESTINATION "${SNAPCRAFT_SOURCE_DIR}/" ) - add_custom_target(snap_TGZ - COMMENT "create TGZ-package" - COMMAND ${CMAKE_CPACK_COMMAND} -G TGZ - ) - add_custom_target(snap COMMENT "create snap-package" COMMAND ${SNAP_EXE} snap COMMAND cp *.snap ${PROJECT_BINARY_DIR} - DEPENDS snap_TGZ + DEPENDS TGZ_package WORKING_DIRECTORY ${SNAPCRAFT_SOURCE_DIR} ) From 20e6bf1b6d0bff0ff519ec3298958fb0083dcc13 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 9 Dec 2016 02:34:47 +0100 Subject: [PATCH 57/60] we don't need the dpkg-shlibdeps workaround anymore. --- cmake/packages/client/deb.txt | 8 +------- cmake/packages/deb.cmake | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/cmake/packages/client/deb.txt b/cmake/packages/client/deb.txt index 4b08be07d4..f6251d77ec 100644 --- a/cmake/packages/client/deb.txt +++ b/cmake/packages/client/deb.txt @@ -65,13 +65,7 @@ set(CPACK_DEBIAN_PACKAGE_SECTION "shell") set(CPACK_PACKAGE_VENDOR ${ARANGODB_PACKAGE_VENDOR}) set(CPACK_PACKAGE_CONTACT ${ARANGODB_PACKAGE_CONTACT}) set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${ARANGODB_URL_INFO_ABOUT}) -# build of dependecies don't work on cross compiling (yet) -if (CROSS_COMPILING) - set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS OFF) - set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.14), libgcc1 (>= 1:3.4), libssl1.0.0 (>= 1.0.1), libstdc++6 (>= 5.2)") -else() - set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) -endif() +set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) set(CPACK_DEBIAN_PACKAGE_CONFLICTS "arangodb, ${CPACKG_PACKAGE_CONFLICTS}, ${CPACKG_PACKAGE_CONFLICTS}-client, ${CPACK_PACKAGE_NAME}") set(CPACK_DEBIAN_COMPRESSION_TYPE "xz") set(CPACK_COMPONENTS_ALL debian-extras) diff --git a/cmake/packages/deb.cmake b/cmake/packages/deb.cmake index fdf7fe1d4f..ad598a8299 100644 --- a/cmake/packages/deb.cmake +++ b/cmake/packages/deb.cmake @@ -10,13 +10,7 @@ set(PACKAGING_HANDLE_CONFIG_FILES true) FILE(READ "${PROJECT_SOURCE_DIR}/Installation/debian/packagedesc.txt" CPACK_DEBIAN_PACKAGE_DESCRIPTION) set(CPACK_DEBIAN_PACKAGE_SECTION "database") set(CPACK_DEBIAN_PACKAGE_CONFLICTS "arangodb, ${CPACKG_PACKAGE_CONFLICTS}, ${CPACKG_PACKAGE_CONFLICTS}-client, ${CPACK_PACKAGE_NAME}-client") -# build of dependecies don't work on cross compiling (yet) -if (CROSS_COMPILING) - set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS OFF) - set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.14), libgcc1 (>= 1:3.4), libssl1.0.0 (>= 1.0.1), libstdc++6 (>= 5.2)") -else() - set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) -endif() +set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) if(NOT DISABLE_XZ_DEB) set(CPACK_DEBIAN_COMPRESSION_TYPE "xz") endif() From eddecc0a4c90a951fef72f43eecfb9881978cbd1 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Fri, 9 Dec 2016 09:29:00 +0100 Subject: [PATCH 58/60] clones method in Jobs more useful --- arangod/Agency/AddFollower.cpp | 24 ++++------------- arangod/Agency/FailedFollower.cpp | 23 +++++----------- arangod/Agency/Job.cpp | 45 ++++++++++++++++++++----------- arangod/Agency/Job.h | 13 ++++++--- 4 files changed, 51 insertions(+), 54 deletions(-) diff --git a/arangod/Agency/AddFollower.cpp b/arangod/Agency/AddFollower.cpp index d247a6d3e2..89c98d0c58 100644 --- a/arangod/Agency/AddFollower.cpp +++ b/arangod/Agency/AddFollower.cpp @@ -100,27 +100,13 @@ bool AddFollower::create() { TRI_ASSERT(current[0].isString()); #endif - std::string planPath = - planColPrefix + _database + "/" + _collection + "/shards"; - - auto const& myClones = clones(_snapshot, _database, _collection); + size_t sub = 0; + auto const& myClones = clones(_snapshot, _database, _collection, _shard); if (!myClones.empty()) { - - size_t sub = 0; - auto myshards = _snapshot(planPath).children(); - auto mpos = std::distance(myshards.begin(), myshards.find(_shard)); - - // Deal with my clones - for (auto const& collection : myClones) { - auto othershards = _snapshot( - planColPrefix + _database + "/" + collection + "/shards").children(); - auto opos = othershards.begin(); - std::advance(opos, mpos); - auto const& shard = opos->first; - + for (auto const& clone : myClones) { AddFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++), - _jobId, _agencyPrefix, _database, collection, shard, - _newFollower); + _jobId, _agencyPrefix, _database, clone.collection, + clone.shard, _newFollower); } } diff --git a/arangod/Agency/FailedFollower.cpp b/arangod/Agency/FailedFollower.cpp index 92c0de7629..2ba8026b8e 100644 --- a/arangod/Agency/FailedFollower.cpp +++ b/arangod/Agency/FailedFollower.cpp @@ -68,25 +68,14 @@ bool FailedFollower::create() { std::string path = _agencyPrefix + toDoPrefix + _jobId; std::string planPath = planColPrefix + _database + "/" + _collection + "/shards"; - - auto const& myClones = clones(_snapshot, _database, _collection); + + size_t sub = 0; + auto const& myClones = clones(_snapshot, _database, _collection, _shard); if (!myClones.empty()) { - - size_t sub = 0; - auto myshards = _snapshot(planPath).children(); - auto mpos = std::distance(myshards.begin(), myshards.find(_shard)); - - // Deal with my clones - for (auto const& collection : myClones) { - auto othershards = _snapshot( - planColPrefix + _database + "/" + collection + "/shards").children(); - auto opos = othershards.begin(); - std::advance(opos, mpos); - auto const& shard = opos->first; - + for (auto const& clone : myClones) { FailedFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++), - _jobId, _agencyPrefix, _database, collection, shard, - _from, _to); + _jobId, _agencyPrefix, _database, clone.collection, + clone.shard, _from, _to); } } diff --git a/arangod/Agency/Job.cpp b/arangod/Agency/Job.cpp index 09de7dac85..eca552a8b8 100644 --- a/arangod/Agency/Job.cpp +++ b/arangod/Agency/Job.cpp @@ -178,22 +178,37 @@ std::vector Job::availableServers(Node const& snapshot) { } -std::vector Job::clones(Node const& snapshot, - std::string const& database, - std::string const& collection) { +std::vector Job::clones( + Node const& snapshot, std::string const& database, + std::string const& collection, std::string const& shard) { - std::vector ret; - std::string databasePath = planColPrefix + database; - try { - for (const auto& colptr : snapshot(databasePath).children()) { // databases - try { - auto const col = *colptr.second; - if (col("distributeShardsLike").slice().copyString() == collection) { - ret.push_back(colptr.first); - } - } catch(...) {} - } - } catch (...) {} + std::vector ret; + + std::string databasePath = planColPrefix + database, + planPath = databasePath + "/" + collection + "/shards"; + + auto myshards = snapshot(planPath).children(); + auto steps = std::distance(myshards.begin(), myshards.find(shard)); + + for (const auto& colptr : snapshot(databasePath).children()) { // collections + + auto const col = *colptr.second; + auto const otherCollection = colptr.first; + + try { + std::string const& prototype = + col("distributeShardsLike").slice().copyString(); + if (otherCollection != collection && prototype == collection) { + auto othershards = col("shards").children(); + auto opos = othershards.begin(); + std::advance(opos, steps); + auto const& otherShard = opos->first; + ret.push_back(shard_t(otherCollection, otherShard)); + } + } catch(...) {} + + } + return ret; } diff --git a/arangod/Agency/Job.h b/arangod/Agency/Job.h index e1406414e3..571223482c 100644 --- a/arangod/Agency/Job.h +++ b/arangod/Agency/Job.h @@ -93,6 +93,13 @@ struct JobCallback { }; struct Job { + + struct shard_t { + std::string collection; + std::string shard; + shard_t (std::string const& c, std::string const& s) : + collection(c), shard(s) {} + }; Job(Node const& snapshot, Agent* agent, std::string const& jobId, std::string const& creator, std::string const& agencyPrefix); @@ -112,9 +119,9 @@ struct Job { static std::vector availableServers( const arangodb::consensus::Node&); - static std::vector clones( - Node const& snapshot, std::string const& database, - std::string const& collection); + static std::vector clones( + Node const& snap, std::string const& db, std::string const& col, + std::string const& shrd); Node const _snapshot; Agent* _agent; From 909335fc61ce3eb1164167b3b464de568c46ea71 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Fri, 9 Dec 2016 09:52:38 +0100 Subject: [PATCH 59/60] fix VS compile errors --- lib/Basics/IndexBucket.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Basics/IndexBucket.h b/lib/Basics/IndexBucket.h index 27883a2a48..3644ff01c7 100644 --- a/lib/Basics/IndexBucket.h +++ b/lib/Basics/IndexBucket.h @@ -166,7 +166,8 @@ struct IndexBucket { TRI_ASSERT(_mmHandle == nullptr); void* data; - int res = TRI_MMFile(nullptr, totalSize, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_POPULATE, _file, &_mmHandle, 0, &data); + int flags = MAP_SHARED; + int res = TRI_MMFile(nullptr, totalSize, PROT_WRITE | PROT_READ, flags, _file, &_mmHandle, 0, &data); if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); From 69ee472a93765c37314b1b0031a01160d1787951 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 9 Dec 2016 14:13:35 +0100 Subject: [PATCH 60/60] properly start the tarball --- Installation/Jenkins/build.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 89f18cf3b2..8c2bf9d9e4 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -575,16 +575,15 @@ if test -n "${TARGET_DIR}"; then mkdir -p ${dir} - cp -a ${SOURCE_DIR}/_CPack_Packages/*/TGZ/${PKG_NAME}/* ${dir} - rm -rf ${dir}/share/arangodb3/js - + (cd _CPack_Packages/*/TGZ/${PKG_NAME}/; rm -rf ${dir}/share/arangodb3/js; tar -c -f ${TARFILE_TMP} *) + (cd ${SOURCE_DIR} touch 3rdParty/.keepme touch arangod/.keepme touch arangosh/.keepme - tar -c -f ${TARFILE_TMP} \ + tar -u -f ${TARFILE_TMP} \ VERSION utils scripts etc/relative etc/testing UnitTests Documentation js \ lib/Basics/errors.dat \ 3rdParty/.keepme \