From 2db0212e4f34e3840101bcef37c80c7bdebc8914 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 30 Jan 2017 17:02:27 +0100 Subject: [PATCH 01/22] supervision tests fail under load --- js/client/modules/@arangodb/testing.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 6af73d1b06..2d87d7c4e1 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -1490,8 +1490,8 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir) usedPorts.push(port); instanceArgs['server.endpoint'] = protocol + '://127.0.0.1:' + port; instanceArgs['agency.my-address'] = protocol + '://127.0.0.1:' + port; - instanceArgs['agency.supervision-grace-period'] = '5.0'; - instanceArgs['agency.supervision-frequency'] = '0.05'; + instanceArgs['agency.supervision-grace-period'] = '10.0'; + instanceArgs['agency.supervision-frequency'] = '1.0'; if (i === N - 1) { let l = []; From cb9bf7c2ff3839a4bb69f2d5ff97609a36559661 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 30 Jan 2017 17:06:02 +0100 Subject: [PATCH 02/22] added obsolete parameter optionality --- arangod/Cluster/ClusterFeature.cpp | 6 ++++-- lib/ProgramOptions/Parameters.h | 5 ++++- lib/ProgramOptions/ProgramOptions.h | 5 +++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/arangod/Cluster/ClusterFeature.cpp b/arangod/Cluster/ClusterFeature.cpp index 8f8c0821e2..d89c7a74e2 100644 --- a/arangod/Cluster/ClusterFeature.cpp +++ b/arangod/Cluster/ClusterFeature.cpp @@ -79,9 +79,11 @@ void ClusterFeature::collectOptions(std::shared_ptr options) { options->addSection("cluster", "Configure the cluster"); options->addObsoleteOption("--cluster.username", - "username used for cluster-internal communication"); + "username used for cluster-internal communication", + true); options->addObsoleteOption("--cluster.password", - "password used for cluster-internal communication"); + "password used for cluster-internal communication", + true); options->addOption("--cluster.agency-endpoint", "agency endpoint to connect to", diff --git a/lib/ProgramOptions/Parameters.h b/lib/ProgramOptions/Parameters.h index 0fd03d6a3e..b813687017 100644 --- a/lib/ProgramOptions/Parameters.h +++ b/lib/ProgramOptions/Parameters.h @@ -418,13 +418,16 @@ struct VectorParameter : public Parameter { // a type that's useful for obsolete parameters that do nothing struct ObsoleteParameter : public Parameter { - bool requiresValue() const override { return false; } + explicit ObsoleteParameter(bool requiresValue) : required(requiresValue) {} + bool requiresValue() const override { return required; } std::string name() const override { return "obsolete"; } std::string valueString() const override { return "-"; } std::string set(std::string const&) override { return ""; } void toVPack(VPackBuilder& builder) const override { builder.add(VPackValue(VPackValueType::Null)); } + + bool required; }; } } diff --git a/lib/ProgramOptions/ProgramOptions.h b/lib/ProgramOptions/ProgramOptions.h index 45a7d4dee7..33e4f95770 100644 --- a/lib/ProgramOptions/ProgramOptions.h +++ b/lib/ProgramOptions/ProgramOptions.h @@ -199,8 +199,9 @@ class ProgramOptions { // adds an obsolete and hidden option to the program options void addObsoleteOption(std::string const& name, - std::string const& description) { - addOption(Option(name, description, new ObsoleteParameter(), true, true)); + std::string const& description, + bool requiresValue) { + addOption(Option(name, description, new ObsoleteParameter(requiresValue), true, true)); } // prints usage information From 2c431ea9f158ed9d247583f95e10a3e4a44bd4cd Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 30 Jan 2017 17:27:13 +0100 Subject: [PATCH 03/22] documentation for ternary operator shortcut --- CHANGELOG | 10 ++++++++++ js/server/tests/aql/aql-ternary.js | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index af19ea4e13..b5978502c6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,16 @@ devel ----- +* added shortcut for AQL ternary operator + instead of `condition ? true-part : false-part` it is now possible to also use a + shortcut variant `condition ? : false-part`, e.g. + + FOR doc IN docs RETURN doc.value ?: 'not present' + + instead of + + FOR doc IN docs RETURN doc.value ? doc.value : 'not present' + * fixed wrong sorting order in cluster, if an index was used to sort with many shards. diff --git a/js/server/tests/aql/aql-ternary.js b/js/server/tests/aql/aql-ternary.js index a32c922c3e..c9c1cfa556 100644 --- a/js/server/tests/aql/aql-ternary.js +++ b/js/server/tests/aql/aql-ternary.js @@ -221,7 +221,27 @@ function ahuacatlTernaryTestSuite () { assertEqual([ 2 ], getQueryResults("RETURN NOOPT([ ] ? 2 : 3)")); assertEqual([ 2 ], getQueryResults("RETURN NOOPT([ 0 ] ? 2 : 3)")); assertEqual([ 2 ], getQueryResults("RETURN NOOPT({ } ? 2 : 3)")); - } + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test ternary operator shortcut +//////////////////////////////////////////////////////////////////////////////// + + testTernaryShortcut : function () { + var expected = [ 'foo', 'foo', 1, 2 ]; + + var actual = getQueryResults("FOR i IN [ null, 0, 1, 2 ] RETURN i ? : 'foo'"); + assertEqual(expected, actual); + + actual = getQueryResults("FOR i IN [ null, 0, 1, 2 ] RETURN i ?: 'foo'"); + assertEqual(expected, actual); + + actual = getQueryResults("FOR i IN [ null, 0, 1, 2 ] RETURN NOOPT(i ? : 'foo')"); + assertEqual(expected, actual); + + actual = getQueryResults("FOR i IN [ null, 0, 1, 2 ] RETURN NOOPT(i ?: 'foo')"); + assertEqual(expected, actual); + }, }; } From d1575670c0c33252514e96ba14b2b74e75ac461d Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 30 Jan 2017 17:30:58 +0100 Subject: [PATCH 04/22] updated documentation --- Documentation/Books/AQL/Operators.mdpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/Books/AQL/Operators.mdpp b/Documentation/Books/AQL/Operators.mdpp index 0c58cb06b6..91d25268bb 100644 --- a/Documentation/Books/AQL/Operators.mdpp +++ b/Documentation/Books/AQL/Operators.mdpp @@ -270,6 +270,17 @@ evaluates to true, and the third operand otherwise. u.age > 15 || u.active == true ? u.userId : null ``` +There is also a shortcut variant of the ternary operator with just two +operands. This variant can be used when the expression for the boolean +condition and the return value should be the same: + +*Examples* + +```js +u.value ? : 'value is null, 0 or not present' +``` + + #### Range operator AQL supports expressing simple numeric ranges with the *..* operator. From 5b736dc3195e642e48ecb65e064f080084c57f99 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 30 Jan 2017 13:05:10 -0500 Subject: [PATCH 05/22] Changed index-filling to dispatch batch insertion tasks to scheduler, obsoleted index-threads option. --- CHANGELOG | 52 +- .../Configuration/GeneralArangod.mdpp | 24 +- Documentation/DocuBlocks/indexThreads.md | 17 - arangod/CMakeLists.txt | 3 +- arangod/Indexes/EdgeIndex.cpp | 213 +++---- arangod/Indexes/EdgeIndex.h | 72 +-- arangod/Indexes/HashIndex.cpp | 270 +++++---- arangod/Indexes/HashIndex.h | 152 ++--- arangod/Indexes/Index.cpp | 110 ++-- arangod/Indexes/Index.h | 37 +- arangod/RestServer/DatabaseFeature.cpp | 46 +- arangod/RestServer/ServerFeature.cpp | 5 +- arangod/RestServer/arangod.cpp | 9 +- arangod/Scheduler/SchedulerFeature.cpp | 3 +- .../StorageEngine/MMFilesWalRecoverState.cpp | 429 +++++++++----- arangod/VocBase/IndexThreadFeature.cpp | 72 --- arangod/VocBase/IndexThreadFeature.h | 51 -- arangod/VocBase/LogicalCollection.cpp | 525 +++++++----------- arangod/VocBase/LogicalCollection.h | 270 ++++----- js/client/modules/@arangodb/testing.js | 12 +- lib/Basics/AssocMulti.h | 334 ++++------- lib/Basics/AssocMultiHelpers.h | 231 ++++++++ lib/Basics/AssocUnique.h | 296 ++++------ lib/Basics/AssocUniqueHelpers.h | 201 +++++++ lib/Basics/LocalTaskQueue.cpp | 207 +++++++ lib/Basics/LocalTaskQueue.h | 190 +++++++ lib/Basics/ThreadPool.cpp | 104 ---- lib/Basics/ThreadPool.h | 95 ---- lib/Basics/WorkerThread.h | 86 --- lib/CMakeLists.txt | 4 +- 30 files changed, 2225 insertions(+), 1895 deletions(-) delete mode 100644 Documentation/DocuBlocks/indexThreads.md delete mode 100644 arangod/VocBase/IndexThreadFeature.cpp delete mode 100644 arangod/VocBase/IndexThreadFeature.h create mode 100644 lib/Basics/AssocMultiHelpers.h create mode 100644 lib/Basics/AssocUniqueHelpers.h create mode 100644 lib/Basics/LocalTaskQueue.cpp create mode 100644 lib/Basics/LocalTaskQueue.h delete mode 100644 lib/Basics/ThreadPool.cpp delete mode 100644 lib/Basics/ThreadPool.h delete mode 100644 lib/Basics/WorkerThread.h diff --git a/CHANGELOG b/CHANGELOG index b5978502c6..c99b3244ad 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,10 @@ devel ----- +* removed IndexThreadFeature, made --database.index-threads option obsolete + +* changed index filling to make it more parallel, dispatch tasks to boost::asio + * added shortcut for AQL ternary operator instead of `condition ? true-part : false-part` it is now possible to also use a shortcut variant `condition ? : false-part`, e.g. @@ -8,7 +12,7 @@ devel FOR doc IN docs RETURN doc.value ?: 'not present' instead of - + FOR doc IN docs RETURN doc.value ? doc.value : 'not present' * fixed wrong sorting order in cluster, if an index was used to sort with many @@ -26,38 +30,38 @@ shards. setting the flag to `true` will make the count operation returned the per-shard counts for the collection: - db._create("test", { numberOfShards: 10 }); - for (i = 0; i < 1000; ++i) { - db.test.insert({value: i}); + db._create("test", { numberOfShards: 10 }); + for (i = 0; i < 1000; ++i) { + db.test.insert({value: i}); } db.test.count(true); - { - "s100058" : 99, - "s100057" : 103, - "s100056" : 100, - "s100050" : 94, - "s100055" : 90, - "s100054" : 122, - "s100051" : 109, - "s100059" : 99, - "s100053" : 95, - "s100052" : 89 + { + "s100058" : 99, + "s100057" : 103, + "s100056" : 100, + "s100050" : 94, + "s100055" : 90, + "s100054" : 122, + "s100051" : 109, + "s100059" : 99, + "s100053" : 95, + "s100052" : 89 } * added optional memory limit for AQL queries: db._query("FOR i IN 1..100000 SORT i RETURN i", {}, { options: { memoryLimit: 100000 } }); - This option limits the default maximum amount of memory (in bytes) that a single + This option limits the default maximum amount of memory (in bytes) that a single AQL query can use. When a single AQL query reaches the specified limit value, the query will be aborted with a *resource limit exceeded* exception. In a cluster, the memory accounting is done per shard, so the limit value is effectively a memory limit per query per shard. - The global limit value can be overriden per query by setting the *memoryLimit* - option value for individual queries when running an AQL query. + The global limit value can be overriden per query by setting the *memoryLimit* + option value for individual queries when running an AQL query. * added server startup option `--query.memory-limit` @@ -180,7 +184,7 @@ v3.1.4 (2016-12-08) * agency RAFT timing estimates more conservative in waitForSync scenario - + * agency RAFT timing estimates capped at maximum 2.0/10.0 for low/high @@ -291,7 +295,7 @@ v3.1.1 (2016-11-15) produced an empty result * disallow updating `_from` and `_to` values of edges in Smart Graphs. Updating these - attributes would lead to potential redistribution of edges to other shards, which must be + attributes would lead to potential redistribution of edges to other shards, which must be avoided. * fixed issue #2148 @@ -1179,7 +1183,7 @@ v2.8.12 (XXXX-XX-XX) v2.8.11 (2016-07-13) -------------------- -* fixed array index batch insertion issues for hash indexes that caused problems when +* fixed array index batch insertion issues for hash indexes that caused problems when no elements remained for insertion * fixed issue #1937 @@ -1621,9 +1625,9 @@ v2.8.0-beta7 (2016-01-06) * fixes for AQL optimizer and traversal * added `--create-collection-type` option to arangoimp - - This allows specifying the type of the collection to be created when - `--create-collection` is set to `true`. + + This allows specifying the type of the collection to be created when + `--create-collection` is set to `true`. * Foxx export cache should no longer break if a broken app is loaded in the web admin interface. diff --git a/Documentation/Books/Manual/Administration/Configuration/GeneralArangod.mdpp b/Documentation/Books/Manual/Administration/Configuration/GeneralArangod.mdpp index 76a8209bdc..f0c3d7cf5f 100644 --- a/Documentation/Books/Manual/Administration/Configuration/GeneralArangod.mdpp +++ b/Documentation/Books/Manual/Administration/Configuration/GeneralArangod.mdpp @@ -132,16 +132,16 @@ daemon. This parameter must be specified if either the flag *daemon* or `--console` -Runs the server in an exclusive emergency console mode. When +Runs the server in an exclusive emergency console mode. When starting the server with this option, the server is started with an interactive JavaScript emergency console, with all networking and HTTP interfaces of the server disabled. No requests can be made to the server in this mode, and the only way to work with the server in this mode is by using the emergency -console. -Note that the server cannot be started in this mode if it is -already running in this or another mode. +console. +Note that the server cannot be started in this mode if it is +already running in this or another mode. ### Random Generator @@ -419,21 +419,6 @@ This option only has an effect if the query cache mode is set to either *on* or *demand*. -### Index threads - -`--database.index-threads` - -Specifies the *number* of background threads for index creation. When a -collection contains extra indexes other than the primary index, these other -indexes can be built by multiple threads in parallel. The index threads are -shared among multiple collections and databases. Specifying a value of *0* will -turn off parallel building, meaning that indexes for each collection are built -sequentially by the thread that opened the collection. If the number of index -threads is greater than 1, it will also be used to built the edge index of a -collection in parallel (this also requires the edge index in the collection to -be split into multiple buckets). - - ### V8 contexts `--javascript.v8-contexts number` @@ -491,4 +476,3 @@ might change in the future if a different version of V8 is being used in ArangoDB. Not all options offered by V8 might be sensible to use in the context of ArangoDB. Use the specific options only if you are sure that they are not harmful for the regular database operation. - diff --git a/Documentation/DocuBlocks/indexThreads.md b/Documentation/DocuBlocks/indexThreads.md deleted file mode 100644 index b9bc150763..0000000000 --- a/Documentation/DocuBlocks/indexThreads.md +++ /dev/null @@ -1,17 +0,0 @@ - - -@brief number of background threads for parallel index creation -`--database.index-threads` - -Specifies the *number* of background threads for index creation. When a -collection contains extra indexes other than the primary index, these -other -indexes can be built by multiple threads in parallel. The index threads -are shared among multiple collections and databases. Specifying a value of -*0* will turn off parallel building, meaning that indexes for each -collection -are built sequentially by the thread that opened the collection. -If the number of index threads is greater than 1, it will also be used to -built the edge index of a collection in parallel (this also requires the -edge index in the collection to be split into multiple buckets). - diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index a099c18178..d996b9600d 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -54,7 +54,7 @@ if (MSVC) VERSION_REVISION ${BUILD_ID} ) endif () - + set(ROCKSDB_FILES Indexes/RocksDBFeature.cpp Indexes/RocksDBIndex.cpp @@ -342,7 +342,6 @@ SET(ARANGOD_SOURCES VocBase/Ditch.cpp VocBase/EdgeCollectionInfo.cpp VocBase/Graphs.cpp - VocBase/IndexThreadFeature.cpp VocBase/KeyGenerator.cpp VocBase/LogicalCollection.cpp VocBase/ManagedDocumentResult.cpp diff --git a/arangod/Indexes/EdgeIndex.cpp b/arangod/Indexes/EdgeIndex.cpp index e4830c121f..5f641280cb 100644 --- a/arangod/Indexes/EdgeIndex.cpp +++ b/arangod/Indexes/EdgeIndex.cpp @@ -25,6 +25,7 @@ #include "Aql/AstNode.h" #include "Aql/SortCondition.h" #include "Basics/Exceptions.h" +#include "Basics/LocalTaskQueue.h" #include "Basics/StaticStrings.h" #include "Basics/StringRef.h" #include "Basics/fasthash.h" @@ -45,9 +46,9 @@ using namespace arangodb; /// @brief hard-coded vector of the index attributes /// note that the attribute names must be hard-coded here to avoid an init-order /// fiasco with StaticStrings::FromString etc. -static std::vector> const IndexAttributes - {{arangodb::basics::AttributeName("_from", false)}, - {arangodb::basics::AttributeName("_to", false)}}; +static std::vector> const + IndexAttributes{{arangodb::basics::AttributeName("_from", false)}, + {arangodb::basics::AttributeName("_to", false)}}; /// @brief hashes an edge key static uint64_t HashElementKey(void*, VPackSlice const* key) { @@ -58,7 +59,8 @@ static uint64_t HashElementKey(void*, VPackSlice const* key) { } /// @brief hashes an edge -static uint64_t HashElementEdge(void*, SimpleIndexElement const& element, bool byKey) { +static uint64_t HashElementEdge(void*, SimpleIndexElement const& element, + bool byKey) { if (byKey) { return element.hash(); } @@ -68,11 +70,12 @@ static uint64_t HashElementEdge(void*, SimpleIndexElement const& element, bool b } /// @brief checks if key and element match -static bool IsEqualKeyEdge(void* userData, VPackSlice const* left, SimpleIndexElement const& right) { +static bool IsEqualKeyEdge(void* userData, VPackSlice const* left, + SimpleIndexElement const& right) { TRI_ASSERT(left != nullptr); IndexLookupContext* context = static_cast(userData); TRI_ASSERT(context != nullptr); - + try { VPackSlice tmp = right.slice(context); TRI_ASSERT(tmp.isString()); @@ -83,17 +86,20 @@ static bool IsEqualKeyEdge(void* userData, VPackSlice const* left, SimpleIndexEl } /// @brief checks for elements are equal -static bool IsEqualElementEdge(void*, SimpleIndexElement const& left, SimpleIndexElement const& right) { +static bool IsEqualElementEdge(void*, SimpleIndexElement const& left, + SimpleIndexElement const& right) { return left.revisionId() == right.revisionId(); } /// @brief checks for elements are equal -static bool IsEqualElementEdgeByKey(void* userData, SimpleIndexElement const& left, SimpleIndexElement const& right) { +static bool IsEqualElementEdgeByKey(void* userData, + SimpleIndexElement const& left, + SimpleIndexElement const& right) { IndexLookupContext* context = static_cast(userData); try { VPackSlice lSlice = left.slice(context); VPackSlice rSlice = right.slice(context); - + TRI_ASSERT(lSlice.isString()); TRI_ASSERT(rSlice.isString()); @@ -102,8 +108,9 @@ static bool IsEqualElementEdgeByKey(void* userData, SimpleIndexElement const& le return false; } } - -EdgeIndexIterator::EdgeIndexIterator(LogicalCollection* collection, arangodb::Transaction* trx, + +EdgeIndexIterator::EdgeIndexIterator(LogicalCollection* collection, + arangodb::Transaction* trx, ManagedDocumentResult* mmdr, arangodb::EdgeIndex const* index, TRI_EdgeIndexHash_t const* indexImpl, @@ -115,13 +122,12 @@ EdgeIndexIterator::EdgeIndexIterator(LogicalCollection* collection, arangodb::Tr _posInBuffer(0), _batchSize(1000), _lastElement() { - - keys.release(); // now we have ownership for _keys + keys.release(); // now we have ownership for _keys } EdgeIndexIterator::~EdgeIndexIterator() { if (_keys != nullptr) { - // return the VPackBuilder to the transaction context + // return the VPackBuilder to the transaction context _trx->transactionContextPtr()->returnBuilder(_keys.release()); } } @@ -144,7 +150,7 @@ IndexLookupResult EdgeIndexIterator::next() { _posInBuffer = 0; _index->lookupByKeyContinue(&_context, _lastElement, _buffer, _batchSize); } - + if (_buffer.empty()) { _lastElement = SimpleIndexElement(); } else { @@ -160,7 +166,8 @@ IndexLookupResult EdgeIndexIterator::next() { return IndexLookupResult(); } -void EdgeIndexIterator::nextBabies(std::vector& buffer, size_t limit) { +void EdgeIndexIterator::nextBabies(std::vector& buffer, + size_t limit) { size_t atMost = _batchSize > limit ? limit : _batchSize; if (atMost == 0) { @@ -189,7 +196,7 @@ void EdgeIndexIterator::nextBabies(std::vector& buffer, size_ for (auto& it : _buffer) { buffer.emplace_back(it.revisionId()); } - + if (_buffer.empty()) { _lastElement = SimpleIndexElement(); } else { @@ -197,7 +204,7 @@ void EdgeIndexIterator::nextBabies(std::vector& buffer, size_ // found something return; } - + // found no result. now go to next lookup value in _keys _iterator.next(); } @@ -211,13 +218,11 @@ void EdgeIndexIterator::reset() { _iterator.reset(); _lastElement = SimpleIndexElement(); } - -AnyDirectionEdgeIndexIterator::AnyDirectionEdgeIndexIterator(LogicalCollection* collection, - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, - arangodb::EdgeIndex const* index, - EdgeIndexIterator* outboundIterator, - EdgeIndexIterator* inboundIterator) + +AnyDirectionEdgeIndexIterator::AnyDirectionEdgeIndexIterator( + LogicalCollection* collection, arangodb::Transaction* trx, + ManagedDocumentResult* mmdr, arangodb::EdgeIndex const* index, + EdgeIndexIterator* outboundIterator, EdgeIndexIterator* inboundIterator) : IndexIterator(collection, trx, mmdr, index), _outbound(outboundIterator), _inbound(inboundIterator), @@ -240,7 +245,8 @@ IndexLookupResult AnyDirectionEdgeIndexIterator::next() { return res; } -void AnyDirectionEdgeIndexIterator::nextBabies(std::vector& result, size_t limit) { +void AnyDirectionEdgeIndexIterator::nextBabies( + std::vector& result, size_t limit) { result.clear(); for (size_t i = 0; i < limit; ++i) { IndexLookupResult res = next(); @@ -261,8 +267,10 @@ void AnyDirectionEdgeIndexIterator::reset() { EdgeIndex::EdgeIndex(TRI_idx_iid_t iid, arangodb::LogicalCollection* collection) : Index(iid, collection, std::vector>( - {{arangodb::basics::AttributeName(StaticStrings::FromString, false)}, - {arangodb::basics::AttributeName(StaticStrings::ToString, false)}}), + {{arangodb::basics::AttributeName(StaticStrings::FromString, + false)}, + {arangodb::basics::AttributeName(StaticStrings::ToString, + false)}}), false, false), _edgesFrom(nullptr), _edgesTo(nullptr), @@ -276,10 +284,9 @@ EdgeIndex::EdgeIndex(TRI_idx_iid_t iid, arangodb::LogicalCollection* collection) auto context = [this]() -> std::string { return this->context(); }; - _edgesFrom = new TRI_EdgeIndexHash_t(HashElementKey, HashElementEdge, - IsEqualKeyEdge, IsEqualElementEdge, - IsEqualElementEdgeByKey, _numBuckets, - 64, context); + _edgesFrom = new TRI_EdgeIndexHash_t( + HashElementKey, HashElementEdge, IsEqualKeyEdge, IsEqualElementEdge, + IsEqualElementEdgeByKey, _numBuckets, 64, context); _edgesTo = new TRI_EdgeIndexHash_t( HashElementKey, HashElementEdge, IsEqualKeyEdge, IsEqualElementEdge, @@ -326,8 +333,8 @@ void EdgeIndex::buildSearchValue(TRI_edge_direction_e dir, builder.close(); } -void EdgeIndex::buildSearchValue(TRI_edge_direction_e dir, - VPackSlice const& id, VPackBuilder& builder) { +void EdgeIndex::buildSearchValue(TRI_edge_direction_e dir, VPackSlice const& id, + VPackBuilder& builder) { TRI_ASSERT(id.isString()); builder.openArray(); switch (dir) { @@ -416,14 +423,14 @@ void EdgeIndex::buildSearchValueFromArray(TRI_edge_direction_e dir, } /// @brief return a selectivity estimate for the index -double EdgeIndex::selectivityEstimate(arangodb::StringRef const* attribute) const { - if (_edgesFrom == nullptr || - _edgesTo == nullptr || +double EdgeIndex::selectivityEstimate( + arangodb::StringRef const* attribute) const { + if (_edgesFrom == nullptr || _edgesTo == nullptr || ServerState::instance()->isCoordinator()) { // use hard-coded selectivity estimate in case of cluster coordinator return 0.1; } - + if (attribute != nullptr) { // the index attribute is given here // now check if we can restrict the selectivity estimation to the correct @@ -470,16 +477,16 @@ void EdgeIndex::toVelocyPackFigures(VPackBuilder& builder) const { builder.add("to", VPackValue(VPackValueType::Object)); _edgesTo->appendToVelocyPack(builder); builder.close(); - //builder.add("buckets", VPackValue(_numBuckets)); + // builder.add("buckets", VPackValue(_numBuckets)); } int EdgeIndex::insert(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, VPackSlice const& doc, bool isRollback) { SimpleIndexElement fromElement(buildFromElement(revisionId, doc)); SimpleIndexElement toElement(buildToElement(revisionId, doc)); - - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, 1); + + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, 1); _edgesFrom->insert(&context, fromElement, true, isRollback); try { @@ -497,11 +504,11 @@ int EdgeIndex::remove(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, VPackSlice const& doc, bool isRollback) { SimpleIndexElement fromElement(buildFromElement(revisionId, doc)); SimpleIndexElement toElement(buildToElement(revisionId, doc)); - - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, 1); - - try { + + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, 1); + + try { _edgesFrom->remove(&context, fromElement); _edgesTo->remove(&context, toElement); return TRI_ERROR_NO_ERROR; @@ -513,15 +520,21 @@ int EdgeIndex::remove(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, } } -int EdgeIndex::batchInsert(arangodb::Transaction* trx, - std::vector> const& documents, - size_t numThreads) { +void EdgeIndex::batchInsert( + arangodb::Transaction* trx, + std::vector> const& documents, + arangodb::basics::LocalTaskQueue* queue) { if (documents.empty()) { - return TRI_ERROR_NO_ERROR; + return; } - - std::vector elements; - elements.reserve(documents.size()); + + std::shared_ptr> fromElements; + fromElements.reset(new std::vector()); + fromElements->reserve(documents.size()); + + std::shared_ptr> toElements; + toElements.reset(new std::vector()); + toElements->reserve(documents.size()); // functions that will be called for each thread auto creator = [&trx, this]() -> void* { @@ -534,32 +547,26 @@ int EdgeIndex::batchInsert(arangodb::Transaction* trx, delete context; }; + // TODO: create parallel tasks for this + // _from for (auto const& it : documents) { VPackSlice value(Transaction::extractFromFromDocument(it.second)); - elements.emplace_back(SimpleIndexElement(it.first, value, static_cast(value.begin() - it.second.begin()))); - } - - int res = _edgesFrom->batchInsert(creator, destroyer, &elements, numThreads); - - if (res != TRI_ERROR_NO_ERROR) { - return res; + fromElements->emplace_back(SimpleIndexElement( + it.first, value, + static_cast(value.begin() - it.second.begin()))); } // _to - elements.clear(); for (auto const& it : documents) { VPackSlice value(Transaction::extractToFromDocument(it.second)); - elements.emplace_back(SimpleIndexElement(it.first, value, static_cast(value.begin() - it.second.begin()))); + toElements->emplace_back(SimpleIndexElement( + it.first, value, + static_cast(value.begin() - it.second.begin()))); } - res = _edgesTo->batchInsert(creator, destroyer, &elements, numThreads); - - if (res != TRI_ERROR_NO_ERROR) { - return res; - } - - return TRI_ERROR_NO_ERROR; + _edgesFrom->batchInsert(creator, destroyer, fromElements, queue); + _edgesTo->batchInsert(creator, destroyer, toElements, queue); } /// @brief unload the index data from memory @@ -578,8 +585,8 @@ int EdgeIndex::sizeHint(arangodb::Transaction* trx, size_t size) { // set an initial size for the index for some new nodes to be created // without resizing - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, 1); + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, 1); int err = _edgesFrom->resize(&context, size + 2049); if (err != TRI_ERROR_NO_ERROR) { @@ -600,7 +607,6 @@ bool EdgeIndex::supportsFilterCondition( arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, size_t itemsInIndex, size_t& estimatedItems, double& estimatedCost) const { - SimpleAttributeEqualityMatcher matcher(IndexAttributes); return matcher.matchOne(this, node, reference, itemsInIndex, estimatedItems, estimatedCost); @@ -608,8 +614,7 @@ bool EdgeIndex::supportsFilterCondition( /// @brief creates an IndexIterator for the given Condition IndexIterator* EdgeIndex::iteratorForCondition( - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, + arangodb::Transaction* trx, ManagedDocumentResult* mmdr, arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, bool reverse) const { TRI_ASSERT(node->type == aql::NODE_TYPE_OPERATOR_NARY_AND); @@ -651,7 +656,6 @@ IndexIterator* EdgeIndex::iteratorForCondition( arangodb::aql::AstNode* EdgeIndex::specializeCondition( arangodb::aql::AstNode* node, arangodb::aql::Variable const* reference) const { - SimpleAttributeEqualityMatcher matcher(IndexAttributes); return matcher.specializeOne(this, node, reference); } @@ -700,13 +704,12 @@ void EdgeIndex::expandInSearchValues(VPackSlice const slice, /// Each key needs to have the following formats: /// /// 1) {"eq": } // The value in index is exactly this -/// +/// /// Reverse is not supported, hence ignored /// NOTE: The iterator is only valid as long as the slice points to /// a valid memory region. IndexIterator* EdgeIndex::iteratorForSlice( - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, + arangodb::Transaction* trx, ManagedDocumentResult* mmdr, arangodb::velocypack::Slice const searchValues, bool) const { if (!searchValues.isArray() || searchValues.length() != 2) { // Invalid searchValue @@ -717,7 +720,7 @@ IndexIterator* EdgeIndex::iteratorForSlice( TRI_ASSERT(it.valid()); VPackSlice const from = it.value(); - + it.next(); TRI_ASSERT(it.valid()); VPackSlice const to = it.value(); @@ -730,20 +733,24 @@ IndexIterator* EdgeIndex::iteratorForSlice( TransactionBuilderLeaser fromBuilder(trx); std::unique_ptr fromKeys(fromBuilder.steal()); fromKeys->add(from); - auto left = std::make_unique(_collection, trx, mmdr, this, _edgesFrom, fromKeys); + auto left = std::make_unique( + _collection, trx, mmdr, this, _edgesFrom, fromKeys); TransactionBuilderLeaser toBuilder(trx); std::unique_ptr toKeys(toBuilder.steal()); toKeys->add(to); - auto right = std::make_unique(_collection, trx, mmdr, this, _edgesTo, toKeys); - return new AnyDirectionEdgeIndexIterator(_collection, trx, mmdr, this, left.release(), right.release()); + auto right = std::make_unique(_collection, trx, mmdr, + this, _edgesTo, toKeys); + return new AnyDirectionEdgeIndexIterator(_collection, trx, mmdr, this, + left.release(), right.release()); } // OUTBOUND search TRI_ASSERT(to.isNull()); TransactionBuilderLeaser builder(trx); std::unique_ptr keys(builder.steal()); keys->add(from); - return new EdgeIndexIterator(_collection, trx, mmdr, this, _edgesFrom, keys); + return new EdgeIndexIterator(_collection, trx, mmdr, this, _edgesFrom, + keys); } else { // INBOUND search TRI_ASSERT(to.isArray()); @@ -756,12 +763,10 @@ IndexIterator* EdgeIndex::iteratorForSlice( /// @brief create the iterator IndexIterator* EdgeIndex::createEqIterator( - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, + arangodb::Transaction* trx, ManagedDocumentResult* mmdr, arangodb::aql::AstNode const* attrNode, arangodb::aql::AstNode const* valNode) const { - - // lease builder, but immediately pass it to the unique_ptr so we don't leak + // lease builder, but immediately pass it to the unique_ptr so we don't leak TransactionBuilderLeaser builder(trx); std::unique_ptr keys(builder.steal()); keys->openArray(); @@ -775,21 +780,20 @@ IndexIterator* EdgeIndex::createEqIterator( // _from or _to? bool const isFrom = (attrNode->stringEquals(StaticStrings::FromString)); - return new EdgeIndexIterator(_collection, trx, mmdr, this, isFrom ? _edgesFrom : _edgesTo, keys); + return new EdgeIndexIterator(_collection, trx, mmdr, this, + isFrom ? _edgesFrom : _edgesTo, keys); } /// @brief create the iterator IndexIterator* EdgeIndex::createInIterator( - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, + arangodb::Transaction* trx, ManagedDocumentResult* mmdr, arangodb::aql::AstNode const* attrNode, arangodb::aql::AstNode const* valNode) const { - - // lease builder, but immediately pass it to the unique_ptr so we don't leak + // lease builder, but immediately pass it to the unique_ptr so we don't leak TransactionBuilderLeaser builder(trx); std::unique_ptr keys(builder.steal()); keys->openArray(); - + size_t const n = valNode->numMembers(); for (size_t i = 0; i < n; ++i) { handleValNode(keys.get(), valNode->getMemberUnchecked(i)); @@ -806,7 +810,8 @@ IndexIterator* EdgeIndex::createInIterator( // _from or _to? bool const isFrom = (attrNode->stringEquals(StaticStrings::FromString)); - return new EdgeIndexIterator(_collection, trx, mmdr, this, isFrom ? _edgesFrom : _edgesTo, keys); + return new EdgeIndexIterator(_collection, trx, mmdr, this, + isFrom ? _edgesFrom : _edgesTo, keys); } /// @brief add a single value node to the iterator's keys @@ -817,24 +822,30 @@ void EdgeIndex::handleValNode(VPackBuilder* keys, } keys->openObject(); - keys->add(StaticStrings::IndexEq, VPackValuePair(valNode->getStringValue(), valNode->getStringLength(), VPackValueType::String)); + keys->add(StaticStrings::IndexEq, + VPackValuePair(valNode->getStringValue(), + valNode->getStringLength(), VPackValueType::String)); keys->close(); - + TRI_IF_FAILURE("EdgeIndex::collectKeys") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } } -SimpleIndexElement EdgeIndex::buildFromElement(TRI_voc_rid_t revisionId, VPackSlice const& doc) const { +SimpleIndexElement EdgeIndex::buildFromElement(TRI_voc_rid_t revisionId, + VPackSlice const& doc) const { TRI_ASSERT(doc.isObject()); VPackSlice value(Transaction::extractFromFromDocument(doc)); TRI_ASSERT(value.isString()); - return SimpleIndexElement(revisionId, value, static_cast(value.begin() - doc.begin())); + return SimpleIndexElement(revisionId, value, + static_cast(value.begin() - doc.begin())); } -SimpleIndexElement EdgeIndex::buildToElement(TRI_voc_rid_t revisionId, VPackSlice const& doc) const { +SimpleIndexElement EdgeIndex::buildToElement(TRI_voc_rid_t revisionId, + VPackSlice const& doc) const { TRI_ASSERT(doc.isObject()); VPackSlice value(Transaction::extractToFromDocument(doc)); TRI_ASSERT(value.isString()); - return SimpleIndexElement(revisionId, value, static_cast(value.begin() - doc.begin())); + return SimpleIndexElement(revisionId, value, + static_cast(value.begin() - doc.begin())); } diff --git a/arangod/Indexes/EdgeIndex.h b/arangod/Indexes/EdgeIndex.h index 2de6275d2b..05a31542a8 100644 --- a/arangod/Indexes/EdgeIndex.h +++ b/arangod/Indexes/EdgeIndex.h @@ -24,22 +24,26 @@ #ifndef ARANGOD_INDEXES_EDGE_INDEX_H #define ARANGOD_INDEXES_EDGE_INDEX_H 1 -#include "Basics/Common.h" #include "Basics/AssocMulti.h" +#include "Basics/Common.h" #include "Indexes/Index.h" #include "Indexes/IndexIterator.h" -#include "VocBase/vocbase.h" #include "VocBase/voc-types.h" +#include "VocBase/vocbase.h" #include #include namespace arangodb { -class EdgeIndex; - -typedef arangodb::basics::AssocMulti TRI_EdgeIndexHash_t; +namespace basics { +class LocalTaskQueue; +} +class EdgeIndex; + +typedef arangodb::basics::AssocMulti + TRI_EdgeIndexHash_t; class EdgeIndexIterator final : public IndexIterator { public: @@ -50,7 +54,7 @@ class EdgeIndexIterator final : public IndexIterator { std::unique_ptr& keys); ~EdgeIndexIterator(); - + char const* typeName() const override { return "edge-index-iterator"; } IndexLookupResult next() override; @@ -82,7 +86,7 @@ class AnyDirectionEdgeIndexIterator final : public IndexIterator { delete _outbound; delete _inbound; } - + char const* typeName() const override { return "any-edge-index-iterator"; } IndexLookupResult next() override; @@ -120,19 +124,18 @@ class EdgeIndex final : public Index { public: /// @brief typedef for hash tables public: - IndexType type() const override { - return Index::TRI_IDX_TYPE_EDGE_INDEX; - } + IndexType type() const override { return Index::TRI_IDX_TYPE_EDGE_INDEX; } bool allowExpansion() const override { return false; } - + bool canBeDropped() const override { return false; } bool isSorted() const override { return false; } bool hasSelectivityEstimate() const override { return true; } - double selectivityEstimate(arangodb::StringRef const* = nullptr) const override; + double selectivityEstimate( + arangodb::StringRef const* = nullptr) const override; size_t memory() const override; @@ -140,12 +143,16 @@ class EdgeIndex final : public Index { void toVelocyPackFigures(VPackBuilder&) const override; - int insert(arangodb::Transaction*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; + int insert(arangodb::Transaction*, TRI_voc_rid_t, + arangodb::velocypack::Slice const&, bool isRollback) override; - int remove(arangodb::Transaction*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; + int remove(arangodb::Transaction*, TRI_voc_rid_t, + arangodb::velocypack::Slice const&, bool isRollback) override; + + void batchInsert(arangodb::Transaction*, + std::vector> const&, + arangodb::basics::LocalTaskQueue*) override; - int batchInsert(arangodb::Transaction*, std::vector> const&, size_t) override; - int unload() override; int sizeHint(arangodb::Transaction*, size_t) override; @@ -188,30 +195,31 @@ class EdgeIndex final : public Index { /// Reverse is not supported, hence ignored /// NOTE: The iterator is only valid as long as the slice points to /// a valid memory region. - IndexIterator* iteratorForSlice(arangodb::Transaction*, + IndexIterator* iteratorForSlice(arangodb::Transaction*, ManagedDocumentResult*, arangodb::velocypack::Slice const, bool) const override; private: /// @brief create the iterator - IndexIterator* createEqIterator( - arangodb::Transaction*, - ManagedDocumentResult*, - arangodb::aql::AstNode const*, - arangodb::aql::AstNode const*) const; - - IndexIterator* createInIterator( - arangodb::Transaction*, - ManagedDocumentResult*, - arangodb::aql::AstNode const*, - arangodb::aql::AstNode const*) const; + IndexIterator* createEqIterator(arangodb::Transaction*, + ManagedDocumentResult*, + arangodb::aql::AstNode const*, + arangodb::aql::AstNode const*) const; + + IndexIterator* createInIterator(arangodb::Transaction*, + ManagedDocumentResult*, + arangodb::aql::AstNode const*, + arangodb::aql::AstNode const*) const; /// @brief add a single value node to the iterator's keys - void handleValNode(VPackBuilder* keys, arangodb::aql::AstNode const* valNode) const; + void handleValNode(VPackBuilder* keys, + arangodb::aql::AstNode const* valNode) const; - SimpleIndexElement buildFromElement(TRI_voc_rid_t, arangodb::velocypack::Slice const& doc) const; - SimpleIndexElement buildToElement(TRI_voc_rid_t, arangodb::velocypack::Slice const& doc) const; + SimpleIndexElement buildFromElement( + TRI_voc_rid_t, arangodb::velocypack::Slice const& doc) const; + SimpleIndexElement buildToElement( + TRI_voc_rid_t, arangodb::velocypack::Slice const& doc) const; private: /// @brief the hash table for _from diff --git a/arangod/Indexes/HashIndex.cpp b/arangod/Indexes/HashIndex.cpp index e8f8b689ac..eacd9903b4 100644 --- a/arangod/Indexes/HashIndex.cpp +++ b/arangod/Indexes/HashIndex.cpp @@ -27,6 +27,7 @@ #include "Aql/SortCondition.h" #include "Basics/Exceptions.h" #include "Basics/FixedSizeAllocator.h" +#include "Basics/LocalTaskQueue.h" #include "Basics/VelocyPackHelper.h" #include "Indexes/IndexLookupContext.h" #include "Indexes/SimpleAttributeEqualityMatcher.h" @@ -44,13 +45,13 @@ LookupBuilder::LookupBuilder( arangodb::aql::Variable const* reference, std::vector> const& fields) : _builder(trx), _usesIn(false), _isEmpty(false), _inStorage(trx) { - TRI_ASSERT(node->type == aql::NODE_TYPE_OPERATOR_NARY_AND); _coveredFields = fields.size(); TRI_ASSERT(node->numMembers() == _coveredFields); std::pair> paramPair; + std::vector> + paramPair; std::vector storageOrder; for (size_t i = 0; i < _coveredFields; ++i) { @@ -88,7 +89,9 @@ LookupBuilder::LookupBuilder( _inStorage->openArray(); } valNode->toVelocyPackValue(*(_inStorage.get())); - _inPosition.emplace(j, std::make_pair(0, std::vector())); + _inPosition.emplace( + j, + std::make_pair(0, std::vector())); _usesIn = true; storageOrder.emplace_back(j); } else { @@ -111,7 +114,7 @@ LookupBuilder::LookupBuilder( auto f = storageOrder.begin(); for (auto const& values : VPackArrayIterator(storageSlice)) { tmp.clear(); - TRI_IF_FAILURE("Index::permutationIN") { + TRI_IF_FAILURE("Index::permutationIN") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } for (auto const& value : VPackArrayIterator(values)) { @@ -132,9 +135,7 @@ LookupBuilder::LookupBuilder( buildNextSearchValue(); } -VPackSlice LookupBuilder::lookup() { - return _builder->slice(); -} +VPackSlice LookupBuilder::lookup() { return _builder->slice(); } bool LookupBuilder::hasAndGetNext() { _builder->clear(); @@ -201,12 +202,11 @@ void LookupBuilder::buildNextSearchValue() { } } } - _builder->close(); // End of search Array + _builder->close(); // End of search Array } /// @brief determines if two elements are equal -static bool IsEqualElementElementUnique(void*, - HashIndexElement const* left, +static bool IsEqualElementElementUnique(void*, HashIndexElement const* left, HashIndexElement const* right) { // this is quite simple return left->revisionId() == right->revisionId(); @@ -218,7 +218,7 @@ static bool IsEqualElementElementMulti(void* userData, HashIndexElement const* right) { TRI_ASSERT(left != nullptr); TRI_ASSERT(right != nullptr); - + if (left->revisionId() != right->revisionId()) { return false; } @@ -228,12 +228,13 @@ static bool IsEqualElementElementMulti(void* userData, IndexLookupContext* context = static_cast(userData); TRI_ASSERT(context != nullptr); - + for (size_t i = 0; i < context->numFields(); ++i) { VPackSlice leftData = left->slice(context, i); VPackSlice rightData = right->slice(context, i); - int res = arangodb::basics::VelocyPackHelper::compare(leftData, rightData, false); + int res = + arangodb::basics::VelocyPackHelper::compare(leftData, rightData, false); if (res != 0) { return false; @@ -249,8 +250,7 @@ static uint64_t HashKey(void*, VPackSlice const* key) { } /// @brief determines if a key corresponds to an element -static bool IsEqualKeyElementMulti(void* userData, - VPackSlice const* left, +static bool IsEqualKeyElementMulti(void* userData, VPackSlice const* left, HashIndexElement const* right) { TRI_ASSERT(left->isArray()); TRI_ASSERT(right->revisionId() != 0); @@ -259,12 +259,13 @@ static bool IsEqualKeyElementMulti(void* userData, // TODO: is it a performance improvement to compare the hash values first? size_t const n = left->length(); - + for (size_t i = 0; i < n; ++i) { VPackSlice const leftVPack = left->at(i); VPackSlice const rightVPack = right->slice(context, i); - - int res = arangodb::basics::VelocyPackHelper::compare(leftVPack, rightVPack, false); + + int res = arangodb::basics::VelocyPackHelper::compare(leftVPack, rightVPack, + false); if (res != 0) { return false; @@ -291,7 +292,7 @@ HashIndexIterator::HashIndexIterator(LogicalCollection* collection, _lookups(trx, node, reference, index->fields()), _buffer(), _posInBuffer(0) { - _index->lookup(_trx, _lookups.lookup(), _buffer); + _index->lookup(_trx, _lookups.lookup(), _buffer); } IndexLookupResult HashIndexIterator::next() { @@ -316,9 +317,10 @@ IndexLookupResult HashIndexIterator::next() { } } -void HashIndexIterator::nextBabies(std::vector& result, size_t atMost) { +void HashIndexIterator::nextBabies(std::vector& result, + size_t atMost) { result.clear(); - + if (atMost == 0) { return; } @@ -361,24 +363,23 @@ void HashIndexIterator::reset() { _lookups.reset(); _index->lookup(_trx, _lookups.lookup(), _buffer); } - -HashIndexIteratorVPack::HashIndexIteratorVPack(LogicalCollection* collection, - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, - HashIndex const* index, - std::unique_ptr& searchValues) + +HashIndexIteratorVPack::HashIndexIteratorVPack( + LogicalCollection* collection, arangodb::Transaction* trx, + ManagedDocumentResult* mmdr, HashIndex const* index, + std::unique_ptr& searchValues) : IndexIterator(collection, trx, mmdr, index), _index(index), _searchValues(searchValues.get()), _iterator(_searchValues->slice()), _buffer(), _posInBuffer(0) { - searchValues.release(); // now we have ownership for searchValues + searchValues.release(); // now we have ownership for searchValues } HashIndexIteratorVPack::~HashIndexIteratorVPack() { if (_searchValues != nullptr) { - // return the VPackBuilder to the transaction context + // return the VPackBuilder to the transaction context _trx->transactionContextPtr()->returnBuilder(_searchValues.release()); } } @@ -419,8 +420,7 @@ void HashIndexIteratorVPack::reset() { /// @brief create the unique array HashIndex::UniqueArray::UniqueArray( - size_t numPaths, - TRI_HashArray_t* hashArray, HashElementFunc* hashElement, + size_t numPaths, TRI_HashArray_t* hashArray, HashElementFunc* hashElement, IsEqualElementElementByKey* isEqualElElByKey) : _hashArray(hashArray), _hashElement(hashElement), @@ -461,7 +461,9 @@ HashIndex::MultiArray::~MultiArray() { HashIndex::HashIndex(TRI_idx_iid_t iid, LogicalCollection* collection, VPackSlice const& info) - : PathBasedIndex(iid, collection, info, sizeof(TRI_voc_rid_t) + sizeof(uint32_t), false), _uniqueArray(nullptr) { + : PathBasedIndex(iid, collection, info, + sizeof(TRI_voc_rid_t) + sizeof(uint32_t), false), + _uniqueArray(nullptr) { uint32_t indexBuckets = 1; if (collection != nullptr) { @@ -469,27 +471,28 @@ HashIndex::HashIndex(TRI_idx_iid_t iid, LogicalCollection* collection, } auto func = std::make_unique(); - auto compare = std::make_unique(_paths.size(), _useExpansion); + auto compare = std::make_unique(_paths.size(), + _useExpansion); if (_unique) { auto array = std::make_unique( - HashKey, *(func.get()), IsEqualKeyElementUnique, IsEqualElementElementUnique, - *(compare.get()), indexBuckets, + HashKey, *(func.get()), IsEqualKeyElementUnique, + IsEqualElementElementUnique, *(compare.get()), indexBuckets, [this]() -> std::string { return this->context(); }); - _uniqueArray = - new HashIndex::UniqueArray(numPaths(), array.get(), func.get(), compare.get()); + _uniqueArray = new HashIndex::UniqueArray(numPaths(), array.get(), + func.get(), compare.get()); array.release(); } else { _multiArray = nullptr; auto array = std::make_unique( - HashKey, *(func.get()), IsEqualKeyElementMulti, IsEqualElementElementMulti, - *(compare.get()), indexBuckets, 64, + HashKey, *(func.get()), IsEqualKeyElementMulti, + IsEqualElementElementMulti, *(compare.get()), indexBuckets, 64, [this]() -> std::string { return this->context(); }); - _multiArray = - new HashIndex::MultiArray(numPaths(), array.get(), func.get(), compare.get()); + _multiArray = new HashIndex::MultiArray(numPaths(), array.get(), func.get(), + compare.get()); array.release(); } @@ -529,8 +532,7 @@ size_t HashIndex::memory() const { size_t elementSize = HashIndexElement::baseMemoryUsage(_paths.size()); if (_unique) { - return static_cast(elementSize * - _uniqueArray->_hashArray->size() + + return static_cast(elementSize * _uniqueArray->_hashArray->size() + _uniqueArray->_hashArray->memoryUsage()); } return static_cast(elementSize * _multiArray->_hashArray->size() + @@ -567,7 +569,7 @@ bool HashIndex::matchesDefinition(VPackSlice const& info) const { auto value = info.get("id"); if (!value.isNone()) { // We already have an id. - if(!value.isString()) { + if (!value.isString()) { // Invalid ID return false; } @@ -608,7 +610,7 @@ bool HashIndex::matchesDefinition(VPackSlice const& info) const { for (size_t i = 0; i < n; ++i) { if (arangodb::basics::AttributeName::isIdentical(_fields[i], translate, - false)) { + false)) { found = true; break; } @@ -661,21 +663,25 @@ int HashIndex::remove(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, return res; } -int HashIndex::batchInsert(arangodb::Transaction* trx, - std::vector> const& documents, - size_t numThreads) { +void HashIndex::batchInsert( + arangodb::Transaction* trx, + std::vector> const& documents, + arangodb::basics::LocalTaskQueue* queue) { + TRI_ASSERT(queue != nullptr); if (_unique) { - return batchInsertUnique(trx, documents, numThreads); + batchInsertUnique(trx, documents, queue); + } else { + batchInsertMulti(trx, documents, queue); } - - return batchInsertMulti(trx, documents, numThreads); } - + int HashIndex::unload() { if (_unique) { - _uniqueArray->_hashArray->truncate([](HashIndexElement*) -> bool { return true; }); + _uniqueArray->_hashArray->truncate( + [](HashIndexElement*) -> bool { return true; }); } else { - _multiArray->_hashArray->truncate([](HashIndexElement*) -> bool { return true; }); + _multiArray->_hashArray->truncate( + [](HashIndexElement*) -> bool { return true; }); } _allocator->deallocateAll(); return TRI_ERROR_NO_ERROR; @@ -688,8 +694,8 @@ int HashIndex::sizeHint(arangodb::Transaction* trx, size_t size) { // than if the index would be fully populated size /= 5; } - - ManagedDocumentResult result(trx); + + ManagedDocumentResult result(trx); IndexLookupContext context(trx, _collection, &result, numPaths()); if (_unique) { @@ -700,15 +706,14 @@ int HashIndex::sizeHint(arangodb::Transaction* trx, size_t size) { } /// @brief locates entries in the hash index given VelocyPack slices -int HashIndex::lookup(arangodb::Transaction* trx, - VPackSlice key, +int HashIndex::lookup(arangodb::Transaction* trx, VPackSlice key, std::vector& documents) const { if (key.isNone()) { return TRI_ERROR_NO_ERROR; } - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, numPaths()); + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, numPaths()); if (_unique) { HashIndexElement* found = @@ -731,8 +736,9 @@ int HashIndex::lookup(arangodb::Transaction* trx, return TRI_ERROR_NO_ERROR; } -int HashIndex::insertUnique(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, - VPackSlice const& doc, bool isRollback) { +int HashIndex::insertUnique(arangodb::Transaction* trx, + TRI_voc_rid_t revisionId, VPackSlice const& doc, + bool isRollback) { std::vector elements; int res = fillElement(elements, revisionId, doc); @@ -744,15 +750,14 @@ int HashIndex::insertUnique(arangodb::Transaction* trx, TRI_voc_rid_t revisionId return res; } - - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, numPaths()); - auto work = - [this, &context](HashIndexElement* element, bool) -> int { - TRI_IF_FAILURE("InsertHashIndex") { return TRI_ERROR_DEBUG; } - return _uniqueArray->_hashArray->insert(&context, element); - }; + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, numPaths()); + + auto work = [this, &context](HashIndexElement* element, bool) -> int { + TRI_IF_FAILURE("InsertHashIndex") { return TRI_ERROR_DEBUG; } + return _uniqueArray->_hashArray->insert(&context, element); + }; size_t const n = elements.size(); @@ -773,28 +778,35 @@ int HashIndex::insertUnique(arangodb::Transaction* trx, TRI_voc_rid_t revisionId return res; } -int HashIndex::batchInsertUnique(arangodb::Transaction* trx, - std::vector> const& documents, size_t numThreads) { - std::vector elements; - elements.reserve(documents.size()); +void HashIndex::batchInsertUnique( + arangodb::Transaction* trx, + std::vector> const& documents, + arangodb::basics::LocalTaskQueue* queue) { + TRI_ASSERT(queue != nullptr); + std::shared_ptr> elements; + elements.reset(new std::vector()); + elements->reserve(documents.size()); + // TODO: create parallel tasks for this for (auto& doc : documents) { - int res = fillElement(elements, doc.first, doc.second); + int res = + fillElement(*(elements.get()), doc.first, doc.second); if (res != TRI_ERROR_NO_ERROR) { - for (auto& it : elements) { + for (auto& it : *(elements.get())) { // free all elements to prevent leak _allocator->deallocate(it); } - return res; + queue->setStatus(res); + return; } } - - if (elements.empty()) { + + if (elements->empty()) { // no elements left to insert - return TRI_ERROR_NO_ERROR; + return; } - + // functions that will be called for each thread auto creator = [&trx, this]() -> void* { ManagedDocumentResult* result = new ManagedDocumentResult(trx); @@ -805,17 +817,23 @@ int HashIndex::batchInsertUnique(arangodb::Transaction* trx, delete context->result(); delete context; }; - - int res = _uniqueArray->_hashArray->batchInsert(creator, destroyer, &elements, numThreads); - if (res != TRI_ERROR_NO_ERROR) { - for (auto& it : elements) { - // free all elements to prevent leak - _allocator->deallocate(it); + // queue the actual insertion tasks + _uniqueArray->_hashArray->batchInsert(creator, destroyer, elements, queue); + + // queue cleanup callback + auto allocator = _allocator.get(); + auto callback = [elements, queue, allocator]() -> void { + if (queue->status() != TRI_ERROR_NO_ERROR) { + for (auto& it : *(elements.get())) { + // free all elements to prevent leak + allocator->deallocate(it); + } } - } - - return res; + }; + std::shared_ptr cbTask; + cbTask.reset(new arangodb::basics::LocalCallbackTask(queue, callback)); + queue->enqueueCallback(cbTask); } int HashIndex::insertMulti(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, @@ -829,9 +847,9 @@ int HashIndex::insertMulti(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, } return res; } - - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, numPaths()); + + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, numPaths()); auto work = [this, &context](HashIndexElement*& element, bool) { TRI_IF_FAILURE("InsertHashIndex") { @@ -879,29 +897,35 @@ int HashIndex::insertMulti(arangodb::Transaction* trx, TRI_voc_rid_t revisionId, return TRI_ERROR_NO_ERROR; } -int HashIndex::batchInsertMulti(arangodb::Transaction* trx, - std::vector> const& documents, size_t numThreads) { - std::vector elements; - elements.reserve(documents.size()); +void HashIndex::batchInsertMulti( + arangodb::Transaction* trx, + std::vector> const& documents, + arangodb::basics::LocalTaskQueue* queue) { + TRI_ASSERT(queue != nullptr); + std::shared_ptr> elements; + elements.reset(new std::vector()); + elements->reserve(documents.size()); + // TODO: create parallel tasks for this for (auto& doc : documents) { - int res = fillElement(elements, doc.first, doc.second); + int res = + fillElement(*(elements.get()), doc.first, doc.second); if (res != TRI_ERROR_NO_ERROR) { // Filling the elements failed for some reason. Assume loading as failed - for (auto& el : elements) { + for (auto& el : *(elements.get())) { // Free all elements that are not yet in the index _allocator->deallocate(el); } - return res; + return; } } - if (elements.empty()) { + if (elements->empty()) { // no elements left to insert - return TRI_ERROR_NO_ERROR; + return; } - + // functions that will be called for each thread auto creator = [&trx, this]() -> void* { ManagedDocumentResult* result = new ManagedDocumentResult(trx); @@ -913,15 +937,29 @@ int HashIndex::batchInsertMulti(arangodb::Transaction* trx, delete context; }; - return _multiArray->_hashArray->batchInsert(creator, destroyer, &elements, numThreads); + // queue actual insertion tasks + _multiArray->_hashArray->batchInsert(creator, destroyer, elements, queue); + + // queue cleanup callback + auto allocator = _allocator.get(); + auto callback = [elements, queue, allocator]() -> void { + if (queue->status() != TRI_ERROR_NO_ERROR) { + // free all elements to prevent leak + for (auto& it : *(elements.get())) { + allocator->deallocate(it); + } + } + }; + std::shared_ptr cbTask; + cbTask.reset(new arangodb::basics::LocalCallbackTask(queue, callback)); + queue->enqueueCallback(cbTask); } int HashIndex::removeUniqueElement(arangodb::Transaction* trx, - HashIndexElement* element, - bool isRollback) { + HashIndexElement* element, bool isRollback) { TRI_IF_FAILURE("RemoveHashIndex") { return TRI_ERROR_DEBUG; } - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, numPaths()); + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, numPaths()); HashIndexElement* old = _uniqueArray->_hashArray->remove(&context, element); if (old == nullptr) { @@ -937,11 +975,10 @@ int HashIndex::removeUniqueElement(arangodb::Transaction* trx, } int HashIndex::removeMultiElement(arangodb::Transaction* trx, - HashIndexElement* element, - bool isRollback) { + HashIndexElement* element, bool isRollback) { TRI_IF_FAILURE("RemoveHashIndex") { return TRI_ERROR_DEBUG; } - ManagedDocumentResult result(trx); - IndexLookupContext context(trx, _collection, &result, numPaths()); + ManagedDocumentResult result(trx); + IndexLookupContext context(trx, _collection, &result, numPaths()); HashIndexElement* old = _multiArray->_hashArray->remove(&context, element); if (old == nullptr) { @@ -961,7 +998,6 @@ bool HashIndex::supportsFilterCondition( arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, size_t itemsInIndex, size_t& estimatedItems, double& estimatedCost) const { - SimpleAttributeEqualityMatcher matcher(_fields); return matcher.matchAll(this, node, reference, itemsInIndex, estimatedItems, estimatedCost); @@ -969,11 +1005,10 @@ bool HashIndex::supportsFilterCondition( /// @brief creates an IndexIterator for the given Condition IndexIterator* HashIndex::iteratorForCondition( - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, + arangodb::Transaction* trx, ManagedDocumentResult* mmdr, arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, bool) const { - TRI_IF_FAILURE("HashIndex::noIterator") { + TRI_IF_FAILURE("HashIndex::noIterator") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } return new HashIndexIterator(_collection, trx, mmdr, this, node, reference); @@ -988,7 +1023,7 @@ IndexIterator* HashIndex::iteratorForSlice(arangodb::Transaction* trx, // Invalid searchValue return nullptr; } - + TransactionBuilderLeaser builder(trx); std::unique_ptr keys(builder.steal()); keys->add(searchValues); @@ -999,7 +1034,6 @@ IndexIterator* HashIndex::iteratorForSlice(arangodb::Transaction* trx, arangodb::aql::AstNode* HashIndex::specializeCondition( arangodb::aql::AstNode* node, arangodb::aql::Variable const* reference) const { - SimpleAttributeEqualityMatcher matcher(_fields); return matcher.specializeAll(this, node, reference); } diff --git a/arangod/Indexes/HashIndex.h b/arangod/Indexes/HashIndex.h index ae7af66607..688993ebae 100644 --- a/arangod/Indexes/HashIndex.h +++ b/arangod/Indexes/HashIndex.h @@ -24,16 +24,16 @@ #ifndef ARANGOD_INDEXES_HASH_INDEX_H #define ARANGOD_INDEXES_HASH_INDEX_H 1 -#include "Basics/Common.h" #include "Basics/AssocMulti.h" #include "Basics/AssocUnique.h" +#include "Basics/Common.h" #include "Basics/VelocyPackHelper.h" #include "Basics/fasthash.h" -#include "Indexes/PathBasedIndex.h" #include "Indexes/IndexIterator.h" +#include "Indexes/PathBasedIndex.h" #include "Utils/Transaction.h" -#include "VocBase/vocbase.h" #include "VocBase/voc-types.h" +#include "VocBase/vocbase.h" #include #include @@ -41,52 +41,53 @@ /// @brief hash index query parameter namespace arangodb { +namespace basics { +class LocalTaskQueue; +} class HashIndex; /// @brief Class to build Slice lookups out of AST Conditions class LookupBuilder { - private: - TransactionBuilderLeaser _builder; - bool _usesIn; - bool _isEmpty; - size_t _coveredFields; - std::unordered_map _mappingFieldCondition; - std::unordered_map< - size_t, std::pair>> - _inPosition; - TransactionBuilderLeaser _inStorage; + private: + TransactionBuilderLeaser _builder; + bool _usesIn; + bool _isEmpty; + size_t _coveredFields; + std::unordered_map + _mappingFieldCondition; + std::unordered_map< + size_t, std::pair>> + _inPosition; + TransactionBuilderLeaser _inStorage; - public: - LookupBuilder( - arangodb::Transaction*, arangodb::aql::AstNode const*, - arangodb::aql::Variable const*, - std::vector> const&); + public: + LookupBuilder( + arangodb::Transaction*, arangodb::aql::AstNode const*, + arangodb::aql::Variable const*, + std::vector> const&); - arangodb::velocypack::Slice lookup(); + arangodb::velocypack::Slice lookup(); - bool hasAndGetNext(); + bool hasAndGetNext(); - void reset(); + void reset(); - private: - - bool incrementInPosition(); - void buildNextSearchValue(); + private: + bool incrementInPosition(); + void buildNextSearchValue(); }; class HashIndexIterator final : public IndexIterator { public: - -/// @brief Construct an HashIndexIterator based on Ast Conditions - HashIndexIterator(LogicalCollection* collection, arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, - HashIndex const* index, + /// @brief Construct an HashIndexIterator based on Ast Conditions + HashIndexIterator(LogicalCollection* collection, arangodb::Transaction* trx, + ManagedDocumentResult* mmdr, HashIndex const* index, arangodb::aql::AstNode const*, arangodb::aql::Variable const*); ~HashIndexIterator() = default; - + char const* typeName() const override { return "hash-index-iterator"; } IndexLookupResult next() override; @@ -104,16 +105,14 @@ class HashIndexIterator final : public IndexIterator { class HashIndexIteratorVPack final : public IndexIterator { public: - -/// @brief Construct an HashIndexIterator based on VelocyPack - HashIndexIteratorVPack(LogicalCollection* collection, - arangodb::Transaction* trx, - ManagedDocumentResult* mmdr, - HashIndex const* index, + /// @brief Construct an HashIndexIterator based on VelocyPack + HashIndexIteratorVPack( + LogicalCollection* collection, arangodb::Transaction* trx, + ManagedDocumentResult* mmdr, HashIndex const* index, std::unique_ptr& searchValues); ~HashIndexIteratorVPack(); - + char const* typeName() const override { return "hash-index-iterator-vpack"; } IndexLookupResult next() override; @@ -141,19 +140,18 @@ class HashIndex final : public PathBasedIndex { ~HashIndex(); public: - IndexType type() const override { - return Index::TRI_IDX_TYPE_HASH_INDEX; - } - + IndexType type() const override { return Index::TRI_IDX_TYPE_HASH_INDEX; } + bool allowExpansion() const override { return true; } - + bool canBeDropped() const override { return true; } bool isSorted() const override { return false; } bool hasSelectivityEstimate() const override { return true; } - double selectivityEstimate(arangodb::StringRef const* = nullptr) const override; + double selectivityEstimate( + arangodb::StringRef const* = nullptr) const override; size_t memory() const override; @@ -162,12 +160,17 @@ class HashIndex final : public PathBasedIndex { bool matchesDefinition(VPackSlice const& info) const override; - int insert(arangodb::Transaction*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; + int insert(arangodb::Transaction*, TRI_voc_rid_t, + arangodb::velocypack::Slice const&, bool isRollback) override; - int remove(arangodb::Transaction*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; + int remove(arangodb::Transaction*, TRI_voc_rid_t, + arangodb::velocypack::Slice const&, bool isRollback) override; + + void batchInsert( + arangodb::Transaction*, + std::vector> const&, + arangodb::basics::LocalTaskQueue* queue = nullptr) override; - int batchInsert(arangodb::Transaction*, std::vector> const&, size_t) override; - int unload() override; int sizeHint(arangodb::Transaction*, size_t) override; @@ -184,12 +187,13 @@ class HashIndex final : public PathBasedIndex { arangodb::aql::Variable const*, bool) const override; -/// @brief creates an IndexIterator for the given VelocyPackSlices -/// Each slice represents the field at the same position. (order matters) -/// And each slice has to be an object of one of the following types: -/// 1) {"eq": } // The value in index is exactly this -/// 2) {"in": } // The value in index os one of them - IndexIterator* iteratorForSlice(arangodb::Transaction*, + /// @brief creates an IndexIterator for the given VelocyPackSlices + /// Each slice represents the field at the same position. (order + /// matters) + /// And each slice has to be an object of one of the following types: + /// 1) {"eq": } // The value in index is exactly this + /// 2) {"in": } // The value in index os one of them + IndexIterator* iteratorForSlice(arangodb::Transaction*, ManagedDocumentResult*, arangodb::velocypack::Slice const, bool) const override; @@ -198,20 +202,25 @@ class HashIndex final : public PathBasedIndex { arangodb::aql::AstNode*, arangodb::aql::Variable const*) const override; private: - /// @brief locates entries in the hash index given a velocypack slice int lookup(arangodb::Transaction*, arangodb::velocypack::Slice, std::vector&) const; - int insertUnique(arangodb::Transaction*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback); + int insertUnique(arangodb::Transaction*, TRI_voc_rid_t, + arangodb::velocypack::Slice const&, bool isRollback); - int batchInsertUnique(arangodb::Transaction*, - std::vector> const&, size_t); + void batchInsertUnique( + arangodb::Transaction*, + std::vector> const&, + arangodb::basics::LocalTaskQueue* queue = nullptr); - int insertMulti(arangodb::Transaction*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback); + int insertMulti(arangodb::Transaction*, TRI_voc_rid_t, + arangodb::velocypack::Slice const&, bool isRollback); - int batchInsertMulti(arangodb::Transaction*, - std::vector> const&, size_t); + void batchInsertMulti( + arangodb::Transaction*, + std::vector> const&, + arangodb::basics::LocalTaskQueue* queue = nullptr); int removeUniqueElement(arangodb::Transaction*, HashIndexElement*, bool); @@ -231,7 +240,7 @@ class HashIndex final : public PathBasedIndex { uint64_t operator()(void* userData, HashIndexElement const* element, bool byKey = true) { uint64_t hash = element->hash(); - + if (byKey) { return hash; } @@ -247,7 +256,8 @@ class HashIndex final : public PathBasedIndex { bool _allowExpansion; public: - IsEqualElementElementByKey(size_t n, bool allowExpansion) : _numFields(n), _allowExpansion(allowExpansion) {} + IsEqualElementElementByKey(size_t n, bool allowExpansion) + : _numFields(n), _allowExpansion(allowExpansion) {} bool operator()(void* userData, HashIndexElement const* left, HashIndexElement const* right) { @@ -259,12 +269,13 @@ class HashIndex final : public PathBasedIndex { } IndexLookupContext* context = static_cast(userData); - + for (size_t i = 0; i < _numFields; ++i) { VPackSlice leftData = left->slice(context, i); VPackSlice rightData = right->slice(context, i); - int res = arangodb::basics::VelocyPackHelper::compare(leftData, rightData, false); + int res = arangodb::basics::VelocyPackHelper::compare(leftData, + rightData, false); if (res != 0) { return false; @@ -276,10 +287,10 @@ class HashIndex final : public PathBasedIndex { }; private: - /// @brief the actual hash index (unique type) typedef arangodb::basics::AssocUnique TRI_HashArray_t; + HashIndexElement*> + TRI_HashArray_t; struct UniqueArray { UniqueArray() = delete; @@ -296,13 +307,12 @@ class HashIndex final : public PathBasedIndex { /// @brief the actual hash index (multi type) typedef arangodb::basics::AssocMulti TRI_HashArrayMulti_t; + HashIndexElement*, uint32_t, false> + TRI_HashArrayMulti_t; struct MultiArray { MultiArray() = delete; - MultiArray(size_t numPaths, - TRI_HashArrayMulti_t*, HashElementFunc*, + MultiArray(size_t numPaths, TRI_HashArrayMulti_t*, HashElementFunc*, IsEqualElementElementByKey*); ~MultiArray(); diff --git a/arangod/Indexes/Index.cpp b/arangod/Indexes/Index.cpp index cdf5a34ae9..3c7e7f0278 100644 --- a/arangod/Indexes/Index.cpp +++ b/arangod/Indexes/Index.cpp @@ -26,6 +26,7 @@ #include "Aql/AstNode.h" #include "Aql/Variable.h" #include "Basics/Exceptions.h" +#include "Basics/LocalTaskQueue.h" #include "Basics/StaticStrings.h" #include "Basics/StringRef.h" #include "Basics/StringUtils.h" @@ -34,9 +35,10 @@ #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" -#include #include #include +#include +#include using namespace arangodb; @@ -53,7 +55,7 @@ Index::Index( } Index::Index(TRI_idx_iid_t iid, arangodb::LogicalCollection* collection, - VPackSlice const& slice) + VPackSlice const& slice) : _iid(iid), _collection(collection), _fields(), @@ -61,9 +63,9 @@ Index::Index(TRI_idx_iid_t iid, arangodb::LogicalCollection* collection, slice, "unique", false)), _sparse(arangodb::basics::VelocyPackHelper::getBooleanValue( slice, "sparse", false)) { - VPackSlice const fields = slice.get("fields"); - setFields(fields, Index::allowExpansion(Index::type(slice.get("type").copyString()))); + setFields(fields, + Index::allowExpansion(Index::type(slice.get("type").copyString()))); } /// @brief create an index stub with a hard-coded selectivity estimate @@ -78,9 +80,9 @@ Index::Index(VPackSlice const& slice) slice, "unique", false)), _sparse(arangodb::basics::VelocyPackHelper::getBooleanValue( slice, "sparse", false)) { - VPackSlice const fields = slice.get("fields"); - setFields(fields, Index::allowExpansion(Index::type(slice.get("type").copyString()))); + setFields(fields, + Index::allowExpansion(Index::type(slice.get("type").copyString()))); } Index::~Index() {} @@ -102,15 +104,17 @@ void Index::setFields(VPackSlice const& fields, bool allowExpansion) { } std::vector parsedAttributes; - TRI_ParseAttributeString(name.copyString(), parsedAttributes, allowExpansion); + TRI_ParseAttributeString(name.copyString(), parsedAttributes, + allowExpansion); _fields.emplace_back(std::move(parsedAttributes)); } } /// @brief validate fields from slice void Index::validateFields(VPackSlice const& slice) { - bool const allowExpansion = Index::allowExpansion(Index::type(slice.get("type").copyString())); - + bool const allowExpansion = + Index::allowExpansion(Index::type(slice.get("type").copyString())); + VPackSlice fields = slice.get("fields"); if (!fields.isArray()) { @@ -125,7 +129,8 @@ void Index::validateFields(VPackSlice const& slice) { } std::vector parsedAttributes; - TRI_ParseAttributeString(name.copyString(), parsedAttributes, allowExpansion); + TRI_ParseAttributeString(name.copyString(), parsedAttributes, + allowExpansion); } } @@ -158,7 +163,7 @@ Index::IndexType Index::type(char const* type) { return TRI_IDX_TYPE_UNKNOWN; } - + Index::IndexType Index::type(std::string const& type) { return Index::type(type.c_str()); } @@ -351,8 +356,8 @@ std::string Index::context() const { result << "index { id: " << id() << ", type: " << typeName() << ", collection: " << _collection->dbName() << "/" - << _collection->name() - << ", unique: " << (_unique ? "true" : "false") << ", fields: "; + << _collection->name() << ", unique: " << (_unique ? "true" : "false") + << ", fields: "; result << "["; for (size_t i = 0; i < _fields.size(); ++i) { if (i > 0) { @@ -432,7 +437,7 @@ bool Index::matchesDefinition(VPackSlice const& info) const { auto value = info.get("id"); if (!value.isNone()) { // We already have an id. - if(!value.isString()) { + if (!value.isString()) { // Invalid ID return false; } @@ -485,12 +490,21 @@ double Index::selectivityEstimate(StringRef const*) const { bool Index::implicitlyUnique() const { // simply return whether the index actually is unique // in this base class, we cannot do anything else - return _unique; + return _unique; } -/// @brief default implementation for selectivityEstimate -int Index::batchInsert(arangodb::Transaction*, std::vector> const&, size_t) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +void Index::batchInsert( + arangodb::Transaction* trx, + std::vector> const& + documents, + arangodb::basics::LocalTaskQueue* queue) { + for (auto const& it : documents) { + int status = insert(trx, it.first, it.second, false); + if (status != TRI_ERROR_NO_ERROR) { + queue->setStatus(status); + break; + } + } } /// @brief default implementation for cleanup @@ -528,8 +542,7 @@ bool Index::supportsFilterCondition(arangodb::aql::AstNode const*, /// @brief default implementation for supportsSortCondition bool Index::supportsSortCondition(arangodb::aql::SortCondition const*, arangodb::aql::Variable const*, - size_t itemsInIndex, - double& estimatedCost, + size_t itemsInIndex, double& estimatedCost, size_t& coveredAttributes) const { // by default, no sort conditions are supported coveredAttributes = 0; @@ -559,12 +572,11 @@ arangodb::aql::AstNode* Index::specializeCondition( } /// @brief perform some base checks for an index condition part -bool Index::canUseConditionPart(arangodb::aql::AstNode const* access, - arangodb::aql::AstNode const* other, - arangodb::aql::AstNode const* op, - arangodb::aql::Variable const* reference, - std::unordered_set& nonNullAttributes, - bool isExecution) const { +bool Index::canUseConditionPart( + arangodb::aql::AstNode const* access, arangodb::aql::AstNode const* other, + arangodb::aql::AstNode const* op, arangodb::aql::Variable const* reference, + std::unordered_set& nonNullAttributes, + bool isExecution) const { if (_sparse) { if (op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_NIN) { return false; @@ -600,19 +612,27 @@ bool Index::canUseConditionPart(arangodb::aql::AstNode const* access, if (!other->isConstant()) { return false; } - if (op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_NE && other->isNullValue()) { // != null. now note that a certain attribute cannot become null - try { nonNullAttributes.emplace(access->toString()); } catch (...) {} + try { + nonNullAttributes.emplace(access->toString()); + } catch (...) { + } } else if (op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT) { // > null. now note that a certain attribute cannot become null - try { nonNullAttributes.emplace(access->toString()); } catch (...) {} + try { + nonNullAttributes.emplace(access->toString()); + } catch (...) { + } } else if (op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE && !other->isNullValue()) { // >= non-null. now note that a certain attribute cannot become null - try { nonNullAttributes.emplace(access->toString()); } catch (...) {} + try { + nonNullAttributes.emplace(access->toString()); + } catch (...) { + } } if (op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT || @@ -621,7 +641,8 @@ bool Index::canUseConditionPart(arangodb::aql::AstNode const* access, // null values try { // check if we've marked this attribute as being non-null already - if (nonNullAttributes.find(access->toString()) == nonNullAttributes.end()) { + if (nonNullAttributes.find(access->toString()) == + nonNullAttributes.end()) { return false; } } catch (...) { @@ -636,7 +657,8 @@ bool Index::canUseConditionPart(arangodb::aql::AstNode const* access, // reason try { // check if we've marked this attribute as being non-null already - if (nonNullAttributes.find(access->toString()) == nonNullAttributes.end()) { + if (nonNullAttributes.find(access->toString()) == + nonNullAttributes.end()) { return false; } } catch (...) { @@ -694,8 +716,8 @@ void Index::expandInSearchValues(VPackSlice const base, VPackBuilder& result) const { TRI_ASSERT(base.isArray()); - VPackArrayBuilder baseGuard(&result); - for (auto const& oneLookup: VPackArrayIterator(base)) { + VPackArrayBuilder baseGuard(&result); + for (auto const& oneLookup : VPackArrayIterator(base)) { TRI_ASSERT(oneLookup.isArray()); bool usesIn = false; @@ -720,11 +742,12 @@ void Index::expandInSearchValues(VPackSlice const base, if (current.hasKey(StaticStrings::IndexIn)) { VPackSlice inList = current.get(StaticStrings::IndexIn); - std::unordered_set - tmp(static_cast(inList.length()), arangodb::basics::VelocyPackHelper::VPackHash(), - arangodb::basics::VelocyPackHelper::VPackEqual()); + std::unordered_set + tmp(static_cast(inList.length()), + arangodb::basics::VelocyPackHelper::VPackHash(), + arangodb::basics::VelocyPackHelper::VPackEqual()); TRI_ASSERT(inList.isArray()); if (inList.length() == 0) { @@ -744,16 +767,16 @@ void Index::expandInSearchValues(VPackSlice const base, } // If there is an entry in elements for one depth it was an in, // all of them are now unique so we simply have to multiply - + size_t level = n - 1; std::vector positions(n, 0); bool done = false; while (!done) { - TRI_IF_FAILURE("Index::permutationIN") { + TRI_IF_FAILURE("Index::permutationIN") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } VPackArrayBuilder guard(&result); - for (size_t i = 0; i < n; ++i) { + for (size_t i = 0; i < n; ++i) { auto list = elements.find(i); if (list == elements.end()) { // Insert @@ -765,7 +788,8 @@ void Index::expandInSearchValues(VPackSlice const base, } while (true) { auto list = elements.find(level); - if (list != elements.end() && ++positions[level] < list->second.size()) { + if (list != elements.end() && + ++positions[level] < list->second.size()) { level = n - 1; // abort inner iteration break; diff --git a/arangod/Indexes/Index.h b/arangod/Indexes/Index.h index e005234000..7f3eed199f 100644 --- a/arangod/Indexes/Index.h +++ b/arangod/Indexes/Index.h @@ -1,3 +1,4 @@ + //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// @@ -24,16 +25,19 @@ #ifndef ARANGOD_INDEXES_INDEX_H #define ARANGOD_INDEXES_INDEX_H 1 -#include "Basics/Common.h" #include "Basics/AttributeNameParser.h" +#include "Basics/Common.h" #include "Basics/Exceptions.h" #include "Indexes/IndexElement.h" -#include "VocBase/vocbase.h" #include "VocBase/voc-types.h" +#include "VocBase/vocbase.h" #include namespace arangodb { +namespace basics { +class LocalTaskQueue; +} class LogicalCollection; class ManagedDocumentResult; @@ -154,9 +158,7 @@ class Index { } /// @brief return the underlying collection - inline LogicalCollection* collection() const { - return _collection; - } + inline LogicalCollection* collection() const { return _collection; } /// @brief return a contextual string for logging std::string context() const; @@ -216,13 +218,15 @@ class Index { /// @brief whether or not the index has a selectivity estimate virtual bool hasSelectivityEstimate() const = 0; - + /// @brief return the selectivity estimate of the index /// must only be called if hasSelectivityEstimate() returns true - virtual double selectivityEstimate(arangodb::StringRef const* = nullptr) const; - + virtual double selectivityEstimate( + arangodb::StringRef const* = nullptr) const; + /// @brief whether or not the index is implicitly unique - /// this can be the case if the index is not declared as unique, but contains a + /// this can be the case if the index is not declared as unique, but contains + /// a /// unique attribute such as _key virtual bool implicitlyUnique() const; @@ -234,11 +238,16 @@ class Index { virtual void toVelocyPackFigures(arangodb::velocypack::Builder&) const; std::shared_ptr toVelocyPackFigures() const; - virtual int insert(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&, bool isRollback) = 0; - virtual int remove(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&, bool isRollback) = 0; - - virtual int batchInsert(arangodb::Transaction*, std::vector> const&, size_t); - + virtual int insert(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&, bool isRollback) = 0; + virtual int remove(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&, bool isRollback) = 0; + + virtual void batchInsert( + arangodb::Transaction*, + std::vector> const&, + arangodb::basics::LocalTaskQueue* queue = nullptr); + virtual int unload() = 0; // a garbage collection function for the index diff --git a/arangod/RestServer/DatabaseFeature.cpp b/arangod/RestServer/DatabaseFeature.cpp index 8cb686c6e7..70002deff6 100644 --- a/arangod/RestServer/DatabaseFeature.cpp +++ b/arangod/RestServer/DatabaseFeature.cpp @@ -22,7 +22,6 @@ #include "DatabaseFeature.h" -#include "Agency/v8-agency.h" #include "Agency/v8-agency.h" #include "ApplicationFeatures/ApplicationServer.h" #include "Aql/QueryCache.h" @@ -83,7 +82,7 @@ void DatabaseManagerThread::run() { StorageEngine* engine = EngineSelectorFeature::ENGINE; while (true) { - try { + try { // check if we have to drop some database TRI_vocbase_t* database = nullptr; @@ -129,7 +128,8 @@ void DatabaseManagerThread::run() { databaseFeature->_databasesProtector.scan(); delete oldLists; - // From now on no other thread can possibly see the old TRI_vocbase_t*, + // From now on no other thread can possibly see the old + // TRI_vocbase_t*, // note that there is only one DatabaseManager thread, so it is // not possible that another thread has seen this very database // and tries to free it at the same time! @@ -143,8 +143,8 @@ void DatabaseManagerThread::run() { RocksDBFeature::dropDatabase(database->id()); LOG(TRACE) << "physically removing database directory '" - << engine->databasePath(database) << "' of database '" - << database->name() << "'"; + << engine->databasePath(database) << "' of database '" + << database->name() << "'"; std::string path; @@ -158,7 +158,7 @@ void DatabaseManagerThread::run() { if (TRI_IsDirectory(path.c_str())) { LOG(TRACE) << "removing app directory '" << path - << "' of database '" << database->name() << "'"; + << "' of database '" << database->name() << "'"; TRI_RemoveDirectory(path.c_str()); } @@ -226,7 +226,7 @@ void DatabaseManagerThread::run() { } catch (...) { } - + // next iteration } } @@ -252,7 +252,6 @@ DatabaseFeature::DatabaseFeature(ApplicationServer* server) startsAfter("EngineSelector"); startsAfter("LogfileManager"); startsAfter("InitDatabase"); - startsAfter("IndexThread"); startsAfter("RevisionCache"); } @@ -298,11 +297,17 @@ void DatabaseFeature::collectOptions(std::shared_ptr options) { "--database.replication-applier", "switch to enable or disable the replication applier", new BooleanParameter(&_replicationApplier)); - - options->addHiddenOption("--database.check-30-revisions", - "check _rev values in collections created before 3.1", - new DiscreteValuesParameter(&_check30Revisions, - std::unordered_set{ "true", "false", "fail" })); + + options->addHiddenOption( + "--database.check-30-revisions", + "check _rev values in collections created before 3.1", + new DiscreteValuesParameter( + &_check30Revisions, + std::unordered_set{"true", "false", "fail"})); + + options->addObsoleteOption( + "--database.index-threads", + "threads to start for parallel background index creation"); } void DatabaseFeature::validateOptions(std::shared_ptr options) { @@ -362,7 +367,7 @@ void DatabaseFeature::start() { } // TODO: handle _upgrade and _checkVersion here - + // activate deadlock detection in case we're not running in cluster mode if (!arangodb::ServerState::instance()->isRunningInCluster()) { enableDeadlockDetection(); @@ -865,10 +870,9 @@ std::vector DatabaseFeature::getDatabaseNamesForUser( TRI_vocbase_t* vocbase = p.second; TRI_ASSERT(vocbase != nullptr); - auto authentication = application_features::ApplicationServer::getFeature( - "Authentication"); - auto level = authentication->canUseDatabase( - username, vocbase->name()); + auto authentication = application_features::ApplicationServer::getFeature< + AuthenticationFeature>("Authentication"); + auto level = authentication->canUseDatabase(username, vocbase->name()); if (level == AuthLevel::NONE) { continue; @@ -1312,8 +1316,8 @@ int DatabaseFeature::writeCreateMarker(TRI_voc_tick_t id, int res = TRI_ERROR_NO_ERROR; try { - MMFilesDatabaseMarker marker(TRI_DF_MARKER_VPACK_CREATE_DATABASE, - id, slice); + MMFilesDatabaseMarker marker(TRI_DF_MARKER_VPACK_CREATE_DATABASE, id, + slice); MMFilesWalSlotInfoCopy slotInfo = arangodb::wal::LogfileManager::instance()->allocateAndWrite(marker, false); @@ -1347,7 +1351,7 @@ int DatabaseFeature::writeDropMarker(TRI_voc_tick_t id) { builder.close(); MMFilesDatabaseMarker marker(TRI_DF_MARKER_VPACK_DROP_DATABASE, id, - builder.slice()); + builder.slice()); MMFilesWalSlotInfoCopy slotInfo = arangodb::wal::LogfileManager::instance()->allocateAndWrite(marker, diff --git a/arangod/RestServer/ServerFeature.cpp b/arangod/RestServer/ServerFeature.cpp index 9f932fa412..3bc03261b0 100644 --- a/arangod/RestServer/ServerFeature.cpp +++ b/arangod/RestServer/ServerFeature.cpp @@ -124,8 +124,7 @@ void ServerFeature::validateOptions(std::shared_ptr) { if (!_restServer) { ApplicationServer::disableFeatures({"Daemon", "Endpoint", "GeneralServer", - "Scheduler", "SslServer", - "Supervisor"}); + "SslServer", "Supervisor"}); DatabaseFeature* database = ApplicationServer::getFeature("Database"); @@ -158,7 +157,7 @@ void ServerFeature::validateOptions(std::shared_ptr) { } void ServerFeature::start() { - if (_operationMode != OperationMode::MODE_CONSOLE && _restServer) { + if (_operationMode != OperationMode::MODE_CONSOLE) { auto scheduler = ApplicationServer::getFeature("Scheduler"); diff --git a/arangod/RestServer/arangod.cpp b/arangod/RestServer/arangod.cpp index dd735dad4f..26ba561fa3 100644 --- a/arangod/RestServer/arangod.cpp +++ b/arangod/RestServer/arangod.cpp @@ -78,7 +78,6 @@ #include "StorageEngine/RocksDBEngine.h" #include "V8Server/FoxxQueuesFeature.h" #include "V8Server/V8DealerFeature.h" -#include "VocBase/IndexThreadFeature.h" #include "Wal/LogfileManager.h" #include "Indexes/RocksDBFeature.h" @@ -112,9 +111,8 @@ static int runServer(int argc, char** argv) { "Cluster", "Daemon", "Dispatcher", "FoxxQueues", "GeneralServer", "LoggerBufferFeature", - "Server", "Scheduler", - "SslServer", "Statistics", - "Supervisor"}; + "Server", "SslServer", + "Statistics", "Supervisor"}; int ret = EXIT_FAILURE; @@ -136,7 +134,6 @@ static int runServer(int argc, char** argv) { server.addFeature(new FrontendFeature(&server)); server.addFeature(new GeneralServerFeature(&server)); server.addFeature(new GreetingsFeature(&server, "arangod")); - server.addFeature(new IndexThreadFeature(&server)); server.addFeature(new InitDatabaseFeature(&server, nonServerFeatures)); server.addFeature(new LanguageFeature(&server)); server.addFeature(new LockfileFeature(&server)); @@ -246,5 +243,5 @@ int main(int argc, char* argv[]) { } } else #endif - return runServer(argc, argv); + return runServer(argc, argv); } diff --git a/arangod/Scheduler/SchedulerFeature.cpp b/arangod/Scheduler/SchedulerFeature.cpp index 851885ae1e..d343c8fa5b 100644 --- a/arangod/Scheduler/SchedulerFeature.cpp +++ b/arangod/Scheduler/SchedulerFeature.cpp @@ -239,7 +239,8 @@ void SchedulerFeature::buildScheduler() { _scheduler->setMinimal(_nrMinimalThreads); _scheduler->setRealMaximum(_nrMaximalThreads); - + + TRI_ASSERT(SCHEDULER == nullptr); SCHEDULER = _scheduler.get(); } diff --git a/arangod/StorageEngine/MMFilesWalRecoverState.cpp b/arangod/StorageEngine/MMFilesWalRecoverState.cpp index 3573dc68e8..44600b97a0 100644 --- a/arangod/StorageEngine/MMFilesWalRecoverState.cpp +++ b/arangod/StorageEngine/MMFilesWalRecoverState.cpp @@ -23,13 +23,13 @@ #include "MMFilesWalRecoverState.h" #include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/Exceptions.h" #include "Basics/FileUtils.h" +#include "Basics/VelocyPackHelper.h" #include "Basics/conversions.h" #include "Basics/files.h" -#include "Basics/Exceptions.h" #include "Basics/memory-map.h" #include "Basics/tri-strings.h" -#include "Basics/VelocyPackHelper.h" #include "RestServer/DatabaseFeature.h" #include "StorageEngine/MMFilesDatafileHelper.h" #include "StorageEngine/MMFilesWalSlots.h" @@ -50,8 +50,10 @@ using namespace arangodb; template static inline T NumericValue(VPackSlice const& slice, char const* attribute) { if (!slice.isObject()) { - LOG(ERR) << "invalid value type when looking for attribute '" << attribute << "': expecting object"; - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "invalid attribute value: expecting object"); + LOG(ERR) << "invalid value type when looking for attribute '" << attribute + << "': expecting object"; + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "invalid attribute value: expecting object"); } VPackSlice v = slice.get(attribute); if (v.isString()) { @@ -60,9 +62,10 @@ static inline T NumericValue(VPackSlice const& slice, char const* attribute) { if (v.isNumber()) { return v.getNumber(); } - + LOG(ERR) << "invalid value for attribute '" << attribute << "'"; - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "invalid attribute value"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "invalid attribute value"); } /// @brief creates the recover state @@ -79,8 +82,9 @@ MMFilesWalRecoverState::MMFilesWalRecoverState(bool ignoreRecoveryErrors) maxRevisionId(0), lastDatabaseId(0), lastCollectionId(0) { - - databaseFeature = application_features::ApplicationServer::getFeature("Database"); + databaseFeature = + application_features::ApplicationServer::getFeature( + "Database"); } /// @brief destroys the recover state @@ -125,7 +129,8 @@ TRI_vocbase_t* MMFilesWalRecoverState::useDatabase(TRI_voc_tick_t databaseId) { } /// @brief release a database (so it can be dropped) -TRI_vocbase_t* MMFilesWalRecoverState::releaseDatabase(TRI_voc_tick_t databaseId) { +TRI_vocbase_t* MMFilesWalRecoverState::releaseDatabase( + TRI_voc_tick_t databaseId) { auto it = openedDatabases.find(databaseId); if (it == openedDatabases.end()) { @@ -163,7 +168,8 @@ TRI_vocbase_t* MMFilesWalRecoverState::releaseDatabase(TRI_voc_tick_t databaseId } /// @brief release a collection (so it can be dropped) -arangodb::LogicalCollection* MMFilesWalRecoverState::releaseCollection(TRI_voc_cid_t collectionId) { +arangodb::LogicalCollection* MMFilesWalRecoverState::releaseCollection( + TRI_voc_cid_t collectionId) { auto it = openedCollections.find(collectionId); if (it == openedCollections.end()) { @@ -191,13 +197,15 @@ arangodb::LogicalCollection* MMFilesWalRecoverState::useCollection( TRI_set_errno(TRI_ERROR_NO_ERROR); TRI_vocbase_col_status_e status; // ignored here - arangodb::LogicalCollection* collection = vocbase->useCollection(collectionId, status); + arangodb::LogicalCollection* collection = + vocbase->useCollection(collectionId, status); if (collection == nullptr) { res = TRI_errno(); if (res == TRI_ERROR_ARANGO_CORRUPTED_COLLECTION) { - LOG(WARN) << "unable to open collection " << collectionId << ". Please check the logs above for errors."; + LOG(WARN) << "unable to open collection " << collectionId + << ". Please check the logs above for errors."; } return nullptr; @@ -225,10 +233,12 @@ LogicalCollection* MMFilesWalRecoverState::getCollection( } int res; - arangodb::LogicalCollection* collection = useCollection(vocbase, collectionId, res); + arangodb::LogicalCollection* collection = + useCollection(vocbase, collectionId, res); if (collection == nullptr) { - LOG(TRACE) << "collection " << collectionId << " of database " << databaseId << " not found"; + LOG(TRACE) << "collection " << collectionId << " of database " << databaseId + << " not found"; return nullptr; } return collection; @@ -238,7 +248,8 @@ LogicalCollection* MMFilesWalRecoverState::getCollection( int MMFilesWalRecoverState::executeSingleOperation( TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId, TRI_df_marker_t const* marker, TRI_voc_fid_t fid, - std::function func) { + std::function + func) { // first find the correct database TRI_vocbase_t* vocbase = useDatabase(databaseId); @@ -248,7 +259,8 @@ int MMFilesWalRecoverState::executeSingleOperation( } int res; - arangodb::LogicalCollection* collection = useCollection(vocbase, collectionId, res); + arangodb::LogicalCollection* collection = + useCollection(vocbase, collectionId, res); if (collection == nullptr) { if (res == TRI_ERROR_ARANGO_CORRUPTED_COLLECTION) { @@ -267,14 +279,17 @@ int MMFilesWalRecoverState::executeSingleOperation( res = TRI_ERROR_INTERNAL; try { - SingleCollectionTransaction trx(arangodb::StandaloneTransactionContext::Create(vocbase), collectionId, TRI_TRANSACTION_WRITE); + SingleCollectionTransaction trx( + arangodb::StandaloneTransactionContext::Create(vocbase), collectionId, + TRI_TRANSACTION_WRITE); trx.addHint(TRI_TRANSACTION_HINT_SINGLE_OPERATION, false); trx.addHint(TRI_TRANSACTION_HINT_NO_BEGIN_MARKER, false); trx.addHint(TRI_TRANSACTION_HINT_NO_ABORT_MARKER, false); trx.addHint(TRI_TRANSACTION_HINT_NO_THROTTLING, false); trx.addHint(TRI_TRANSACTION_HINT_LOCK_NEVER, false); - trx.addHint(TRI_TRANSACTION_HINT_RECOVERY, false); // to turn off waitForSync! + trx.addHint(TRI_TRANSACTION_HINT_RECOVERY, + false); // to turn off waitForSync! res = trx.begin(); @@ -294,13 +309,16 @@ int MMFilesWalRecoverState::executeSingleOperation( // commit the operation res = trx.commit(); } catch (arangodb::basics::Exception const& ex) { - LOG(ERR) << "caught exception during recovery of marker type " << TRI_NameMarkerDatafile(marker) << ": " << ex.what(); + LOG(ERR) << "caught exception during recovery of marker type " + << TRI_NameMarkerDatafile(marker) << ": " << ex.what(); res = ex.code(); } catch (std::exception const& ex) { - LOG(ERR) << "caught exception during recovery of marker type " << TRI_NameMarkerDatafile(marker) << ": " << ex.what(); + LOG(ERR) << "caught exception during recovery of marker type " + << TRI_NameMarkerDatafile(marker) << ": " << ex.what(); res = TRI_ERROR_INTERNAL; } catch (...) { - LOG(ERR) << "caught unknown exception during recovery of marker type " << TRI_NameMarkerDatafile(marker); + LOG(ERR) << "caught unknown exception during recovery of marker type " + << TRI_NameMarkerDatafile(marker); res = TRI_ERROR_INTERNAL; } @@ -309,9 +327,11 @@ int MMFilesWalRecoverState::executeSingleOperation( /// @brief callback to handle one marker during recovery /// this function only builds up state and does not change any data -bool MMFilesWalRecoverState::InitialScanMarker(TRI_df_marker_t const* marker, void* data, - MMFilesDatafile* datafile) { - MMFilesWalRecoverState* state = reinterpret_cast(data); +bool MMFilesWalRecoverState::InitialScanMarker(TRI_df_marker_t const* marker, + void* data, + MMFilesDatafile* datafile) { + MMFilesWalRecoverState* state = + reinterpret_cast(data); TRI_ASSERT(marker != nullptr); @@ -328,9 +348,11 @@ bool MMFilesWalRecoverState::InitialScanMarker(TRI_df_marker_t const* marker, vo switch (type) { case TRI_DF_MARKER_VPACK_DOCUMENT: { - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); if (payloadSlice.isObject()) { - TRI_voc_rid_t revisionId = Transaction::extractRevFromDocument(payloadSlice); + TRI_voc_rid_t revisionId = + Transaction::extractRevFromDocument(payloadSlice); if (revisionId > state->maxRevisionId) { state->maxRevisionId = revisionId; } @@ -344,7 +366,8 @@ bool MMFilesWalRecoverState::InitialScanMarker(TRI_df_marker_t const* marker, vo // transaction, // we'll have it in the failed list at the end of the scan and can ignore // it - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); TRI_voc_tid_t const tid = MMFilesDatafileHelper::TransactionId(marker); state->failedTransactions.emplace(tid, std::make_pair(databaseId, false)); break; @@ -359,22 +382,25 @@ bool MMFilesWalRecoverState::InitialScanMarker(TRI_df_marker_t const* marker, vo case TRI_DF_MARKER_VPACK_ABORT_TRANSACTION: { // insert this transaction into the list of failed transactions - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); TRI_voc_tid_t const tid = MMFilesDatafileHelper::TransactionId(marker); state->failedTransactions[tid] = std::make_pair(databaseId, true); break; } - + case TRI_DF_MARKER_VPACK_DROP_DATABASE: { // note that the database was dropped and doesn't need to be recovered - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); state->totalDroppedDatabases.emplace(databaseId); break; } case TRI_DF_MARKER_VPACK_DROP_COLLECTION: { // note that the collection was dropped and doesn't need to be recovered - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); state->totalDroppedCollections.emplace(collectionId); break; } @@ -389,24 +415,29 @@ bool MMFilesWalRecoverState::InitialScanMarker(TRI_df_marker_t const* marker, vo /// @brief callback to replay one marker during recovery /// this function modifies indexes etc. -bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* data, - MMFilesDatafile* datafile) { - MMFilesWalRecoverState* state = reinterpret_cast(data); +bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, + void* data, + MMFilesDatafile* datafile) { + MMFilesWalRecoverState* state = + reinterpret_cast(data); #ifdef ARANGODB_ENABLE_FAILURE_TESTS LOG(TRACE) << "replaying marker of type " << TRI_NameMarkerDatafile(marker); #endif - + TRI_df_marker_type_t const type = marker->getType(); try { switch (type) { case TRI_DF_MARKER_PROLOGUE: { // simply note the last state - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); - LOG(TRACE) << "found prologue marker. databaseId: " << databaseId << ", collectionId: " << collectionId; + LOG(TRACE) << "found prologue marker. databaseId: " << databaseId + << ", collectionId: " << collectionId; state->resetCollection(databaseId, collectionId); return true; } @@ -417,33 +448,41 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d case TRI_DF_MARKER_VPACK_DOCUMENT: { // re-insert the document/edge into the collection - TRI_voc_tick_t const databaseId = state->lastDatabaseId; // from prologue - TRI_voc_cid_t const collectionId = state->lastCollectionId; // from prologue - + TRI_voc_tick_t const databaseId = + state->lastDatabaseId; // from prologue + TRI_voc_cid_t const collectionId = + state->lastCollectionId; // from prologue + if (state->isDropped(databaseId, collectionId)) { return true; } TRI_voc_tid_t const tid = MMFilesDatafileHelper::TransactionId(marker); - + if (state->ignoreTransaction(tid)) { // transaction was aborted return true; } - - LOG(TRACE) << "found document marker. databaseId: " << databaseId << ", collectionId: " << collectionId << ", transactionId: " << tid; + + LOG(TRACE) << "found document marker. databaseId: " << databaseId + << ", collectionId: " << collectionId + << ", transactionId: " << tid; int res = state->executeSingleOperation( databaseId, collectionId, marker, datafile->fid(), - [&](SingleCollectionTransaction* trx, MMFilesMarkerEnvelope* envelope) -> int { + [&](SingleCollectionTransaction* trx, + MMFilesMarkerEnvelope* envelope) -> int { if (trx->documentCollection()->isVolatile()) { return TRI_ERROR_NO_ERROR; } - TRI_df_marker_t const* marker = static_cast(envelope->mem()); + TRI_df_marker_t const* marker = + static_cast(envelope->mem()); - std::string const collectionName = trx->documentCollection()->name(); - uint8_t const* ptr = reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type); + std::string const collectionName = + trx->documentCollection()->name(); + uint8_t const* ptr = reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type); OperationOptions options; options.silent = true; @@ -454,7 +493,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d // try an insert first TRI_ASSERT(VPackSlice(ptr).isObject()); - OperationResult opRes = trx->insert(collectionName, VPackSlice(ptr), options); + OperationResult opRes = + trx->insert(collectionName, VPackSlice(ptr), options); int res = opRes.code; if (res == TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED) { @@ -469,7 +509,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d if (res != TRI_ERROR_NO_ERROR && res != TRI_ERROR_ARANGO_CONFLICT && res != TRI_ERROR_ARANGO_DATABASE_NOT_FOUND && res != TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND) { - LOG(WARN) << "unable to insert document in collection " << collectionId << " of database " << databaseId << ": " << TRI_errno_string(res); + LOG(WARN) << "unable to insert document in collection " + << collectionId << " of database " << databaseId << ": " + << TRI_errno_string(res); ++state->errorCount; return state->canContinue(); } @@ -478,8 +520,10 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d case TRI_DF_MARKER_VPACK_REMOVE: { // re-apply the remove operation - TRI_voc_tick_t const databaseId = state->lastDatabaseId; // from prologue - TRI_voc_cid_t const collectionId = state->lastCollectionId; // from prologue + TRI_voc_tick_t const databaseId = + state->lastDatabaseId; // from prologue + TRI_voc_cid_t const collectionId = + state->lastCollectionId; // from prologue TRI_ASSERT(databaseId > 0); TRI_ASSERT(collectionId > 0); @@ -489,24 +533,30 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } TRI_voc_tid_t const tid = MMFilesDatafileHelper::TransactionId(marker); - + if (state->ignoreTransaction(tid)) { return true; } - - LOG(TRACE) << "found remove marker. databaseId: " << databaseId << ", collectionId: " << collectionId << ", transactionId: " << tid; + + LOG(TRACE) << "found remove marker. databaseId: " << databaseId + << ", collectionId: " << collectionId + << ", transactionId: " << tid; int res = state->executeSingleOperation( databaseId, collectionId, marker, datafile->fid(), - [&](SingleCollectionTransaction* trx, MMFilesMarkerEnvelope* envelope) -> int { + [&](SingleCollectionTransaction* trx, + MMFilesMarkerEnvelope* envelope) -> int { if (trx->documentCollection()->isVolatile()) { return TRI_ERROR_NO_ERROR; } - - TRI_df_marker_t const* marker = static_cast(envelope->mem()); - std::string const collectionName = trx->documentCollection()->name(); - uint8_t const* ptr = reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type); + TRI_df_marker_t const* marker = + static_cast(envelope->mem()); + + std::string const collectionName = + trx->documentCollection()->name(); + uint8_t const* ptr = reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type); OperationOptions options; options.silent = true; @@ -515,20 +565,23 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d options.ignoreRevs = true; try { - OperationResult opRes = trx->remove(collectionName, VPackSlice(ptr), options); + OperationResult opRes = + trx->remove(collectionName, VPackSlice(ptr), options); if (opRes.code == TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND) { - // document to delete is not present. this error can be ignored + // document to delete is not present. this error can be + // ignored return TRI_ERROR_NO_ERROR; } return opRes.code; } catch (arangodb::basics::Exception const& ex) { if (ex.code() == TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND) { - // document to delete is not present. this error can be ignored + // document to delete is not present. this error can be + // ignored return TRI_ERROR_NO_ERROR; } return ex.code(); } - // should not get here... + // should not get here... return TRI_ERROR_INTERNAL; }); @@ -536,7 +589,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d res != TRI_ERROR_ARANGO_DATABASE_NOT_FOUND && res != TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND && res != TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND) { - LOG(WARN) << "unable to remove document in collection " << collectionId << " of database " << databaseId << ": " << TRI_errno_string(res); + LOG(WARN) << "unable to remove document in collection " + << collectionId << " of database " << databaseId << ": " + << TRI_errno_string(res); ++state->errorCount; return state->canContinue(); } @@ -548,10 +603,13 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d // ----------------------------------------------------------------------------- case TRI_DF_MARKER_VPACK_RENAME_COLLECTION: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); + if (!payloadSlice.isObject()) { LOG(WARN) << "cannot rename collection: invalid marker"; ++state->errorCount; @@ -561,8 +619,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d if (state->isDropped(databaseId)) { return true; } - - LOG(TRACE) << "found collection rename marker. databaseId: " << databaseId << ", collectionId: " << collectionId; + + LOG(TRACE) << "found collection rename marker. databaseId: " + << databaseId << ", collectionId: " << collectionId; TRI_vocbase_t* vocbase = state->useDatabase(databaseId); @@ -572,7 +631,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d return true; } - arangodb::LogicalCollection* collection = state->releaseCollection(collectionId); + arangodb::LogicalCollection* collection = + state->releaseCollection(collectionId); if (collection == nullptr) { collection = vocbase->lookupCollection(collectionId); @@ -586,7 +646,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d VPackSlice nameSlice = payloadSlice.get("name"); if (!nameSlice.isString()) { - LOG(WARN) << "cannot rename collection " << collectionId << " in database " << databaseId << ": name attribute is no string"; + LOG(WARN) << "cannot rename collection " << collectionId + << " in database " << databaseId + << ": name attribute is no string"; ++state->errorCount; return state->canContinue(); } @@ -604,7 +666,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d int res = vocbase->renameCollection(collection, name, true, false); if (res != TRI_ERROR_NO_ERROR) { - LOG(WARN) << "cannot rename collection " << collectionId << " in database " << databaseId << " to '" << name << "': " << TRI_errno_string(res); + LOG(WARN) << "cannot rename collection " << collectionId + << " in database " << databaseId << " to '" << name + << "': " << TRI_errno_string(res); ++state->errorCount; return state->canContinue(); } @@ -612,10 +676,13 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } case TRI_DF_MARKER_VPACK_CHANGE_COLLECTION: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); + if (!payloadSlice.isObject()) { LOG(WARN) << "cannot change properties of collection: invalid marker"; ++state->errorCount; @@ -625,8 +692,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d if (state->isDropped(databaseId)) { return true; } - - LOG(TRACE) << "found collection change marker. databaseId: " << databaseId << ", collectionId: " << collectionId; + + LOG(TRACE) << "found collection change marker. databaseId: " + << databaseId << ", collectionId: " << collectionId; TRI_vocbase_t* vocbase = state->useDatabase(databaseId); @@ -641,17 +709,22 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d if (collection == nullptr) { // if the underlying collection is gone, we can go on - LOG(TRACE) << "cannot change properties of collection " << collectionId << " in database " << databaseId << ": " << TRI_errno_string(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND); + LOG(TRACE) << "cannot change properties of collection " + << collectionId << " in database " << databaseId << ": " + << TRI_errno_string(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND); return true; } - // turn off sync temporarily if the database or collection are going to be + // turn off sync temporarily if the database or collection are going to + // be // dropped later bool const forceSync = state->willBeDropped(databaseId, collectionId); int res = collection->update(payloadSlice, forceSync); if (res != TRI_ERROR_NO_ERROR) { - LOG(WARN) << "cannot change collection properties for collection " << collectionId << " in database " << databaseId << ": " << TRI_errno_string(res); + LOG(WARN) << "cannot change collection properties for collection " + << collectionId << " in database " << databaseId << ": " + << TRI_errno_string(res); ++state->errorCount; return state->canContinue(); } @@ -660,51 +733,63 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } case TRI_DF_MARKER_VPACK_CREATE_INDEX: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); + if (!payloadSlice.isObject()) { LOG(WARN) << "cannot create index for collection: invalid marker"; ++state->errorCount; return state->canContinue(); } - + TRI_idx_iid_t indexId = NumericValue(payloadSlice, "id"); if (state->isDropped(databaseId, collectionId)) { return true; } - - LOG(TRACE) << "found create index marker. databaseId: " << databaseId << ", collectionId: " << collectionId; + + LOG(TRACE) << "found create index marker. databaseId: " << databaseId + << ", collectionId: " << collectionId; TRI_vocbase_t* vocbase = state->useDatabase(databaseId); if (vocbase == nullptr) { // if the underlying database is gone, we can go on - LOG(TRACE) << "cannot create index for collection " << collectionId << " in database " << databaseId << ": " << TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); + LOG(TRACE) << "cannot create index for collection " << collectionId + << " in database " << databaseId << ": " + << TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); return true; } - arangodb::LogicalCollection* col = vocbase->lookupCollection(collectionId); + arangodb::LogicalCollection* col = + vocbase->lookupCollection(collectionId); if (col == nullptr) { // if the underlying collection gone, we can go on - LOG(TRACE) << "cannot create index for collection " << collectionId << " in database " << databaseId << ": " << TRI_errno_string(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND); + LOG(TRACE) << "cannot create index for collection " << collectionId + << " in database " << databaseId << ": " + << TRI_errno_string(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND); return true; } RocksDBFeature::dropIndex(databaseId, collectionId, indexId); - std::string const indexName("index-" + std::to_string(indexId) + ".json"); - std::string const filename(arangodb::basics::FileUtils::buildFilename(col->path(), indexName)); + std::string const indexName("index-" + std::to_string(indexId) + + ".json"); + std::string const filename( + arangodb::basics::FileUtils::buildFilename(col->path(), indexName)); bool const forceSync = state->willBeDropped(databaseId, collectionId); bool ok = arangodb::basics::VelocyPackHelper::velocyPackToFile( filename, payloadSlice, forceSync); if (!ok) { - LOG(WARN) << "cannot create index " << indexId << ", collection " << collectionId << " in database " << databaseId; + LOG(WARN) << "cannot create index " << indexId << ", collection " + << collectionId << " in database " << databaseId; ++state->errorCount; return state->canContinue(); } else { @@ -715,7 +800,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d int res = col->restoreIndex(&trx, payloadSlice, unused); if (res != TRI_ERROR_NO_ERROR) { - LOG(WARN) << "cannot create index " << indexId << ", collection " << collectionId << " in database " << databaseId; + LOG(WARN) << "cannot create index " << indexId << ", collection " + << collectionId << " in database " << databaseId; ++state->errorCount; return state->canContinue(); } @@ -725,18 +811,22 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } case TRI_DF_MARKER_VPACK_CREATE_COLLECTION: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); + if (!payloadSlice.isObject()) { LOG(WARN) << "cannot create collection: invalid marker"; ++state->errorCount; return state->canContinue(); } - - LOG(TRACE) << "found create collection marker. databaseId: " << databaseId << ", collectionId: " << collectionId; - + + LOG(TRACE) << "found create collection marker. databaseId: " + << databaseId << ", collectionId: " << collectionId; + // remove the drop marker state->droppedCollections.erase(collectionId); @@ -752,12 +842,13 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d return true; } - arangodb::LogicalCollection* collection = state->releaseCollection(collectionId); + arangodb::LogicalCollection* collection = + state->releaseCollection(collectionId); if (collection == nullptr) { collection = vocbase->lookupCollection(collectionId); } - + if (collection != nullptr) { // drop an existing collection vocbase->dropCollection(collection, true, false); @@ -765,7 +856,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d RocksDBFeature::dropCollection(databaseId, collectionId); - // check if there is another collection with the same name as the one that + // check if there is another collection with the same name as the one + // that // we attempt to create VPackSlice const nameSlice = payloadSlice.get("name"); std::string name = ""; @@ -774,14 +866,16 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d name = nameSlice.copyString(); collection = vocbase->lookupCollection(name); - if (collection != nullptr) { + if (collection != nullptr) { TRI_voc_cid_t otherCid = collection->cid(); state->releaseCollection(otherCid); vocbase->dropCollection(collection, true, false); } } else { - LOG(WARN) << "empty name attribute in create collection marker for collection " << collectionId << " and database " << databaseId; + LOG(WARN) << "empty name attribute in create collection marker for " + "collection " + << collectionId << " and database " << databaseId; ++state->errorCount; return state->canContinue(); } @@ -802,15 +896,18 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d int res = TRI_ERROR_NO_ERROR; try { if (state->willBeDropped(collectionId)) { - // in case we detect that this collection is going to be deleted anyway, + // in case we detect that this collection is going to be deleted + // anyway, // set the sync properties to false temporarily bool oldSync = state->databaseFeature->forceSyncProperties(); state->databaseFeature->forceSyncProperties(false); - collection = vocbase->createCollection(b2.slice(), collectionId, false); + collection = + vocbase->createCollection(b2.slice(), collectionId, false); state->databaseFeature->forceSyncProperties(oldSync); } else { // collection will be kept - collection = vocbase->createCollection(b2.slice(), collectionId, false); + collection = + vocbase->createCollection(b2.slice(), collectionId, false); } TRI_ASSERT(collection != nullptr); } catch (basics::Exception const& ex) { @@ -820,7 +917,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } if (res != TRI_ERROR_NO_ERROR) { - LOG(WARN) << "cannot create collection " << collectionId << " in database " << databaseId << ": " << TRI_errno_string(res); + LOG(WARN) << "cannot create collection " << collectionId + << " in database " << databaseId << ": " + << TRI_errno_string(res); ++state->errorCount; return state->canContinue(); } @@ -828,17 +927,20 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } case TRI_DF_MARKER_VPACK_CREATE_DATABASE: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); + if (!payloadSlice.isObject()) { LOG(WARN) << "cannot create database: invalid marker"; ++state->errorCount; return state->canContinue(); } - - LOG(TRACE) << "found create database marker. databaseId: " << databaseId; - + + LOG(TRACE) << "found create database marker. databaseId: " + << databaseId; + // remove the drop marker state->droppedDatabases.erase(databaseId); TRI_vocbase_t* vocbase = state->releaseDatabase(databaseId); @@ -852,7 +954,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d VPackSlice const nameSlice = payloadSlice.get("name"); if (!nameSlice.isString()) { - LOG(WARN) << "cannot unpack database properties for database " << databaseId; + LOG(WARN) << "cannot unpack database properties for database " + << databaseId; ++state->errorCount; return state->canContinue(); } @@ -869,18 +972,20 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d // TODO: how to signal a dropDatabase failure here? state->databaseFeature->dropDatabase(nameString, false, true, false); } - + RocksDBFeature::dropDatabase(databaseId); vocbase = nullptr; - /* TODO: check what TRI_ERROR_ARANGO_DATABASE_NOT_FOUND means here + /* TODO: check what TRI_ERROR_ARANGO_DATABASE_NOT_FOUND means here WaitForDeletion(state->server, databaseId, TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); */ - int res = state->databaseFeature->createDatabase(databaseId, nameString, false, vocbase); + int res = state->databaseFeature->createDatabase(databaseId, nameString, + false, vocbase); if (res != TRI_ERROR_NO_ERROR) { - LOG(WARN) << "cannot create database " << databaseId << ": " << TRI_errno_string(res); + LOG(WARN) << "cannot create database " << databaseId << ": " + << TRI_errno_string(res); ++state->errorCount; return state->canContinue(); } @@ -889,19 +994,24 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } case TRI_DF_MARKER_VPACK_DROP_INDEX: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); - VPackSlice const payloadSlice(reinterpret_cast(marker) + MMFilesDatafileHelper::VPackOffset(type)); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); + VPackSlice const payloadSlice(reinterpret_cast(marker) + + MMFilesDatafileHelper::VPackOffset(type)); + if (!payloadSlice.isObject()) { LOG(WARN) << "cannot drop index for collection: invalid marker"; ++state->errorCount; return state->canContinue(); } - + TRI_idx_iid_t indexId = NumericValue(payloadSlice, "id"); - - LOG(TRACE) << "found drop index marker. databaseId: " << databaseId << ", collectionId: " << collectionId << ", indexId: " << indexId; + + LOG(TRACE) << "found drop index marker. databaseId: " << databaseId + << ", collectionId: " << collectionId + << ", indexId: " << indexId; if (state->isDropped(databaseId, collectionId)) { return true; @@ -915,7 +1025,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d return true; } - arangodb::LogicalCollection* col = vocbase->lookupCollection(collectionId); + arangodb::LogicalCollection* col = + vocbase->lookupCollection(collectionId); if (col == nullptr) { // if the underlying collection gone, we can go on @@ -928,21 +1039,26 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d RocksDBFeature::dropIndex(databaseId, collectionId, indexId); // additionally remove the index file - std::string const indexName("index-" + std::to_string(indexId) + ".json"); - std::string const filename(arangodb::basics::FileUtils::buildFilename(col->path(), indexName)); + std::string const indexName("index-" + std::to_string(indexId) + + ".json"); + std::string const filename( + arangodb::basics::FileUtils::buildFilename(col->path(), indexName)); TRI_UnlinkFile(filename.c_str()); break; } case TRI_DF_MARKER_VPACK_DROP_COLLECTION: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); - TRI_voc_cid_t const collectionId = MMFilesDatafileHelper::CollectionId(marker); - + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_cid_t const collectionId = + MMFilesDatafileHelper::CollectionId(marker); + // insert the drop marker state->droppedCollections.emplace(collectionId); - - LOG(TRACE) << "found drop collection marker. databaseId: " << databaseId << ", collectionId: " << collectionId; + + LOG(TRACE) << "found drop collection marker. databaseId: " << databaseId + << ", collectionId: " << collectionId; TRI_vocbase_t* vocbase = state->useDatabase(databaseId); @@ -952,7 +1068,8 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } // ignore any potential error returned by this call - arangodb::LogicalCollection* collection = state->releaseCollection(collectionId); + arangodb::LogicalCollection* collection = + state->releaseCollection(collectionId); if (collection == nullptr) { collection = vocbase->lookupCollection(collectionId); @@ -966,11 +1083,12 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d } case TRI_DF_MARKER_VPACK_DROP_DATABASE: { - TRI_voc_tick_t const databaseId = MMFilesDatafileHelper::DatabaseId(marker); + TRI_voc_tick_t const databaseId = + MMFilesDatafileHelper::DatabaseId(marker); // insert the drop marker state->droppedDatabases.emplace(databaseId); - + LOG(TRACE) << "found drop database marker. databaseId: " << databaseId; TRI_vocbase_t* vocbase = state->releaseDatabase(databaseId); @@ -983,9 +1101,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* d RocksDBFeature::dropDatabase(databaseId); break; } - - case TRI_DF_MARKER_HEADER: - case TRI_DF_MARKER_COL_HEADER: + + case TRI_DF_MARKER_HEADER: + case TRI_DF_MARKER_COL_HEADER: case TRI_DF_MARKER_FOOTER: { // new datafile or end of datafile. forget state! state->resetCollection(); @@ -1015,7 +1133,8 @@ int MMFilesWalRecoverState::replayLogfile(wal::Logfile* logfile, int number) { int const n = static_cast(logfilesToProcess.size()); - LOG(INFO) << "replaying WAL logfile '" << logfileName << "' (" << (number + 1) << " of " << n << ")"; + LOG(INFO) << "replaying WAL logfile '" << logfileName << "' (" << (number + 1) + << " of " << n << ")"; MMFilesDatafile* df = logfile->df(); @@ -1023,8 +1142,10 @@ int MMFilesWalRecoverState::replayLogfile(wal::Logfile* logfile, int number) { df->sequentialAccess(); df->willNeed(); - if (!TRI_IterateDatafile(df, &MMFilesWalRecoverState::ReplayMarker, static_cast(this))) { - LOG(WARN) << "WAL inspection failed when scanning logfile '" << logfileName << "'"; + if (!TRI_IterateDatafile(df, &MMFilesWalRecoverState::ReplayMarker, + static_cast(this))) { + LOG(WARN) << "WAL inspection failed when scanning logfile '" << logfileName + << "'"; return TRI_ERROR_ARANGO_RECOVERY; } @@ -1061,7 +1182,7 @@ int MMFilesWalRecoverState::abortOpenTransactions() { LOG(TRACE) << "writing abort markers for still open transactions"; int res = TRI_ERROR_NO_ERROR; - + VPackBuilder builder; try { @@ -1076,10 +1197,12 @@ int MMFilesWalRecoverState::abortOpenTransactions() { } TRI_voc_tick_t databaseId = (*it).second.first; - - MMFilesTransactionMarker marker(TRI_DF_MARKER_VPACK_ABORT_TRANSACTION, databaseId, transactionId); + + MMFilesTransactionMarker marker(TRI_DF_MARKER_VPACK_ABORT_TRANSACTION, + databaseId, transactionId); MMFilesWalSlotInfoCopy slotInfo = - arangodb::wal::LogfileManager::instance()->allocateAndWrite(marker, false); + arangodb::wal::LogfileManager::instance()->allocateAndWrite(marker, + false); if (slotInfo.errorCode != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(slotInfo.errorCode); @@ -1132,7 +1255,7 @@ int MMFilesWalRecoverState::fillIndexes() { arangodb::StandaloneTransactionContext::Create(collection->vocbase()), collection->cid(), TRI_TRANSACTION_WRITE); - int res = collection->fillIndexes(&trx); + int res = collection->fillIndexes(&trx, *(collection->indexList())); if (res != TRI_ERROR_NO_ERROR) { return res; diff --git a/arangod/VocBase/IndexThreadFeature.cpp b/arangod/VocBase/IndexThreadFeature.cpp deleted file mode 100644 index 8853385023..0000000000 --- a/arangod/VocBase/IndexThreadFeature.cpp +++ /dev/null @@ -1,72 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2016 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Jan Steemann -//////////////////////////////////////////////////////////////////////////////// - -#include "IndexThreadFeature.h" - -#include "ApplicationFeatures/ApplicationServer.h" -#include "Basics/ThreadPool.h" -#include "ProgramOptions/ProgramOptions.h" - -using namespace arangodb; -using namespace arangodb::application_features; -using namespace arangodb::basics; -using namespace arangodb::options; - -IndexThreadFeature::IndexThreadFeature(ApplicationServer* server) - : ApplicationFeature(server, "IndexThread"), - _indexThreads(2) { - setOptional(false); - requiresElevatedPrivileges(false); - startsAfter("DatabasePath"); - startsAfter("EngineSelector"); -} - -void IndexThreadFeature::collectOptions(std::shared_ptr options) { - options->addSection("database", "Configure the database"); - - options->addHiddenOption( - "--database.index-threads", - "threads to start for parallel background index creation", - new UInt64Parameter(&_indexThreads)); -} - -void IndexThreadFeature::validateOptions(std::shared_ptr options) { - // some arbitrary limit - if (_indexThreads > 128) { - _indexThreads = 128; - } -} - -void IndexThreadFeature::start() { - // create the index thread pool - if (_indexThreads > 0) { - _indexPool.reset(new ThreadPool(static_cast(_indexThreads), "IndexBuilder")); - } - LOG(TRACE) << "starting " << _indexThreads << " index thread(s)"; -} - -void IndexThreadFeature::unprepare() { - LOG(TRACE) << "stopping index thread(s)"; - // turn off index threads - _indexPool.reset(); -} - diff --git a/arangod/VocBase/IndexThreadFeature.h b/arangod/VocBase/IndexThreadFeature.h deleted file mode 100644 index 6aa5f76af5..0000000000 --- a/arangod/VocBase/IndexThreadFeature.h +++ /dev/null @@ -1,51 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2016 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Jan Steemann -//////////////////////////////////////////////////////////////////////////////// - -#ifndef APPLICATION_FEATURES_INDEX_THREAD_FEATURE_H -#define APPLICATION_FEATURES_INDEX_THREAD_FEATURE_H 1 - -#include "ApplicationFeatures/ApplicationFeature.h" - -namespace arangodb { -namespace basics { -class ThreadPool; -} - -class IndexThreadFeature final : public application_features::ApplicationFeature { - public: - explicit IndexThreadFeature(application_features::ApplicationServer* server); - - public: - void collectOptions(std::shared_ptr) override final; - void validateOptions(std::shared_ptr) override final; - void start() override final; - void unprepare() override final; - - basics::ThreadPool* getThreadPool() const { return _indexPool.get(); } - - private: - uint64_t _indexThreads; - std::unique_ptr _indexPool; -}; -} - -#endif diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 4368406229..2ca8cafdf1 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -19,16 +19,17 @@ /// Copyright holder is ArangoDB GmbH, Cologne, Germany /// /// @author Michael Hackstein +/// @author Daniel H. Larkin //////////////////////////////////////////////////////////////////////////////// #include "LogicalCollection.h" #include "Aql/QueryCache.h" #include "Basics/Barrier.h" +#include "Basics/LocalTaskQueue.h" #include "Basics/ReadLocker.h" #include "Basics/StaticStrings.h" #include "Basics/StringUtils.h" -#include "Basics/ThreadPool.h" #include "Basics/Timers.h" #include "Basics/VelocyPackHelper.h" #include "Basics/WriteLocker.h" @@ -46,6 +47,8 @@ #include "Indexes/SkiplistIndex.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/RevisionCacheFeature.h" +#include "Scheduler/Scheduler.h" +#include "Scheduler/SchedulerFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/MMFilesDocumentOperation.h" #include "StorageEngine/MMFilesWalMarker.h" @@ -59,7 +62,6 @@ #include "Utils/StandaloneTransactionContext.h" #include "VocBase/CollectionRevisionsCache.h" #include "VocBase/DatafileStatisticsContainer.h" -#include "VocBase/IndexThreadFeature.h" #include "VocBase/KeyGenerator.h" #include "VocBase/ManagedDocumentResult.h" #include "VocBase/PhysicalCollection.h" @@ -79,31 +81,31 @@ int TRI_AddOperationTransaction(TRI_transaction_t*, TRI_voc_rid_t, MMFilesDocumentOperation&, MMFilesWalMarker const* marker, bool&); -/// @brief helper struct for filling indexes -class IndexFiller { +/// @brief helper class for filling indexes +class IndexFillerTask : public arangodb::basics::LocalTask { public: - IndexFiller(arangodb::Transaction* trx, - arangodb::LogicalCollection* collection, arangodb::Index* idx, - std::function callback) - : _trx(trx), _collection(collection), _idx(idx), _callback(callback) {} + IndexFillerTask( + arangodb::basics::LocalTaskQueue* queue, arangodb::Transaction* trx, + arangodb::Index* idx, + std::vector> const& documents) + : LocalTask(queue), _trx(trx), _idx(idx), _documents(documents) {} - void operator()() { - int res = TRI_ERROR_INTERNAL; + void run() { TRI_ASSERT(_idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); try { - res = _collection->fillIndex(_trx, _idx); - } catch (...) { + _idx->batchInsert(_trx, _documents, _queue); + } catch (std::exception& e) { + _queue->setStatus(TRI_ERROR_INTERNAL); } - _callback(res); + _queue->join(); } private: arangodb::Transaction* _trx; - arangodb::LogicalCollection* _collection; arangodb::Index* _idx; - std::function _callback; + std::vector> const& _documents; }; namespace { @@ -1106,7 +1108,7 @@ void LogicalCollection::drop() { if (_revisionsCache != nullptr) { _revisionsCache->clear(); } - + // make sure collection has been closed this->close(); @@ -1386,8 +1388,9 @@ void LogicalCollection::open(bool ignoreErrors) { if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE( - res, std::string("cannot open document collection from path '") + - path() + "': " + TRI_errno_string(res)); + res, + std::string("cannot open document collection from path '") + path() + + "': " + TRI_errno_string(res)); } arangodb::SingleCollectionTransaction trx( @@ -1414,8 +1417,9 @@ void LogicalCollection::open(bool ignoreErrors) { if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION_MESSAGE( - res, std::string("cannot iterate data of document collection: ") + - TRI_errno_string(res)); + res, + std::string("cannot iterate data of document collection: ") + + TRI_errno_string(res)); } _isInitialIteration = false; @@ -1452,7 +1456,7 @@ void LogicalCollection::open(bool ignoreErrors) { if (!arangodb::wal::LogfileManager::instance()->isInRecovery()) { // build the index structures, and fill the indexes - fillIndexes(&trx); + fillIndexes(&trx, *(indexList())); } _revisionsCache->allowInvalidation(true); @@ -1584,9 +1588,12 @@ std::shared_ptr LogicalCollection::createIndex(Transaction* trx, } TRI_ASSERT(idx.get()->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); - int res = fillIndex(trx, idx.get(), false); + std::vector> indexListLocal; + indexListLocal.emplace_back(idx); + int res = fillIndexes(trx, indexListLocal, false); if (res != TRI_ERROR_NO_ERROR) { + usleep(1000000); THROW_ARANGO_EXCEPTION(res); } @@ -1605,7 +1612,8 @@ std::shared_ptr LogicalCollection::createIndex(Transaction* trx, VPackBuilder builder; bool const doSync = application_features::ApplicationServer::getFeature( - "Database")->forceSyncProperties(); + "Database") + ->forceSyncProperties(); toVelocyPack(builder, false); update(builder.slice(), doSync); } @@ -1638,7 +1646,9 @@ int LogicalCollection::restoreIndex(Transaction* trx, VPackSlice const& info, TRI_ASSERT(newIdx.get()->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); - int res = fillIndex(trx, newIdx.get()); + std::vector> indexListLocal; + indexListLocal.emplace_back(newIdx); + int res = fillIndexes(trx, indexListLocal, false); if (res != TRI_ERROR_NO_ERROR) { return res; @@ -1675,8 +1685,7 @@ int LogicalCollection::saveIndex(arangodb::Index* idx, bool writeMarker) { try { MMFilesCollectionMarker marker(TRI_DF_MARKER_VPACK_CREATE_INDEX, - _vocbase->id(), cid(), - builder->slice()); + _vocbase->id(), cid(), builder->slice()); MMFilesWalSlotInfoCopy slotInfo = arangodb::wal::LogfileManager::instance()->allocateAndWrite(marker, @@ -1754,7 +1763,8 @@ bool LogicalCollection::dropIndex(TRI_idx_iid_t iid, bool writeMarker) { VPackBuilder builder; bool const doSync = application_features::ApplicationServer::getFeature( - "Database")->forceSyncProperties(); + "Database") + ->forceSyncProperties(); toVelocyPack(builder, false); update(builder.slice(), doSync); } @@ -1769,8 +1779,8 @@ bool LogicalCollection::dropIndex(TRI_idx_iid_t iid, bool writeMarker) { markerBuilder.close(); MMFilesCollectionMarker marker(TRI_DF_MARKER_VPACK_DROP_INDEX, - _vocbase->id(), cid(), - markerBuilder.slice()); + _vocbase->id(), cid(), + markerBuilder.slice()); MMFilesWalSlotInfoCopy slotInfo = arangodb::wal::LogfileManager::instance()->allocateAndWrite(marker, @@ -1855,17 +1865,47 @@ int LogicalCollection::detectIndexes(arangodb::Transaction* trx) { return TRI_ERROR_NO_ERROR; } -int LogicalCollection::fillIndexes(arangodb::Transaction* trx) { +std::vector> const* +LogicalCollection::indexList() const { + return &_indexes; +} + +int LogicalCollection::fillIndexes( + arangodb::Transaction* trx, + std::vector> const& indexes, + bool skipPersistent) { // distribute the work to index threads plus this thread TRI_ASSERT(!ServerState::instance()->isCoordinator()); - size_t const n = _indexes.size(); + size_t const n = indexes.size(); - if (n == 1) { + if (n == 0 || (n == 1 && + indexes[0].get()->type() == + Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX)) { return TRI_ERROR_NO_ERROR; } + bool rolledBack = false; + auto rollbackAll = [&]() -> void { + for (size_t i = 0; i < n; i++) { + auto idx = indexes[i].get(); + if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { + continue; + } + if (idx->isPersistent()) { + continue; + } + idx->unload(); // TODO: check is this safe? truncate not necessarily + // feasible + } + }; + double start = TRI_microtime(); + TRI_ASSERT(SchedulerFeature::SCHEDULER != nullptr); + auto ioService = SchedulerFeature::SCHEDULER->ioService(); + TRI_ASSERT(ioService != nullptr); + arangodb::basics::LocalTaskQueue queue(ioService); + // only log performance infos for indexes with more than this number of // entries static size_t const NotificationSizeThreshold = 131072; @@ -1877,73 +1917,107 @@ int LogicalCollection::fillIndexes(arangodb::Transaction* trx) { << "/" << name() << " }, indexes: " << (n - 1); } - TRI_ASSERT(n > 1); + TRI_ASSERT(n > 0); - std::atomic result(TRI_ERROR_NO_ERROR); + try { + TRI_ASSERT(!ServerState::instance()->isCoordinator()); - { - arangodb::basics::Barrier barrier(n - 1); + // give the index a size hint + auto nrUsed = primaryIndex->size(); + for (size_t i = 0; i < n; i++) { + auto idx = indexes[i]; + if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { + continue; + } + idx.get()->sizeHint(trx, nrUsed); + } - auto indexPool = - application_features::ApplicationServer::getFeature( - "IndexThread") - ->getThreadPool(); + // process documents a million at a time + size_t blockSize = 1024 * 1024 * 1; - auto callback = [&barrier, &result](int res) -> void { - // update the error code - if (res != TRI_ERROR_NO_ERROR) { - int expected = TRI_ERROR_NO_ERROR; - result.compare_exchange_strong(expected, res, - std::memory_order_acquire); + if (nrUsed < blockSize) { + blockSize = nrUsed; + } + if (blockSize == 0) { + blockSize = 1; + } + + ManagedDocumentResult mmdr(trx); + + std::vector> documents; + documents.reserve(blockSize); + + auto insertInAllIndexes = [&]() -> void { + for (size_t i = 0; i < n; ++i) { + auto idx = indexes[i]; + if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { + continue; + } + fillIndex(&queue, trx, idx.get(), documents, skipPersistent); } - barrier.join(); + queue.dispatchAndWait(); + + if (queue.status() != TRI_ERROR_NO_ERROR) { + rollbackAll(); + rolledBack = true; + } }; - // now actually fill the secondary indexes - for (size_t i = 1; i < n; ++i) { - auto idx = _indexes[i]; - TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); + if (nrUsed > 0) { + arangodb::basics::BucketPosition position; + uint64_t total = 0; - // index threads must come first, otherwise this thread will block the - // loop and - // prevent distribution to threads - if (indexPool != nullptr && i != (n - 1)) { - try { - // move task into thread pool - IndexFiller indexTask(trx, this, idx.get(), callback); + while (true) { + SimpleIndexElement element = + primaryIndex->lookupSequential(trx, position, total); - static_cast(indexPool)->enqueue( - indexTask); - } catch (...) { - // set error code - int expected = TRI_ERROR_NO_ERROR; - result.compare_exchange_strong(expected, TRI_ERROR_INTERNAL, - std::memory_order_acquire); - - barrier.join(); - } - } else { - // fill index in this thread - int res; - - try { - res = fillIndex(trx, idx.get()); - } catch (...) { - res = TRI_ERROR_INTERNAL; + if (!element) { + break; } - if (res != TRI_ERROR_NO_ERROR) { - int expected = TRI_ERROR_NO_ERROR; - result.compare_exchange_strong(expected, res, - std::memory_order_acquire); - } + TRI_voc_rid_t revisionId = element.revisionId(); - barrier.join(); + if (readRevision(trx, mmdr, revisionId)) { + uint8_t const* vpack = mmdr.vpack(); + TRI_ASSERT(vpack != nullptr); + documents.emplace_back(std::make_pair(revisionId, VPackSlice(vpack))); + + if (documents.size() == blockSize) { + // now actually fill the secondary indexes + insertInAllIndexes(); + if (queue.status() != TRI_ERROR_NO_ERROR) { + break; + }; + documents.clear(); + } + } } } - // barrier waits here until all threads have joined + // process the remainder of the documents + if (queue.status() == TRI_ERROR_NO_ERROR && !documents.empty()) { + insertInAllIndexes(); + } + + // TODO: fix perf logging? + } catch (arangodb::basics::Exception const& ex) { + queue.setStatus(ex.code()); + } catch (std::bad_alloc const&) { + queue.setStatus(TRI_ERROR_OUT_OF_MEMORY); + } catch (std::exception const& ex) { + LOG(WARN) << "caught exception while filling indexes: " << ex.what(); + queue.setStatus(TRI_ERROR_INTERNAL); + } catch (...) { + LOG(WARN) << "caught unknown exception while filling indexes"; + queue.setStatus(TRI_ERROR_INTERNAL); + } + + if (queue.status() != TRI_ERROR_NO_ERROR && !rolledBack) { + try { + rollbackAll(); + } catch (...) { + } } LOG_TOPIC(TRACE, Logger::PERFORMANCE) @@ -1951,7 +2025,7 @@ int LogicalCollection::fillIndexes(arangodb::Transaction* trx) { << " s, fill-indexes-document-collection { collection: " << _vocbase->name() << "/" << name() << " }, indexes: " << (n - 1); - return result.load(); + return queue.status(); } void LogicalCollection::addIndex(std::shared_ptr idx) { @@ -1959,7 +2033,7 @@ void LogicalCollection::addIndex(std::shared_ptr idx) { TRI_ASSERT(idx->type() != arangodb::Index::TRI_IDX_TYPE_PRIMARY_INDEX || _indexes.empty()); - auto const id = idx->id(); + auto const id = idx->id(); for (auto const& it : _indexes) { if (it->id() == id) { // already have this particular index. do not add it again @@ -1982,8 +2056,7 @@ void LogicalCollection::addIndex(std::shared_ptr idx) { void LogicalCollection::addIndexCoordinator( std::shared_ptr idx, bool distribute) { - - auto const id = idx->id(); + auto const id = idx->id(); for (auto const& it : _indexes) { if (it->id() == id) { // already have this particular index. do not add it again @@ -2126,9 +2199,9 @@ int LogicalCollection::insert(Transaction* trx, VPackSlice const slice, } // create marker - MMFilesCrudMarker insertMarker( - TRI_DF_MARKER_VPACK_DOCUMENT, - TRI_MarkerIdTransaction(trx->getInternals()), newSlice); + MMFilesCrudMarker insertMarker(TRI_DF_MARKER_VPACK_DOCUMENT, + TRI_MarkerIdTransaction(trx->getInternals()), + newSlice); MMFilesWalMarker const* marker; if (options.recoveryMarker == nullptr) { @@ -2143,8 +2216,7 @@ int LogicalCollection::insert(Transaction* trx, VPackSlice const slice, return TRI_ERROR_DEBUG; } - MMFilesDocumentOperation operation(this, - TRI_VOC_DOCUMENT_OPERATION_INSERT); + MMFilesDocumentOperation operation(this, TRI_VOC_DOCUMENT_OPERATION_INSERT); TRI_IF_FAILURE("InsertDocumentNoHeader") { // test what happens if no header can be acquired @@ -2177,13 +2249,13 @@ int LogicalCollection::insert(Transaction* trx, VPackSlice const slice, bool const useDeadlockDetector = (lock && !trx->isSingleOperationTransaction()); try { - arangodb::CollectionWriteLocker collectionLocker(this, useDeadlockDetector, - lock); + arangodb::CollectionWriteLocker collectionLocker( + this, useDeadlockDetector, lock); try { // insert into indexes res = insertDocument(trx, revisionId, doc, operation, marker, - options.waitForSync); + options.waitForSync); } catch (basics::Exception const& ex) { res = ex.code(); } catch (std::bad_alloc const&) { @@ -2194,7 +2266,7 @@ int LogicalCollection::insert(Transaction* trx, VPackSlice const slice, } catch (...) { // the collectionLocker may have thrown in its constructor... // if it did, then we need to manually remove the revision id - // from the list of revisions + // from the list of revisions try { removeRevision(revisionId, false); } catch (...) { @@ -2212,7 +2284,7 @@ int LogicalCollection::insert(Transaction* trx, VPackSlice const slice, // store the tick that was used for writing the document resultMarkerTick = operation.tick(); - } + } return res; } @@ -2323,9 +2395,9 @@ int LogicalCollection::update(Transaction* trx, VPackSlice const newSlice, } // create marker - MMFilesCrudMarker updateMarker( - TRI_DF_MARKER_VPACK_DOCUMENT, - TRI_MarkerIdTransaction(trx->getInternals()), builder->slice()); + MMFilesCrudMarker updateMarker(TRI_DF_MARKER_VPACK_DOCUMENT, + TRI_MarkerIdTransaction(trx->getInternals()), + builder->slice()); MMFilesWalMarker const* marker; if (options.recoveryMarker == nullptr) { @@ -2336,8 +2408,7 @@ int LogicalCollection::update(Transaction* trx, VPackSlice const newSlice, VPackSlice const newDoc(marker->vpack()); - MMFilesDocumentOperation operation(this, - TRI_VOC_DOCUMENT_OPERATION_UPDATE); + MMFilesDocumentOperation operation(this, TRI_VOC_DOCUMENT_OPERATION_UPDATE); try { insertRevision(revisionId, marker->vpack(), 0, true); @@ -2486,9 +2557,9 @@ int LogicalCollection::replace(Transaction* trx, VPackSlice const newSlice, } // create marker - MMFilesCrudMarker replaceMarker( - TRI_DF_MARKER_VPACK_DOCUMENT, - TRI_MarkerIdTransaction(trx->getInternals()), builder->slice()); + MMFilesCrudMarker replaceMarker(TRI_DF_MARKER_VPACK_DOCUMENT, + TRI_MarkerIdTransaction(trx->getInternals()), + builder->slice()); MMFilesWalMarker const* marker; if (options.recoveryMarker == nullptr) { @@ -2499,8 +2570,7 @@ int LogicalCollection::replace(Transaction* trx, VPackSlice const newSlice, VPackSlice const newDoc(marker->vpack()); - MMFilesDocumentOperation operation( - this, TRI_VOC_DOCUMENT_OPERATION_REPLACE); + MMFilesDocumentOperation operation(this, TRI_VOC_DOCUMENT_OPERATION_REPLACE); try { insertRevision(revisionId, marker->vpack(), 0, true); @@ -2585,9 +2655,9 @@ int LogicalCollection::remove(arangodb::Transaction* trx, } // create marker - MMFilesCrudMarker removeMarker( - TRI_DF_MARKER_VPACK_REMOVE, TRI_MarkerIdTransaction(trx->getInternals()), - builder->slice()); + MMFilesCrudMarker removeMarker(TRI_DF_MARKER_VPACK_REMOVE, + TRI_MarkerIdTransaction(trx->getInternals()), + builder->slice()); MMFilesWalMarker const* marker; if (options.recoveryMarker == nullptr) { @@ -2609,8 +2679,7 @@ int LogicalCollection::remove(arangodb::Transaction* trx, } TRI_ASSERT(!key.isNone()); - MMFilesDocumentOperation operation(this, - TRI_VOC_DOCUMENT_OPERATION_REMOVE); + MMFilesDocumentOperation operation(this, TRI_VOC_DOCUMENT_OPERATION_REMOVE); bool const useDeadlockDetector = (lock && !trx->isSingleOperationTransaction()); @@ -2720,9 +2789,9 @@ int LogicalCollection::remove(arangodb::Transaction* trx, } // create marker - MMFilesCrudMarker removeMarker( - TRI_DF_MARKER_VPACK_REMOVE, TRI_MarkerIdTransaction(trx->getInternals()), - builder->slice()); + MMFilesCrudMarker removeMarker(TRI_DF_MARKER_VPACK_REMOVE, + TRI_MarkerIdTransaction(trx->getInternals()), + builder->slice()); MMFilesWalMarker const* marker = &removeMarker; @@ -2734,8 +2803,7 @@ int LogicalCollection::remove(arangodb::Transaction* trx, VPackSlice key = Transaction::extractKeyFromDocument(oldDoc); TRI_ASSERT(!key.isNone()); - MMFilesDocumentOperation operation(this, - TRI_VOC_DOCUMENT_OPERATION_REMOVE); + MMFilesDocumentOperation operation(this, TRI_VOC_DOCUMENT_OPERATION_REMOVE); bool const useDeadlockDetector = (lock && !trx->isSingleOperationTransaction()); @@ -2873,206 +2941,33 @@ void LogicalCollection::sizeHint(Transaction* trx, int64_t hint) { _revisionsCache->sizeHint(hint); } -/// @brief initializes an index with all existing documents -int LogicalCollection::fillIndex(arangodb::Transaction* trx, - arangodb::Index* idx, bool skipPersistent) { +/// @brief initializes an index with a set of existing documents +void LogicalCollection::fillIndex( + arangodb::basics::LocalTaskQueue* queue, arangodb::Transaction* trx, + arangodb::Index* idx, + std::vector> const& documents, + bool skipPersistent) { TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); TRI_ASSERT(!ServerState::instance()->isCoordinator()); if (!useSecondaryIndexes()) { - return TRI_ERROR_NO_ERROR; + return; // TRI_ERROR_NO_ERROR; } if (idx->isPersistent() && skipPersistent) { - return TRI_ERROR_NO_ERROR; + return; // TRI_ERROR_NO_ERROR; } try { - size_t nrUsed = primaryIndex()->size(); - auto indexPool = - application_features::ApplicationServer::getFeature( - "IndexThread") - ->getThreadPool(); - - int res; - - if (indexPool != nullptr && idx->hasBatchInsert() && nrUsed > 256 * 1024 && - _indexBuckets > 1) { - // use batch insert if there is an index pool, - // the collection has more than one index bucket - // and it contains a significant amount of documents - res = fillIndexBatch(trx, idx); - } else { - res = fillIndexSequential(trx, idx); - } - - return res; - } catch (arangodb::basics::Exception const& ex) { - return ex.code(); - } catch (std::bad_alloc const&) { - return TRI_ERROR_OUT_OF_MEMORY; - } catch (std::exception const& ex) { - LOG(WARN) << "caught exception while filling indexes: " << ex.what(); - return TRI_ERROR_INTERNAL; + // move task into thread pool + std::shared_ptr worker; + worker.reset(new IndexFillerTask(queue, trx, idx, documents)); + queue->enqueue(worker); } catch (...) { - LOG(WARN) << "caught unknown exception while filling indexes"; - return TRI_ERROR_INTERNAL; + // set error code + queue->setStatus(TRI_ERROR_INTERNAL); } } -/// @brief fill an index in batches -int LogicalCollection::fillIndexBatch(arangodb::Transaction* trx, - arangodb::Index* idx) { - TRI_ASSERT(!ServerState::instance()->isCoordinator()); - auto indexPool = - application_features::ApplicationServer::getFeature( - "IndexThread") - ->getThreadPool(); - - double start = TRI_microtime(); - - LOG_TOPIC(TRACE, Logger::PERFORMANCE) - << "fill-index-batch { collection: " << _vocbase->name() << "/" << name() - << " }, " << idx->context() << ", threads: " << indexPool->numThreads() - << ", buckets: " << indexBuckets(); - - // give the index a size hint - auto primaryIndex = this->primaryIndex(); - - auto nrUsed = primaryIndex->size(); - - idx->sizeHint(trx, nrUsed); - - // process documents a million at a time - size_t blockSize = 1024 * 1024 * 1; - - if (nrUsed < blockSize) { - blockSize = nrUsed; - } - if (blockSize == 0) { - blockSize = 1; - } - - int res = TRI_ERROR_NO_ERROR; - - ManagedDocumentResult mmdr(trx); - - std::vector> documents; - documents.reserve(blockSize); - - if (nrUsed > 0) { - arangodb::basics::BucketPosition position; - uint64_t total = 0; - - while (true) { - SimpleIndexElement element = - primaryIndex->lookupSequential(trx, position, total); - - if (!element) { - break; - } - - TRI_voc_rid_t revisionId = element.revisionId(); - - if (readRevision(trx, mmdr, revisionId)) { - uint8_t const* vpack = mmdr.vpack(); - TRI_ASSERT(vpack != nullptr); - documents.emplace_back(std::make_pair(revisionId, VPackSlice(vpack))); - - if (documents.size() == blockSize) { - res = idx->batchInsert(trx, documents, indexPool->numThreads()); - documents.clear(); - - // some error occurred - if (res != TRI_ERROR_NO_ERROR) { - break; - } - } - } - } - } - - // process the remainder of the documents - if (res == TRI_ERROR_NO_ERROR && !documents.empty()) { - res = idx->batchInsert(trx, documents, indexPool->numThreads()); - } - - LOG_TOPIC(TRACE, Logger::PERFORMANCE) - << "[timer] " << Logger::FIXED(TRI_microtime() - start) - << " s, fill-index-batch { collection: " << _vocbase->name() << "/" - << name() << " }, " << idx->context() - << ", threads: " << indexPool->numThreads() - << ", buckets: " << indexBuckets(); - - return res; -} - -/// @brief fill an index sequentially -int LogicalCollection::fillIndexSequential(arangodb::Transaction* trx, - arangodb::Index* idx) { - TRI_ASSERT(!ServerState::instance()->isCoordinator()); - double start = TRI_microtime(); - - LOG_TOPIC(TRACE, Logger::PERFORMANCE) - << "fill-index-sequential { collection: " << _vocbase->name() << "/" - << name() << " }, " << idx->context() << ", buckets: " << indexBuckets(); - - // give the index a size hint - auto primaryIndex = this->primaryIndex(); - size_t nrUsed = primaryIndex->size(); - - TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); - idx->sizeHint(trx, nrUsed); - - if (nrUsed > 0) { -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - static int const LoopSize = 10000; - int counter = 0; - int loops = 0; -#endif - - arangodb::basics::BucketPosition position; - uint64_t total = 0; - ManagedDocumentResult result(trx); - - while (true) { - SimpleIndexElement element = - primaryIndex->lookupSequential(trx, position, total); - - if (!element) { - break; - } - - TRI_voc_rid_t revisionId = element.revisionId(); - if (readRevision(trx, result, revisionId)) { - uint8_t const* vpack = result.vpack(); - TRI_ASSERT(vpack != nullptr); - int res = idx->insert(trx, revisionId, VPackSlice(vpack), false); - - if (res != TRI_ERROR_NO_ERROR) { - return res; - } - } else { - return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND; // oops - } -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - if (++counter == LoopSize) { - counter = 0; - ++loops; - LOG(TRACE) << "indexed " << (LoopSize * loops) - << " documents of collection " << cid(); - } -#endif - } - } - - LOG_TOPIC(TRACE, Logger::PERFORMANCE) - << "[timer] " << Logger::FIXED(TRI_microtime() - start) - << " s, fill-index-sequential { collection: " << _vocbase->name() << "/" - << name() << " }, " << idx->context() << ", buckets: " << indexBuckets(); - - return TRI_ERROR_NO_ERROR; -} - /// @brief read unlocks a collection int LogicalCollection::endRead(bool useDeadlockDetector) { if (arangodb::Transaction::_makeNolockHeaders != nullptr) { @@ -3414,10 +3309,12 @@ int LogicalCollection::updateDocument( /// @brief insert a document, low level worker /// the caller must make sure the write lock on the collection is held -int LogicalCollection::insertDocument( - arangodb::Transaction* trx, TRI_voc_rid_t revisionId, VPackSlice const& doc, - MMFilesDocumentOperation& operation, - MMFilesWalMarker const* marker, bool& waitForSync) { +int LogicalCollection::insertDocument(arangodb::Transaction* trx, + TRI_voc_rid_t revisionId, + VPackSlice const& doc, + MMFilesDocumentOperation& operation, + MMFilesWalMarker const* marker, + bool& waitForSync) { // insert into primary index first int res = insertPrimaryIndex(trx, revisionId, doc); @@ -3588,7 +3485,8 @@ int LogicalCollection::newObjectForInsert( *p++ = 0xf3; // custom type for _id if (ServerState::isDBServer(trx->serverRole()) && !_isSystem) { // db server in cluster, note: the local collections _statistics, - // _statisticsRaw and _statistics15 (which are the only system collections) + // _statisticsRaw and _statistics15 (which are the only system + // collections) // must not be treated as shards but as local collections MMFilesDatafileHelper::StoreNumber(p, _planId, sizeof(uint64_t)); } else { @@ -3634,7 +3532,8 @@ int LogicalCollection::newObjectForInsert( return TRI_ERROR_NO_ERROR; } -/// @brief new object for replace, oldValue must have _key and _id correctly set +/// @brief new object for replace, oldValue must have _key and _id correctly +/// set void LogicalCollection::newObjectForReplace( Transaction* trx, VPackSlice const& oldValue, VPackSlice const& newValue, VPackSlice const& fromSlice, VPackSlice const& toSlice, @@ -3840,7 +3739,8 @@ bool LogicalCollection::readRevisionConditional(Transaction* trx, void LogicalCollection::insertRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, TRI_voc_fid_t fid, bool isInWal) { - // note: there is no need to insert into the cache here as the data points to + // note: there is no need to insert into the cache here as the data points + // to // temporary storage getPhysical()->insertRevision(revisionId, dataptr, fid, isInWal, true); } @@ -3848,7 +3748,8 @@ void LogicalCollection::insertRevision(TRI_voc_rid_t revisionId, void LogicalCollection::updateRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, TRI_voc_fid_t fid, bool isInWal) { - // note: there is no need to modify the cache entry here as insertRevision has + // note: there is no need to modify the cache entry here as insertRevision + // has // not inserted the document into the cache getPhysical()->updateRevision(revisionId, dataptr, fid, isInWal); } diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index 7e90f998fd..fb211783e0 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -34,6 +34,10 @@ struct TRI_df_marker_t; namespace arangodb { +namespace basics { +class LocalTaskQueue; +} + namespace velocypack { class Slice; } @@ -63,32 +67,31 @@ class LogicalCollection { friend struct ::TRI_vocbase_t; public: - LogicalCollection(TRI_vocbase_t*, arangodb::velocypack::Slice const&, bool isPhysical); + LogicalCollection(TRI_vocbase_t*, arangodb::velocypack::Slice const&, + bool isPhysical); virtual ~LogicalCollection(); - - enum CollectionVersions { - VERSION_30 = 5, - VERSION_31 = 6 - }; + + enum CollectionVersions { VERSION_30 = 5, VERSION_31 = 6 }; protected: // If you need a copy outside the class, use clone below. explicit LogicalCollection(LogicalCollection const&); private: LogicalCollection& operator=(LogicalCollection const&) = delete; + public: LogicalCollection() = delete; - + virtual std::unique_ptr clone() { auto p = new LogicalCollection(*this); return std::unique_ptr(p); } /// @brief hard-coded minimum version number for collections - static constexpr uint32_t minimumVersion() { return VERSION_30; } + static constexpr uint32_t minimumVersion() { return VERSION_30; } /// @brief current version for collections - static constexpr uint32_t currentVersion() { return VERSION_31; } + static constexpr uint32_t currentVersion() { return VERSION_31; } /// @brief determine whether a collection name is a system collection name static inline bool IsSystemName(std::string const& name) { @@ -102,13 +105,15 @@ class LogicalCollection { static bool IsAllowedName(bool isSystem, std::string const& name); void ensureRevisionsCache(); - + void isInitialIteration(bool value) { _isInitialIteration = value; } - // TODO: MOVE TO PHYSICAL? + // TODO: MOVE TO PHYSICAL? bool isFullyCollected(); - int64_t uncollectedLogfileEntries() const { return _uncollectedLogfileEntries.load(); } - + int64_t uncollectedLogfileEntries() const { + return _uncollectedLogfileEntries.load(); + } + void increaseUncollectedLogfileEntries(int64_t value) { _uncollectedLogfileEntries += value; } @@ -127,20 +132,15 @@ class LogicalCollection { void lastCompactionStamp(double value) { _lastCompactionStamp = value; } void setRevisionError() { _revisionError = true; } - // SECTION: Meta Information - uint32_t version() const { - return _version; - } - + uint32_t version() const { return _version; } + void setVersion(CollectionVersions version) { _version = version; } uint32_t internalVersion() const; - inline TRI_voc_cid_t cid() const { - return _cid; - } + inline TRI_voc_cid_t cid() const { return _cid; } std::string cid_as_string() const; @@ -151,7 +151,7 @@ class LogicalCollection { inline bool useSecondaryIndexes() const { return _useSecondaryIndexes; } void useSecondaryIndexes(bool value) { _useSecondaryIndexes = value; } - + std::string name() const; std::string dbName() const; std::string const& path() const; @@ -159,18 +159,18 @@ class LogicalCollection { void distributeShardsLike(std::string const&); std::vector const& avoidServers() const; - void avoidServers(std::vector const&) ; + void avoidServers(std::vector const&); // For normal collections the realNames is just a vector of length 1 // with its name. For smart edge collections (enterprise only) this is // different. virtual std::vector realNames() const { - std::vector res {name()}; + std::vector res{name()}; return res; } // Same here, this is for reading in AQL: virtual std::vector realNamesForRead() const { - std::vector res {name()}; + std::vector res{name()}; return res; } @@ -182,7 +182,8 @@ class LogicalCollection { /// @brief try to fetch the collection status under a lock /// the boolean value will be set to true if the lock could be acquired - /// if the boolean is false, the return value is always TRI_VOC_COL_STATUS_CORRUPTED + /// if the boolean is false, the return value is always + /// TRI_VOC_COL_STATUS_CORRUPTED TRI_vocbase_col_status_e tryFetchStatus(bool&); std::string statusString(); @@ -203,16 +204,14 @@ class LogicalCollection { bool isVolatile() const; bool waitForSync() const; bool isSmart() const; - + void waitForSync(bool value) { _waitForSync = value; } std::unique_ptr const& followers() const; - + void setDeleted(bool); - Ditches* ditches() const { - return getPhysical()->ditches(); - } + Ditches* ditches() const { return getPhysical()->ditches(); } void setRevision(TRI_voc_rid_t, bool); @@ -224,7 +223,7 @@ class LogicalCollection { inline arangodb::KeyGenerator* keyGenerator() const { return _keyGenerator.get(); } - + PhysicalCollection* getPhysical() const { return _physical.get(); } // SECTION: Indexes @@ -247,7 +246,6 @@ class LogicalCollection { int replicationFactor() const; bool isSatellite() const; - // SECTION: Sharding int numberOfShards() const; bool allowUserKeys() const; @@ -255,7 +253,7 @@ class LogicalCollection { std::vector const& shardKeys() const; std::shared_ptr shardIds() const; void setShardMap(std::shared_ptr& map); - + /// @brief a method to skip certain documents in AQL write operations, /// this is only used in the enterprise edition for smart graphs virtual bool skipForAqlWrite(arangodb::velocypack::Slice document, @@ -278,16 +276,14 @@ class LogicalCollection { /// The builder has to be an opened Type::Object void toVelocyPack(arangodb::velocypack::Builder&, bool, TRI_voc_tick_t); - inline TRI_vocbase_t* vocbase() const { - return _vocbase; - } + inline TRI_vocbase_t* vocbase() const { return _vocbase; } // Update this collection. virtual int update(arangodb::velocypack::Slice const&, bool); /// @brief return the figures for a collection virtual std::shared_ptr figures(); - + /// @brief opens an existing collection void open(bool ignoreErrors); @@ -297,20 +293,21 @@ class LogicalCollection { /// datafile management /// @brief rotate the active journal - will do nothing if there is no journal - int rotateActiveJournal() { - return getPhysical()->rotateActiveJournal(); - } - + int rotateActiveJournal() { return getPhysical()->rotateActiveJournal(); } + /// @brief increase dead stats for a datafile, if it exists - void updateStats(TRI_voc_fid_t fid, DatafileStatisticsContainer const& values) { + void updateStats(TRI_voc_fid_t fid, + DatafileStatisticsContainer const& values) { return getPhysical()->updateStats(fid, values); } - - bool applyForTickRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax, - std::function const& callback) { + + bool applyForTickRange( + TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax, + std::function const& callback) { return getPhysical()->applyForTickRange(dataMin, dataMax, callback); } - + /// @brief disallow starting the compaction of the collection void preventCompaction() { getPhysical()->preventCompaction(); } bool tryPreventCompaction() { return getPhysical()->tryPreventCompaction(); } @@ -345,8 +342,13 @@ class LogicalCollection { int restoreIndex(arangodb::Transaction*, arangodb::velocypack::Slice const&, std::shared_ptr&); + /// @brief Exposes a pointer to index list + std::vector> const* indexList() const; + /// @brief Fill indexes used in recovery - int fillIndexes(arangodb::Transaction*); + int fillIndexes(arangodb::Transaction*, + std::vector> const&, + bool skipPersistent = true); /// @brief Saves Index to file int saveIndex(arangodb::Index* idx, bool writeMarker); @@ -356,48 +358,61 @@ class LogicalCollection { int cleanupIndexes(); // SECTION: Index access (local only) - - int read(arangodb::Transaction*, std::string const&, ManagedDocumentResult& result, bool); - int read(arangodb::Transaction*, arangodb::StringRef const&, ManagedDocumentResult& result, bool); + + int read(arangodb::Transaction*, std::string const&, + ManagedDocumentResult& result, bool); + int read(arangodb::Transaction*, arangodb::StringRef const&, + ManagedDocumentResult& result, bool); /// @brief processes a truncate operation (note: currently this only clears /// the read-cache int truncate(Transaction* trx); int insert(arangodb::Transaction*, arangodb::velocypack::Slice const, - ManagedDocumentResult& result, arangodb::OperationOptions&, TRI_voc_tick_t&, bool); + ManagedDocumentResult& result, arangodb::OperationOptions&, + TRI_voc_tick_t&, bool); int update(arangodb::Transaction*, arangodb::velocypack::Slice const, - ManagedDocumentResult& result, arangodb::OperationOptions&, TRI_voc_tick_t&, bool, - TRI_voc_rid_t& prevRev, ManagedDocumentResult& previous); + ManagedDocumentResult& result, arangodb::OperationOptions&, + TRI_voc_tick_t&, bool, TRI_voc_rid_t& prevRev, + ManagedDocumentResult& previous); int replace(arangodb::Transaction*, arangodb::velocypack::Slice const, - ManagedDocumentResult& result, arangodb::OperationOptions&, TRI_voc_tick_t&, bool, - TRI_voc_rid_t& prevRev, ManagedDocumentResult& previous); + ManagedDocumentResult& result, arangodb::OperationOptions&, + TRI_voc_tick_t&, bool, TRI_voc_rid_t& prevRev, + ManagedDocumentResult& previous); int remove(arangodb::Transaction*, arangodb::velocypack::Slice const, - arangodb::OperationOptions&, TRI_voc_tick_t&, bool, + arangodb::OperationOptions&, TRI_voc_tick_t&, bool, TRI_voc_rid_t& prevRev, ManagedDocumentResult& previous); - /// @brief removes a document or edge, fast path function for database documents - int remove(arangodb::Transaction*, TRI_voc_rid_t oldRevisionId, arangodb::velocypack::Slice const, - arangodb::OperationOptions&, TRI_voc_tick_t&, bool); + /// @brief removes a document or edge, fast path function for database + /// documents + int remove(arangodb::Transaction*, TRI_voc_rid_t oldRevisionId, + arangodb::velocypack::Slice const, arangodb::OperationOptions&, + TRI_voc_tick_t&, bool); - int rollbackOperation(arangodb::Transaction*, TRI_voc_document_operation_e, - TRI_voc_rid_t oldRevisionId, arangodb::velocypack::Slice const& oldDoc, - TRI_voc_rid_t newRevisionId, arangodb::velocypack::Slice const& newDoc); - - // TODO Make Private and IndexFiller as friend - /// @brief initializes an index with all existing documents - int fillIndex(arangodb::Transaction*, arangodb::Index*, - bool skipPersistent = true); + int rollbackOperation(arangodb::Transaction*, TRI_voc_document_operation_e, + TRI_voc_rid_t oldRevisionId, + arangodb::velocypack::Slice const& oldDoc, + TRI_voc_rid_t newRevisionId, + arangodb::velocypack::Slice const& newDoc); int beginReadTimed(bool useDeadlockDetector, double timeout = 0.0); int beginWriteTimed(bool useDeadlockDetector, double timeout = 0.0); int endRead(bool useDeadlockDetector); int endWrite(bool useDeadlockDetector); - - bool readRevision(arangodb::Transaction*, ManagedDocumentResult& result, TRI_voc_rid_t revisionId); - bool readRevisionConditional(arangodb::Transaction*, ManagedDocumentResult& result, TRI_voc_rid_t revisionId, TRI_voc_tick_t maxTick, bool excludeWal); - void insertRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, TRI_voc_fid_t fid, bool isInWal); - void updateRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, TRI_voc_fid_t fid, bool isInWal); - bool updateRevisionConditional(TRI_voc_rid_t revisionId, TRI_df_marker_t const* oldPosition, TRI_df_marker_t const* newPosition, TRI_voc_fid_t newFid, bool isInWal); + bool readRevision(arangodb::Transaction*, ManagedDocumentResult& result, + TRI_voc_rid_t revisionId); + bool readRevisionConditional(arangodb::Transaction*, + ManagedDocumentResult& result, + TRI_voc_rid_t revisionId, TRI_voc_tick_t maxTick, + bool excludeWal); + + void insertRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, + TRI_voc_fid_t fid, bool isInWal); + void updateRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, + TRI_voc_fid_t fid, bool isInWal); + bool updateRevisionConditional(TRI_voc_rid_t revisionId, + TRI_df_marker_t const* oldPosition, + TRI_df_marker_t const* newPosition, + TRI_voc_fid_t newFid, bool isInWal); void removeRevision(TRI_voc_rid_t revisionId, bool updateStats); void removeRevisionCacheEntry(TRI_voc_rid_t revisionId); @@ -416,84 +431,87 @@ class LogicalCollection { // SECTION: Indexes (local only) + // TODO Make Private and IndexFiller as friend + /// @brief initializes an index with all existing documents + void fillIndex(arangodb::basics::LocalTaskQueue*, arangodb::Transaction*, + arangodb::Index*, + std::vector> const&, + bool); + // @brief create index with the given definition. - bool openIndex(arangodb::velocypack::Slice const&, arangodb::Transaction*); - /// @brief fill an index in batches - int fillIndexBatch(arangodb::Transaction* trx, arangodb::Index* idx); - - /// @brief fill an index sequentially - int fillIndexSequential(arangodb::Transaction* trx, arangodb::Index* idx); // SECTION: Index access (local only) int lookupDocument(arangodb::Transaction*, VPackSlice const, ManagedDocumentResult& result); - int checkRevision(arangodb::Transaction*, TRI_voc_rid_t expected, TRI_voc_rid_t found); + int checkRevision(arangodb::Transaction*, TRI_voc_rid_t expected, + TRI_voc_rid_t found); - int updateDocument(arangodb::Transaction*, - TRI_voc_rid_t oldRevisionId, arangodb::velocypack::Slice const& oldDoc, - TRI_voc_rid_t newRevisionId, arangodb::velocypack::Slice const& newDoc, + int updateDocument(arangodb::Transaction*, TRI_voc_rid_t oldRevisionId, + arangodb::velocypack::Slice const& oldDoc, + TRI_voc_rid_t newRevisionId, + arangodb::velocypack::Slice const& newDoc, MMFilesDocumentOperation&, MMFilesWalMarker const*, bool& waitForSync); - int insertDocument(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&, + int insertDocument(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&, MMFilesDocumentOperation&, MMFilesWalMarker const*, bool& waitForSync); - int insertPrimaryIndex(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&); - - int deletePrimaryIndex(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&); + int insertPrimaryIndex(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&); - int insertSecondaryIndexes(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&, + int deletePrimaryIndex(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&); + + int insertSecondaryIndexes(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&, bool isRollback); - int deleteSecondaryIndexes(arangodb::Transaction*, TRI_voc_rid_t revisionId, arangodb::velocypack::Slice const&, + int deleteSecondaryIndexes(arangodb::Transaction*, TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const&, bool isRollback); // SECTION: Document pre commit preperation (only local) /// @brief new object for insert, value must have _key set correctly. - int newObjectForInsert( - arangodb::Transaction* trx, - arangodb::velocypack::Slice const& value, - arangodb::velocypack::Slice const& fromSlice, - arangodb::velocypack::Slice const& toSlice, - bool isEdgeCollection, - arangodb::velocypack::Builder& builder, - bool isRestore); + int newObjectForInsert(arangodb::Transaction* trx, + arangodb::velocypack::Slice const& value, + arangodb::velocypack::Slice const& fromSlice, + arangodb::velocypack::Slice const& toSlice, + bool isEdgeCollection, + arangodb::velocypack::Builder& builder, + bool isRestore); /// @brief new object for replace - void newObjectForReplace( - arangodb::Transaction* trx, - arangodb::velocypack::Slice const& oldValue, - arangodb::velocypack::Slice const& newValue, - arangodb::velocypack::Slice const& fromSlice, - arangodb::velocypack::Slice const& toSlice, - bool isEdgeCollection, - std::string const& rev, - arangodb::velocypack::Builder& builder); + void newObjectForReplace(arangodb::Transaction* trx, + arangodb::velocypack::Slice const& oldValue, + arangodb::velocypack::Slice const& newValue, + arangodb::velocypack::Slice const& fromSlice, + arangodb::velocypack::Slice const& toSlice, + bool isEdgeCollection, std::string const& rev, + arangodb::velocypack::Builder& builder); /// @brief merge two objects for update - void mergeObjectsForUpdate( - arangodb::Transaction* trx, - arangodb::velocypack::Slice const& oldValue, - arangodb::velocypack::Slice const& newValue, - bool isEdgeCollection, - std::string const& rev, - bool mergeObjects, bool keepNull, - arangodb::velocypack::Builder& b); + void mergeObjectsForUpdate(arangodb::Transaction* trx, + arangodb::velocypack::Slice const& oldValue, + arangodb::velocypack::Slice const& newValue, + bool isEdgeCollection, std::string const& rev, + bool mergeObjects, bool keepNull, + arangodb::velocypack::Builder& b); /// @brief new object for remove, must have _key set - void newObjectForRemove( - arangodb::Transaction* trx, - arangodb::velocypack::Slice const& oldValue, - std::string const& rev, - arangodb::velocypack::Builder& builder); + void newObjectForRemove(arangodb::Transaction* trx, + arangodb::velocypack::Slice const& oldValue, + std::string const& rev, + arangodb::velocypack::Builder& builder); void increaseInternalVersion(); protected: - void toVelocyPackInObject(arangodb::velocypack::Builder& result, bool translateCids) const; + void toVelocyPackInObject(arangodb::velocypack::Builder& result, + bool translateCids) const; // SECTION: Meta Information // @@ -542,7 +560,7 @@ class LogicalCollection { // TODO Really VPack? std::shared_ptr const> _keyOptions; // options for key creation - + uint32_t _version; // SECTION: Indexes @@ -578,21 +596,21 @@ class LogicalCollection { TRI_voc_tick_t _maxTick; std::unique_ptr _keyGenerator; - + mutable arangodb::basics::ReadWriteLock _lock; // lock protecting the status and name - + mutable arangodb::basics::ReadWriteLock _idxLock; // lock protecting the indexes mutable arangodb::basics::ReadWriteLock _infoLock; // lock protecting the info - + arangodb::Mutex _compactionStatusLock; size_t _nextCompactionStartIndex; char const* _lastCompactionStatus; double _lastCompactionStamp; - + std::atomic _uncollectedLogfileEntries; /// @brief: flag that is set to true when the documents are diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 2d87d7c4e1..28a6741e15 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -3881,17 +3881,23 @@ testFuncs.upgrade = function (options) { result.upgrade.first = executeAndWait(ARANGOD_BIN, argv, options, 'upgrade'); - if (result.upgrade.first !== 0 && !options.force) { + if (result.upgrade.first.status !== true) { print('not removing ' + tmpDataDir); - return result; + return result.upgrade; } ++result.upgrade.total; result.upgrade.second = executeAndWait(ARANGOD_BIN, argv, options, 'upgrade'); + + if (result.upgrade.second.status !== true) { + print('not removing ' + tmpDataDir); + return result.upgrade; + } cleanupDirectories.push(tmpDataDir); - + + result.upgrade.status = true; return result; }; diff --git a/lib/Basics/AssocMulti.h b/lib/Basics/AssocMulti.h index c744d73698..a83dd0e7ce 100644 --- a/lib/Basics/AssocMulti.h +++ b/lib/Basics/AssocMulti.h @@ -21,6 +21,7 @@ /// @author Dr. Frank Celler /// @author Martin Schoenert /// @author Max Neunhoeffer +/// @author Daniel H. Larkin //////////////////////////////////////////////////////////////////////////////// #ifndef ARANGODB_BASICS_ASSOC_MULTI_H @@ -29,19 +30,21 @@ // Activate for additional debugging: // #define TRI_CHECK_MULTI_POINTER_HASH 1 +#include "Basics/AssocMultiHelpers.h" #include "Basics/Common.h" #include "Basics/AssocHelpers.h" #include "Basics/IndexBucket.h" +#include "Basics/LocalTaskQueue.h" #include "Basics/Mutex.h" #include "Basics/MutexLocker.h" #include "Basics/prime-numbers.h" #include "Logger/Logger.h" -#include #include #include +#include -#ifdef TRI_CHECK_MULTI_POINTER_HASH +#ifdef TRI_CHECK_MULTI_POINTER_HASH #include #endif @@ -90,44 +93,6 @@ namespace basics { /// //////////////////////////////////////////////////////////////////////////////// -template -struct Entry { - private: - uint64_t hashCache; // cache the hash value, this stores the - // hashByKey for the first element in the - // linked list and the hashByElm for all - // others - public: - Element value; // the data stored in this slot - IndexType next; // index of the data following in the linked - // list of all items with the same key - IndexType prev; // index of the data preceding in the linked - // list of all items with the same key - uint64_t readHashCache() const { return hashCache; } - void writeHashCache(uint64_t v) { hashCache = v; } - - Entry() : hashCache(0), value(), next(INVALID_INDEX), prev(INVALID_INDEX) {} - - private: - static IndexType const INVALID_INDEX = ((IndexType)0) - 1; -}; - -template -struct Entry { - Element value; // the data stored in this slot - IndexType next; // index of the data following in the linked - // list of all items with the same key - IndexType prev; // index of the data preceding in the linked - // list of all items with the same key - uint64_t readHashCache() const { return 0; } - void writeHashCache(uint64_t v) { TRI_ASSERT(false); } - - Entry() : value(), next(INVALID_INDEX), prev(INVALID_INDEX) {} - - private: - static IndexType const INVALID_INDEX = ((IndexType)0) - 1; -}; - template class AssocMulti { @@ -145,12 +110,12 @@ class AssocMulti { typedef std::function IsEqualElementElementFuncType; typedef std::function CallbackElementFuncType; - + private: typedef Entry EntryType; - + typedef arangodb::basics::IndexBucket Bucket; - + std::vector _buckets; size_t _bucketsMask; @@ -213,7 +178,7 @@ class AssocMulti { } numberBuckets = nr; _bucketsMask = nr - 1; - + _buckets.resize(numberBuckets); try { @@ -226,9 +191,7 @@ class AssocMulti { } } - ~AssocMulti() { - _buckets.clear(); - } + ~AssocMulti() { _buckets.clear(); } ////////////////////////////////////////////////////////////////////////////// /// @brief return the memory used by the hash table @@ -325,179 +288,102 @@ class AssocMulti { /// @brief adds multiple elements to the array ////////////////////////////////////////////////////////////////////////////// - int batchInsert(std::function const& contextCreator, - std::function const& contextDestroyer, - std::vector const* data, - size_t numThreads) { + void batchInsert(std::function const& contextCreator, + std::function const& contextDestroyer, + std::shared_ptr const> data, + LocalTaskQueue* queue) { if (data->empty()) { // nothing to do - return TRI_ERROR_NO_ERROR; + return; } - std::atomic res(TRI_ERROR_NO_ERROR); - - std::vector const& elements = *(data); + std::vector const& elements = *(data.get()); + // set the number of partitioners sensibly + size_t numThreads = _buckets.size(); if (elements.size() < numThreads) { numThreads = elements.size(); } - if (numThreads > _buckets.size()) { - numThreads = _buckets.size(); - } - - TRI_ASSERT(numThreads > 0); size_t const chunkSize = elements.size() / numThreads; typedef std::vector> DocumentsPerBucket; + typedef MultiInserterTask Inserter; + typedef MultiPartitionerTask Partitioner; - arangodb::Mutex bucketMapLocker; + // allocate working space and coordination tools for tasks - std::vector> allBuckets; - allBuckets.resize(_bucketsMask + 1); // initialize to number of buckets + std::shared_ptr> bucketMapLocker; + bucketMapLocker.reset(new std::vector(_buckets.size())); - // partition the work into some buckets - { - std::function partitioner; - partitioner = [&](size_t lower, size_t upper, void* userData) -> void { - try { - std::vector partitions; - partitions.resize(_bucketsMask + 1); // initialize to number of buckets - - for (size_t i = lower; i < upper; ++i) { - uint64_t hashByKey = _hashElement(userData, elements[i], true); - auto bucketId = hashByKey & _bucketsMask; - - partitions[bucketId].emplace_back(elements[i], hashByKey); - } - - // transfer ownership to the central map - MUTEX_LOCKER(mutexLocker, bucketMapLocker); - - for (size_t i = 0; i < partitions.size(); ++i) { - allBuckets[i].emplace_back(std::move(partitions[i])); - } - } catch (...) { - res = TRI_ERROR_INTERNAL; - } - - contextDestroyer(userData); - }; - - std::vector threads; - threads.reserve(numThreads); - - try { - for (size_t i = 0; i < numThreads; ++i) { - size_t lower = i * chunkSize; - size_t upper = (i + 1) * chunkSize; - - if (i + 1 == numThreads) { - // last chunk. account for potential rounding errors - upper = elements.size(); - } else if (upper > elements.size()) { - upper = elements.size(); - } - - threads.emplace_back(std::thread(partitioner, lower, upper, contextCreator())); - } - } catch (...) { - res = TRI_ERROR_INTERNAL; - } - - for (size_t i = 0; i < threads.size(); ++i) { - // must join threads, otherwise the program will crash - threads[i].join(); - } - - // sort vectors in vectors so that we have a deterministics insertion order - for (size_t i = 0; i < allBuckets.size(); ++i) { - std::sort(allBuckets[i].begin(), allBuckets[i].end(), [](DocumentsPerBucket const& lhs, DocumentsPerBucket const& rhs) -> bool { - if (lhs.empty() && rhs.empty()) { - return false; - } - if (lhs.empty() && !rhs.empty()) { - return true; - } - if (rhs.empty() && !lhs.empty()) { - return false; - } - - return lhs[0].first < rhs[0].first; - }); - } + std::shared_ptr>> bucketFlags; + bucketFlags.reset(new std::vector>(_buckets.size())); + for (size_t i = 0; i < bucketFlags->size(); i++) { + (*bucketFlags)[i] = numThreads; } - if (res.load() != TRI_ERROR_NO_ERROR) { - return res.load(); - } + std::shared_ptr>> inserters; + inserters.reset(new std::vector>); + inserters->reserve(_buckets.size()); - // now the data is partitioned... + std::shared_ptr>> allBuckets; + allBuckets.reset( + new std::vector>(_buckets.size())); - // now insert the bucket data in parallel - { - auto inserter = [&](size_t chunk, void* userData) -> void { - try { - for (size_t i = 0; i < allBuckets.size(); ++i) { - uint64_t bucketId = i; + auto doInsertBinding = [&]( + UserData* userData, Element const& element, uint64_t hashByKey, + Bucket& b, bool const overwrite, bool const checkEquality) -> Element { + return doInsert(userData, element, hashByKey, b, overwrite, + checkEquality); + }; - if (bucketId % numThreads != chunk) { - // we're not responsible for this bucket! - continue; - } + try { + // create inserter tasks to be dispatched later by partitioners + for (size_t i = 0; i < allBuckets->size(); i++) { + std::shared_ptr worker; + worker.reset(new Inserter(queue, contextDestroyer, &_buckets, + doInsertBinding, i, contextCreator(), + allBuckets)); + inserters->emplace_back(worker); + } + // enqueue partitioner tasks + for (size_t i = 0; i < numThreads; ++i) { + size_t lower = i * chunkSize; + size_t upper = (i + 1) * chunkSize; - // we're responsible for this bucket! - Bucket& b = _buckets[static_cast(bucketId)]; - - for (auto const& it : allBuckets[i]) { - for (auto const& it2 : it) { - doInsert(userData, it2.first, it2.second, b, true, false); - } - } - } - } catch (...) { - res = TRI_ERROR_INTERNAL; + if (i + 1 == numThreads) { + // last chunk. account for potential rounding errors + upper = elements.size(); + } else if (upper > elements.size()) { + upper = elements.size(); } - contextDestroyer(userData); - }; - - std::vector threads; - threads.reserve(numThreads); - - try { - for (size_t i = 0; i < numThreads; ++i) { - threads.emplace_back(std::thread(inserter, i, contextCreator())); - } - } catch (...) { - res = TRI_ERROR_INTERNAL; - } - - for (size_t i = 0; i < threads.size(); ++i) { - // must join threads, otherwise the program will crash - threads[i].join(); + std::shared_ptr worker; + worker.reset(new Partitioner(queue, _hashElement, contextDestroyer, + data, lower, upper, contextCreator(), + bucketFlags, bucketMapLocker, allBuckets, + inserters)); + queue->enqueue(worker); } + } catch (...) { + queue->setStatus(TRI_ERROR_INTERNAL); } #ifdef TRI_CHECK_MULTI_POINTER_HASH { - void* userData = contextCreator(); - check(userData, true, true); - contextDestroyer(userData); + auto& checkFn = check; + auto callback = [&contextCreator, &contextDestroyer, &checkFn]() -> void { + if (queue->status() == TRI_ERROR_NO_ERROR) { + void* userData = contextCreator(); + checkFn(userData, true, true); + contextDestroyer(userData); + } + }; + std::shared_ptr cbTask; + cbTask.reset(new arangodb::basics::LocalCallbackTask(queue, callback)); + queue->enqueueCallback(cbTask); } #endif - if (res.load() != TRI_ERROR_NO_ERROR) { - // Rollback such that the data can be deleted outside - void* userData = contextCreator(); - try { - for (auto const& d : *data) { - remove(userData, d); - } - } catch (...) { - } - contextDestroyer(userData); - } - return res.load(); } void truncate(CallbackElementFuncType callback) { @@ -539,8 +425,9 @@ class AssocMulti { /// @brief adds a key/element to the array ////////////////////////////////////////////////////////////////////////////// - Element doInsert(UserData* userData, Element const& element, uint64_t hashByKey, - Bucket& b, bool const overwrite, bool const checkEquality) { + Element doInsert(UserData* userData, Element const& element, + uint64_t hashByKey, Bucket& b, bool const overwrite, + bool const checkEquality) { // if the checkEquality flag is not set, we do not check for element // equality we use this flag to speed up initial insertion into the // index, i.e. when the index is built for a collection and we know @@ -574,10 +461,11 @@ class AssocMulti { // Now find the first slot with an entry with the same key // that is the start of a linked list, or a free slot: - while (b._table[i].value && - (b._table[i].prev != INVALID_INDEX || - (useHashCache && b._table[i].readHashCache() != hashByKey) || - !_isEqualElementElementByKey(userData, element, b._table[i].value))) { + while ( + b._table[i].value && + (b._table[i].prev != INVALID_INDEX || + (useHashCache && b._table[i].readHashCache() != hashByKey) || + !_isEqualElementElementByKey(userData, element, b._table[i].value))) { i = incr(b, i); #ifdef TRI_INTERNAL_STATS // update statistics @@ -879,10 +767,11 @@ class AssocMulti { #endif // search the table - while (b._table[i].value && - (b._table[i].prev != INVALID_INDEX || - (useHashCache && b._table[i].readHashCache() != hashByKey) || - !_isEqualElementElementByKey(userData, element, b._table[i].value))) { + while ( + b._table[i].value && + (b._table[i].prev != INVALID_INDEX || + (useHashCache && b._table[i].readHashCache() != hashByKey) || + !_isEqualElementElementByKey(userData, element, b._table[i].value))) { i = incr(b, i); #ifdef TRI_INTERNAL_STATS _nrProbesF++; @@ -905,8 +794,9 @@ class AssocMulti { /// continuation ////////////////////////////////////////////////////////////////////////////// - std::vector* lookupWithElementByKeyContinue( - UserData* userData, Element const& element, size_t limit = 0) const { + std::vector* lookupWithElementByKeyContinue(UserData* userData, + Element const& element, + size_t limit = 0) const { auto result = std::make_unique>(); lookupWithElementByKeyContinue(userData, element, *result.get(), limit); return result.release(); @@ -944,11 +834,11 @@ class AssocMulti { // Now find the first slot with an entry with the same key // that is the start of a linked list, or a free slot: - while ( - b._table[i].value && - (b._table[i].prev != INVALID_INDEX || - (useHashCache && b._table[i].readHashCache() != hashByKey) || - !_isEqualElementElementByKey(userData, element, b._table[i].value))) { + while (b._table[i].value && + (b._table[i].prev != INVALID_INDEX || + (useHashCache && b._table[i].readHashCache() != hashByKey) || + !_isEqualElementElementByKey(userData, element, + b._table[i].value))) { i = incr(b, i); #ifdef TRI_INTERNAL_STATS _nrProbes++; @@ -1002,7 +892,8 @@ class AssocMulti { } ////////////////////////////////////////////////////////////////////////////// - /// @brief removes an element from the array, caller is responsible to free it + /// @brief removes an element from the array, caller is responsible to free + /// it ////////////////////////////////////////////////////////////////////////////// Element remove(UserData* userData, Element const& element) { @@ -1149,13 +1040,13 @@ class AssocMulti { LOG(TRACE) << "resizing hash " << cb << ", target size: " << targetSize; - LOG_TOPIC(TRACE, Logger::PERFORMANCE) << - "hash-resize " << cb << ", target size: " << targetSize; + LOG_TOPIC(TRACE, Logger::PERFORMANCE) << "hash-resize " << cb + << ", target size: " << targetSize; double start = TRI_microtime(); - + targetSize = TRI_NearPrime(targetSize); - + Bucket copy; copy.allocate(targetSize); @@ -1193,19 +1084,21 @@ class AssocMulti { } else { hashByElm = _hashElement(userData, oldTable[k].value, false); } - insertFurther(userData, copy, oldTable[k].value, hashByKey, hashByElm, - insertPosition); + insertFurther(userData, copy, oldTable[k].value, hashByKey, + hashByElm, insertPosition); k = oldTable[k].prev; } } } } - + b = std::move(copy); LOG(TRACE) << "resizing hash " << cb << " done"; - LOG_TOPIC(TRACE, Logger::PERFORMANCE) << "[timer] " << Logger::FIXED(TRI_microtime() - start) << " s, hash-resize, " << cb << ", target size: " << targetSize; + LOG_TOPIC(TRACE, Logger::PERFORMANCE) + << "[timer] " << Logger::FIXED(TRI_microtime() - start) + << " s, hash-resize, " << cb << ", target size: " << targetSize; } #ifdef TRI_CHECK_MULTI_POINTER_HASH @@ -1359,10 +1252,11 @@ class AssocMulti { // Now find the first slot with an entry with the same key // that is the start of a linked list, or a free slot: - while (b._table[i].value && - (b._table[i].prev != INVALID_INDEX || - (useHashCache && b._table[i].readHashCache() != hashByKey) || - !_isEqualElementElementByKey(userData, element, b._table[i].value))) { + while ( + b._table[i].value && + (b._table[i].prev != INVALID_INDEX || + (useHashCache && b._table[i].readHashCache() != hashByKey) || + !_isEqualElementElementByKey(userData, element, b._table[i].value))) { i = incr(b, i); #ifdef TRI_INTERNAL_STATS _nrProbes++; diff --git a/lib/Basics/AssocMultiHelpers.h b/lib/Basics/AssocMultiHelpers.h new file mode 100644 index 0000000000..1548ebadec --- /dev/null +++ b/lib/Basics/AssocMultiHelpers.h @@ -0,0 +1,231 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Dr. Frank Celler +/// @author Martin Schoenert +/// @author Max Neunhoeffer +/// @author Daniel H. Larkin +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_BASICS_ASSOC_MULTI_HELPERS_H +#define ARANGODB_BASICS_ASSOC_MULTI_HELPERS_H 1 + +#include "Basics/Common.h" +#include "Basics/IndexBucket.h" +#include "Basics/LocalTaskQueue.h" +#include "Basics/Mutex.h" +#include "Basics/MutexLocker.h" + +namespace arangodb { +namespace basics { + +template +struct Entry { + private: + uint64_t hashCache; // cache the hash value, this stores the + // hashByKey for the first element in the + // linked list and the hashByElm for all + // others + public: + Element value; // the data stored in this slot + IndexType next; // index of the data following in the linked + // list of all items with the same key + IndexType prev; // index of the data preceding in the linked + // list of all items with the same key + uint64_t readHashCache() const { return hashCache; } + void writeHashCache(uint64_t v) { hashCache = v; } + + Entry() : hashCache(0), value(), next(INVALID_INDEX), prev(INVALID_INDEX) {} + + private: + static IndexType const INVALID_INDEX = ((IndexType)0) - 1; +}; + +template +struct Entry { + Element value; // the data stored in this slot + IndexType next; // index of the data following in the linked + // list of all items with the same key + IndexType prev; // index of the data preceding in the linked + // list of all items with the same key + uint64_t readHashCache() const { return 0; } + void writeHashCache(uint64_t v) { TRI_ASSERT(false); } + + Entry() : value(), next(INVALID_INDEX), prev(INVALID_INDEX) {} + + private: + static IndexType const INVALID_INDEX = ((IndexType)0) - 1; +}; + +template +class MultiInserterTask : public LocalTask { + private: + typedef Entry EntryType; + typedef arangodb::basics::IndexBucket Bucket; + typedef std::vector> DocumentsPerBucket; + + std::function _contextDestroyer; + std::vector* _buckets; + std::function + _doInsert; + + size_t _i; + void* _userData; + + std::shared_ptr>> _allBuckets; + + public: + MultiInserterTask( + LocalTaskQueue* queue, std::function contextDestroyer, + std::vector* buckets, + std::function + doInsert, + size_t i, void* userData, + std::shared_ptr>> allBuckets) + : LocalTask(queue), + _contextDestroyer(contextDestroyer), + _buckets(buckets), + _doInsert(doInsert), + _i(i), + _userData(userData), + _allBuckets(allBuckets) {} + + void run() { + // sort first so we have a deterministic insertion order + std::sort((*_allBuckets)[_i].begin(), (*_allBuckets)[_i].end(), + [](DocumentsPerBucket const& lhs, + DocumentsPerBucket const& rhs) -> bool { + if (lhs.empty() && rhs.empty()) { + return false; + } + if (lhs.empty() && !rhs.empty()) { + return true; + } + if (rhs.empty() && !lhs.empty()) { + return false; + } + + return lhs[0].first < rhs[0].first; + }); + // now actually insert them + try { + Bucket& b = (*_buckets)[static_cast(_i)]; + + for (auto const& it : (*_allBuckets)[_i]) { + for (auto const& it2 : it) { + _doInsert(_userData, it2.first, it2.second, b, true, false); + } + } + } catch (...) { + _queue->setStatus(TRI_ERROR_INTERNAL); + } + + _contextDestroyer(_userData); + _queue->join(); + } +}; + +template +class MultiPartitionerTask : public LocalTask { + private: + typedef MultiInserterTask Inserter; + typedef std::vector> DocumentsPerBucket; + + std::function _hashElement; + std::function _contextDestroyer; + std::shared_ptr const> _data; + std::vector const* _elements; + + size_t _lower; + size_t _upper; + void* _userData; + + std::shared_ptr>> _bucketFlags; + std::shared_ptr> _bucketMapLocker; + std::shared_ptr>> _allBuckets; + std::shared_ptr>> _inserters; + + uint64_t _bucketsMask; + + public: + MultiPartitionerTask( + LocalTaskQueue* queue, + std::function hashElement, + std::function const& contextDestroyer, + std::shared_ptr const> data, size_t lower, + size_t upper, void* userData, + std::shared_ptr>> bucketFlags, + std::shared_ptr> bucketMapLocker, + std::shared_ptr>> allBuckets, + std::shared_ptr>> inserters) + : LocalTask(queue), + _hashElement(hashElement), + _contextDestroyer(contextDestroyer), + _data(data), + _elements(_data.get()), + _lower(lower), + _upper(upper), + _userData(userData), + _bucketFlags(bucketFlags), + _bucketMapLocker(bucketMapLocker), + _allBuckets(allBuckets), + _inserters(inserters), + _bucketsMask(_allBuckets->size() - 1) {} + + void run() { + try { + std::vector partitions; + partitions.resize( + _allBuckets->size()); // initialize to number of buckets + + for (size_t i = _lower; i < _upper; ++i) { + uint64_t hashByKey = _hashElement(_userData, (*_elements)[i], true); + auto bucketId = hashByKey & _bucketsMask; + + partitions[bucketId].emplace_back((*_elements)[i], hashByKey); + } + + // transfer ownership to the central map + for (size_t i = 0; i < partitions.size(); ++i) { + { + MUTEX_LOCKER(mutexLocker, (*_bucketMapLocker)[i]); + (*_allBuckets)[i].emplace_back(std::move(partitions[i])); + (*_bucketFlags)[i]--; + if ((*_bucketFlags)[i].load() == 0) { + // queue inserter for bucket i + _queue->enqueue((*_inserters)[i]); + } + } + } + } catch (...) { + _queue->setStatus(TRI_ERROR_INTERNAL); + } + + _contextDestroyer(_userData); + _queue->join(); + } +}; + +} // namespace arangodb::basics +} // namespace arangodb + +#endif diff --git a/lib/Basics/AssocUnique.h b/lib/Basics/AssocUnique.h index d5284ef2a9..036568a30b 100644 --- a/lib/Basics/AssocUnique.h +++ b/lib/Basics/AssocUnique.h @@ -21,19 +21,22 @@ /// @author Dr. Frank Celler /// @author Martin Schoenert /// @author Michael Hackstein +/// @author Daniel H. Larkin //////////////////////////////////////////////////////////////////////////////// #ifndef ARANGODB_BASICS_ASSOC_UNIQUE_H #define ARANGODB_BASICS_ASSOC_UNIQUE_H 1 +#include "Basics/AssocUniqueHelpers.h" #include "Basics/Common.h" -#include #include #include +#include #include "Basics/AssocHelpers.h" #include "Basics/IndexBucket.h" +#include "Basics/LocalTaskQueue.h" #include "Basics/MutexLocker.h" #include "Basics/gcd.h" #include "Basics/prime-numbers.h" @@ -43,22 +46,6 @@ namespace arangodb { namespace basics { -struct BucketPosition { - size_t bucketId; - uint64_t position; - - BucketPosition() : bucketId(SIZE_MAX), position(0) {} - - void reset() { - bucketId = SIZE_MAX - 1; - position = 0; - } - - bool operator==(BucketPosition const& other) const { - return position == other.position && bucketId == other.bucketId; - } -}; - //////////////////////////////////////////////////////////////////////////////// /// @brief associative array //////////////////////////////////////////////////////////////////////////////// @@ -67,13 +54,15 @@ template class AssocUnique { private: typedef void UserData; + typedef arangodb::basics::BucketPosition BucketPosition; public: typedef std::function HashKeyFuncType; typedef std::function HashElementFuncType; typedef std::function IsEqualKeyElementFuncType; + Element const&)> + IsEqualKeyElementFuncType; typedef std::function IsEqualElementElementFuncType; @@ -131,9 +120,7 @@ class AssocUnique { } } - ~AssocUnique() { - _buckets.clear(); - } + ~AssocUnique() { _buckets.clear(); } ////////////////////////////////////////////////////////////////////////////// /// @brief adhere to the rule of five @@ -163,8 +150,7 @@ class AssocUnique { if (b._nrAlloc > targetSize && !allowShrink) { return; } - if (allowShrink && - b._nrAlloc >= targetSize && + if (allowShrink && b._nrAlloc >= targetSize && b._nrAlloc < 1.25 * targetSize) { // no need to shrink return; @@ -180,8 +166,8 @@ class AssocUnique { double start = TRI_microtime(); if (targetSize > NotificationSizeThreshold) { - LOG_TOPIC(TRACE, Logger::PERFORMANCE) << - "hash-resize " << cb << ", target size: " << targetSize; + LOG_TOPIC(TRACE, Logger::PERFORMANCE) << "hash-resize " << cb + << ", target size: " << targetSize; } TRI_ASSERT(targetSize > 0); @@ -223,7 +209,9 @@ class AssocUnique { LOG(TRACE) << "resizing hash " << cb << " done"; - LOG_TOPIC(TRACE, Logger::PERFORMANCE) << "[timer] " << Logger::FIXED(TRI_microtime() - start) << " s, hash-resize, " << cb << ", target size: " << targetSize; + LOG_TOPIC(TRACE, Logger::PERFORMANCE) + << "[timer] " << Logger::FIXED(TRI_microtime() - start) + << " s, hash-resize, " << cb << ", target size: " << targetSize; } ////////////////////////////////////////////////////////////////////////////// @@ -272,19 +260,19 @@ class AssocUnique { /// This does not resize and expects to have enough space ////////////////////////////////////////////////////////////////////////////// - int doInsert(UserData* userData, Element const& element, Bucket& b, uint64_t hash) { + int doInsert(UserData* userData, Element const& element, Bucket& b, + uint64_t hash) { uint64_t const n = b._nrAlloc; uint64_t i = hash % n; uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualElementElementByKey(userData, element, b._table[i]); + !_isEqualElementElementByKey(userData, element, b._table[i]); ++i) ; if (i == n) { - for (i = 0; - i < k && b._table[i] && - !_isEqualElementElementByKey(userData, element, b._table[i]); + for (i = 0; i < k && b._table[i] && + !_isEqualElementElementByKey(userData, element, b._table[i]); ++i) ; } @@ -310,9 +298,7 @@ class AssocUnique { } } - size_t buckets() const { - return _buckets.size(); - } + size_t buckets() const { return _buckets.size(); } ////////////////////////////////////////////////////////////////////////////// /// @brief checks if this index is empty @@ -350,7 +336,7 @@ class AssocUnique { } return sum; } - + size_t capacity() const { size_t sum = 0; for (auto& b : _buckets) { @@ -409,13 +395,12 @@ class AssocUnique { uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualElementElementByKey(userData, element, b._table[i]); + !_isEqualElementElementByKey(userData, element, b._table[i]); ++i) ; if (i == n) { - for (i = 0; - i < k && b._table[i] && - !_isEqualElementElementByKey(userData, element, b._table[i]); + for (i = 0; i < k && b._table[i] && + !_isEqualElementElementByKey(userData, element, b._table[i]); ++i) ; } @@ -444,12 +429,12 @@ class AssocUnique { uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; if (i == n) { for (i = 0; i < k && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; } @@ -461,7 +446,7 @@ class AssocUnique { return b._table[i]; } - + Element* findByKeyRef(UserData* userData, Key const* key) const { uint64_t hash = _hashKey(userData, key); uint64_t i = hash; @@ -473,12 +458,12 @@ class AssocUnique { uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; if (i == n) { for (i = 0; i < k && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; } @@ -510,12 +495,12 @@ class AssocUnique { uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; if (i == n) { for (i = 0; i < k && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; } @@ -581,194 +566,109 @@ class AssocUnique { /// @brief adds multiple elements to the array ////////////////////////////////////////////////////////////////////////////// - int batchInsert(std::function const& contextCreator, - std::function const& contextDestroyer, - std::vector const* data, - size_t numThreads) { + void batchInsert(std::function const& contextCreator, + std::function const& contextDestroyer, + std::shared_ptr const> data, + arangodb::basics::LocalTaskQueue* queue) { + TRI_ASSERT(queue != nullptr); if (data->empty()) { // nothing to do - return TRI_ERROR_NO_ERROR; + return; } - std::atomic res(TRI_ERROR_NO_ERROR); - std::vector const& elements = *(data); + std::vector const& elements = *(data.get()); + // set number of partitioners sensibly + size_t numThreads = _buckets.size(); if (elements.size() < numThreads) { numThreads = elements.size(); } - if (numThreads > _buckets.size()) { - numThreads = _buckets.size(); - } - - TRI_ASSERT(numThreads > 0); size_t const chunkSize = elements.size() / numThreads; typedef std::vector> DocumentsPerBucket; - arangodb::Mutex bucketMapLocker; + typedef UniqueInserterTask Inserter; + typedef UniquePartitionerTask Partitioner; - std::unordered_map> allBuckets; + // allocate working space and coordination tools for tasks - // partition the work into some buckets - { - auto partitioner = [&](size_t lower, size_t upper, void* userData) -> void { - try { - std::unordered_map partitions; + std::shared_ptr> bucketMapLocker; + bucketMapLocker.reset(new std::vector(_buckets.size())); - for (size_t i = lower; i < upper; ++i) { - uint64_t hash = _hashElement(userData, elements[i]); - auto bucketId = hash & _bucketsMask; - - auto it = partitions.find(bucketId); - - if (it == partitions.end()) { - it = partitions.emplace(bucketId, DocumentsPerBucket()).first; - } - - (*it).second.emplace_back(elements[i], hash); - } - - // transfer ownership to the central map - MUTEX_LOCKER(mutexLocker, bucketMapLocker); - - for (auto& it : partitions) { - auto it2 = allBuckets.find(it.first); - - if (it2 == allBuckets.end()) { - it2 = allBuckets.emplace(it.first, - std::vector()).first; - } - - (*it2).second.emplace_back(std::move(it.second)); - } - } catch (...) { - res = TRI_ERROR_OUT_OF_MEMORY; - } - - contextDestroyer(userData); - }; - - std::vector threads; - threads.reserve(numThreads); - - try { - for (size_t i = 0; i < numThreads; ++i) { - size_t lower = i * chunkSize; - size_t upper = (i + 1) * chunkSize; - - if (i + 1 == numThreads) { - // last chunk. account for potential rounding errors - upper = elements.size(); - } else if (upper > elements.size()) { - upper = elements.size(); - } - - threads.emplace_back(std::thread(partitioner, lower, upper, contextCreator())); - } - } catch (...) { - res = TRI_ERROR_OUT_OF_MEMORY; - } - - for (size_t i = 0; i < threads.size(); ++i) { - // must join threads, otherwise the program will crash - threads[i].join(); - } + std::shared_ptr>> bucketFlags; + bucketFlags.reset(new std::vector>(_buckets.size())); + for (size_t i = 0; i < bucketFlags->size(); i++) { + (*bucketFlags)[i] = numThreads; } - if (res.load() != TRI_ERROR_NO_ERROR) { - return res.load(); - } + std::shared_ptr>> inserters; + inserters.reset(new std::vector>); + inserters->reserve(_buckets.size()); - // now the data is partitioned... + std::shared_ptr>> allBuckets; + allBuckets.reset( + new std::vector>(_buckets.size())); - // now insert the bucket data in parallel - { - auto inserter = [&](size_t chunk, void* userData) -> void { - try { - for (auto const& it : allBuckets) { - uint64_t bucketId = it.first; + auto doInsertBinding = [&](UserData* userData, Element const& element, + Bucket& b, uint64_t hashByKey) -> int { + return doInsert(userData, element, b, hashByKey); + }; + auto checkResizeBinding = [&](UserData* userData, Bucket& b, + uint64_t expected) -> bool { + return checkResize(userData, b, expected); + }; - if (bucketId % numThreads != chunk) { - // we're not responsible for this bucket! - continue; - } + try { + // generate inserter tasks to be dispatched later by partitioners + for (size_t i = 0; i < allBuckets->size(); i++) { + std::shared_ptr worker; + worker.reset(new Inserter(queue, contextDestroyer, &_buckets, + doInsertBinding, checkResizeBinding, i, + contextCreator(), allBuckets)); + inserters->emplace_back(worker); + } + // queue partitioner tasks + for (size_t i = 0; i < numThreads; ++i) { + size_t lower = i * chunkSize; + size_t upper = (i + 1) * chunkSize; - // we're responsible for this bucket! - Bucket& b = _buckets[static_cast(bucketId)]; - uint64_t expected = 0; - - for (auto const& it2 : it.second) { - expected += it2.size(); - } - - if (!checkResize(userData, b, expected)) { - res = TRI_ERROR_OUT_OF_MEMORY; - return; - } - - for (auto const& it2 : it.second) { - for (auto const& it3 : it2) { - doInsert(userData, it3.first, b, it3.second); - } - } - } - } catch (...) { - res = TRI_ERROR_OUT_OF_MEMORY; + if (i + 1 == numThreads) { + // last chunk. account for potential rounding errors + upper = elements.size(); + } else if (upper > elements.size()) { + upper = elements.size(); } - contextDestroyer(userData); - }; - - std::vector threads; - threads.reserve(numThreads); - - try { - for (size_t i = 0; i < numThreads; ++i) { - threads.emplace_back(std::thread(inserter, i, contextCreator())); - } - } catch (...) { - res = TRI_ERROR_OUT_OF_MEMORY; - } - - for (size_t i = 0; i < threads.size(); ++i) { - // must join threads, otherwise the program will crash - threads[i].join(); + std::shared_ptr worker; + worker.reset(new Partitioner(queue, _hashElement, contextDestroyer, + data, lower, upper, contextCreator(), + bucketFlags, bucketMapLocker, allBuckets, + inserters)); + queue->enqueue(worker); } + } catch (...) { + queue->setStatus(TRI_ERROR_INTERNAL); } - - if (res.load() != TRI_ERROR_NO_ERROR) { - // Rollback such that the data can be deleted outside - void* userData = contextCreator(); - try { - for (auto const& d : *data) { - remove(userData, d); - } - } catch (...) { - } - contextDestroyer(userData); - } - return res.load(); } - ////////////////////////////////////////////////////////////////////////////// /// @brief helper to heal a hole where we deleted something ////////////////////////////////////////////////////////////////////////////// void healHole(UserData* userData, Bucket& b, uint64_t i) { - // ........................................................................... + // // remove item - destroy any internal memory associated with the // element structure - // ........................................................................... + // b._table[i] = Element(); b._nrUsed--; uint64_t const n = b._nrAlloc; - // ........................................................................... + // // and now check the following places for items to move closer together // so that there are no gaps in the array - // ........................................................................... + // uint64_t k = TRI_IncModU64(i, n); @@ -805,12 +705,12 @@ class AssocUnique { uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; if (i == n) { for (i = 0; i < k && b._table[i] && - !_isEqualKeyElement(userData, key, hash, b._table[i]); + !_isEqualKeyElement(userData, key, hash, b._table[i]); ++i) ; } @@ -837,12 +737,12 @@ class AssocUnique { uint64_t k = i; for (; i < n && b._table[i] && - !_isEqualElementElement(userData, element, b._table[i]); + !_isEqualElementElement(userData, element, b._table[i]); ++i) ; if (i == n) { for (i = 0; i < k && b._table[i] && - !_isEqualElementElement(userData, element, b._table[i]); + !_isEqualElementElement(userData, element, b._table[i]); ++i) ; } @@ -868,7 +768,7 @@ class AssocUnique { } } } - + /// @brief a method to iterate over all elements in a bucket. this method /// can NOT be used for deleting elements bool invokeOnAllElements(CallbackElementFuncType const& callback, Bucket& b) { diff --git a/lib/Basics/AssocUniqueHelpers.h b/lib/Basics/AssocUniqueHelpers.h new file mode 100644 index 0000000000..b51e0cb40a --- /dev/null +++ b/lib/Basics/AssocUniqueHelpers.h @@ -0,0 +1,201 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Dr. Frank Celler +/// @author Martin Schoenert +/// @author Michael Hackstein +/// @author Daniel H. Larkin +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_BASICS_ASSOC_UNIQUE_HELPERS_H +#define ARANGODB_BASICS_ASSOC_UNIQUE_HELPERS_H 1 + +#include "Basics/Common.h" + +#include "Basics/IndexBucket.h" +#include "Basics/LocalTaskQueue.h" +#include "Basics/MutexLocker.h" + +namespace arangodb { +namespace basics { + +struct BucketPosition { + size_t bucketId; + uint64_t position; + + BucketPosition() : bucketId(SIZE_MAX), position(0) {} + + void reset() { + bucketId = SIZE_MAX - 1; + position = 0; + } + + bool operator==(BucketPosition const& other) const { + return position == other.position && bucketId == other.bucketId; + } +}; + +template +class UniqueInserterTask : public LocalTask { + private: + typedef arangodb::basics::IndexBucket Bucket; + typedef std::vector> DocumentsPerBucket; + + std::function _contextDestroyer; + std::vector* _buckets; + std::function _doInsert; + std::function _checkResize; + + size_t _i; + void* _userData; + + std::shared_ptr>> _allBuckets; + + public: + UniqueInserterTask( + LocalTaskQueue* queue, std::function contextDestroyer, + std::vector* buckets, + std::function doInsert, + std::function checkResize, size_t i, + void* userData, + std::shared_ptr>> allBuckets) + : LocalTask(queue), + _contextDestroyer(contextDestroyer), + _buckets(buckets), + _doInsert(doInsert), + _checkResize(checkResize), + _i(i), + _userData(userData), + _allBuckets(allBuckets) {} + + void run() { + // actually insert them + try { + Bucket& b = (*_buckets)[static_cast(_i)]; + + for (auto const& it : (*_allBuckets)[_i]) { + uint64_t expected = it.size(); + if (!_checkResize(_userData, b, expected)) { + _queue->setStatus(TRI_ERROR_OUT_OF_MEMORY); + _queue->join(); + return; + } + for (auto const& it2 : it) { + int status = _doInsert(_userData, it2.first, b, it2.second); + if (status != TRI_ERROR_NO_ERROR) { + _queue->setStatus(status); + _queue->join(); + _contextDestroyer(_userData); + return; + } + } + } + } catch (...) { + _queue->setStatus(TRI_ERROR_INTERNAL); + } + + _contextDestroyer(_userData); + _queue->join(); + } +}; + +template +class UniquePartitionerTask : public LocalTask { + private: + typedef UniqueInserterTask Inserter; + typedef std::vector> DocumentsPerBucket; + + std::function _hashElement; + std::function _contextDestroyer; + std::shared_ptr const> _data; + std::vector const* _elements; + + size_t _lower; + size_t _upper; + void* _userData; + + std::shared_ptr>> _bucketFlags; + std::shared_ptr> _bucketMapLocker; + std::shared_ptr>> _allBuckets; + std::shared_ptr>> _inserters; + + uint64_t _bucketsMask; + + public: + UniquePartitionerTask( + LocalTaskQueue* queue, + std::function hashElement, + std::function const& contextDestroyer, + std::shared_ptr const> data, size_t lower, + size_t upper, void* userData, + std::shared_ptr>> bucketFlags, + std::shared_ptr> bucketMapLocker, + std::shared_ptr>> allBuckets, + std::shared_ptr>> inserters) + : LocalTask(queue), + _hashElement(hashElement), + _contextDestroyer(contextDestroyer), + _data(data), + _elements(_data.get()), + _lower(lower), + _upper(upper), + _userData(userData), + _bucketFlags(bucketFlags), + _bucketMapLocker(bucketMapLocker), + _allBuckets(allBuckets), + _inserters(inserters), + _bucketsMask(_allBuckets->size() - 1) {} + + void run() { + try { + std::vector partitions; + partitions.resize( + _allBuckets->size()); // initialize to number of buckets + + for (size_t i = _lower; i < _upper; ++i) { + uint64_t hashByKey = _hashElement(_userData, (*_elements)[i]); + auto bucketId = hashByKey & _bucketsMask; + + partitions[bucketId].emplace_back((*_elements)[i], hashByKey); + } + + // transfer ownership to the central map + for (size_t i = 0; i < partitions.size(); ++i) { + MUTEX_LOCKER(mutexLocker, (*_bucketMapLocker)[i]); + (*_allBuckets)[i].emplace_back(std::move(partitions[i])); + (*_bucketFlags)[i]--; + if ((*_bucketFlags)[i].load() == 0) { + // queue inserter for bucket i + _queue->enqueue((*_inserters)[i]); + } + } + } catch (...) { + _queue->setStatus(TRI_ERROR_INTERNAL); + } + + _contextDestroyer(_userData); + _queue->join(); + } +}; + +} // namespace basics +} // namespace arangodb + +#endif diff --git a/lib/Basics/LocalTaskQueue.cpp b/lib/Basics/LocalTaskQueue.cpp new file mode 100644 index 0000000000..e20a3e1307 --- /dev/null +++ b/lib/Basics/LocalTaskQueue.cpp @@ -0,0 +1,207 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Daniel H. Larkin +//////////////////////////////////////////////////////////////////////////////// + +#include "LocalTaskQueue.h" +#include "Basics/ConditionLocker.h" +#include "Basics/MutexLocker.h" +#include "Basics/asio-helper.h" + +using namespace arangodb::basics; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief create a task tied to the specified queue +//////////////////////////////////////////////////////////////////////////////// + +LocalTask::LocalTask(LocalTaskQueue* queue) : _queue(queue) {} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief dispatch this task to the underlying io_service +//////////////////////////////////////////////////////////////////////////////// + +void LocalTask::dispatch() { + auto self = shared_from_this(); + _queue->ioService()->post([self, this]() { run(); }); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief create a callback task tied to the specified queue +//////////////////////////////////////////////////////////////////////////////// + +LocalCallbackTask::LocalCallbackTask(LocalTaskQueue* queue, + std::function cb) + : _queue(queue), _cb(cb) {} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief run the callback and join +//////////////////////////////////////////////////////////////////////////////// + +void LocalCallbackTask::run() { + try { + _cb(); + } catch (...) { + } + _queue->join(); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief dispatch this task to the underlying io_service +//////////////////////////////////////////////////////////////////////////////// + +void LocalCallbackTask::dispatch() { + auto self = shared_from_this(); + _queue->ioService()->post([self, this]() { run(); }); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief create a queue using the specified io_service +//////////////////////////////////////////////////////////////////////////////// + +LocalTaskQueue::LocalTaskQueue(boost::asio::io_service* ioService) + : _ioService(ioService), + _queue(), + _callbackQueue(), + _condition(), + _mutex(), + _missing(0), + _status(TRI_ERROR_NO_ERROR) { + TRI_ASSERT(_ioService != nullptr); +} + +////////////////////////////////////////////////////////////////////////////// +/// @brief exposes underlying io_service +////////////////////////////////////////////////////////////////////////////// + +boost::asio::io_service* LocalTaskQueue::ioService() { return _ioService; } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroy the queue. +//////////////////////////////////////////////////////////////////////////////// + +LocalTaskQueue::~LocalTaskQueue() {} + +////////////////////////////////////////////////////////////////////////////// +/// @brief enqueue a task to be run +////////////////////////////////////////////////////////////////////////////// + +void LocalTaskQueue::enqueue(std::shared_ptr task) { + MUTEX_LOCKER(locker, _mutex); + _queue.push(task); +} + +////////////////////////////////////////////////////////////////////////////// +/// @brief enqueue a callback task to be run after all normal tasks finish; +/// useful for cleanup tasks +////////////////////////////////////////////////////////////////////////////// + +void LocalTaskQueue::enqueueCallback(std::shared_ptr task) { + MUTEX_LOCKER(locker, _mutex); + _callbackQueue.push(task); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief join a single task. reduces the number of waiting tasks and wakes +/// up the queue's dispatchAndWait() routine +//////////////////////////////////////////////////////////////////////////////// + +void LocalTaskQueue::join() { + CONDITION_LOCKER(guard, _condition); + TRI_ASSERT(_missing > 0); + --_missing; + _condition.signal(); +} + +////////////////////////////////////////////////////////////////////////////// +/// @brief dispatch all tasks, including those that are queued while running, +/// and wait for all tasks to join; then dispatch all callback tasks and wait +/// for them to join +////////////////////////////////////////////////////////////////////////////// + +void LocalTaskQueue::dispatchAndWait() { + // regular task loop + if (!_queue.empty()) { + while (true) { + CONDITION_LOCKER(guard, _condition); + + { + MUTEX_LOCKER(locker, _mutex); + // dispatch all newly queued tasks + if (_status == TRI_ERROR_NO_ERROR) { + while (!_queue.empty()) { + auto task = _queue.front(); + task->dispatch(); + _queue.pop(); + ++_missing; + } + } + } + + if (_missing == 0) { + break; + } + + guard.wait(100000); + } + } + + // callback task loop + if (!_callbackQueue.empty()) { + while (true) { + CONDITION_LOCKER(guard, _condition); + + { + MUTEX_LOCKER(locker, _mutex); + // dispatch all newly queued callbacks + while (!_callbackQueue.empty()) { + auto task = _callbackQueue.front(); + task->dispatch(); + _callbackQueue.pop(); + ++_missing; + } + } + + if (_missing == 0) { + break; + } + + guard.wait(100000); + } + } +} + +////////////////////////////////////////////////////////////////////////////// +/// @brief set status of queue +////////////////////////////////////////////////////////////////////////////// + +void LocalTaskQueue::setStatus(int status) { + MUTEX_LOCKER(locker, _mutex); + _status = status; +} + +////////////////////////////////////////////////////////////////////////////// +/// @brief return overall status of queue tasks +////////////////////////////////////////////////////////////////////////////// + +int LocalTaskQueue::status() { + MUTEX_LOCKER(locker, _mutex); + return _status; +} diff --git a/lib/Basics/LocalTaskQueue.h b/lib/Basics/LocalTaskQueue.h new file mode 100644 index 0000000000..9dc957f81a --- /dev/null +++ b/lib/Basics/LocalTaskQueue.h @@ -0,0 +1,190 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Daniel H. Larkin +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_BASICS_LOCAL_TASK_QUEUE_H +#define ARANGODB_BASICS_LOCAL_TASK_QUEUE_H 1 + +#include +#include + +#include "Basics/Common.h" +#include "Basics/ConditionVariable.h" +#include "Basics/Mutex.h" +#include "Basics/asio-helper.h" + +namespace arangodb { +namespace basics { + +class LocalTaskQueue; + +class LocalTask : public std::enable_shared_from_this { + public: + LocalTask() = delete; + LocalTask(LocalTask const&) = delete; + LocalTask& operator=(LocalTask const&) = delete; + + explicit LocalTask(LocalTaskQueue* queue); + virtual ~LocalTask() {} + + virtual void run() = 0; + void dispatch(); + + protected: + ////////////////////////////////////////////////////////////////////////////// + /// @brief the underlying queue + ////////////////////////////////////////////////////////////////////////////// + + LocalTaskQueue* _queue; +}; + +class LocalCallbackTask + : public std::enable_shared_from_this { + public: + LocalCallbackTask() = delete; + LocalCallbackTask(LocalCallbackTask const&) = delete; + LocalCallbackTask& operator=(LocalCallbackTask const&) = delete; + + LocalCallbackTask(LocalTaskQueue* queue, std::function cb); + virtual ~LocalCallbackTask() {} + + virtual void run(); + void dispatch(); + + protected: + ////////////////////////////////////////////////////////////////////////////// + /// @brief the underlying queue + ////////////////////////////////////////////////////////////////////////////// + + LocalTaskQueue* _queue; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief the callback executed by run() (any exceptions will be caught and + /// ignored; must not call queue->setStatus() or queue->enqueue()) + ////////////////////////////////////////////////////////////////////////////// + + std::function _cb; +}; + +class LocalTaskQueue { + public: + LocalTaskQueue(LocalTaskQueue const&) = delete; + LocalTaskQueue& operator=(LocalTaskQueue const&) = delete; + + explicit LocalTaskQueue(boost::asio::io_service*); + + ~LocalTaskQueue(); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief exposes underlying io_service + ////////////////////////////////////////////////////////////////////////////// + + boost::asio::io_service* ioService(); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief enqueue a task to be run + ////////////////////////////////////////////////////////////////////////////// + + void enqueue(std::shared_ptr); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief enqueue a callback task to be run after all normal tasks finish; + /// useful for cleanup tasks + ////////////////////////////////////////////////////////////////////////////// + + void enqueueCallback(std::shared_ptr); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief join a single task. reduces the number of waiting tasks and wakes + /// up the queues's dispatchAndWait() routine + ////////////////////////////////////////////////////////////////////////////// + + void join(); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief dispatch all tasks, including those that are queued while running, + /// and wait for all tasks to join; then dispatch all callback tasks and wait + /// for them to join + ////////////////////////////////////////////////////////////////////////////// + + void dispatchAndWait(); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief set status of queue + ////////////////////////////////////////////////////////////////////////////// + + void setStatus(int); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief return overall status of queue tasks + ////////////////////////////////////////////////////////////////////////////// + + int status(); + + private: + ////////////////////////////////////////////////////////////////////////////// + /// @brief io_service to dispatch tasks to + ////////////////////////////////////////////////////////////////////////////// + + boost::asio::io_service* _ioService; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief internal task queue + ////////////////////////////////////////////////////////////////////////////// + + std::queue> _queue; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief internal callback task queue + ////////////////////////////////////////////////////////////////////////////// + + std::queue> _callbackQueue; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief condition variable + ////////////////////////////////////////////////////////////////////////////// + + arangodb::basics::ConditionVariable _condition; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief internal mutex + ////////////////////////////////////////////////////////////////////////////// + + Mutex _mutex; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief number of dispatched, non-joined tasks + ////////////////////////////////////////////////////////////////////////////// + + size_t _missing; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief overall status of queue tasks + ////////////////////////////////////////////////////////////////////////////// + + int _status; +}; + +} // namespace arangodb::basics +} // namespace arangodb + +#endif diff --git a/lib/Basics/ThreadPool.cpp b/lib/Basics/ThreadPool.cpp deleted file mode 100644 index c587df602b..0000000000 --- a/lib/Basics/ThreadPool.cpp +++ /dev/null @@ -1,104 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Jan Steemann -//////////////////////////////////////////////////////////////////////////////// - -#include "ThreadPool.h" - -#include "Basics/WorkerThread.h" - -using namespace arangodb::basics; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief create a pool with the specified size of worker threads -//////////////////////////////////////////////////////////////////////////////// - -ThreadPool::ThreadPool(size_t size, std::string const& name) - : _condition(), _threads(), _tasks(), _name(name), _stopping(false) { - _threads.reserve(size); - - for (size_t i = 0; i < size; ++i) { - auto workerThread = new WorkerThread(this); - - try { - _threads.emplace_back(workerThread); - } catch (...) { - // clean up - delete workerThread; - for (auto& it : _threads) { - delete it; - } - _threads.clear(); - throw; - } - } - - // now start them all - for (auto& it : _threads) { - it->start(); - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief destroy the pool -//////////////////////////////////////////////////////////////////////////////// - -ThreadPool::~ThreadPool() { - _stopping = true; - { - CONDITION_LOCKER(guard, _condition); - _condition.broadcast(); - } - - for (auto* it : _threads) { - it->waitForDone(); - } - - for (auto* it : _threads) { - delete it; - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief dequeue a task -//////////////////////////////////////////////////////////////////////////////// - -bool ThreadPool::dequeue(std::function& result) { - while (!_stopping) { - CONDITION_LOCKER(guard, _condition); - - if (_tasks.empty()) { - guard.wait(1000000); - } - - if (_stopping) { - break; - } - - if (!_tasks.empty()) { - result = _tasks.front(); - _tasks.pop_front(); - return true; - } - } - - return false; -} diff --git a/lib/Basics/ThreadPool.h b/lib/Basics/ThreadPool.h deleted file mode 100644 index 83c57e4593..0000000000 --- a/lib/Basics/ThreadPool.h +++ /dev/null @@ -1,95 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Jan Steemann -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGODB_BASICS_THREAD_POOL_H -#define ARANGODB_BASICS_THREAD_POOL_H 1 - -#include "Basics/Common.h" -#include "Basics/ConditionLocker.h" -#include "Basics/ConditionVariable.h" - -#include - -namespace arangodb { -namespace basics { - -class WorkerThread; - -class ThreadPool { - public: - ThreadPool(ThreadPool const&) = delete; - ThreadPool& operator=(ThreadPool const&) = delete; - - ThreadPool(size_t, std::string const&); - - ~ThreadPool(); - - public: - ////////////////////////////////////////////////////////////////////////////// - /// @brief return the number of threads in the pool - ////////////////////////////////////////////////////////////////////////////// - - size_t numThreads() const { return _threads.size(); } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief return the name of the pool - ////////////////////////////////////////////////////////////////////////////// - - char const* name() const { return _name.c_str(); } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief dequeue a task - ////////////////////////////////////////////////////////////////////////////// - - bool dequeue(std::function&); - - ////////////////////////////////////////////////////////////////////////////// - /// @brief enqueue a task - ////////////////////////////////////////////////////////////////////////////// - - template - void enqueue(T task) { - { - CONDITION_LOCKER(guard, _condition); - _tasks.emplace_back(std::function(task)); - } - - _condition.signal(); - } - - private: - arangodb::basics::ConditionVariable _condition; - - std::vector _threads; - - std::deque> _tasks; - - std::string _name; - - std::atomic _stopping; -}; - -} // namespace arangodb::basics -} // namespace arangodb - -#endif diff --git a/lib/Basics/WorkerThread.h b/lib/Basics/WorkerThread.h deleted file mode 100644 index e1d05d27d3..0000000000 --- a/lib/Basics/WorkerThread.h +++ /dev/null @@ -1,86 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Jan Steemann -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGODB_BASICS_WORKER_THREAD_H -#define ARANGODB_BASICS_WORKER_THREAD_H 1 - -#include "Basics/Common.h" -#include "Basics/Thread.h" - -namespace arangodb { -namespace basics { - -class WorkerThread : public Thread { - public: - WorkerThread(WorkerThread const&) = delete; - WorkerThread operator=(WorkerThread const&) = delete; - - WorkerThread(ThreadPool* pool) - : Thread(pool->name()), _pool(pool), _status(0) {} - - ~WorkerThread() { - waitForDone(); - shutdown(); - } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief stops the worker thread - ////////////////////////////////////////////////////////////////////////////// - - void waitForDone() { - int expected = 0; - _status.compare_exchange_strong(expected, 1, std::memory_order_relaxed); - - while (_status != 2) { - usleep(5000); - } - } - - protected: - void run() { - while (_status == 0) { - if (isStopping()) { - break; - } - - std::function task; - - if (!_pool->dequeue(task)) { - break; - } - - task(); - } - - _status = 2; - } - - private: - ThreadPool* _pool; - - std::atomic _status; -}; -} -} - -#endif diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index a4e088f5a4..896741299f 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -21,7 +21,7 @@ if (USE_MAINTAINER_MODE AND NOT MSVC) add_custom_target(clean_lib_autogenerated COMMAND rm -f ${CMAKE_SOURCE_DIR}/lib/V8/v8-json.cpp) - + list(APPEND CLEAN_AUTOGENERATED_FILES clean_lib_autogenerated) set(CLEAN_AUTOGENERATED_FILES ${CLEAN_AUTOGENERATED_FILES} PARENT_SCOPE) endif () @@ -133,6 +133,7 @@ add_library(${LIB_ARANGO} STATIC Basics/Exceptions.cpp Basics/FileUtils.cpp Basics/HybridLogicalClock.cpp + Basics/LocalTaskQueue.cpp Basics/Mutex.cpp Basics/MutexLocker.cpp Basics/Nonce.cpp @@ -144,7 +145,6 @@ add_library(${LIB_ARANGO} STATIC Basics/StringRef.cpp Basics/StringUtils.cpp Basics/Thread.cpp - Basics/ThreadPool.cpp Basics/Timers.cpp Basics/Utf8Helper.cpp Basics/VelocyPackDumper.cpp From b4ec71d33a0abd08b136985fbc1f9a354164b57d Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 30 Jan 2017 14:26:06 -0500 Subject: [PATCH 06/22] Fixed compilation bug --- arangod/RestServer/DatabaseFeature.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arangod/RestServer/DatabaseFeature.cpp b/arangod/RestServer/DatabaseFeature.cpp index 70002deff6..25532b84c0 100644 --- a/arangod/RestServer/DatabaseFeature.cpp +++ b/arangod/RestServer/DatabaseFeature.cpp @@ -174,8 +174,10 @@ void DatabaseManagerThread::run() { usleep(10000); }; while (!arangodb::wal::LogfileManager::instance() - ->executeWhileNothingQueued(callback)) { - LOG(DEBUG) << "Trying to shutdown dropped database, waiting for phase in which the collector thread does not have queued operations."; + ->executeWhileNothingQueued(callback)) { + LOG(DEBUG) << "Trying to shutdown dropped database, waiting for " + "phase in which the collector thread does not have " + "queued operations."; usleep(500000); } @@ -307,7 +309,7 @@ void DatabaseFeature::collectOptions(std::shared_ptr options) { options->addObsoleteOption( "--database.index-threads", - "threads to start for parallel background index creation"); + "threads to start for parallel background index creation", true); } void DatabaseFeature::validateOptions(std::shared_ptr options) { From af3c206d89297db32d74eb9ba17b72d7b27d560a Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Mon, 30 Jan 2017 22:32:05 +0100 Subject: [PATCH 07/22] Try to solve sporadic shutdown blockage in heartbeat thread. --- arangod/Cluster/HeartbeatThread.cpp | 71 ++++++++++++++--------------- arangod/Cluster/HeartbeatThread.h | 8 +--- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index 3e70fb53da..589c660ef0 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -98,35 +98,44 @@ HeartbeatThread::~HeartbeatThread() { shutdown(); } /// watching the command key, it will wake up and apply the change locally. //////////////////////////////////////////////////////////////////////////////// +class HeartbeatBackgroundJob { + std::shared_ptr _heartbeatThread; + public: + HeartbeatBackgroundJob(std::shared_ptr hbt) + : _heartbeatThread(hbt) {} + + void operator()() { + _heartbeatThread->runBackgroundJob(); + } +}; + +void HeartbeatThread::runBackgroundJob() { + uint64_t jobNr = ++_backgroundJobsLaunched; + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback started " << jobNr; + { + DBServerAgencySync job(this); + job.work(); + } + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback ended " << jobNr; + + { + MUTEX_LOCKER(mutexLocker, *_statusLock); + if (_launchAnotherBackgroundJob) { + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " + << ++_backgroundJobsPosted; + _launchAnotherBackgroundJob = false; + _ioService->post(HeartbeatBackgroundJob(shared_from_this())); + } else { + _backgroundJobScheduledOrRunning = false; + _launchAnotherBackgroundJob = false; + } + } +} + void HeartbeatThread::run() { if (ServerState::instance()->isCoordinator()) { runCoordinator(); } else { - // Set the member variable that holds a closure to run background - // jobs in JS: - auto self = shared_from_this(); - _backgroundJob = [self, this]() { - uint64_t jobNr = ++_backgroundJobsLaunched; - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback started " << jobNr; - { - DBServerAgencySync job(this); - job.work(); - } - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback ended " << jobNr; - - { - MUTEX_LOCKER(mutexLocker, *_statusLock); - if (_launchAnotherBackgroundJob) { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " - << ++_backgroundJobsPosted; - _launchAnotherBackgroundJob = false; - _ioService->post(_backgroundJob); - } else { - _backgroundJobScheduledOrRunning = false; - _launchAnotherBackgroundJob = false; - } - } - }; runDBServer(); } } @@ -338,16 +347,6 @@ void HeartbeatThread::runDBServer() { } _agencyCallbackRegistry->unregisterCallback(planAgencyCallback); - int count = 0; - while (++count < 3000) { - { - MUTEX_LOCKER(mutexLocker, *_statusLock); - if (!_backgroundJobScheduledOrRunning) { - break; - } - } - usleep(100000); - } LOG_TOPIC(TRACE, Logger::HEARTBEAT) << "stopped heartbeat thread (DBServer version)"; } @@ -764,7 +763,7 @@ void HeartbeatThread::syncDBServerStatusQuo() { LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " << ++_backgroundJobsPosted; _backgroundJobScheduledOrRunning = true; - _ioService->post(_backgroundJob); + _ioService->post(HeartbeatBackgroundJob(shared_from_this())); } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Cluster/HeartbeatThread.h b/arangod/Cluster/HeartbeatThread.h index 9e17cf8c4c..e02c008bd5 100644 --- a/arangod/Cluster/HeartbeatThread.h +++ b/arangod/Cluster/HeartbeatThread.h @@ -74,6 +74,8 @@ class HeartbeatThread : public Thread, void setReady() { _ready.store(true); } + void runBackgroundJob(); + void dispatchedJobResult(DBServerAgencySyncResult); ////////////////////////////////////////////////////////////////////////////// @@ -253,12 +255,6 @@ class HeartbeatThread : public Thread, ////////////////////////////////////////////////////////////////////////////// bool _launchAnotherBackgroundJob; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief _backgroundJob, the closure that does the work - ////////////////////////////////////////////////////////////////////////////// - - std::function _backgroundJob; }; } From 4172b3e0e46f3cc0bf793a444846b4cd6aa76057 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 30 Jan 2017 22:54:00 +0100 Subject: [PATCH 08/22] turn down loglevel a bit --- arangod/Wal/LogfileManager.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/arangod/Wal/LogfileManager.cpp b/arangod/Wal/LogfileManager.cpp index 865eed1ca0..b4ec42e78c 100644 --- a/arangod/Wal/LogfileManager.cpp +++ b/arangod/Wal/LogfileManager.cpp @@ -876,7 +876,7 @@ int LogfileManager::flush(bool waitForSync, bool waitForCollector, res = this->waitForCollector(lastOpenLogfileId, maxWaitTime); if (res == TRI_ERROR_LOCK_TIMEOUT) { - LOG(ERR) << "got lock timeout when waiting for WAL flush. lastOpenLogfileId: " << lastOpenLogfileId; + LOG(DEBUG) << "got lock timeout when waiting for WAL flush. lastOpenLogfileId: " << lastOpenLogfileId; } } else if (res == TRI_ERROR_ARANGO_DATAFILE_EMPTY) { // current logfile is empty and cannot be collected @@ -887,7 +887,7 @@ int LogfileManager::flush(bool waitForSync, bool waitForCollector, res = this->waitForCollector(lastSealedLogfileId, maxWaitTime); if (res == TRI_ERROR_LOCK_TIMEOUT) { - LOG(ERR) << "got lock timeout when waiting for WAL flush. lastSealedLogfileId: " << lastSealedLogfileId; + LOG(DEBUG) << "got lock timeout when waiting for WAL flush. lastSealedLogfileId: " << lastSealedLogfileId; } } } @@ -1731,8 +1731,7 @@ int LogfileManager::waitForCollector(Logfile::IdType logfileId, // try again } - // TODO: remove debug info here - LOG(ERR) << "going into lock timeout. having waited for logfile: " << logfileId << ", maxWaitTime: " << maxWaitTime; + LOG(DEBUG) << "going into lock timeout. having waited for logfile: " << logfileId << ", maxWaitTime: " << maxWaitTime; logStatus(); // waited for too long @@ -1740,12 +1739,11 @@ int LogfileManager::waitForCollector(Logfile::IdType logfileId, } void LogfileManager::logStatus() { - // TODO: remove debug info here - LOG(ERR) << "logfile manager status report: lastCollectedId: " << _lastCollectedId.load() << ", lastSealedId: " << _lastSealedId.load(); + LOG(DEBUG) << "logfile manager status report: lastCollectedId: " << _lastCollectedId.load() << ", lastSealedId: " << _lastSealedId.load(); READ_LOCKER(locker, _logfilesLock); for (auto logfile : _logfiles) { - LOG(ERR) << "- logfile " << logfile.second->id() << ", filename '" << logfile.second->filename() - << "', status " << logfile.second->statusText(); + LOG(DEBUG) << "- logfile " << logfile.second->id() << ", filename '" << logfile.second->filename() + << "', status " << logfile.second->statusText(); } } From 16c19ad927dc274f2d22754a0f36bae233cfd889 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 30 Jan 2017 23:02:05 +0100 Subject: [PATCH 09/22] cppcheck --- arangod/Cluster/HeartbeatThread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index 589c660ef0..d0e05a7650 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -101,7 +101,7 @@ HeartbeatThread::~HeartbeatThread() { shutdown(); } class HeartbeatBackgroundJob { std::shared_ptr _heartbeatThread; public: - HeartbeatBackgroundJob(std::shared_ptr hbt) + explicit HeartbeatBackgroundJob(std::shared_ptr hbt) : _heartbeatThread(hbt) {} void operator()() { From 9525f46b2b0514881e2b361e79219530c6a1d993 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Mon, 30 Jan 2017 23:22:58 +0100 Subject: [PATCH 10/22] Some cleanup of comments for better understandability. --- arangod/Cluster/HeartbeatThread.cpp | 39 +++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index d0e05a7650..17c2c873cb 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -86,16 +86,18 @@ HeartbeatThread::HeartbeatThread(AgencyCallbackRegistry* agencyCallbackRegistry, HeartbeatThread::~HeartbeatThread() { shutdown(); } //////////////////////////////////////////////////////////////////////////////// -/// @brief heartbeat main loop -/// the heartbeat thread constantly reports the current server status to the -/// agency. it does so by sending the current state string to the key -/// "Sync/ServerStates/" + my-id. -/// after transferring the current state to the agency, the heartbeat thread -/// will wait for changes on the "Sync/Commands/" + my-id key. If no changes -/// occur, -/// then the request it aborted and the heartbeat thread will go on with -/// reporting its state to the agency again. If it notices a change when -/// watching the command key, it will wake up and apply the change locally. +/// @brief running of heartbeat background jobs (in JavaScript), we run +/// these by instantiating an object in class HeartbeatBackgroundJob, +/// which is a std::function and holds a shared_ptr to the +/// HeartbeatThread singleton itself. This instance is then posted to +/// the io_service for execution in the thread pool. Should the heartbeat +/// thread itself terminate during shutdown, then the HeartbeatThread +/// singleton itself is still kept alive by the shared_ptr in the instance +/// of HeartbeatBackgroundJob. The operator() method simply calls the +/// runBackgroundJob() method of the heartbeat thread. Should this have +/// to schedule another background job, then it can simply create a new +/// HeartbeatBackgroundJob instance, again using shared_from_this() to +/// create a new shared_ptr keeping the HeartbeatThread object alive. //////////////////////////////////////////////////////////////////////////////// class HeartbeatBackgroundJob { @@ -109,6 +111,10 @@ class HeartbeatBackgroundJob { } }; +//////////////////////////////////////////////////////////////////////////////// +/// @brief method runBackgroundJob() +//////////////////////////////////////////////////////////////////////////////// + void HeartbeatThread::runBackgroundJob() { uint64_t jobNr = ++_backgroundJobsLaunched; LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback started " << jobNr; @@ -132,6 +138,19 @@ void HeartbeatThread::runBackgroundJob() { } } +//////////////////////////////////////////////////////////////////////////////// +/// @brief heartbeat main loop +/// the heartbeat thread constantly reports the current server status to the +/// agency. it does so by sending the current state string to the key +/// "Sync/ServerStates/" + my-id. +/// after transferring the current state to the agency, the heartbeat thread +/// will wait for changes on the "Sync/Commands/" + my-id key. If no changes +/// occur, +/// then the request it aborted and the heartbeat thread will go on with +/// reporting its state to the agency again. If it notices a change when +/// watching the command key, it will wake up and apply the change locally. +//////////////////////////////////////////////////////////////////////////////// + void HeartbeatThread::run() { if (ServerState::instance()->isCoordinator()) { runCoordinator(); From 91c97d301a91a8d4f4d1529c1dba1e66ce2d8332 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Mon, 30 Jan 2017 23:59:46 +0100 Subject: [PATCH 11/22] Lower timeout in query registry to 10min from 1h. --- arangod/Aql/ExecutionEngine.cpp | 2 +- arangod/Aql/QueryRegistry.h | 2 +- arangod/Aql/RestAqlHandler.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index 4507235bc5..58f79e8536 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -1042,7 +1042,7 @@ struct CoordinatorInstanciator : public WalkerWorker { id = TRI_NewTickServer(); try { - queryRegistry->insert(id, engine->getQuery(), 3600.0); + queryRegistry->insert(id, engine->getQuery(), 600.0); } catch (...) { delete engine->getQuery(); // This deletes the new query as well as the engine diff --git a/arangod/Aql/QueryRegistry.h b/arangod/Aql/QueryRegistry.h index e67bf9eaa2..071a985900 100644 --- a/arangod/Aql/QueryRegistry.h +++ b/arangod/Aql/QueryRegistry.h @@ -45,7 +45,7 @@ class QueryRegistry { /// a query for this and combination and an exception will /// be thrown in that case. The time to live is in seconds and the /// query will be deleted if it is not opened for that amount of time. - void insert(QueryId id, Query* query, double ttl = 3600.0); + void insert(QueryId id, Query* query, double ttl = 600.0); /// @brief open, find a query in the registry, if none is found, a nullptr /// is returned, otherwise, ownership of the query is transferred to the diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 22c5ee0928..8e19b2cb5c 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -105,7 +105,7 @@ void RestAqlHandler::createQueryFromVelocyPack() { } // Now the query is ready to go, store it in the registry and return: - double ttl = 3600.0; + double ttl = 600.0; bool found; std::string const& ttlstring = _request->header("ttl", found); @@ -317,7 +317,7 @@ void RestAqlHandler::createQueryFromString() { } // Now the query is ready to go, store it in the registry and return: - double ttl = 3600.0; + double ttl = 600.0; bool found; std::string const& ttlstring = _request->header("ttl", found); From f4a521eebfde476decceb0beef25f89599aa4199 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 31 Jan 2017 08:36:36 +0100 Subject: [PATCH 12/22] updated CHANGELOG --- CHANGELOG | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index c99b3244ad..c0f3e96983 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,12 @@ devel * changed index filling to make it more parallel, dispatch tasks to boost::asio +* updated versions of bundled node modules: + - joi: from 8.4.2 to 9.2.0 + - joi-to-json-schema: from 2.2.0 to 2.3.0 + - sinon: from 1.17.4 to 1.17.6 + - lodash: from 4.13.1 to 4.16.6 + * added shortcut for AQL ternary operator instead of `condition ? true-part : false-part` it is now possible to also use a shortcut variant `condition ? : false-part`, e.g. From 8c1441740a41134c12833ba92c2c741fc51e6cff Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 31 Jan 2017 08:36:52 +0100 Subject: [PATCH 13/22] removed libev --- LICENSES-OTHER-COMPONENTS.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/LICENSES-OTHER-COMPONENTS.md b/LICENSES-OTHER-COMPONENTS.md index 4fb9f1f10c..adaabcff75 100644 --- a/LICENSES-OTHER-COMPONENTS.md +++ b/LICENSES-OTHER-COMPONENTS.md @@ -51,11 +51,6 @@ * Project Home: http://site.icu-project.org/ * License: [ICU License](http://source.icu-project.org/repos/icu/icu/trunk/license.html) -### libev 4.11 - -* Project Home: http://software.schmorp.de/pkg/libev.html -* License: Dual-License [BSD-style 2-Clause License](http://cvs.schmorp.de/libev/LICENSE?revision=1.11&view=markup) - ### linenoise-ng * GitHub: https://github.com/arangodb/linenoise-ng From 5d267037f46924df65892116e3cbfbd86f034257 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 31 Jan 2017 08:47:25 +0100 Subject: [PATCH 14/22] updated CHANGELOG --- CHANGELOG | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index c0f3e96983..b2dd39ddc5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -80,6 +80,13 @@ edge attribute `label`. * added option -D to define a configuration file environment key=value +* changed encoding behavior for URLs encoded in the C++ code of ArangoDB: + previously the special characters `-`, `_`, `~` and `.` were returned as-is + after URL-encoding, now `.` will be encoded to be `%2e`. + This also changes the behavior of how incoming URIs are processed: previously + occurrences of `..` in incoming request URIs were collapsed (e.g. `a/../b/` was + collapsed to a plain `b/`). Now `..` in incoming request URIs are not collapsed. + * Foxx request URL suffix is no longer unescaped * @arangodb/request option json now defaults to `true` if the response body is not empty and encoding is not explicitly set to `null` (binary). From fe130289090b95e8cb765d716e4150592f8f57fd Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 31 Jan 2017 08:50:09 +0100 Subject: [PATCH 15/22] removed unused file --- Documentation/DocuBlocks/schedulerBackend.md | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 Documentation/DocuBlocks/schedulerBackend.md diff --git a/Documentation/DocuBlocks/schedulerBackend.md b/Documentation/DocuBlocks/schedulerBackend.md deleted file mode 100644 index 541ae1d21c..0000000000 --- a/Documentation/DocuBlocks/schedulerBackend.md +++ /dev/null @@ -1,10 +0,0 @@ - - -@brief scheduler backend -`--scheduler.backend arg` - -The I/O method used by the event handler. The default (if this option is -not specified) is to try all recommended backends. This is platform -specific. See libev for further details and the meaning of select, poll -and epoll. - From d8d8ef9755493732dd8eb5e9f8aa00cab12936b0 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Tue, 31 Jan 2017 08:56:19 +0100 Subject: [PATCH 16/22] Add an assertion. --- arangod/Cluster/HeartbeatThread.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index 17c2c873cb..d38b72c200 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -126,9 +126,10 @@ void HeartbeatThread::runBackgroundJob() { { MUTEX_LOCKER(mutexLocker, *_statusLock); + TRI_assert(_backgroundJobScheduledOrRunning); if (_launchAnotherBackgroundJob) { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " - << ++_backgroundJobsPosted; + jobNr = ++_backgroundJobsPosted; + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr; _launchAnotherBackgroundJob = false; _ioService->post(HeartbeatBackgroundJob(shared_from_this())); } else { @@ -779,8 +780,8 @@ void HeartbeatThread::syncDBServerStatusQuo() { } // schedule a job for the change: - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " - << ++_backgroundJobsPosted; + uint64_t jobNr = ++_backgroundJobsPosted; + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " << jobNr; _backgroundJobScheduledOrRunning = true; _ioService->post(HeartbeatBackgroundJob(shared_from_this())); } From d8171651bf5979620099469c3a0b452ef808c0d8 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Tue, 31 Jan 2017 08:59:20 +0100 Subject: [PATCH 17/22] Fix assert. --- arangod/Cluster/HeartbeatThread.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp index d38b72c200..972829eec0 100644 --- a/arangod/Cluster/HeartbeatThread.cpp +++ b/arangod/Cluster/HeartbeatThread.cpp @@ -117,19 +117,19 @@ class HeartbeatBackgroundJob { void HeartbeatThread::runBackgroundJob() { uint64_t jobNr = ++_backgroundJobsLaunched; - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback started " << jobNr; + LOG_TOPIC(INFO, Logger::HEARTBEAT) << "sync callback started " << jobNr; { DBServerAgencySync job(this); job.work(); } - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback ended " << jobNr; + LOG_TOPIC(INFO, Logger::HEARTBEAT) << "sync callback ended " << jobNr; { MUTEX_LOCKER(mutexLocker, *_statusLock); - TRI_assert(_backgroundJobScheduledOrRunning); + TRI_ASSERT(_backgroundJobScheduledOrRunning); if (_launchAnotherBackgroundJob) { jobNr = ++_backgroundJobsPosted; - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr; + LOG_TOPIC(INFO, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr; _launchAnotherBackgroundJob = false; _ioService->post(HeartbeatBackgroundJob(shared_from_this())); } else { @@ -596,7 +596,7 @@ bool HeartbeatThread::init() { //////////////////////////////////////////////////////////////////////////////// void HeartbeatThread::dispatchedJobResult(DBServerAgencySyncResult result) { - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Dispatched job returned!"; + LOG_TOPIC(INFO, Logger::HEARTBEAT) << "Dispatched job returned!"; bool doSleep = false; { MUTEX_LOCKER(mutexLocker, *_statusLock); @@ -781,7 +781,7 @@ void HeartbeatThread::syncDBServerStatusQuo() { // schedule a job for the change: uint64_t jobNr = ++_backgroundJobsPosted; - LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " << jobNr; + LOG_TOPIC(INFO, Logger::HEARTBEAT) << "dispatching sync " << jobNr; _backgroundJobScheduledOrRunning = true; _ioService->post(HeartbeatBackgroundJob(shared_from_this())); } From 66fbc966486f05e682e6c0840af1277409ad16ba Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 31 Jan 2017 09:07:22 +0100 Subject: [PATCH 18/22] fix CHANGELOG for devel --- CHANGELOG | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index b2dd39ddc5..158c97fd2f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,12 @@ devel * changed index filling to make it more parallel, dispatch tasks to boost::asio +* more detailed stacktraces in Foxx apps + + +v3.1.10 (2017-XX-XX) +-------------------- + * updated versions of bundled node modules: - joi: from 8.4.2 to 9.2.0 - joi-to-json-schema: from 2.2.0 to 2.3.0 From d7f2ee7e2f90f90a238b9376dd28420460d195be Mon Sep 17 00:00:00 2001 From: jsteemann Date: Tue, 31 Jan 2017 09:07:34 +0100 Subject: [PATCH 19/22] fix comment --- arangod/Indexes/PathBasedIndex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Indexes/PathBasedIndex.cpp b/arangod/Indexes/PathBasedIndex.cpp index db8969e475..0028d59316 100644 --- a/arangod/Indexes/PathBasedIndex.cpp +++ b/arangod/Indexes/PathBasedIndex.cpp @@ -124,7 +124,7 @@ int PathBasedIndex::fillElement(std::vector& elements, auto slices = buildIndexValue(doc); if (slices.size() == n) { - // if shapes.size() != n, then the value is not inserted into the index + // if slices.size() != n, then the value is not inserted into the index // because of index sparsity! T* element = static_cast(_allocator->allocate()); TRI_ASSERT(element != nullptr); From a5b4eb3c3daef77382af201a17808924b448a250 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Tue, 31 Jan 2017 09:31:52 +0100 Subject: [PATCH 20/22] Lower log level for local shard operations to debug. --- js/server/modules/@arangodb/cluster.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/js/server/modules/@arangodb/cluster.js b/js/server/modules/@arangodb/cluster.js index a5d542cdb9..0b203f2fc8 100644 --- a/js/server/modules/@arangodb/cluster.js +++ b/js/server/modules/@arangodb/cluster.js @@ -745,7 +745,7 @@ function executePlanForCollections(plannedCollections) { let collection; if (!localCollections.hasOwnProperty(shardName)) { // must create this shard - console.info("creating local shard '%s/%s' for central '%s/%s'", + console.debug("creating local shard '%s/%s' for central '%s/%s'", database, shardName, database, @@ -813,7 +813,7 @@ function executePlanForCollections(plannedCollections) { }, {}); if (Object.keys(properties).length > 0) { - console.info("updating properties for local shard '%s/%s'", + console.debug("updating properties for local shard '%s/%s'", database, shardName); @@ -831,17 +831,17 @@ function executePlanForCollections(plannedCollections) { // Now check whether the status is OK: if (collectionStatus !== collectionInfo.status) { - console.info("detected status change for local shard '%s/%s'", + console.debug("detected status change for local shard '%s/%s'", database, shardName); if (collectionInfo.status === ArangoCollection.STATUS_UNLOADED) { - console.info("unloading local shard '%s/%s'", + console.debug("unloading local shard '%s/%s'", database, shardName); collection.unload(); } else if (collectionInfo.status === ArangoCollection.STATUS_LOADED) { - console.info("loading local shard '%s/%s'", + console.debug("loading local shard '%s/%s'", database, shardName); collection.load(); @@ -1264,7 +1264,7 @@ function executePlanForDatabases(plannedDatabases) { if (!plannedDatabases.hasOwnProperty(name) && name.substr(0, 1) !== '_') { // must drop database - console.info("dropping local database '%s'", name); + console.debug("dropping local database '%s'", name); // Do we have to stop a replication applier first? if (ArangoServerState.role() === 'SECONDARY') { @@ -1273,7 +1273,7 @@ function executePlanForDatabases(plannedDatabases) { var rep = require('@arangodb/replication'); var state = rep.applier.state(); if (state.state.running === true) { - console.info('stopping replication applier first'); + console.debug('stopping replication applier first'); rep.applier.stop(); } } From b7b8a6cf88ca2b6247bd77b87f7b9f22827bef87 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Tue, 31 Jan 2017 09:37:47 +0100 Subject: [PATCH 21/22] lowering log output in agencycomm --- arangod/Agency/AgencyComm.cpp | 44 ++++++++++++------- .../tests/resilience/moving-shards-cluster.js | 2 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/arangod/Agency/AgencyComm.cpp b/arangod/Agency/AgencyComm.cpp index ba0d79e811..a51d53c90c 100644 --- a/arangod/Agency/AgencyComm.cpp +++ b/arangod/Agency/AgencyComm.cpp @@ -612,7 +612,7 @@ std::string AgencyCommManager::redirect( << specification << ", url = " << rest; if (endpoint == specification) { - LOG_TOPIC(WARN, Logger::AGENCYCOMM) + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << "got an agency redirect back to the old agency '" << endpoint << "'"; failedNonLocking(std::move(connection), endpoint); return ""; @@ -632,7 +632,7 @@ std::string AgencyCommManager::redirect( std::remove(_endpoints.begin(), _endpoints.end(), specification), _endpoints.end()); - LOG_TOPIC(WARN, Logger::AGENCYCOMM) + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << "Got an agency redirect from '" << endpoint << "' to '" << specification << "'"; @@ -1359,17 +1359,11 @@ AgencyCommResult AgencyComm::sendWithFailover( LOG_TOPIC(ERR, Logger::AGENCYCOMM) << result._message; return result; } - - if (1 < tries) { - LOG_TOPIC(WARN, Logger::AGENCYCOMM) - << "Retrying agency communication at '" << endpoint - << "', tries: " << tries << " (" - << 1.e-2 * ( - std::round( - 1.e+2 * std::chrono::duration( - std::chrono::steady_clock::now() - started).count())) << "s)"; - } - + + double elapsed = 1.e-2 * ( + std::round(1.e+2 * std::chrono::duration( + std::chrono::steady_clock::now() - started).count())); + // try to send; if we fail completely, do not retry try { result = send(connection.get(), method, conTimeout, url, body, clientId); @@ -1379,7 +1373,7 @@ AgencyCommResult AgencyComm::sendWithFailover( connection = AgencyCommManager::MANAGER->acquire(endpoint); continue; } - + // got a result, we are done if (result.successful()) { AgencyCommManager::MANAGER->release(std::move(connection), endpoint); @@ -1389,7 +1383,7 @@ AgencyCommResult AgencyComm::sendWithFailover( // break on a watch timeout (drop connection) if (!clientId.empty() && result._sent && (result._statusCode == 0 || result._statusCode == 503)) { - + VPackBuilder b; { VPackArrayBuilder ab(&b); @@ -1470,6 +1464,26 @@ AgencyCommResult AgencyComm::sendWithFailover( break; } + if (tries%50 == 0) { + LOG_TOPIC(WARN, Logger::AGENCYCOMM) + << "Bad agency communiction! Unsuccessful consecutive tries:" + << tries << " (" << elapsed << "s). Network checks needed!"; + } else if (tries%15 == 0) { + LOG_TOPIC(INFO, Logger::AGENCYCOMM) + << "Flaky agency communication. Unsuccessful consecutive tries: " + << tries << " (" << elapsed << "s). Network checks advised."; + } + + if (1 < tries) { + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) + << "Retrying agency communication at '" << endpoint + << "', tries: " << tries << " (" + << 1.e-2 * ( + std::round( + 1.e+2 * std::chrono::duration( + std::chrono::steady_clock::now() - started).count())) << "s)"; + } + // here we have failed and want to try next endpoint AgencyCommManager::MANAGER->failed(std::move(connection), endpoint); endpoint.clear(); diff --git a/js/server/tests/resilience/moving-shards-cluster.js b/js/server/tests/resilience/moving-shards-cluster.js index 97f3e81537..2c6dc62f63 100644 --- a/js/server/tests/resilience/moving-shards-cluster.js +++ b/js/server/tests/resilience/moving-shards-cluster.js @@ -163,7 +163,7 @@ function MovingShardsSuite () { } if (!ok) { - console.info( + console.error( "Failed: Server " + id + " was not cleaned out. List of cleaned servers: [" + obj.cleanedServers + "]"); } From 3b2f0bac6aa0a3021d047cb840727fc55eb4051c Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Tue, 31 Jan 2017 10:06:36 +0100 Subject: [PATCH 22/22] remove "junk" from test code --- js/server/tests/aql/aql-optimizer-geoindex.js | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/js/server/tests/aql/aql-optimizer-geoindex.js b/js/server/tests/aql/aql-optimizer-geoindex.js index 76ae0fbb93..ec37685c5d 100644 --- a/js/server/tests/aql/aql-optimizer-geoindex.js +++ b/js/server/tests/aql/aql-optimizer-geoindex.js @@ -53,23 +53,8 @@ function optimizerRuleTestSuite() { sorted : true }; - var ruleName = "use-geoindex"; - var secondRuleName = "use-geoindexes"; - var removeCalculationNodes = "remove-unnecessary-calculations-2"; + var ruleName = "geoindex"; var colName = "UnitTestsAqlOptimizer" + ruleName.replace(/-/g, "_"); - var colNameOther = colName + "_XX"; - - // various choices to control the optimizer: - var paramNone = { optimizer: { rules: [ "-all" ] } }; - var paramIndexFromSort = { optimizer: { rules: [ "-all", "+" + ruleName ] } }; - var paramIndexRange = { optimizer: { rules: [ "-all", "+" + secondRuleName ] } }; - var paramIndexFromSort_IndexRange = { optimizer: { rules: [ "-all", "+" + ruleName, "+" + secondRuleName ] } }; - var paramIndexFromSort_IndexRange_RemoveCalculations = { - optimizer: { rules: [ "-all", "+" + ruleName, "+" + secondRuleName, "+" + removeCalculationNodes ] } - }; - var paramIndexFromSort_RemoveCalculations = { - optimizer: { rules: [ "-all", "+" + ruleName, "+" + removeCalculationNodes ] } - }; var geocol; var sortArray = function (l, r) { @@ -113,19 +98,6 @@ function optimizerRuleTestSuite() { }; var geodistance = function(latitude1, longitude1, latitude2, longitude2) { - //if (TYPEWEIGHT(latitude1) !== TYPEWEIGHT_NUMBER || - // TYPEWEIGHT(longitude1) !== TYPEWEIGHT_NUMBER || - // TYPEWEIGHT(latitude2) !== TYPEWEIGHT_NUMBER || - // TYPEWEIGHT(longitude2) !== TYPEWEIGHT_NUMBER) { - // WARN('DISTANCE', INTERNAL.errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH); - // return null; - //} - - //var p1 = AQL_TO_NUMBER(latitude1) * (Math.PI / 180.0); - //var p2 = AQL_TO_NUMBER(latitude2) * (Math.PI / 180.0); - //var d1 = AQL_TO_NUMBER(latitude2 - latitude1) * (Math.PI / 180.0); - //var d2 = AQL_TO_NUMBER(longitude2 - longitude1) * (Math.PI / 180.0); - var p1 = (latitude1) * (Math.PI / 180.0); var p2 = (latitude2) * (Math.PI / 180.0); var d1 = (latitude2 - latitude1) * (Math.PI / 180.0); @@ -165,7 +137,6 @@ function optimizerRuleTestSuite() { tearDown : function () { internal.db._drop(colName); - internal.db._drop(colNameOther); geocol = null; }, @@ -215,14 +186,6 @@ function optimizerRuleTestSuite() { queries.forEach(function(query) { var result = AQL_EXPLAIN(query.string); - // //optimized on cluster - // if (query[1]) { - // assertNotEqual(-1, removeAlwaysOnClusterRules(result.plan.rules).indexOf(ruleName), query[0]); - // } - // else { - // assertEqual(-1, removeAlwaysOnClusterRules(result.plan.rules).indexOf(ruleName), query[0]); - // } - //sort nodes if (query.sort) { hasSortNode(result,query); @@ -268,7 +231,6 @@ function optimizerRuleTestSuite() { var pairs = result.json.map(function(res){ return [res.lat,res.lon]; }); - //internal.print(pairs) assertEqual(expected[qindex].sort(),pairs.sort()); //expect(expected[qindex].sort()).to.be.equal(result.json.sort()) });