From 45ee2431b35209d78c6792288cc914cd312e1dad Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Tue, 26 Nov 2019 16:25:39 -0500 Subject: [PATCH] Persistent tree beginnings. --- arangod/Replication/DatabaseInitialSyncer.cpp | 2 +- arangod/RestHandler/RestReplicationHandler.cpp | 5 +++-- arangod/RocksDBEngine/RocksDBCollection.cpp | 10 ++++++---- arangod/RocksDBEngine/RocksDBCollection.h | 1 - arangod/StorageEngine/PhysicalCollection.cpp | 2 +- arangod/StorageEngine/PhysicalCollection.h | 2 +- arangod/VocBase/LogicalCollection.cpp | 6 ++++++ arangod/VocBase/LogicalCollection.h | 3 +++ arangod/VocBase/Methods/Collections.cpp | 2 ++ lib/Basics/StaticStrings.cpp | 1 + lib/Basics/StaticStrings.h | 1 + lib/Containers/MerkleTree.cpp | 9 +++++++++ 12 files changed, 34 insertions(+), 10 deletions(-) diff --git a/arangod/Replication/DatabaseInitialSyncer.cpp b/arangod/Replication/DatabaseInitialSyncer.cpp index 39ba81e78e..cfc44e0cd0 100644 --- a/arangod/Replication/DatabaseInitialSyncer.cpp +++ b/arangod/Replication/DatabaseInitialSyncer.cpp @@ -1382,7 +1382,7 @@ Result DatabaseInitialSyncer::fetchCollectionSyncByRevisions(arangodb::LogicalCo // diff with local tree std::pair fullRange = treeMaster->range(); std::unique_ptr treeLocal = - physical->revisionTree(*trx, fullRange.first, fullRange.second); + physical->revisionTree(*trx, fullRange.second); std::vector> ranges = treeMaster->diff(*treeLocal); if (ranges.empty()) { // no differences, done! diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index ff52adc85d..294388ee1e 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -33,6 +33,7 @@ #include "Basics/VelocyPackHelper.h" #include "Basics/WriteLocker.h" #include "Basics/StaticStrings.h" +#include "Basics/hashes.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterHelpers.h" #include "Cluster/ClusterMethods.h" @@ -2901,11 +2902,11 @@ void RestReplicationHandler::handleCommandRevisionTree() { *static_cast(ctx.iter.get()); std::size_t constexpr maxDepth = 6; - std::size_t const rangeMin = it.hasMore() ? it.revision() : 0; + std::size_t const rangeMin = static_cast(ctx.collection->minRevision()); containers::RevisionTree tree(maxDepth, rangeMin); while (it.hasMore()) { - tree.insert(it.revision(), it.document().hashString()); + tree.insert(it.revision(), TRI_FnvHashPod(it.revision())/*it.document().hashString()*/); it.next(); } diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index 0c77adcf45..281ddbe82c 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -27,6 +27,7 @@ #include "Basics/StaticStrings.h" #include "Basics/StringUtils.h" #include "Basics/VelocyPackHelper.h" +#include "Basics/hashes.h" #include "Cache/CacheManagerFeature.h" #include "Cache/Common.h" #include "Cache/Manager.h" @@ -1190,19 +1191,20 @@ Result RocksDBCollection::remove(transaction::Methods& trx, velocypack::Slice sl } std::unique_ptr RocksDBCollection::revisionTree( - transaction::Methods& trx, std::size_t rangeMin, std::size_t rangeMax) { + transaction::Methods& trx, std::size_t rangeMax) { std::unique_ptr iter = getReplicationIterator(ReplicationIterator::Ordering::Revision, trx); RevisionReplicationIterator& it = *static_cast(iter.get()); + TRI_voc_rid_t rangeMin = _logicalCollection.minRevision(); std::size_t constexpr maxDepth = 6; std::unique_ptr tree = - std::make_unique(maxDepth, rangeMin, rangeMax); + std::make_unique(maxDepth, static_cast(rangeMin), rangeMax); - it.seek(static_cast(rangeMin)); + it.seek(rangeMin); while (it.hasMore() && it.revision() <= static_cast(rangeMax)) { - tree->insert(it.revision(), it.document().hashString()); + tree->insert(it.revision(), TRI_FnvHashPod(it.revision())/*it.document().hashString()*/); it.next(); } diff --git a/arangod/RocksDBEngine/RocksDBCollection.h b/arangod/RocksDBEngine/RocksDBCollection.h index 12a275744c..55a4426f4c 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.h +++ b/arangod/RocksDBEngine/RocksDBCollection.h @@ -138,7 +138,6 @@ class RocksDBCollection final : public RocksDBMetaCollection { inline bool cacheEnabled() const { return _cacheEnabled; } std::unique_ptr revisionTree(transaction::Methods& trx, - std::size_t rangeMin, std::size_t rangeMax) override; void adjustNumberDocuments(transaction::Methods&, int64_t) override; diff --git a/arangod/StorageEngine/PhysicalCollection.cpp b/arangod/StorageEngine/PhysicalCollection.cpp index 94baa9de6e..85990f0317 100644 --- a/arangod/StorageEngine/PhysicalCollection.cpp +++ b/arangod/StorageEngine/PhysicalCollection.cpp @@ -529,7 +529,7 @@ Result PhysicalCollection::newObjectForReplace(transaction::Methods*, } std::unique_ptr PhysicalCollection::revisionTree( - transaction::Methods& trx, std::size_t rangeMin, std::size_t rangeMax) { + transaction::Methods& trx, std::size_t rangeMax) { return nullptr; } diff --git a/arangod/StorageEngine/PhysicalCollection.h b/arangod/StorageEngine/PhysicalCollection.h index b67f0c2a87..40c679467f 100644 --- a/arangod/StorageEngine/PhysicalCollection.h +++ b/arangod/StorageEngine/PhysicalCollection.h @@ -223,7 +223,7 @@ class PhysicalCollection { bool isRestore, TRI_voc_rid_t& revisionId) const; virtual std::unique_ptr revisionTree( - transaction::Methods& trx, std::size_t rangeMin, std::size_t rangeMax); + transaction::Methods& trx, std::size_t rangeMax); protected: PhysicalCollection(LogicalCollection& collection, arangodb::velocypack::Slice const& info); diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index ce3e0fa4db..0b4366f371 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -152,6 +152,7 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t& vocbase, VPackSlice const& i #endif _usesRevisionsAsDocumentIds( Helper::getBooleanValue(info, StaticStrings::UsesRevisionsAsDocumentIds, false)), + _minRevision(Helper::getNumericValue(info, StaticStrings::MinRevision, 0)), _waitForSync(Helper::getBooleanValue(info, StaticStrings::WaitForSyncString, false)), _allowUserKeys(Helper::getBooleanValue(info, "allowUserKeys", true)), #ifdef USE_ENTERPRISE @@ -439,6 +440,10 @@ bool LogicalCollection::usesRevisionsAsDocumentIds() const { return _usesRevisionsAsDocumentIds; } +TRI_voc_rid_t LogicalCollection::minRevision() const { + return _minRevision; +} + std::unique_ptr const& LogicalCollection::followers() const { return _followers; } @@ -685,6 +690,7 @@ arangodb::Result LogicalCollection::appendVelocyPack(arangodb::velocypack::Build result.add(StaticStrings::IsSmartChild, VPackValue(isSmartChild())); result.add(StaticStrings::UsesRevisionsAsDocumentIds, VPackValue(usesRevisionsAsDocumentIds())); + result.add(StaticStrings::MinRevision, VPackValue(minRevision())); if (hasSmartJoinAttribute()) { result.add(StaticStrings::SmartJoinAttribute, VPackValue(_smartJoinAttribute)); diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index bc95125dc9..061949869a 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -146,6 +146,7 @@ class LogicalCollection : public LogicalDataSource { bool isSmartChild() const { return false; } #endif bool usesRevisionsAsDocumentIds() const; + TRI_voc_rid_t minRevision() const; /// @brief is this a cluster-wide Plan (ClusterInfo) collection bool isAStub() const { return _isAStub; } /// @brief is this a cluster-wide Plan (ClusterInfo) collection @@ -379,6 +380,8 @@ class LogicalCollection : public LogicalDataSource { bool const _usesRevisionsAsDocumentIds; + TRI_voc_rid_t const _minRevision; + // SECTION: Properties bool _waitForSync; diff --git a/arangod/VocBase/Methods/Collections.cpp b/arangod/VocBase/Methods/Collections.cpp index 53a92ddd94..2d561d93e9 100644 --- a/arangod/VocBase/Methods/Collections.cpp +++ b/arangod/VocBase/Methods/Collections.cpp @@ -294,6 +294,8 @@ Result Collections::create(TRI_vocbase_t& vocbase, if (addUseRevs) { helper.add(arangodb::StaticStrings::UsesRevisionsAsDocumentIds, arangodb::velocypack::Value(useRevs)); + helper.add(arangodb::StaticStrings::MinRevision, + arangodb::velocypack::Value(TRI_HybridLogicalClock())); } if (ServerState::instance()->isCoordinator()) { diff --git a/lib/Basics/StaticStrings.cpp b/lib/Basics/StaticStrings.cpp index 90f13c8d6f..df360e4fd3 100644 --- a/lib/Basics/StaticStrings.cpp +++ b/lib/Basics/StaticStrings.cpp @@ -244,6 +244,7 @@ std::string const StaticStrings::Sharding("sharding"); std::string const StaticStrings::Satellite("satellite"); std::string const StaticStrings::UsesRevisionsAsDocumentIds( "usesRevisionsAsDocumentIds"); +std::string const StaticStrings::MinRevision("minRevision"); // graph attribute names std::string const StaticStrings::GraphCollection("_graphs"); diff --git a/lib/Basics/StaticStrings.h b/lib/Basics/StaticStrings.h index 85e9c9e333..6e266a4a2b 100644 --- a/lib/Basics/StaticStrings.h +++ b/lib/Basics/StaticStrings.h @@ -225,6 +225,7 @@ class StaticStrings { static std::string const Sharding; static std::string const Satellite; static std::string const UsesRevisionsAsDocumentIds; + static std::string const MinRevision; // graph attribute names static std::string const GraphCollection; diff --git a/lib/Containers/MerkleTree.cpp b/lib/Containers/MerkleTree.cpp index 2dfb9c3f88..b6ac069a10 100644 --- a/lib/Containers/MerkleTree.cpp +++ b/lib/Containers/MerkleTree.cpp @@ -37,6 +37,9 @@ #include #include +#include +#include + #include "MerkleTree.h" #include "Basics/debugging.h" @@ -336,6 +339,12 @@ std::size_t MerkleTree::index(std::size_t key, // not thread-safe, lock buffer from outside TRI_ASSERT(depth <= meta().maxDepth); TRI_ASSERT(key >= meta().rangeMin); + if (key >= meta().rangeMax) { + while (true) { + LOG_DEVEL << "WAITING FOR DEBUG"; + std::this_thread::sleep_for(std::chrono::seconds(10)); + } + } TRI_ASSERT(key < meta().rangeMax); // special fast case