From 163b3d45bfd943839f78378fe62abadc24818f57 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Mon, 13 Feb 2017 11:47:22 +0100 Subject: [PATCH 1/6] remove fullyCollected and friends from LogicalCollection --- arangod/MMFiles/MMFilesCollection.cpp | 11 +++++++++- arangod/MMFiles/MMFilesCollection.h | 21 +++++++++++++++++++ arangod/MMFiles/MMFilesCollectorThread.cpp | 3 ++- arangod/MMFiles/MMFilesEngine.cpp | 8 +++---- arangod/MMFiles/MMFilesEngine.h | 2 +- .../MMFiles/MMFilesTransactionCollection.cpp | 4 +++- arangod/MMFiles/MMFilesTransactionState.cpp | 4 +++- arangod/VocBase/LogicalCollection.cpp | 14 +++++++------ arangod/VocBase/LogicalCollection.h | 17 --------------- arangod/VocBase/PhysicalCollection.h | 2 ++ 10 files changed, 54 insertions(+), 32 deletions(-) diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index 43117df386..e94c84c61f 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -295,7 +295,11 @@ bool MMFilesCollection::OpenIterator(TRI_df_marker_t const* marker, MMFilesColle } MMFilesCollection::MMFilesCollection(LogicalCollection* collection) - : PhysicalCollection(collection), _ditches(collection), _initialCount(0), _lastRevision(0) {} + : PhysicalCollection(collection) + , _ditches(collection) + , _initialCount(0), _lastRevision(0) + , _uncollectedLogfileEntries(0) + {} MMFilesCollection::~MMFilesCollection() { try { @@ -1134,6 +1138,11 @@ int MMFilesCollection::iterateMarkersOnLoad(arangodb::Transaction* trx) { return TRI_ERROR_NO_ERROR; } +bool MMFilesCollection::isFullyCollected() const { + int64_t uncollected = _uncollectedLogfileEntries.load(); + return (uncollected == 0); +} + MMFilesDocumentPosition MMFilesCollection::lookupRevision(TRI_voc_rid_t revisionId) const { TRI_ASSERT(revisionId != 0); MMFilesDocumentPosition const old = _revisionsCache.lookup(revisionId); diff --git a/arangod/MMFiles/MMFilesCollection.h b/arangod/MMFiles/MMFilesCollection.h index e8c9acf7f5..dc20ddf4af 100644 --- a/arangod/MMFiles/MMFilesCollection.h +++ b/arangod/MMFiles/MMFilesCollection.h @@ -162,6 +162,24 @@ class MMFilesCollection final : public PhysicalCollection { /// @brief iterate all markers of a collection on load int iterateMarkersOnLoad(arangodb::Transaction* trx) override; + + virtual bool isFullyCollected() const override; + + int64_t uncollectedLogfileEntries() const { + return _uncollectedLogfileEntries.load(); + } + + void increaseUncollectedLogfileEntries(int64_t value) { + _uncollectedLogfileEntries += value; + } + + void decreaseUncollectedLogfileEntries(int64_t value) { + _uncollectedLogfileEntries -= value; + if (_uncollectedLogfileEntries < 0) { + _uncollectedLogfileEntries = 0; + } + } + private: static int OpenIteratorHandleDocumentMarker(TRI_df_marker_t const* marker, @@ -221,6 +239,9 @@ class MMFilesCollection final : public PhysicalCollection { TRI_voc_rid_t _lastRevision; MMFilesRevisionsCache _revisionsCache; + + std::atomic _uncollectedLogfileEntries; + }; } diff --git a/arangod/MMFiles/MMFilesCollectorThread.cpp b/arangod/MMFiles/MMFilesCollectorThread.cpp index f763b4f7a6..83550e8096 100644 --- a/arangod/MMFiles/MMFilesCollectorThread.cpp +++ b/arangod/MMFiles/MMFilesCollectorThread.cpp @@ -30,6 +30,7 @@ #include "Basics/hashes.h" #include "Basics/memory-map.h" #include "Logger/Logger.h" +#include "MMFiles/MMFilesCollection.h" #include "MMFiles/MMFilesDatafileHelper.h" #include "MMFiles/MMFilesLogfileManager.h" #include "MMFiles/MMFilesIndexElement.h" @@ -678,7 +679,7 @@ int MMFilesCollectorThread::processCollectionOperations(MMFilesCollectorCache* c << collection->name() << "'"; updateDatafileStatistics(collection, cache); - collection->decreaseUncollectedLogfileEntries(cache->totalOperationsCount); + static_cast(collection->getPhysical())->decreaseUncollectedLogfileEntries(cache->totalOperationsCount); res = TRI_ERROR_NO_ERROR; } catch (arangodb::basics::Exception const& ex) { diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp index 529af7a81b..3e4e721995 100644 --- a/arangod/MMFiles/MMFilesEngine.cpp +++ b/arangod/MMFiles/MMFilesEngine.cpp @@ -125,10 +125,10 @@ std::string const MMFilesEngine::FeatureName("MMFilesEngine"); // create the storage engine MMFilesEngine::MMFilesEngine(application_features::ApplicationServer* server) - : StorageEngine(server, EngineName, FeatureName, new MMFilesIndexFactory()), - _isUpgrade(false), - _maxTick(0) { -} + : StorageEngine(server, EngineName, FeatureName, new MMFilesIndexFactory()) + , _isUpgrade(false) + , _maxTick(0) + {} MMFilesEngine::~MMFilesEngine() { } diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h index 1fb4178efd..acf3a68236 100644 --- a/arangod/MMFiles/MMFilesEngine.h +++ b/arangod/MMFiles/MMFilesEngine.h @@ -257,7 +257,7 @@ public: /// @brief Add engine specific AQL functions. void addAqlFunctions() const override; - + private: /// @brief: check the initial markers in a datafile bool checkDatafileHeader(MMFilesDatafile* datafile, std::string const& filename) const; diff --git a/arangod/MMFiles/MMFilesTransactionCollection.cpp b/arangod/MMFiles/MMFilesTransactionCollection.cpp index 136752db4c..3a670d2802 100644 --- a/arangod/MMFiles/MMFilesTransactionCollection.cpp +++ b/arangod/MMFiles/MMFilesTransactionCollection.cpp @@ -25,6 +25,7 @@ #include "Basics/Exceptions.h" #include "Logger/Logger.h" #include "MMFiles/MMFilesDocumentOperation.h" +#include "MMFiles/MMFilesCollection.h" #include "StorageEngine/TransactionState.h" #include "Utils/Transaction.h" #include "Utils/TransactionHints.h" @@ -140,7 +141,8 @@ void MMFilesTransactionCollection::freeOperations(Transaction* activeTrx, bool m _collection->setRevision(_originalRevision, true); } else if (!_collection->isVolatile() && !isSingleOperationTransaction) { // only count logfileEntries if the collection is durable - _collection->increaseUncollectedLogfileEntries(_operations->size()); + arangodb::PhysicalCollection* collPtr = _collection->getPhysical(); + static_cast(collPtr)->increaseUncollectedLogfileEntries(_operations->size()); } delete _operations; diff --git a/arangod/MMFiles/MMFilesTransactionState.cpp b/arangod/MMFiles/MMFilesTransactionState.cpp index b7efe712d2..e8265cd790 100644 --- a/arangod/MMFiles/MMFilesTransactionState.cpp +++ b/arangod/MMFiles/MMFilesTransactionState.cpp @@ -25,6 +25,7 @@ #include "Aql/QueryCache.h" #include "Logger/Logger.h" #include "Basics/Exceptions.h" +#include "MMFiles/MMFilesCollection.h" #include "MMFiles/MMFilesDatafileHelper.h" #include "MMFiles/MMFilesDocumentOperation.h" #include "MMFiles/MMFilesLogfileManager.h" @@ -298,7 +299,8 @@ int MMFilesTransactionState::addOperation(TRI_voc_rid_t revisionId, arangodb::aql::QueryCache::instance()->invalidate( _vocbase, collection->name()); - collection->increaseUncollectedLogfileEntries(1); + auto cptr = collection->getPhysical(); + static_cast(cptr)->increaseUncollectedLogfileEntries(1); } else { // operation is buffered and might be rolled back TransactionCollection* trxCollection = this->collection(collection->cid(), AccessMode::Type::WRITE); diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 3e143489bd..bedfe1bc70 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -44,6 +44,7 @@ #include "StorageEngine/EngineSelectorFeature.h" #include "MMFiles/MMFilesDocumentOperation.h" //#include "MMFiles/MMFilesLogfileManager.h" +#include "MMFiles/MMFilesCollection.h" //remove #include "MMFiles/MMFilesPrimaryIndex.h" #include "MMFiles/MMFilesIndexElement.h" #include "MMFiles/MMFilesToken.h" @@ -230,7 +231,6 @@ LogicalCollection::LogicalCollection(LogicalCollection const& other) _nextCompactionStartIndex(0), _lastCompactionStatus(nullptr), _lastCompactionStamp(0.0), - _uncollectedLogfileEntries(0), _isInitialIteration(false), _revisionError(false) { _keyGenerator.reset(KeyGenerator::factory(other.keyOptions())); @@ -295,7 +295,6 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, _nextCompactionStartIndex(0), _lastCompactionStatus(nullptr), _lastCompactionStamp(0.0), - _uncollectedLogfileEntries(0), _isInitialIteration(false), _revisionError(false) { if (!IsAllowedName(info)) { @@ -602,9 +601,7 @@ bool LogicalCollection::IsAllowedName(bool allowSystem, /// @brief whether or not a collection is fully collected bool LogicalCollection::isFullyCollected() { - int64_t uncollected = _uncollectedLogfileEntries.load(); - - return (uncollected == 0); + return getPhysical()->isFullyCollected(); } void LogicalCollection::setNextCompactionStartIndex(size_t index) { @@ -1193,7 +1190,12 @@ std::shared_ptr LogicalCollection::figures() { builder->add("lastTick", VPackValue(_maxTick)); builder->add("uncollectedLogfileEntries", - VPackValue(_uncollectedLogfileEntries)); + VPackValue( + //MOVE TO PHYSICAL + static_cast(getPhysical()) + ->uncollectedLogfileEntries() + ) + ); // fills in compaction status char const* lastCompactionStatus = "-"; diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index edeea98077..d30613e0e1 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -109,21 +109,6 @@ class LogicalCollection { // TODO: MOVE TO PHYSICAL? bool isFullyCollected(); - int64_t uncollectedLogfileEntries() const { - return _uncollectedLogfileEntries.load(); - } - - void increaseUncollectedLogfileEntries(int64_t value) { - _uncollectedLogfileEntries += value; - } - - void decreaseUncollectedLogfileEntries(int64_t value) { - _uncollectedLogfileEntries -= value; - if (_uncollectedLogfileEntries < 0) { - _uncollectedLogfileEntries = 0; - } - } - void setNextCompactionStartIndex(size_t); size_t getNextCompactionStartIndex(); void setCompactionStatus(char const*); @@ -611,8 +596,6 @@ class LogicalCollection { char const* _lastCompactionStatus; double _lastCompactionStamp; - std::atomic _uncollectedLogfileEntries; - /// @brief: flag that is set to true when the documents are /// initial enumerated and the primary index is built bool _isInitialIteration; diff --git a/arangod/VocBase/PhysicalCollection.h b/arangod/VocBase/PhysicalCollection.h index c7251573eb..2469b88dbb 100644 --- a/arangod/VocBase/PhysicalCollection.h +++ b/arangod/VocBase/PhysicalCollection.h @@ -101,6 +101,8 @@ class PhysicalCollection { virtual void updateRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, TRI_voc_fid_t fid, bool isInWal) = 0; virtual bool updateRevisionConditional(TRI_voc_rid_t revisionId, TRI_df_marker_t const* oldPosition, TRI_df_marker_t const* newPosition, TRI_voc_fid_t newFid, bool isInWal) = 0; virtual void removeRevision(TRI_voc_rid_t revisionId, bool updateStats) = 0; + + virtual bool isFullyCollected() const = 0; protected: LogicalCollection* _logicalCollection; From 95615f6ce5c4bda17a24c50ea78e38fafc61560c Mon Sep 17 00:00:00 2001 From: baslr Date: Mon, 13 Feb 2017 14:52:35 +0100 Subject: [PATCH 2/6] arangoexport: continue documentation --- Documentation/Books/Manual/Administration/Arangoexport.mdpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/Books/Manual/Administration/Arangoexport.mdpp b/Documentation/Books/Manual/Administration/Arangoexport.mdpp index e9cf830eed..c1826260d8 100644 --- a/Documentation/Books/Manual/Administration/Arangoexport.mdpp +++ b/Documentation/Books/Manual/Administration/Arangoexport.mdpp @@ -52,6 +52,9 @@ This exports the collection *test* into the output directory *export* as jsonl. Export XGMML ------------ + +[XGMML](https://en.wikipedia.org/wiki/XGMML) is an XML application based on [GML](https://en.wikipedia.org/wiki/Graph_Modelling_Language). To view the XGMML file you can use for example [Cytoscape](http://cytoscape.org). + ## XGMML specific options *--xgmml-label-attribute* specify the name of the attribute that will become the label in the xgmml file. From be38052db16b39c753866403d9384a8f81bd76d7 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 13 Feb 2017 15:15:02 +0100 Subject: [PATCH 3/6] don't allow to run against pre-running server - it may fail --- README_maintainers.md | 5 +++-- utils/generateExamples.js | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README_maintainers.md b/README_maintainers.md index a22f9207d3..f109935efa 100644 --- a/README_maintainers.md +++ b/README_maintainers.md @@ -500,8 +500,9 @@ generate - `./utils/generateExamples.sh --onlyThisOne geoIndexSelect` will only produce one example - *geoIndexSelect* - `./utils/generateExamples.sh --onlyThisOne 'MOD.*'` will only produce the examples matching that regex; Note that examples with enumerations in their name may base on others in their series - so you should generate the whole group. - - `./utils/generateExamples.sh --server.endpoint tcp://127.0.0.1:8529` will utilize an existing arangod instead of starting a new one. - this does seriously cut down the execution time. + - running `onlyThisOne` in conjunction with a pre-started server cuts down the execution time even more. + In addition to the `--onlyThisOne ...` specify i.e. `--server.endpoint tcp://127.0.0.1:8529` to utilize your already running arangod. + Please note that examples may collide with existing collections like 'test' - you need to make sure your server is clean enough. - you can use generateExamples like that: `./utils/generateExamples.sh \ --server.endpoint 'tcp://127.0.0.1:8529' \ diff --git a/utils/generateExamples.js b/utils/generateExamples.js index 88cbebe10f..86d8fc6139 100644 --- a/utils/generateExamples.js +++ b/utils/generateExamples.js @@ -111,8 +111,12 @@ function main(argv) { } if (options.hasOwnProperty('server.endpoint')) { + if (scriptArguments.hasOwnProperty('onlyThisOne')) { + throw("don't run the full suite on pre-existing servers"); + } startServer = false; serverEndpoint = options['server.endpoint']; + } let args = [theScript].concat(internal.toArgv(scriptArguments)); From 82f91cfa1bd28e355e648bfd168e396ac5a5ad15 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Mon, 13 Feb 2017 15:22:38 +0100 Subject: [PATCH 4/6] prepare move of compaction to physicalcollection --- arangod/VocBase/LogicalCollection.h | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index d30613e0e1..3829e8d3db 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -108,13 +108,27 @@ class LogicalCollection { void isInitialIteration(bool value) { _isInitialIteration = value; } // TODO: MOVE TO PHYSICAL? - bool isFullyCollected(); - void setNextCompactionStartIndex(size_t); - size_t getNextCompactionStartIndex(); - void setCompactionStatus(char const*); + bool isFullyCollected(); //should not be exposed + + void setNextCompactionStartIndex(size_t index){ + MUTEX_LOCKER(mutexLocker, _compactionStatusLock); + _nextCompactionStartIndex = index; + } + + size_t getNextCompactionStartIndex(){ + MUTEX_LOCKER(mutexLocker, _compactionStatusLock); + return _nextCompactionStartIndex; + } + + void setCompactionStatus(char const* reason){ + TRI_ASSERT(reason != nullptr); + MUTEX_LOCKER(mutexLocker, _compactionStatusLock); + _lastCompactionStatus = reason; + } double lastCompactionStamp() const { return _lastCompactionStamp; } void lastCompactionStamp(double value) { _lastCompactionStamp = value; } + void setRevisionError() { _revisionError = true; } // SECTION: Meta Information From e3d8f19368ea6997c0790cc28cf3dc153ce9d8d3 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Mon, 13 Feb 2017 15:21:09 +0100 Subject: [PATCH 5/6] Fix unused variables --- arangod/Cluster/ServerState.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arangod/Cluster/ServerState.cpp b/arangod/Cluster/ServerState.cpp index 6a1c6949e3..4053688daf 100644 --- a/arangod/Cluster/ServerState.cpp +++ b/arangod/Cluster/ServerState.cpp @@ -401,12 +401,6 @@ bool ServerState::integrateIntoCluster(ServerState::RoleEnum role, FATAL_ERROR_EXIT(); } - const std::string agencyKey = roleToAgencyKey(role); - const std::string planKey = "Plan/" + agencyKey + "/" + id; - const std::string currentKey = "Current/" + agencyKey + "/" + id; - - _id = id; - findAndSetRoleBlocking(); LOG_TOPIC(DEBUG, Logger::CLUSTER) << "We successfully announced ourselves as " << roleToString(role) << " and our id is " From 41ee789fccd976fe2b6df4aee8cc39c9bb59bc40 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 13 Feb 2017 15:24:30 +0100 Subject: [PATCH 6/6] abort uf submodules aren't properly existing --- 3rdParty/CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/3rdParty/CMakeLists.txt b/3rdParty/CMakeLists.txt index c2eb58d50a..7df396f6d8 100644 --- a/3rdParty/CMakeLists.txt +++ b/3rdParty/CMakeLists.txt @@ -4,6 +4,17 @@ # External Projects used by ArangoDB # ------------------------------------------------------------------------------ + +if(NOT EXISTS V8/v8/LICENSE OR + NOT EXISTS V8/v8/testing/gtest/LICENSE) + message(FATAL_ERROR "GIT sumbodules not checked out properly - aborting! Run: +git submodule update --recursive +git submodule update --init --recursive +On Windows you need to make sure git is recent enough and may create symlinks! +") +endif() + + include(ExternalProject) ################################################################################