diff --git a/3rdParty/CMakeLists.txt b/3rdParty/CMakeLists.txt index c2eb58d50a..7df396f6d8 100644 --- a/3rdParty/CMakeLists.txt +++ b/3rdParty/CMakeLists.txt @@ -4,6 +4,17 @@ # External Projects used by ArangoDB # ------------------------------------------------------------------------------ + +if(NOT EXISTS V8/v8/LICENSE OR + NOT EXISTS V8/v8/testing/gtest/LICENSE) + message(FATAL_ERROR "GIT sumbodules not checked out properly - aborting! Run: +git submodule update --recursive +git submodule update --init --recursive +On Windows you need to make sure git is recent enough and may create symlinks! +") +endif() + + include(ExternalProject) ################################################################################ diff --git a/Documentation/Books/Manual/Administration/Arangoexport.mdpp b/Documentation/Books/Manual/Administration/Arangoexport.mdpp index e9cf830eed..c1826260d8 100644 --- a/Documentation/Books/Manual/Administration/Arangoexport.mdpp +++ b/Documentation/Books/Manual/Administration/Arangoexport.mdpp @@ -52,6 +52,9 @@ This exports the collection *test* into the output directory *export* as jsonl. Export XGMML ------------ + +[XGMML](https://en.wikipedia.org/wiki/XGMML) is an XML application based on [GML](https://en.wikipedia.org/wiki/Graph_Modelling_Language). To view the XGMML file you can use for example [Cytoscape](http://cytoscape.org). + ## XGMML specific options *--xgmml-label-attribute* specify the name of the attribute that will become the label in the xgmml file. diff --git a/README_maintainers.md b/README_maintainers.md index a22f9207d3..f109935efa 100644 --- a/README_maintainers.md +++ b/README_maintainers.md @@ -500,8 +500,9 @@ generate - `./utils/generateExamples.sh --onlyThisOne geoIndexSelect` will only produce one example - *geoIndexSelect* - `./utils/generateExamples.sh --onlyThisOne 'MOD.*'` will only produce the examples matching that regex; Note that examples with enumerations in their name may base on others in their series - so you should generate the whole group. - - `./utils/generateExamples.sh --server.endpoint tcp://127.0.0.1:8529` will utilize an existing arangod instead of starting a new one. - this does seriously cut down the execution time. + - running `onlyThisOne` in conjunction with a pre-started server cuts down the execution time even more. + In addition to the `--onlyThisOne ...` specify i.e. `--server.endpoint tcp://127.0.0.1:8529` to utilize your already running arangod. + Please note that examples may collide with existing collections like 'test' - you need to make sure your server is clean enough. - you can use generateExamples like that: `./utils/generateExamples.sh \ --server.endpoint 'tcp://127.0.0.1:8529' \ diff --git a/arangod/Cluster/ServerState.cpp b/arangod/Cluster/ServerState.cpp index 6a1c6949e3..4053688daf 100644 --- a/arangod/Cluster/ServerState.cpp +++ b/arangod/Cluster/ServerState.cpp @@ -401,12 +401,6 @@ bool ServerState::integrateIntoCluster(ServerState::RoleEnum role, FATAL_ERROR_EXIT(); } - const std::string agencyKey = roleToAgencyKey(role); - const std::string planKey = "Plan/" + agencyKey + "/" + id; - const std::string currentKey = "Current/" + agencyKey + "/" + id; - - _id = id; - findAndSetRoleBlocking(); LOG_TOPIC(DEBUG, Logger::CLUSTER) << "We successfully announced ourselves as " << roleToString(role) << " and our id is " diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index c788491741..1a141b1e9e 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -299,7 +299,11 @@ bool MMFilesCollection::OpenIterator(TRI_df_marker_t const* marker, MMFilesColle } MMFilesCollection::MMFilesCollection(LogicalCollection* collection) - : PhysicalCollection(collection), _ditches(collection), _initialCount(0), _lastRevision(0) {} + : PhysicalCollection(collection) + , _ditches(collection) + , _initialCount(0), _lastRevision(0) + , _uncollectedLogfileEntries(0) + {} MMFilesCollection::~MMFilesCollection() { try { @@ -1241,6 +1245,11 @@ int MMFilesCollection::insert(transaction::Methods* trx, return res; } +bool MMFilesCollection::isFullyCollected() const { + int64_t uncollected = _uncollectedLogfileEntries.load(); + return (uncollected == 0); +} + MMFilesDocumentPosition MMFilesCollection::lookupRevision(TRI_voc_rid_t revisionId) const { TRI_ASSERT(revisionId != 0); MMFilesDocumentPosition const old = _revisionsCache.lookup(revisionId); diff --git a/arangod/MMFiles/MMFilesCollection.h b/arangod/MMFiles/MMFilesCollection.h index 265c630151..a60433e049 100644 --- a/arangod/MMFiles/MMFilesCollection.h +++ b/arangod/MMFiles/MMFilesCollection.h @@ -161,9 +161,6 @@ class MMFilesCollection final : public PhysicalCollection { Ditches* ditches() const override { return &_ditches; } - /// @brief iterate all markers of a collection on load - int iterateMarkersOnLoad(transaction::Methods* trx) override; - //////////////////////////////////// // -- SECTION DML Operations -- /////////////////////////////////// @@ -173,6 +170,26 @@ class MMFilesCollection final : public PhysicalCollection { arangodb::ManagedDocumentResult& result, OperationOptions& options, TRI_voc_tick_t& resultMarkerTick, bool lock) override; + + /// @brief iterate all markers of a collection on load + int iterateMarkersOnLoad(arangodb::transaction::Methods* trx) override; + + virtual bool isFullyCollected() const override; + + int64_t uncollectedLogfileEntries() const { + return _uncollectedLogfileEntries.load(); + } + + void increaseUncollectedLogfileEntries(int64_t value) { + _uncollectedLogfileEntries += value; + } + + void decreaseUncollectedLogfileEntries(int64_t value) { + _uncollectedLogfileEntries -= value; + if (_uncollectedLogfileEntries < 0) { + _uncollectedLogfileEntries = 0; + } + } private: static int OpenIteratorHandleDocumentMarker(TRI_df_marker_t const* marker, @@ -258,6 +275,9 @@ class MMFilesCollection final : public PhysicalCollection { TRI_voc_rid_t _lastRevision; MMFilesRevisionsCache _revisionsCache; + + std::atomic _uncollectedLogfileEntries; + }; } diff --git a/arangod/MMFiles/MMFilesCollectorThread.cpp b/arangod/MMFiles/MMFilesCollectorThread.cpp index ffe4927f31..4afd112008 100644 --- a/arangod/MMFiles/MMFilesCollectorThread.cpp +++ b/arangod/MMFiles/MMFilesCollectorThread.cpp @@ -30,6 +30,7 @@ #include "Basics/hashes.h" #include "Basics/memory-map.h" #include "Logger/Logger.h" +#include "MMFiles/MMFilesCollection.h" #include "MMFiles/MMFilesDatafileHelper.h" #include "MMFiles/MMFilesLogfileManager.h" #include "MMFiles/MMFilesIndexElement.h" @@ -678,7 +679,7 @@ int MMFilesCollectorThread::processCollectionOperations(MMFilesCollectorCache* c << collection->name() << "'"; updateDatafileStatistics(collection, cache); - collection->decreaseUncollectedLogfileEntries(cache->totalOperationsCount); + static_cast(collection->getPhysical())->decreaseUncollectedLogfileEntries(cache->totalOperationsCount); res = TRI_ERROR_NO_ERROR; } catch (arangodb::basics::Exception const& ex) { diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp index 529af7a81b..3e4e721995 100644 --- a/arangod/MMFiles/MMFilesEngine.cpp +++ b/arangod/MMFiles/MMFilesEngine.cpp @@ -125,10 +125,10 @@ std::string const MMFilesEngine::FeatureName("MMFilesEngine"); // create the storage engine MMFilesEngine::MMFilesEngine(application_features::ApplicationServer* server) - : StorageEngine(server, EngineName, FeatureName, new MMFilesIndexFactory()), - _isUpgrade(false), - _maxTick(0) { -} + : StorageEngine(server, EngineName, FeatureName, new MMFilesIndexFactory()) + , _isUpgrade(false) + , _maxTick(0) + {} MMFilesEngine::~MMFilesEngine() { } diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h index 1fb4178efd..acf3a68236 100644 --- a/arangod/MMFiles/MMFilesEngine.h +++ b/arangod/MMFiles/MMFilesEngine.h @@ -257,7 +257,7 @@ public: /// @brief Add engine specific AQL functions. void addAqlFunctions() const override; - + private: /// @brief: check the initial markers in a datafile bool checkDatafileHeader(MMFilesDatafile* datafile, std::string const& filename) const; diff --git a/arangod/MMFiles/MMFilesTransactionCollection.cpp b/arangod/MMFiles/MMFilesTransactionCollection.cpp index d2ec61d848..4633b8bdaf 100644 --- a/arangod/MMFiles/MMFilesTransactionCollection.cpp +++ b/arangod/MMFiles/MMFilesTransactionCollection.cpp @@ -25,6 +25,7 @@ #include "Basics/Exceptions.h" #include "Logger/Logger.h" #include "MMFiles/MMFilesDocumentOperation.h" +#include "MMFiles/MMFilesCollection.h" #include "StorageEngine/TransactionState.h" #include "Transaction/Methods.h" #include "Transaction/Hints.h" @@ -140,7 +141,8 @@ void MMFilesTransactionCollection::freeOperations(transaction::Methods* activeTr _collection->setRevision(_originalRevision, true); } else if (!_collection->isVolatile() && !isSingleOperationTransaction) { // only count logfileEntries if the collection is durable - _collection->increaseUncollectedLogfileEntries(_operations->size()); + arangodb::PhysicalCollection* collPtr = _collection->getPhysical(); + static_cast(collPtr)->increaseUncollectedLogfileEntries(_operations->size()); } delete _operations; diff --git a/arangod/MMFiles/MMFilesTransactionState.cpp b/arangod/MMFiles/MMFilesTransactionState.cpp index 5b3bb1608b..3f66a5fcef 100644 --- a/arangod/MMFiles/MMFilesTransactionState.cpp +++ b/arangod/MMFiles/MMFilesTransactionState.cpp @@ -25,6 +25,7 @@ #include "Aql/QueryCache.h" #include "Logger/Logger.h" #include "Basics/Exceptions.h" +#include "MMFiles/MMFilesCollection.h" #include "MMFiles/MMFilesDatafileHelper.h" #include "MMFiles/MMFilesDocumentOperation.h" #include "MMFiles/MMFilesLogfileManager.h" @@ -298,7 +299,8 @@ int MMFilesTransactionState::addOperation(TRI_voc_rid_t revisionId, arangodb::aql::QueryCache::instance()->invalidate( _vocbase, collection->name()); - collection->increaseUncollectedLogfileEntries(1); + auto cptr = collection->getPhysical(); + static_cast(cptr)->increaseUncollectedLogfileEntries(1); } else { // operation is buffered and might be rolled back TransactionCollection* trxCollection = this->collection(collection->cid(), AccessMode::Type::WRITE); diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index a9da6ed9d7..3ef5b48936 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -44,6 +44,7 @@ #include "StorageEngine/EngineSelectorFeature.h" #include "MMFiles/MMFilesDocumentOperation.h" //#include "MMFiles/MMFilesLogfileManager.h" +#include "MMFiles/MMFilesCollection.h" //remove #include "MMFiles/MMFilesPrimaryIndex.h" #include "MMFiles/MMFilesIndexElement.h" #include "MMFiles/MMFilesToken.h" @@ -230,7 +231,6 @@ LogicalCollection::LogicalCollection(LogicalCollection const& other) _nextCompactionStartIndex(0), _lastCompactionStatus(nullptr), _lastCompactionStamp(0.0), - _uncollectedLogfileEntries(0), _isInitialIteration(false), _revisionError(false) { _keyGenerator.reset(KeyGenerator::factory(other.keyOptions())); @@ -295,7 +295,6 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, _nextCompactionStartIndex(0), _lastCompactionStatus(nullptr), _lastCompactionStamp(0.0), - _uncollectedLogfileEntries(0), _isInitialIteration(false), _revisionError(false) { if (!IsAllowedName(info)) { @@ -602,9 +601,7 @@ bool LogicalCollection::IsAllowedName(bool allowSystem, /// @brief whether or not a collection is fully collected bool LogicalCollection::isFullyCollected() { - int64_t uncollected = _uncollectedLogfileEntries.load(); - - return (uncollected == 0); + return getPhysical()->isFullyCollected(); } void LogicalCollection::setNextCompactionStartIndex(size_t index) { @@ -1193,7 +1190,12 @@ std::shared_ptr LogicalCollection::figures() { builder->add("lastTick", VPackValue(_maxTick)); builder->add("uncollectedLogfileEntries", - VPackValue(_uncollectedLogfileEntries)); + VPackValue( + //MOVE TO PHYSICAL + static_cast(getPhysical()) + ->uncollectedLogfileEntries() + ) + ); // fills in compaction status char const* lastCompactionStatus = "-"; diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index 925ffc0d8a..55cfdc3d69 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -111,28 +111,27 @@ class LogicalCollection { void isInitialIteration(bool value) { _isInitialIteration = value; } // TODO: MOVE TO PHYSICAL? - bool isFullyCollected(); - int64_t uncollectedLogfileEntries() const { - return _uncollectedLogfileEntries.load(); + bool isFullyCollected(); //should not be exposed + + void setNextCompactionStartIndex(size_t index){ + MUTEX_LOCKER(mutexLocker, _compactionStatusLock); + _nextCompactionStartIndex = index; } - void increaseUncollectedLogfileEntries(int64_t value) { - _uncollectedLogfileEntries += value; + size_t getNextCompactionStartIndex(){ + MUTEX_LOCKER(mutexLocker, _compactionStatusLock); + return _nextCompactionStartIndex; } - void decreaseUncollectedLogfileEntries(int64_t value) { - _uncollectedLogfileEntries -= value; - if (_uncollectedLogfileEntries < 0) { - _uncollectedLogfileEntries = 0; - } + void setCompactionStatus(char const* reason){ + TRI_ASSERT(reason != nullptr); + MUTEX_LOCKER(mutexLocker, _compactionStatusLock); + _lastCompactionStatus = reason; } - - void setNextCompactionStartIndex(size_t); - size_t getNextCompactionStartIndex(); - void setCompactionStatus(char const*); double lastCompactionStamp() const { return _lastCompactionStamp; } void lastCompactionStamp(double value) { _lastCompactionStamp = value; } + void setRevisionError() { _revisionError = true; } // SECTION: Meta Information @@ -615,8 +614,6 @@ class LogicalCollection { char const* _lastCompactionStatus; double _lastCompactionStamp; - std::atomic _uncollectedLogfileEntries; - /// @brief: flag that is set to true when the documents are /// initial enumerated and the primary index is built bool _isInitialIteration; diff --git a/arangod/VocBase/PhysicalCollection.h b/arangod/VocBase/PhysicalCollection.h index 4f3bb2b9b0..0fe672b064 100644 --- a/arangod/VocBase/PhysicalCollection.h +++ b/arangod/VocBase/PhysicalCollection.h @@ -106,6 +106,8 @@ class PhysicalCollection { virtual void updateRevision(TRI_voc_rid_t revisionId, uint8_t const* dataptr, TRI_voc_fid_t fid, bool isInWal) = 0; virtual bool updateRevisionConditional(TRI_voc_rid_t revisionId, TRI_df_marker_t const* oldPosition, TRI_df_marker_t const* newPosition, TRI_voc_fid_t newFid, bool isInWal) = 0; virtual void removeRevision(TRI_voc_rid_t revisionId, bool updateStats) = 0; + + virtual bool isFullyCollected() const = 0; virtual int insert(arangodb::transaction::Methods* trx, arangodb::velocypack::Slice const newSlice, diff --git a/utils/generateExamples.js b/utils/generateExamples.js index 88cbebe10f..86d8fc6139 100644 --- a/utils/generateExamples.js +++ b/utils/generateExamples.js @@ -111,8 +111,12 @@ function main(argv) { } if (options.hasOwnProperty('server.endpoint')) { + if (scriptArguments.hasOwnProperty('onlyThisOne')) { + throw("don't run the full suite on pre-existing servers"); + } startServer = false; serverEndpoint = options['server.endpoint']; + } let args = [theScript].concat(internal.toArgv(scriptArguments));