diff --git a/CHANGELOG b/CHANGELOG index 7f648b47d0..aecfbee00b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ devel ----- +* fix potential port number over-/underruns + * added startup option `--log.shorten-filenames` for controlling whether filenames in log message should be shortened to just the filename with the absolute path diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c182c80f8..59d019f9ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -490,6 +490,8 @@ if (USE_MAINTAINER_MODE) find_program(AWK_EXECUTABLE awk) endif () +find_program(FILE_EXECUTABLE file) + ################################################################################ ## FAILURE TESTS ################################################################################ diff --git a/Documentation/Books/Manual/Indexing/Geo.mdpp b/Documentation/Books/Manual/Indexing/Geo.mdpp index 2357cd97d3..99212e8fe1 100644 --- a/Documentation/Books/Manual/Indexing/Geo.mdpp +++ b/Documentation/Books/Manual/Indexing/Geo.mdpp @@ -90,6 +90,42 @@ Create a geo index for a hash array attribute: @END_EXAMPLE_ARANGOSH_OUTPUT @endDocuBlock geoIndexCreateForArrayAttribute2 +Use GeoIndex with AQL SORT statement: + + @startDocuBlockInline geoIndexSortOptimization + @EXAMPLE_ARANGOSH_OUTPUT{geoIndexSortOptimization} + ~db._create("geoSort") + db.geoSort.ensureIndex({ type: "geo", fields: [ "latitude", "longitude" ] }); + | for (i = -90; i <= 90; i += 10) { + | for (j = -180; j <= 180; j += 10) { + | db.geoSort.save({ name : "Name/" + i + "/" + j, latitude : i, longitude : j }); + | } + } + var query = "FOR doc in geoSort SORT distance(doc.latitude, doc.longitude, 0, 0) LIMIT 5 RETURN doc" + db._explain(query, {}, {colors: false}); + db._query(query); + ~db._drop("geoSort") + @END_EXAMPLE_ARANGOSH_OUTPUT + @endDocuBlock geoIndexSortOptimization + +Use GeoIndex with AQL FILTER statement: + + @startDocuBlockInline geoIndexFilterOptimization + @EXAMPLE_ARANGOSH_OUTPUT{geoIndexFilterOptimization} + ~db._create("geoFilter") + db.geoFilter.ensureIndex({ type: "geo", fields: [ "latitude", "longitude" ] }); + | for (i = -90; i <= 90; i += 10) { + | for (j = -180; j <= 180; j += 10) { + | db.geoFilter.save({ name : "Name/" + i + "/" + j, latitude : i, longitude : j }); + | } + } + var query = "FOR doc in geoFilter FILTER distance(doc.latitude, doc.longitude, 0, 0) < 2000 RETURN doc" + db._explain(query, {}, {colors: false}); + db._query(query); + ~db._drop("geoFilter") + @END_EXAMPLE_ARANGOSH_OUTPUT + @endDocuBlock geoIndexFilterOptimization + @startDocuBlock collectionGeo diff --git a/Documentation/Books/Manual/Indexing/IndexBasics.mdpp b/Documentation/Books/Manual/Indexing/IndexBasics.mdpp index ffd5b1d413..93c255ba2d 100644 --- a/Documentation/Books/Manual/Indexing/IndexBasics.mdpp +++ b/Documentation/Books/Manual/Indexing/IndexBasics.mdpp @@ -273,8 +273,10 @@ The geo index provides operations to find documents with coordinates nearest to comparison coordinate, and to find documents with coordinates that are within a specifiable radius around a comparison coordinate. -The geo index is used via dedicated functions in AQL or the simple queries functions, -but will not be used for other types of queries or conditions. +The geo index is used via dedicated functions in AQL, the simple queries +functions and it is implicitly applied when in AQL a SORT or FILTER is used with +the distance function. Otherwise it will not be used for other types of queries +or conditions. ### Fulltext Index diff --git a/Documentation/Books/Manual/Indexing/WhichIndex.mdpp b/Documentation/Books/Manual/Indexing/WhichIndex.mdpp index b1aead5a0c..a5be484995 100644 --- a/Documentation/Books/Manual/Indexing/WhichIndex.mdpp +++ b/Documentation/Books/Manual/Indexing/WhichIndex.mdpp @@ -71,7 +71,10 @@ different usage scenarios: { "coords": [ 50.9406645, 6.9599115 ] } - Geo indexes will only be invoked via special functions. + Geo indexes will be invoked via special functions or AQL optimization. The + optimization can be triggered when a collection with geo index is enumerated + and a SORT or FILTER statement is used in conjunction with the distance + function. - fulltext index: a fulltext index can be used to index all words contained in a specific attribute of all documents in a collection. Only words with a diff --git a/UnitTests/Basics/EndpointTest.cpp b/UnitTests/Basics/EndpointTest.cpp index b8b3f2dbfd..5feedae00a 100644 --- a/UnitTests/Basics/EndpointTest.cpp +++ b/UnitTests/Basics/EndpointTest.cpp @@ -49,8 +49,6 @@ BOOST_TEST_DONT_PRINT_LOG_VALUE(arangodb::Endpoint::EndpointType) // --SECTION-- macros // ----------------------------------------------------------------------------- -#define DELETE_ENDPOINT(e) if (e != 0) delete e; - #define FACTORY_NAME(name) name ## Factory #define FACTORY(name, specification) arangodb::Endpoint::FACTORY_NAME(name)(specification) @@ -58,12 +56,12 @@ BOOST_TEST_DONT_PRINT_LOG_VALUE(arangodb::Endpoint::EndpointType) #define CHECK_ENDPOINT_FEATURE(type, specification, feature, expected) \ e = FACTORY(type, specification); \ BOOST_CHECK_EQUAL((expected), (e->feature())); \ - DELETE_ENDPOINT(e); + delete e; #define CHECK_ENDPOINT_SERVER_FEATURE(type, specification, feature, expected) \ e = arangodb::Endpoint::serverFactory(specification, 1, true); \ BOOST_CHECK_EQUAL((expected), (e->feature())); \ - DELETE_ENDPOINT(e); + delete e; // ----------------------------------------------------------------------------- // --SECTION-- setup / tear-down @@ -118,6 +116,11 @@ BOOST_AUTO_TEST_CASE (EndpointInvalid) { BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("ssl@tcp://127.0.0.1:8529")); BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("https@tcp://127.0.0.1:8529")); BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("https@tcp://127.0.0.1:")); + + BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:65536")); + BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:65537")); + BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:-1")); + BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:6555555555")); } //////////////////////////////////////////////////////////////////////////////// @@ -491,7 +494,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedServer1) { e = arangodb::Endpoint::serverFactory("tcp://127.0.0.1", 1, true); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } //////////////////////////////////////////////////////////////////////////////// @@ -503,7 +506,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedServer2) { e = arangodb::Endpoint::serverFactory("ssl://127.0.0.1", 1, true); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } //////////////////////////////////////////////////////////////////////////////// @@ -516,7 +519,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedServer3) { e = arangodb::Endpoint::serverFactory("unix:///tmp/socket", 1, true); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } #endif @@ -529,7 +532,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedClient1) { e = arangodb::Endpoint::clientFactory("tcp://127.0.0.1"); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } //////////////////////////////////////////////////////////////////////////////// @@ -541,7 +544,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedClient2) { e = arangodb::Endpoint::clientFactory("ssl://127.0.0.1"); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } //////////////////////////////////////////////////////////////////////////////// @@ -554,7 +557,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedClient3) { e = arangodb::Endpoint::clientFactory("unix:///tmp/socket"); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } #endif @@ -575,7 +578,7 @@ BOOST_AUTO_TEST_CASE (EndpointServerTcpIpv4WithPort) { BOOST_CHECK_EQUAL(667, e->port()); BOOST_CHECK_EQUAL("127.0.0.1:667", e->hostAndPort()); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } //////////////////////////////////////////////////////////////////////////////// @@ -596,7 +599,7 @@ BOOST_AUTO_TEST_CASE (EndpointServerUnix) { BOOST_CHECK_EQUAL(0, e->port()); BOOST_CHECK_EQUAL("localhost", e->hostAndPort()); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } #endif @@ -617,7 +620,7 @@ BOOST_AUTO_TEST_CASE (EndpointClientSslIpV6WithPortHttp) { BOOST_CHECK_EQUAL(43425, e->port()); BOOST_CHECK_EQUAL("[0001:0002:0003:0004:0005:0006:0007:0008]:43425", e->hostAndPort()); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } //////////////////////////////////////////////////////////////////////////////// @@ -637,7 +640,7 @@ BOOST_AUTO_TEST_CASE (EndpointClientTcpIpv6WithoutPort) { BOOST_CHECK_EQUAL(8529, e->port()); BOOST_CHECK_EQUAL("[::]:8529", e->hostAndPort()); BOOST_CHECK_EQUAL(false, e->isConnected()); - DELETE_ENDPOINT(e); + delete e; } BOOST_AUTO_TEST_SUITE_END() diff --git a/UnitTests/Basics/files-test.cpp b/UnitTests/Basics/files-test.cpp index abcb98ca69..39ca381a58 100644 --- a/UnitTests/Basics/files-test.cpp +++ b/UnitTests/Basics/files-test.cpp @@ -38,6 +38,7 @@ using namespace arangodb::basics; static bool Initialized = false; +static uint64_t counter = 0; // ----------------------------------------------------------------------------- // --SECTION-- setup / tear-down @@ -73,8 +74,6 @@ struct CFilesSetup { } StringBuffer* writeFile (const char* blob) { - static uint64_t counter = 0; - StringBuffer* filename = new StringBuffer(TRI_UNKNOWN_MEM_ZONE); filename->appendText(_directory); filename->appendText("/tmp-"); @@ -108,6 +107,71 @@ struct CFilesSetup { BOOST_FIXTURE_TEST_SUITE(CFilesTest, CFilesSetup) +BOOST_AUTO_TEST_CASE (tst_createdirectory) { + std::ostringstream out; + out << _directory.c_str() << "/tmp-" << ++counter << "-dir"; + + std::string filename = out.str(); + long unused1; + std::string unused2; + int res = TRI_CreateDirectory(filename.c_str(), unused1, unused2); + BOOST_CHECK_EQUAL(0, res); + BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename.c_str())); + BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename.c_str())); + + res = TRI_RemoveDirectory(filename.c_str()); + BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename.c_str())); + BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename.c_str())); +} + +BOOST_AUTO_TEST_CASE (tst_createdirectoryrecursive) { + std::ostringstream out; + out << _directory.c_str() << "/tmp-" << ++counter << "-dir"; + + std::string filename1 = out.str(); + out << "/abc"; + std::string filename2 = out.str(); + + long unused1; + std::string unused2; + int res = TRI_CreateRecursiveDirectory(filename2.c_str(), unused1, unused2); + BOOST_CHECK_EQUAL(0, res); + BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename1.c_str())); + BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename1.c_str())); + BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename2.c_str())); + BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename2.c_str())); + + res = TRI_RemoveDirectory(filename1.c_str()); + BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename1.c_str())); + BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename1.c_str())); + BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename2.c_str())); + BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename2.c_str())); +} + +BOOST_AUTO_TEST_CASE (tst_removedirectorydeterministic) { + std::ostringstream out; + out << _directory.c_str() << "/tmp-" << ++counter << "-dir"; + + std::string filename1 = out.str(); + out << "/abc"; + std::string filename2 = out.str(); + + long unused1; + std::string unused2; + int res = TRI_CreateRecursiveDirectory(filename2.c_str(), unused1, unused2); + BOOST_CHECK_EQUAL(0, res); + BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename1.c_str())); + BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename1.c_str())); + BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename2.c_str())); + BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename2.c_str())); + + res = TRI_RemoveDirectoryDeterministic(filename1.c_str()); + BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename1.c_str())); + BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename1.c_str())); + BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename2.c_str())); + BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename2.c_str())); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief test file exists //////////////////////////////////////////////////////////////////////////////// @@ -116,6 +180,7 @@ BOOST_AUTO_TEST_CASE (tst_existsfile) { StringBuffer* filename = writeFile(""); BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename->c_str())); TRI_UnlinkFile(filename->c_str()); + BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename->c_str())); delete filename; } diff --git a/arangod/Agency/AddFollower.cpp b/arangod/Agency/AddFollower.cpp index da7e50082d..5e7a4f283e 100644 --- a/arangod/Agency/AddFollower.cpp +++ b/arangod/Agency/AddFollower.cpp @@ -25,11 +25,7 @@ #include "Agency/Agent.h" #include "Agency/Job.h" -#include -#include - using namespace arangodb::consensus; -using namespace arangodb::velocypack; AddFollower::AddFollower(Node const& snapshot, Agent* agent, std::string const& jobId, std::string const& creator, diff --git a/arangod/Agency/AgencyComm.cpp b/arangod/Agency/AgencyComm.cpp index bdf56bf018..f2a867a6c4 100644 --- a/arangod/Agency/AgencyComm.cpp +++ b/arangod/Agency/AgencyComm.cpp @@ -1,4 +1,4 @@ -/////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// /// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany @@ -54,7 +54,6 @@ using namespace arangodb; using namespace arangodb::application_features; -using namespace arangodb::basics; using namespace arangodb::httpclient; using namespace arangodb::rest; @@ -450,7 +449,7 @@ std::string AgencyCommManager::path(std::string const& p1) { return ""; } - return MANAGER->_prefix + "/" + StringUtils::trim(p1, "/"); + return MANAGER->_prefix + "/" + basics::StringUtils::trim(p1, "/"); } std::string AgencyCommManager::path(std::string const& p1, @@ -459,8 +458,8 @@ std::string AgencyCommManager::path(std::string const& p1, return ""; } - return MANAGER->_prefix + "/" + StringUtils::trim(p1, "/") + "/" + - StringUtils::trim(p2, "/"); + return MANAGER->_prefix + "/" + basics::StringUtils::trim(p1, "/") + "/" + + basics::StringUtils::trim(p2, "/"); } std::string AgencyCommManager::generateStamp() { @@ -674,7 +673,7 @@ void AgencyCommManager::removeEndpoint(std::string const& endpoint) { } std::string AgencyCommManager::endpointsString() const { - return StringUtils::join(endpoints(), ", "); + return basics::StringUtils::join(endpoints(), ", "); } std::vector AgencyCommManager::endpoints() const { @@ -1280,7 +1279,7 @@ void AgencyComm::updateEndpoints(arangodb::velocypack::Slice const& current) { for (const auto& i : VPackObjectIterator(current)) { auto const endpoint = Endpoint::unifiedForm(i.value.copyString()); if (std::find(stored.begin(), stored.end(), endpoint) == stored.end()) { - LOG_TOPIC(INFO, Logger::CLUSTER) + LOG_TOPIC(DEBUG, Logger::CLUSTER) << "Adding endpoint " << endpoint << " to agent pool"; AgencyCommManager::MANAGER->addEndpoint(endpoint); } @@ -1391,7 +1390,7 @@ AgencyCommResult AgencyComm::sendWithFailover( b.add(VPackValue(clientId)); } - LOG_TOPIC(INFO, Logger::AGENCYCOMM) << + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << "Failed agency comm (" << result._statusCode << ")! " << "Inquiring about clientId " << clientId << "."; @@ -1410,25 +1409,25 @@ AgencyCommResult AgencyComm::sendWithFailover( for (auto const& i : VPackArrayIterator(inner)) { if (i.isUInt()) { if (i.getUInt() == 0) { - LOG_TOPIC(INFO, Logger::AGENCYCOMM) + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << body << " failed: " << outer.toJson(); return result; } else { success = true; } } else { - LOG_TOPIC(INFO, Logger::AGENCYCOMM) + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << body << " failed with " << outer.toJson(); } } } } if (success) { - LOG_TOPIC(INFO, Logger::AGENCYCOMM) + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << body << " succeeded (" << outer.toJson() << ")"; return inq; } else { - LOG_TOPIC(INFO, Logger::AGENCYCOMM) + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << body << " failed (" << outer.toJson() << ")"; return result; } @@ -1437,7 +1436,7 @@ AgencyCommResult AgencyComm::sendWithFailover( } return inq; } else { - LOG_TOPIC(INFO, Logger::AGENCYCOMM) << + LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) << "Inquiry failed (" << inq._statusCode << "). Keep trying ..."; continue; } diff --git a/arangod/Agency/AgencyFeature.cpp b/arangod/Agency/AgencyFeature.cpp index 74057e4475..74edfd92e6 100644 --- a/arangod/Agency/AgencyFeature.cpp +++ b/arangod/Agency/AgencyFeature.cpp @@ -232,7 +232,7 @@ void AgencyFeature::start() { _agent.reset(new consensus::Agent(consensus::config_t( _size, _poolSize, _minElectionTimeout, _maxElectionTimeout, endpoint, - _agencyEndpoints, _supervision, _waitForSync, _supervisionFrequency, + _agencyEndpoints, _supervision, false, _supervisionFrequency, _compactionStepSize, _compactionKeepSize, _supervisionGracePeriod, _cmdLineTimings))); diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp index 291666392d..78ada76632 100644 --- a/arangod/Agency/Agent.cpp +++ b/arangod/Agency/Agent.cpp @@ -257,7 +257,7 @@ bool Agent::recvAppendEntriesRPC( term_t term, std::string const& leaderId, index_t prevIndex, term_t prevTerm, index_t leaderCommitIndex, query_t const& queries) { - LOG_TOPIC(DEBUG, Logger::AGENCY) << "Got AppendEntriesRPC from " + LOG_TOPIC(TRACE, Logger::AGENCY) << "Got AppendEntriesRPC from " << leaderId << " with term " << term; // Update commit index @@ -276,40 +276,42 @@ bool Agent::recvAppendEntriesRPC( size_t nqs = queries->slice().length(); // State machine, _lastCommitIndex to advance atomically - MUTEX_LOCKER(mutexLocker, _ioLock); - if (nqs > 0) { + + MUTEX_LOCKER(mutexLocker, _ioLock); + size_t ndups = _state.removeConflicts(queries); - + if (nqs > ndups) { - LOG_TOPIC(TRACE, Logger::AGENCY) + LOG_TOPIC(DEBUG, Logger::AGENCY) << "Appending " << nqs - ndups << " entries to state machine. (" << nqs << ", " << ndups << ")"; - + try { - _state.log(queries, ndups); + + auto last = _state.log(queries, ndups); + + _spearhead.apply( + _state.slices(last-ndups, last), last, _constituent.term()); + + _readDB.apply( + _state.slices(last-ndups, last), last, _constituent.term()); + + _lastCommitIndex = last; + + if (_lastCommitIndex >= _nextCompationAfter) { + _state.compact(_lastCommitIndex); + _nextCompationAfter += _config.compactionStepSize(); + } + } catch (std::exception const&) { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Malformed query: " << __FILE__ << __LINE__; } + } } - - - _spearhead.apply( - _state.slices(_lastCommitIndex + 1, leaderCommitIndex), _lastCommitIndex, - _constituent.term()); - _readDB.apply( - _state.slices(_lastCommitIndex + 1, leaderCommitIndex), _lastCommitIndex, - _constituent.term()); - - _lastCommitIndex = leaderCommitIndex; - - if (_lastCommitIndex >= _nextCompationAfter) { - _state.compact(_lastCommitIndex); - _nextCompationAfter += _config.compactionStepSize(); - } return true; } @@ -348,7 +350,7 @@ void Agent::sendAppendEntriesRPC() { duration m = system_clock::now() - _lastSent[followerId]; if (highest == _lastHighest[followerId] && - m.count() < 0.5 * _config.minPing()) { + m.count() < 0.25 * _config.minPing()) { continue; } diff --git a/arangod/Agency/CleanOutServer.cpp b/arangod/Agency/CleanOutServer.cpp index 9fa87616b0..a49ce97f8c 100644 --- a/arangod/Agency/CleanOutServer.cpp +++ b/arangod/Agency/CleanOutServer.cpp @@ -28,7 +28,6 @@ #include "Agency/MoveShard.h" using namespace arangodb::consensus; -using namespace arangodb::velocypack; CleanOutServer::CleanOutServer(Node const& snapshot, Agent* agent, std::string const& jobId, diff --git a/arangod/Agency/Constituent.cpp b/arangod/Agency/Constituent.cpp index a0b303eb03..d2a03cc8dc 100644 --- a/arangod/Agency/Constituent.cpp +++ b/arangod/Agency/Constituent.cpp @@ -468,11 +468,13 @@ void Constituent::callElection() { void Constituent::update(std::string const& leaderID, term_t t) { MUTEX_LOCKER(guard, _castLock); _term = t; + if (_leaderID != leaderID) { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Constituent::update: setting _leaderID to " << leaderID << " in term " << _term; _leaderID = leaderID; + _role = FOLLOWER; } } @@ -546,6 +548,11 @@ void Constituent::run() { LOG_TOPIC(DEBUG, Logger::AGENCY) << "Set _leaderID to " << _leaderID << " in term " << _term; } else { + + { + MUTEX_LOCKER(guard, _castLock); + _role = FOLLOWER; + } while (!this->isStopping()) { if (_role == FOLLOWER) { static double const M = 1.0e6; diff --git a/arangod/Agency/FailedFollower.cpp b/arangod/Agency/FailedFollower.cpp index f76cf6e162..2f7ed71078 100644 --- a/arangod/Agency/FailedFollower.cpp +++ b/arangod/Agency/FailedFollower.cpp @@ -75,7 +75,7 @@ bool FailedFollower::create() { } } - _jb = std::make_shared(); + _jb = std::make_shared(); _jb->openArray(); _jb->openObject(); @@ -128,7 +128,7 @@ bool FailedFollower::start() { // Copy todo to pending - velocypack::Builder todo, pending; + Builder todo, pending; // Get todo entry todo.openArray(); @@ -254,7 +254,7 @@ JOB_STATUS FailedFollower::status() { if (compareServerLists(planned.slice(), current.slice())) { // Remove shard from /arango/Target/FailedServers/ array - velocypack::Builder del; + Builder del; del.openArray(); del.openObject(); std::string path = _agencyPrefix + failedServersPrefix + "/" + _from; diff --git a/arangod/Agency/FailedLeader.cpp b/arangod/Agency/FailedLeader.cpp index 28dcb016d8..afad0e5c37 100644 --- a/arangod/Agency/FailedLeader.cpp +++ b/arangod/Agency/FailedLeader.cpp @@ -27,7 +27,6 @@ #include "Agency/Job.h" using namespace arangodb::consensus; -using namespace arangodb::velocypack; FailedLeader::FailedLeader(Node const& snapshot, Agent* agent, std::string const& jobId, std::string const& creator, diff --git a/arangod/Agency/FailedServer.cpp b/arangod/Agency/FailedServer.cpp index 2821198a6b..79f63d0164 100644 --- a/arangod/Agency/FailedServer.cpp +++ b/arangod/Agency/FailedServer.cpp @@ -58,7 +58,7 @@ bool FailedServer::start() { << "Start FailedServer job " + _jobId + " for server " + _server; // Copy todo to pending - velocypack::Builder todo, pending; + Builder todo, pending; // Get todo entry todo.openArray(); @@ -210,7 +210,7 @@ bool FailedServer::create() { std::string path = _agencyPrefix + toDoPrefix + _jobId; - _jb = std::make_shared(); + _jb = std::make_shared(); _jb->openArray(); _jb->openObject(); @@ -271,7 +271,7 @@ JOB_STATUS FailedServer::status() { // mop: ohhh...server is healthy again! bool serverHealthy = serverHealth == Supervision::HEALTH_STATUS_GOOD; - std::shared_ptr deleteTodos; + std::shared_ptr deleteTodos; Node::Children const todos = _snapshot(toDoPrefix).children(); Node::Children const pends = _snapshot(pendingPrefix).children(); @@ -281,7 +281,7 @@ JOB_STATUS FailedServer::status() { if (!subJob.first.compare(0, _jobId.size() + 1, _jobId + "-")) { if (serverHealthy) { if (!deleteTodos) { - deleteTodos.reset(new velocypack::Builder()); + deleteTodos.reset(new Builder()); deleteTodos->openArray(); deleteTodos->openObject(); } diff --git a/arangod/Agency/Inception.cpp b/arangod/Agency/Inception.cpp index 4e56a74dc5..f355f789f5 100644 --- a/arangod/Agency/Inception.cpp +++ b/arangod/Agency/Inception.cpp @@ -66,7 +66,7 @@ void Inception::gossip() { auto const version = config.version(); // Build gossip message - auto out = std::make_shared(); + auto out = std::make_shared(); out->openObject(); out->add("endpoint", VPackValue(config.endpoint())); out->add("id", VPackValue(config.id())); @@ -169,7 +169,7 @@ bool Inception::restartingActiveAgent() { auto const& clientEp = myConfig.endpoint(); auto const majority = (myConfig.size()+1)/2; - velocypack::Builder greeting; + Builder greeting; { VPackObjectBuilder b(&greeting); greeting.add(clientId, VPackValue(clientEp)); @@ -259,7 +259,7 @@ bool Inception::restartingActiveAgent() { } } - auto agency = std::make_shared(); + auto agency = std::make_shared(); agency->openObject(); agency->add("term", theirConfig.get("term")); agency->add("id", VPackValue(theirLeaderId)); @@ -435,7 +435,7 @@ bool Inception::estimateRAFTInterval() { LOG_TOPIC(DEBUG, Logger::AGENCY) << "mean(" << mean << ") stdev(" << stdev<< ")"; - velocypack::Builder measurement; + Builder measurement; measurement.openObject(); measurement.add("mean", VPackValue(mean)); measurement.add("stdev", VPackValue(stdev)); @@ -541,8 +541,10 @@ void Inception::run() { LOG_TOPIC(INFO, Logger::AGENCY) << "Activating agent."; _agent->ready(true); } else { + if (!this->isStopping()) { LOG_TOPIC(FATAL, Logger::AGENCY) << "Unable to restart with persisted pool. Fatal exit."; + } FATAL_ERROR_EXIT(); // FATAL ERROR } diff --git a/arangod/Agency/Job.cpp b/arangod/Agency/Job.cpp index 6a9f401a6a..24e56d14f6 100644 --- a/arangod/Agency/Job.cpp +++ b/arangod/Agency/Job.cpp @@ -25,7 +25,7 @@ using namespace arangodb::consensus; -bool arangodb::consensus::compareServerLists(velocypack::Slice plan, velocypack::Slice current) { +bool arangodb::consensus::compareServerLists(Slice plan, Slice current) { if (!plan.isArray() || !current.isArray()) { return false; } @@ -80,7 +80,7 @@ JOB_STATUS Job::exists() const { bool Job::finish(std::string const& type, bool success, std::string const& reason) const { - velocypack::Builder pending, finished; + Builder pending, finished; // Get todo entry pending.openArray(); diff --git a/arangod/Agency/Job.h b/arangod/Agency/Job.h index 4ae6ea5d4e..8826f90de1 100644 --- a/arangod/Agency/Job.h +++ b/arangod/Agency/Job.h @@ -28,7 +28,6 @@ #include "Node.h" #include "Supervision.h" -#include #include #include #include @@ -42,7 +41,7 @@ namespace consensus { // and all others followers. Both arguments must be arrays. Returns true, // if the first items in both slice are equal and if both arrays contain // the same set of strings. -bool compareServerLists(velocypack::Slice plan, velocypack::Slice current); +bool compareServerLists(Slice plan, Slice current); enum JOB_STATUS { TODO, PENDING, FINISHED, FAILED, NOTFOUND }; const std::vector pos({"/Target/ToDo/", "/Target/Pending/", @@ -64,9 +63,9 @@ static std::string const plannedServers = "/Plan/DBServers"; static std::string const healthPrefix = "/Supervision/Health/"; inline arangodb::consensus::write_ret_t transact(Agent* _agent, - velocypack::Builder const& transaction, + Builder const& transaction, bool waitForCommit = true) { - query_t envelope = std::make_shared(); + query_t envelope = std::make_shared(); try { envelope->openArray(); @@ -138,7 +137,7 @@ struct Job { std::string _creator; std::string _agencyPrefix; - std::shared_ptr _jb; + std::shared_ptr _jb; }; diff --git a/arangod/Agency/MoveShard.cpp b/arangod/Agency/MoveShard.cpp index 7a10d2740b..c38a08686d 100644 --- a/arangod/Agency/MoveShard.cpp +++ b/arangod/Agency/MoveShard.cpp @@ -29,7 +29,6 @@ static std::string const DBServer = "DBServer"; using namespace arangodb::consensus; -using namespace arangodb::velocypack; MoveShard::MoveShard(Node const& snapshot, Agent* agent, std::string const& jobId, std::string const& creator, diff --git a/arangod/Agency/Node.cpp b/arangod/Agency/Node.cpp index 0242a6a60e..1dab13d96b 100644 --- a/arangod/Agency/Node.cpp +++ b/arangod/Agency/Node.cpp @@ -33,9 +33,8 @@ #include #include -using namespace arangodb; -using namespace arangodb::basics; using namespace arangodb::consensus; +using namespace arangodb::basics; struct NotEmpty { bool operator()(const std::string& s) { return !s.empty(); } @@ -89,16 +88,16 @@ Node::Node(std::string const& name, Store* store) Node::~Node() {} /// Get slice to value buffer -velocypack::Slice Node::slice() const { +Slice Node::slice() const { // Some array if (_isArray) { rebuildVecBuf(); - return velocypack::Slice(_vecBuf.data()); + return Slice(_vecBuf.data()); } // Some value if (!_value.empty()) { - return velocypack::Slice(_value.front().data()); + return Slice(_value.front().data()); } // Empty object @@ -107,10 +106,10 @@ velocypack::Slice Node::slice() const { void Node::rebuildVecBuf() const { if (_vecBufDirty) { // Dirty vector buffer - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); for (auto const& i : _value) { - tmp.add(velocypack::Slice(i.data())); + tmp.add(Slice(i.data())); } tmp.close(); _vecBuf = *tmp.steal(); @@ -324,7 +323,7 @@ Store& Node::store() { return *(root()._store); } Store const& Node::store() const { return *(root()._store); } // velocypack value type of this node -velocypack::ValueType Node::valueType() const { return slice().type(); } +ValueType Node::valueType() const { return slice().type(); } // file time to live entry for this node to now + millis bool Node::addTimeToLive(long millis) { @@ -359,7 +358,7 @@ namespace consensus { /// Set value template <> bool Node::handle(VPackSlice const& slice) { - VPackSlice val = slice.get("new"); + Slice val = slice.get("new"); if (val.isObject()) { if (val.hasKey("op")) { // No longer a keyword but a regular key "op" @@ -394,12 +393,12 @@ bool Node::handle(VPackSlice const& slice) { /// Increment integer value or set 1 template <> bool Node::handle(VPackSlice const& slice) { - velocypack::Builder tmp; + Builder tmp; tmp.openObject(); try { - tmp.add("tmp", velocypack::Value(this->slice().getInt() + 1)); + tmp.add("tmp", Value(this->slice().getInt() + 1)); } catch (std::exception const&) { - tmp.add("tmp", velocypack::Value(1)); + tmp.add("tmp", Value(1)); } tmp.close(); *this = tmp.slice().get("tmp"); @@ -409,12 +408,12 @@ bool Node::handle(VPackSlice const& slice) { /// Decrement integer value or set -1 template <> bool Node::handle(VPackSlice const& slice) { - velocypack::Builder tmp; + Builder tmp; tmp.openObject(); try { - tmp.add("tmp", velocypack::Value(this->slice().getInt() - 1)); + tmp.add("tmp", Value(this->slice().getInt() - 1)); } catch (std::exception const&) { - tmp.add("tmp", velocypack::Value(-1)); + tmp.add("tmp", Value(-1)); } tmp.close(); *this = tmp.slice().get("tmp"); @@ -429,7 +428,7 @@ bool Node::handle(VPackSlice const& slice) { << slice.toJson(); return false; } - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); if (this->slice().isArray()) { for (auto const& old : VPackArrayIterator(this->slice())) tmp.add(old); @@ -448,7 +447,7 @@ bool Node::handle(VPackSlice const& slice) { << "Operator erase without value to be erased: " << slice.toJson(); return false; } - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); if (this->slice().isArray()) { for (auto const& old : VPackArrayIterator(this->slice())) { @@ -475,7 +474,7 @@ bool Node::handle(VPackSlice const& slice) { << slice.toJson(); return false; } - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); if (this->slice().isArray()) { for (auto const& old : VPackArrayIterator(this->slice())) { @@ -494,7 +493,7 @@ bool Node::handle(VPackSlice const& slice) { /// Remove element from end of array. template <> bool Node::handle(VPackSlice const& slice) { - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); if (this->slice().isArray()) { VPackArrayIterator it(this->slice()); @@ -519,7 +518,7 @@ bool Node::handle(VPackSlice const& slice) { << slice.toJson(); return false; } - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); tmp.add(slice.get("new")); if (this->slice().isArray()) { @@ -533,7 +532,7 @@ bool Node::handle(VPackSlice const& slice) { /// Remove element from front of array template <> bool Node::handle(VPackSlice const& slice) { - velocypack::Builder tmp; + Builder tmp; tmp.openArray(); if (this->slice().isArray()) { // If a VPackArrayIterator it(this->slice()); @@ -678,7 +677,7 @@ bool Node::applies(VPackSlice const& slice) { return true; } -void Node::toBuilder(velocypack::Builder& builder, bool showHidden) const { +void Node::toBuilder(Builder& builder, bool showHidden) const { try { if (type() == NODE) { VPackObjectBuilder guard(&builder); @@ -729,7 +728,7 @@ Node::Children& Node::children() { return _children; } Node::Children const& Node::children() const { return _children; } std::string Node::toJson() const { - velocypack::Builder builder; + Builder builder; builder.openArray(); toBuilder(builder); builder.close(); @@ -796,7 +795,7 @@ std::string Node::getString() const { return slice().copyString(); } -velocypack::Slice Node::getArray() const { +Slice Node::getArray() const { if (type() == NODE) { throw StoreException("Must not convert NODE type to array"); } @@ -804,6 +803,6 @@ velocypack::Slice Node::getArray() const { throw StoreException("Not an array type"); } rebuildVecBuf(); - return velocypack::Slice(_vecBuf.data()); + return Slice(_vecBuf.data()); } diff --git a/arangod/Agency/Node.h b/arangod/Agency/Node.h index 43f9442f65..4cf67d5e31 100644 --- a/arangod/Agency/Node.h +++ b/arangod/Agency/Node.h @@ -50,6 +50,8 @@ enum Operation { REPLACE }; +using namespace arangodb::velocypack; + class StoreException : public std::exception { public: explicit StoreException(std::string const& message) : _message(message) {} @@ -159,7 +161,7 @@ class Node { bool handle(arangodb::velocypack::Slice const&); /// @brief Create Builder representing this store - void toBuilder(velocypack::Builder&, bool showHidden = false) const; + void toBuilder(Builder&, bool showHidden = false) const; /// @brief Access children Children& children(); @@ -168,10 +170,10 @@ class Node { Children const& children() const; /// @brief Create slice from value - velocypack::Slice slice() const; + Slice slice() const; /// @brief Get value type - velocypack::ValueType valueType() const; + ValueType valueType() const; /// @brief Add observer for this node bool addObserver(std::string const&); @@ -216,7 +218,7 @@ class Node { std::string getString() const; /// @brief Get array value - velocypack::Slice getArray() const; + Slice getArray() const; protected: /// @brief Add time to live entry @@ -232,8 +234,8 @@ class Node { Store* _store; ///< @brief Store Children _children; ///< @brief child nodes TimePoint _ttl; ///< @brief my expiry - std::vector> _value; ///< @brief my value - mutable velocypack::Buffer _vecBuf; + std::vector> _value; ///< @brief my value + mutable Buffer _vecBuf; mutable bool _vecBufDirty; bool _isArray; }; diff --git a/arangod/Agency/RemoveServer.cpp b/arangod/Agency/RemoveServer.cpp index 8357a4aaf0..d2180a97da 100644 --- a/arangod/Agency/RemoveServer.cpp +++ b/arangod/Agency/RemoveServer.cpp @@ -27,7 +27,6 @@ #include "Agency/Job.h" using namespace arangodb::consensus; -using namespace arangodb::velocypack; RemoveServer::RemoveServer(Node const& snapshot, Agent* agent, std::string const& jobId, std::string const& creator, diff --git a/arangod/Agency/RestAgencyHandler.cpp b/arangod/Agency/RestAgencyHandler.cpp index 281e0a6c6e..152e7b360d 100644 --- a/arangod/Agency/RestAgencyHandler.cpp +++ b/arangod/Agency/RestAgencyHandler.cpp @@ -35,10 +35,10 @@ #include "Rest/Version.h" using namespace arangodb; + using namespace arangodb::basics; -using namespace arangodb::consensus; using namespace arangodb::rest; -using namespace arangodb::velocypack; +using namespace arangodb::consensus; //////////////////////////////////////////////////////////////////////////////// /// @brief ArangoDB server diff --git a/arangod/Agency/State.h b/arangod/Agency/State.h index ae7917b817..95224de2cc 100644 --- a/arangod/Agency/State.h +++ b/arangod/Agency/State.h @@ -184,7 +184,7 @@ class State { size_t _cur; /// @brief Operation options - OperationOptions _options; + arangodb::OperationOptions _options; /// @brief Empty log entry; static log_t emptyLog; diff --git a/arangod/Agency/Store.cpp b/arangod/Agency/Store.cpp index f265c6b65c..fcd606999e 100644 --- a/arangod/Agency/Store.cpp +++ b/arangod/Agency/Store.cpp @@ -40,9 +40,8 @@ #include #include -using namespace arangodb::basics; using namespace arangodb::consensus; -using namespace arangodb::velocypack; +using namespace arangodb::basics; /// Non-Emptyness of string struct NotEmpty { diff --git a/arangod/Agency/Store.h b/arangod/Agency/Store.h index 658a10993c..dceee37f78 100644 --- a/arangod/Agency/Store.h +++ b/arangod/Agency/Store.h @@ -28,9 +28,6 @@ #include "Basics/Thread.h" #include "Node.h" -#include -#include - namespace arangodb { namespace consensus { @@ -61,10 +58,10 @@ class Store : public arangodb::Thread { std::vector apply(query_t const& query, bool verbose = false); /// @brief Apply single entry in query - bool apply(velocypack::Slice const& query, bool verbose = false); + bool apply(Slice const& query, bool verbose = false); /// @brief Apply entry in query - std::vector apply(std::vector const& query, + std::vector apply(std::vector const& query, index_t lastCommitIndex, term_t term, bool inform = true); @@ -82,7 +79,7 @@ class Store : public arangodb::Thread { bool start(); /// @brief Dump everything to builder - void dumpToBuilder(velocypack::Builder&) const; + void dumpToBuilder(Builder&) const; /// @brief Notify observers void notifyObservers() const; @@ -93,7 +90,7 @@ class Store : public arangodb::Thread { Store& operator=(VPackSlice const& slice); /// @brief Create Builder representing this store - void toBuilder(velocypack::Builder&, bool showHidden = false) const; + void toBuilder(Builder&, bool showHidden = false) const; /// @brief Copy out a node Node get(std::string const& path) const; diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index b1f02b3312..e78fe1e81a 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -41,9 +41,9 @@ #include "Basics/MutexLocker.h" using namespace arangodb; -using namespace arangodb::application_features; + using namespace arangodb::consensus; -using namespace arangodb::velocypack; +using namespace arangodb::application_features; std::string Supervision::_agencyPrefix = "/arango"; diff --git a/arangod/Agency/UnassumedLeadership.cpp b/arangod/Agency/UnassumedLeadership.cpp index 569a30f99e..1aa7768685 100644 --- a/arangod/Agency/UnassumedLeadership.cpp +++ b/arangod/Agency/UnassumedLeadership.cpp @@ -27,7 +27,6 @@ #include "Agency/Job.h" using namespace arangodb::consensus; -using namespace arangodb::velocypack; UnassumedLeadership::UnassumedLeadership( Node const& snapshot, Agent* agent, std::string const& jobId, diff --git a/arangod/Agency/v8-agency.cpp b/arangod/Agency/v8-agency.cpp index 46264ba53b..3e14fc7a8b 100644 --- a/arangod/Agency/v8-agency.cpp +++ b/arangod/Agency/v8-agency.cpp @@ -39,7 +39,6 @@ using namespace arangodb; using namespace arangodb::application_features; using namespace arangodb::basics; using namespace arangodb::consensus; -using namespace arangodb::velocypack; static void JS_EnabledAgent(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_BEGIN(isolate); diff --git a/arangod/Aql/AstNode.h b/arangod/Aql/AstNode.h index c0b0c1b8ab..d9602e07f3 100644 --- a/arangod/Aql/AstNode.h +++ b/arangod/Aql/AstNode.h @@ -418,7 +418,6 @@ struct AstNode { bool isAttributeAccessForVariable(Variable const* variable, bool allowIndexedAccess) const { auto node = getAttributeAccessForVariable(allowIndexedAccess); - if (node == nullptr) { return false; } diff --git a/arangod/Aql/ClusterBlocks.cpp b/arangod/Aql/ClusterBlocks.cpp index c077b3adbc..1cb85a8f5f 100644 --- a/arangod/Aql/ClusterBlocks.cpp +++ b/arangod/Aql/ClusterBlocks.cpp @@ -33,6 +33,7 @@ #include "Aql/AqlValue.h" #include "Aql/BlockCollector.h" #include "Aql/ExecutionEngine.h" +#include "Aql/ExecutionStats.h" #include "Basics/Exceptions.h" #include "Basics/StaticStrings.h" #include "Basics/StringBuffer.h" @@ -1328,7 +1329,7 @@ int RemoteBlock::initializeCursor(AqlItemBlock* items, size_t pos) { responseBodyBuf.c_str(), responseBodyBuf.length()); VPackSlice slice = builder->slice(); - + if (slice.hasKey("code")) { return slice.get("code").getNumericValue(); } @@ -1362,9 +1363,14 @@ int RemoteBlock::shutdown(int errorCode) { std::shared_ptr builder = VPackParser::fromJson(responseBodyBuf.c_str(), responseBodyBuf.length()); VPackSlice slice = builder->slice(); - - // read "warnings" attribute if present and add it to our query + if (slice.isObject()) { + if (slice.hasKey("stats")) { + ExecutionStats newStats(slice.get("stats")); + _engine->_stats.add(newStats); + } + + // read "warnings" attribute if present and add it to our query VPackSlice warnings = slice.get("warnings"); if (warnings.isArray()) { auto query = _engine->getQuery(); @@ -1415,19 +1421,14 @@ AqlItemBlock* RemoteBlock::getSome(size_t atLeast, size_t atMost) { res->result->getBodyVelocyPack(); VPackSlice responseBody = responseBodyBuilder->slice(); - ExecutionStats newStats(responseBody.get("stats")); - - _engine->_stats.addDelta(_deltaStats, newStats); - _deltaStats = newStats; - if (VelocyPackHelper::getBooleanValue(responseBody, "exhausted", true)) { traceGetSomeEnd(nullptr); return nullptr; } - auto r = new arangodb::aql::AqlItemBlock(_engine->getQuery()->resourceMonitor(), responseBody); - traceGetSomeEnd(r); - return r; + auto r = std::make_unique(_engine->getQuery()->resourceMonitor(), responseBody); + traceGetSomeEnd(r.get()); + return r.release(); // cppcheck-suppress style DEBUG_END_BLOCK(); diff --git a/arangod/Aql/ClusterBlocks.h b/arangod/Aql/ClusterBlocks.h index 947b19613e..4194bb994b 100644 --- a/arangod/Aql/ClusterBlocks.h +++ b/arangod/Aql/ClusterBlocks.h @@ -28,7 +28,6 @@ #include "Aql/ClusterNodes.h" #include "Aql/ExecutionBlock.h" #include "Aql/ExecutionNode.h" -#include "Aql/ExecutionStats.h" #include "Rest/GeneralRequest.h" namespace arangodb { @@ -339,9 +338,6 @@ class RemoteBlock : public ExecutionBlock { /// @brief the ID of the query on the server as a string std::string _queryId; - /// @brief the ID of the query on the server as a string - ExecutionStats _deltaStats; - /// @brief whether or not this block will forward initialize, /// initializeCursor or shutDown requests bool const _isResponsibleForInitializeCursor; diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index c4dd24bbc5..fea5cee911 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -529,6 +529,7 @@ struct CoordinatorInstanciator : public WalkerWorker { VPackBuilder tmp; query->ast()->variables()->toVelocyPack(tmp); + result.add("initialize", VPackValue(false)); result.add("variables", tmp.slice()); result.add("collections", VPackValue(VPackValueType::Array)); @@ -1133,7 +1134,7 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan( bool const isCoordinator = arangodb::ServerState::instance()->isCoordinator(role); bool const isDBServer = arangodb::ServerState::instance()->isDBServer(role); - + TRI_ASSERT(queryRegistry != nullptr); ExecutionEngine* engine = nullptr; @@ -1354,8 +1355,11 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan( } engine->_root = root; - root->initialize(); - root->initializeCursor(nullptr, 0); + + if (plan->isResponsibleForInitialize()) { + root->initialize(); + root->initializeCursor(nullptr, 0); + } return engine; } catch (...) { diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index 77231ded83..e0be4a6c44 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -177,6 +177,7 @@ ExecutionPlan::ExecutionPlan(Ast* ast) : _ids(), _root(nullptr), _varUsageComputed(false), + _isResponsibleForInitialize(true), _nextId(0), _ast(ast), _lastLimitNode(nullptr), @@ -280,6 +281,7 @@ ExecutionPlan* ExecutionPlan::clone() { plan->_root = _root->clone(plan.get(), true, false); plan->_nextId = _nextId; plan->_appliedRules = _appliedRules; + plan->_isResponsibleForInitialize = _isResponsibleForInitialize; CloneNodeAdder adder(plan.get()); plan->_root->walk(&adder); @@ -348,6 +350,7 @@ void ExecutionPlan::toVelocyPack(VPackBuilder& builder, Ast* ast, bool verbose) size_t nrItems = 0; builder.add("estimatedCost", VPackValue(_root->getCost(nrItems))); builder.add("estimatedNrItems", VPackValue(nrItems)); + builder.add("initialize", VPackValue(_isResponsibleForInitialize)); builder.close(); } @@ -1882,17 +1885,22 @@ void ExecutionPlan::insertDependency(ExecutionNode* oldNode, /// @brief create a plan from VPack ExecutionNode* ExecutionPlan::fromSlice(VPackSlice const& slice) { - ExecutionNode* ret = nullptr; - if (!slice.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "plan slice is not an object"); } + if (slice.hasKey("initialize")) { + // whether or not this plan (or fragment) is responsible for calling initialize + _isResponsibleForInitialize = slice.get("initialize").getBoolean(); + } + VPackSlice nodes = slice.get("nodes"); if (!nodes.isArray()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "plan \"nodes\" attribute is not an array"); } + + ExecutionNode* ret = nullptr; // first, re-create all nodes from the Slice, using the node ids // no dependency links will be set up in this step diff --git a/arangod/Aql/ExecutionPlan.h b/arangod/Aql/ExecutionPlan.h index 63b04cb2cb..39de6a2c35 100644 --- a/arangod/Aql/ExecutionPlan.h +++ b/arangod/Aql/ExecutionPlan.h @@ -75,6 +75,8 @@ class ExecutionPlan { /// @brief check if the plan is empty inline bool empty() const { return (_root == nullptr); } + + bool isResponsibleForInitialize() const { return _isResponsibleForInitialize; } /// @brief note that an optimizer rule was applied inline void addAppliedRule(int level) { _appliedRules.emplace_back(level); } @@ -299,6 +301,8 @@ class ExecutionPlan { /// @brief flag to indicate whether the variable usage is computed bool _varUsageComputed; + bool _isResponsibleForInitialize; + /// @brief auto-increment sequence for node ids size_t _nextId; diff --git a/arangod/Aql/ExecutionStats.cpp b/arangod/Aql/ExecutionStats.cpp index 84f6d3a5b0..eb007e3f70 100644 --- a/arangod/Aql/ExecutionStats.cpp +++ b/arangod/Aql/ExecutionStats.cpp @@ -74,9 +74,7 @@ ExecutionStats::ExecutionStats() executionTime(0.0) {} ExecutionStats::ExecutionStats(VPackSlice const& slice) - : httpRequests(0), - fullCount(-1), - executionTime(0.0) { + : ExecutionStats() { if (!slice.isObject()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "stats is not an object"); @@ -88,7 +86,6 @@ ExecutionStats::ExecutionStats(VPackSlice const& slice) scannedIndex = slice.get("scannedIndex").getNumber(); filtered = slice.get("filtered").getNumber(); - if (slice.hasKey("httpRequests")) { httpRequests = slice.get("httpRequests").getNumber(); } diff --git a/arangod/Aql/ExecutionStats.h b/arangod/Aql/ExecutionStats.h index 2e92edb4a3..4aaf4b5b45 100644 --- a/arangod/Aql/ExecutionStats.h +++ b/arangod/Aql/ExecutionStats.h @@ -58,21 +58,22 @@ struct ExecutionStats { scannedIndex += summand.scannedIndex; filtered += summand.filtered; httpRequests += summand.httpRequests; - fullCount += summand.fullCount; + if (summand.fullCount > 0) { + // fullCount may be negative, don't add it then + fullCount += summand.fullCount; + } // intentionally no modification of executionTime } - /// @brief sumarize the delta of two other sets of ExecutionStats to us - void addDelta(ExecutionStats const& lastStats, - ExecutionStats const& newStats) { - writesExecuted += newStats.writesExecuted - lastStats.writesExecuted; - writesIgnored += newStats.writesIgnored - lastStats.writesIgnored; - scannedFull += newStats.scannedFull - lastStats.scannedFull; - scannedIndex += newStats.scannedIndex - lastStats.scannedIndex; - filtered += newStats.filtered - lastStats.filtered; - httpRequests += newStats.httpRequests - lastStats.httpRequests; - fullCount += newStats.fullCount - lastStats.fullCount; - // intentionally no modification of executionTime + void clear() { + writesExecuted = 0; + writesIgnored = 0; + scannedFull = 0; + scannedIndex = 0; + filtered = 0; + httpRequests = 0; + fullCount = -1; + executionTime = 0.0; } /// @brief number of successfully executed write operations diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 46c384c7e9..97619e32b3 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -2350,13 +2350,15 @@ void arangodb::aql::scatterInClusterRule(Optimizer* opt, ExecutionPlan* plan, // Using Index for sort only works if all indexes are equal. auto first = allIndexes[0].getIndex(); - for (auto const& path : first->fieldNames()) { - elements.emplace_back(sortVariable, !isSortReverse, path); - } - for (auto const& it : allIndexes) { - if (first != it.getIndex()) { - elements.clear(); - break; + if (first->isSorted()) { + for (auto const& path : first->fieldNames()) { + elements.emplace_back(sortVariable, !isSortReverse, path); + } + for (auto const& it : allIndexes) { + if (first != it.getIndex()) { + elements.clear(); + break; + } } } } else if (nodeType == ExecutionNode::INSERT || @@ -4098,47 +4100,54 @@ MMFilesGeoIndexInfo iterativePreorderWithCondition(EN::NodeType type, AstNode* r return MMFilesGeoIndexInfo{}; } -MMFilesGeoIndexInfo geoDistanceFunctionArgCheck(std::pair const& pair, ExecutionPlan* plan, MMFilesGeoIndexInfo info){ - using SV = std::vector; +MMFilesGeoIndexInfo geoDistanceFunctionArgCheck(std::pair const& pair, + ExecutionPlan* plan, MMFilesGeoIndexInfo info){ + std::pair> attributeAccess1; + std::pair> attributeAccess2; + // first and second should be based on the same document - need to provide the document // in order to see which collection is bound to it and if that collections supports geo-index - if( !pair.first->isAttributeAccessForVariable() || !pair.second->isAttributeAccessForVariable()){ + if (!pair.first->isAttributeAccessForVariable(attributeAccess1) || + !pair.second->isAttributeAccessForVariable(attributeAccess2)) { info.invalidate(); return info; } + TRI_ASSERT(attributeAccess1.first != nullptr); + TRI_ASSERT(attributeAccess2.first != nullptr); + // expect access of the for doc.attribute - // TODO: more complex access path have to be added: loop until REFERENCE TYPE IS FOUND - auto setter1 = plan->getVarSetBy(static_cast(pair.first->getMember(0)->getData())->id); - auto setter2 = plan->getVarSetBy(static_cast(pair.second->getMember(0)->getData())->id); - SV accessPath1{pair.first->getString()}; - SV accessPath2{pair.second->getString()}; + auto setter1 = plan->getVarSetBy(attributeAccess1.first->id); + auto setter2 = plan->getVarSetBy(attributeAccess2.first->id); - if(setter1 == setter2){ - if(setter1->getType() == EN::ENUMERATE_COLLECTION){ - auto collNode = reinterpret_cast(setter1); + if (setter1 != nullptr && + setter2 != nullptr && + setter1 == setter2 && + setter1->getType() == EN::ENUMERATE_COLLECTION) { + auto collNode = reinterpret_cast(setter1); + auto coll = collNode->collection(); //what kind of indexes does it have on what attributes + auto lcoll = coll->getCollection(); + // TODO - check collection for suitable geo-indexes + for(auto indexShardPtr : lcoll->getIndexes()){ + // get real index + arangodb::Index& index = *indexShardPtr.get(); - auto coll = collNode->collection(); //what kind of indexes does it have on what attributes - auto lcoll = coll->getCollection(); - // TODO - check collection for suitable geo-indexes - for(auto indexShardPtr : lcoll->getIndexes()){ - // get real index - arangodb::Index& index = *indexShardPtr.get(); + // check if current index is a geo-index + if( index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX + && index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO2_INDEX) { + continue; + } - // check if current index is a geo-index - if( index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX - && index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO2_INDEX){ - continue; - } + TRI_ASSERT(index.fields().size() == 2); - //check access paths of attributes in ast and those in index match - if( index.fieldNames()[0] == accessPath1 && index.fieldNames()[1] == accessPath2 ){ - info.collectionNode = collNode; - info.index = indexShardPtr; - info.longitude = std::move(accessPath1); - info.latitude = std::move(accessPath2); - return info; - } + //check access paths of attributes in ast and those in index match + if (index.fields()[0] == attributeAccess1.second && + index.fields()[1] == attributeAccess2.second) { + info.collectionNode = collNode; + info.index = indexShardPtr; + TRI_AttributeNamesJoinNested(attributeAccess1.second, info.longitude, true); + TRI_AttributeNamesJoinNested(attributeAccess2.second, info.latitude, true); + return info; } } } diff --git a/arangod/Aql/Query.cpp b/arangod/Aql/Query.cpp index 535b9d9fbd..ad973ad648 100644 --- a/arangod/Aql/Query.cpp +++ b/arangod/Aql/Query.cpp @@ -731,14 +731,12 @@ QueryResult Query::execute(QueryRegistry* registry) { } _trx->commit(); + + result.context = _trx->transactionContext(); _engine->_stats.setExecutionTime(TRI_microtime() - _startTime); auto stats = std::make_shared(); - _engine->_stats.toVelocyPack(*(stats.get())); - - result.context = _trx->transactionContext(); - - cleanupPlanAndEngine(TRI_ERROR_NO_ERROR); + cleanupPlanAndEngine(TRI_ERROR_NO_ERROR, stats.get()); enterState(FINALIZATION); @@ -913,18 +911,15 @@ QueryResultV8 Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry) { _trx->commit(); - _engine->_stats.setExecutionTime(TRI_microtime() - _startTime); - auto stats = std::make_shared(); - _engine->_stats.toVelocyPack(*(stats.get())); - - result.context = _trx->transactionContext(); - LOG_TOPIC(DEBUG, Logger::QUERIES) << TRI_microtime() - _startTime << " " << "Query::executeV8: before cleanupPlanAndEngine" << " this: " << (uintptr_t) this; - cleanupPlanAndEngine(TRI_ERROR_NO_ERROR); + result.context = _trx->transactionContext(); + _engine->_stats.setExecutionTime(TRI_microtime() - _startTime); + auto stats = std::make_shared(); + cleanupPlanAndEngine(TRI_ERROR_NO_ERROR, stats.get()); enterState(FINALIZATION); @@ -1387,10 +1382,13 @@ std::string Query::getStateString() const { } /// @brief cleanup plan and engine for current query -void Query::cleanupPlanAndEngine(int errorCode) { +void Query::cleanupPlanAndEngine(int errorCode, VPackBuilder* statsBuilder) { if (_engine != nullptr) { try { _engine->shutdown(errorCode); + if (statsBuilder != nullptr) { + _engine->_stats.toVelocyPack(*statsBuilder); + } } catch (...) { // shutdown may fail but we must not throw here // (we're also called from the destructor) diff --git a/arangod/Aql/Query.h b/arangod/Aql/Query.h index 1dea32cd8c..6717a2af71 100644 --- a/arangod/Aql/Query.h +++ b/arangod/Aql/Query.h @@ -378,7 +378,7 @@ class Query { void enterState(ExecutionState); /// @brief cleanup plan and engine for current query - void cleanupPlanAndEngine(int); + void cleanupPlanAndEngine(int, VPackBuilder* statsBuilder = nullptr); /// @brief create a TransactionContext std::shared_ptr createTransactionContext(); diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 8e19b2cb5c..c4aac31141 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -697,7 +697,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, try { res = query->trx()->lockCollections(); } catch (...) { - LOG(ERR) << "lock lead to an exception"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "lock lead to an exception"); @@ -726,15 +725,10 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, if (items.get() == nullptr) { answerBuilder.add("exhausted", VPackValue(true)); answerBuilder.add("error", VPackValue(false)); - answerBuilder.add(VPackValue("stats")); - query->getStats(answerBuilder); } else { try { items->toVelocyPack(query->trx(), answerBuilder); - answerBuilder.add(VPackValue("stats")); - query->getStats(answerBuilder); } catch (...) { - LOG(ERR) << "cannot transform AqlItemBlock to VelocyPack"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "cannot transform AqlItemBlock to VelocyPack"); @@ -760,7 +754,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, skipped = block->skipSomeForShard(atLeast, atMost, shardId); } } catch (...) { - LOG(ERR) << "skipSome lead to an exception"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "skipSome lead to an exception"); @@ -768,8 +761,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, } answerBuilder.add("skipped", VPackValue(static_cast(skipped))); answerBuilder.add("error", VPackValue(false)); - answerBuilder.add(VPackValue("stats")); - query->getStats(answerBuilder); } else if (operation == "skip") { auto number = VelocyPackHelper::getNumericValue(querySlice, "number", 1); @@ -789,10 +780,7 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, } answerBuilder.add("exhausted", VPackValue(exhausted)); answerBuilder.add("error", VPackValue(false)); - answerBuilder.add(VPackValue("stats")); - query->getStats(answerBuilder); } catch (...) { - LOG(ERR) << "skip lead to an exception"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "skip lead to an exception"); @@ -803,7 +791,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, try { res = query->engine()->initialize(); } catch (...) { - LOG(ERR) << "initialize lead to an exception"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "initialize lead to an exception"); @@ -825,7 +812,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, res = query->engine()->initializeCursor(items.get(), pos); } } catch (...) { - LOG(ERR) << "initializeCursor lead to an exception"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "initializeCursor lead to an exception"); @@ -833,8 +819,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, } answerBuilder.add("error", VPackValue(res != TRI_ERROR_NO_ERROR)); answerBuilder.add("code", VPackValue(static_cast(res))); - answerBuilder.add(VPackValue("stats")); - query->getStats(answerBuilder); } else if (operation == "shutdown") { int res = TRI_ERROR_INTERNAL; int errorCode = VelocyPackHelper::getNumericValue( @@ -854,7 +838,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, _queryRegistry->destroy(_vocbase, _qId, errorCode); _qId = 0; } catch (...) { - LOG(ERR) << "shutdown lead to an exception"; generateError(rest::ResponseCode::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, "shutdown lead to an exception"); @@ -863,7 +846,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, answerBuilder.add("error", VPackValue(res != TRI_ERROR_NO_ERROR)); answerBuilder.add("code", VPackValue(res)); } else { - LOG(ERR) << "Unknown operation!"; generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND); return; @@ -875,7 +857,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query, generateError(rest::ResponseCode::BAD, e.code()); return; } catch (...) { - LOG(ERR) << "OUT OF MEMORY when handling query."; generateError(rest::ResponseCode::BAD, TRI_ERROR_OUT_OF_MEMORY); return; } diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 4e28253aa3..d26962c82d 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -2033,6 +2033,7 @@ int flushWalOnAllDBServers(bool waitForSync, bool waitForCollector) { } if (nrok != (int)DBservers.size()) { + LOG(WARN) << "could not flush WAL on all servers. confirmed: " << nrok << ", expected: " << DBservers.size(); return TRI_ERROR_INTERNAL; } diff --git a/arangod/Cluster/DBServerAgencySync.cpp b/arangod/Cluster/DBServerAgencySync.cpp index d1f741caad..2769b52eff 100644 --- a/arangod/Cluster/DBServerAgencySync.cpp +++ b/arangod/Cluster/DBServerAgencySync.cpp @@ -54,6 +54,8 @@ void DBServerAgencySync::work() { DBServerAgencySyncResult DBServerAgencySync::execute() { // default to system database + double startTime = TRI_microtime(); + LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "DBServerAgencySync::execute starting"; DatabaseFeature* database = ApplicationServer::getFeature("Database"); @@ -80,6 +82,11 @@ DBServerAgencySyncResult DBServerAgencySync::execute() { return result; } + double now = TRI_microtime(); + if (now - startTime > 5.0) { + LOG(INFO) << "DBServerAgencySync::execute took more than 5s to get free V8 context, starting handle-plan-change now"; + } + TRI_DEFER(V8DealerFeature::DEALER->exitContext(context)); auto isolate = context->_isolate; diff --git a/arangod/MMFiles/MMFilesGeoIndex.h b/arangod/MMFiles/MMFilesGeoIndex.h index fd30736849..afbc706b97 100644 --- a/arangod/MMFiles/MMFilesGeoIndex.h +++ b/arangod/MMFiles/MMFilesGeoIndex.h @@ -118,7 +118,7 @@ friend class MMFilesGeoIndexIterator; bool canBeDropped() const override { return true; } - bool isSorted() const override { return false; } + bool isSorted() const override { return true; } bool hasSelectivityEstimate() const override { return false; } diff --git a/arangod/dbg.cmake b/arangod/dbg.cmake index d81497f0b0..763351e0d1 100644 --- a/arangod/dbg.cmake +++ b/arangod/dbg.cmake @@ -1,17 +1,12 @@ # -*- mode: CMAKE; -*- # these are the install targets for the client package. # we can't use RUNTIME DESTINATION here. +# include(/tmp/dump_vars.cmake) +message( "CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME}/ CMAKE_INSTALL_SBINDIR ${CMAKE_INSTALL_SBINDIR}") -set(STRIP_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip") -execute_process(COMMAND mkdir -p ${STRIP_DIR}) - -set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX}) -set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX}) -if (NOT MSVC AND CMAKE_STRIP) - execute_process(COMMAND "rm" -f ${STRIP_FILE}) - execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) - set(FILE ${STRIP_FILE}) -endif() -install( - PROGRAMS ${FILE} - DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_SBINDIR}) +install_debinfo( + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" + "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_SBINDIR}" + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX}" + "${STRIP_DIR}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX}" + ) diff --git a/arangosh/dbg.cmake b/arangosh/dbg.cmake index 641e35f2ad..aea97a5800 100644 --- a/arangosh/dbg.cmake +++ b/arangosh/dbg.cmake @@ -2,63 +2,30 @@ # these are the install targets for the client package. # we can't use RUNTIME DESTINATION here. -set(STRIP_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip") -execute_process(COMMAND mkdir -p ${STRIP_DIR}) -set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX}) -set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX}) -if (NOT MSVC AND CMAKE_STRIP) - execute_process(COMMAND "rm" -f ${STRIP_FILE}) - execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) - set(FILE ${STRIP_FILE}) -endif() - -install( - PROGRAMS ${FILE} - DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR}) - - -set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX}) -set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX}) -if (NOT MSVC AND CMAKE_STRIP) - execute_process(COMMAND "rm" -f ${STRIP_FILE}) - execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) - set(FILE ${STRIP_FILE}) -endif() -install( - PROGRAMS ${FILE} - DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR}) - -set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX}) -set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX}) -if (NOT MSVC AND CMAKE_STRIP) - execute_process(COMMAND "rm" -f ${STRIP_FILE}) - execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) - set(FILE ${STRIP_FILE}) -endif() -install( - PROGRAMS ${FILE} - DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR}) - -set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX}) -set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX}) -if (NOT MSVC AND CMAKE_STRIP) - execute_process(COMMAND "rm" -f ${STRIP_FILE}) - execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) - set(FILE ${STRIP_FILE}) -endif() -install( - PROGRAMS ${FILE} - DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR}) - -set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX}) -set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX}) -if (NOT MSVC AND CMAKE_STRIP) - execute_process(COMMAND "rm" -f ${STRIP_FILE}) - execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) - set(FILE ${STRIP_FILE}) -endif() -install( - PROGRAMS ${FILE} - DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR}) +install_debinfo( + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" + "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}" + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX}" + "${STRIP_DIR}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX}") +install_debinfo( + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" + "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}" + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX}" + "${STRIP_DIR}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX}") +install_debinfo( + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" + "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}" + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX}" + "${STRIP_DIR}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX}") +install_debinfo( + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" + "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}" + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX}" + "${STRIP_DIR}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX}") +install_debinfo( + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" + "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}" + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX}" + "${STRIP_DIR}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX}") diff --git a/cmake/ArangoDBInstall.cmake b/cmake/ArangoDBInstall.cmake index 75f9eee60e..171f43dcf1 100644 --- a/cmake/ArangoDBInstall.cmake +++ b/cmake/ArangoDBInstall.cmake @@ -19,9 +19,9 @@ endif() # debug info directory: if (${CMAKE_INSTALL_LIBDIR} STREQUAL "usr/lib64") # some systems have weird places for usr/lib: - set(CMAKE_INSTALL_DEBINFO_DIR "usr/lib/debug/${CMAKE_PROJECT_NAME}") + set(CMAKE_INSTALL_DEBINFO_DIR "usr/lib/debug/") else () - set(CMAKE_INSTALL_DEBINFO_DIR "${CMAKE_INSTALL_LIBDIR}/debug/${CMAKE_PROJECT_NAME}") + set(CMAKE_INSTALL_DEBINFO_DIR "${CMAKE_INSTALL_LIBDIR}/debug/") endif () set(CMAKE_INSTALL_SYSCONFDIR_ARANGO "${CMAKE_INSTALL_SYSCONFDIR}/${CMAKE_PROJECT_NAME}") diff --git a/cmake/InstallMacros.cmake b/cmake/InstallMacros.cmake index 03f8f7eac0..2f59bae529 100644 --- a/cmake/InstallMacros.cmake +++ b/cmake/InstallMacros.cmake @@ -157,3 +157,39 @@ macro(to_native_path sourceVarName) endif() set("INC_${sourceVarName}" ${myVar}) endmacro() + +macro(install_debinfo + STRIP_DIR + USER_SUB_DEBINFO_DIR + USER_FILE + USER_STRIP_FILE) + + set(SUB_DEBINFO_DIR ${USER_SUB_DEBINFO_DIR}) + set(FILE ${USER_FILE}) + set(STRIP_FILE ${USER_STRIP_FILE}) + execute_process(COMMAND mkdir -p ${STRIP_DIR}) + if (NOT MSVC AND CMAKE_STRIP) + execute_process(COMMAND "rm" -f ${STRIP_FILE}) + + execute_process( + COMMAND ${FILE_EXECUTABLE} ${FILE} + OUTPUT_VARIABLE FILE_RESULT) + + string(REGEX + REPLACE ".*=([a-z0-9]*),.*" "\\1" + FILE_CHECKSUM + ${FILE_RESULT} + ) + + if (NOT ${FILE_CHECKSUM} STREQUAL "") + string(SUBSTRING ${FILE_CHECKSUM} 0 2 SUB_DIR) + string(SUBSTRING ${FILE_CHECKSUM} 2 -1 STRIP_FILE) + set(SUB_DEBINFO_DIR .build-id/${SUB_DIR}) + endif() + execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE}) + set(FILE ${STRIP_FILE}) + endif() + install( + PROGRAMS ${FILE} + DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${SUB_DEBINFO_DIR}) +endmacro() diff --git a/cmake/packages/dbg/deb.txt b/cmake/packages/dbg/deb.txt index 5efe953f33..a3b5ad71a2 100644 --- a/cmake/packages/dbg/deb.txt +++ b/cmake/packages/dbg/deb.txt @@ -1,7 +1,7 @@ ################################################################################ # the client package is a complete cmake sub package. ################################################################################ -project(PACKAGE-DBG) +project(@CMAKE_PROJECT_NAME@) cmake_minimum_required(VERSION 2.8) ################################################################################ @@ -15,6 +15,9 @@ set(CROSS_COMPILING @CROSS_COMPILING@) set(CMAKE_INSTALL_BINDIR @CMAKE_INSTALL_BINDIR@) set(CMAKE_INSTALL_FULL_BINDIR @CMAKE_INSTALL_FULL_BINDIR@) +set(CMAKE_INSTALL_SBINDIR @CMAKE_INSTALL_SBINDIR@) +set(CMAKE_INSTALL_FULL_SBINDIR @CMAKE_INSTALL_FULL_SBINDIR@) + set(CMAKE_INSTALL_DATAROOTDIR @CMAKE_INSTALL_DATAROOTDIR@) set(CMAKE_INSTALL_DATAROOTDIR_ARANGO @CMAKE_INSTALL_DATAROOTDIR_ARANGO@) set(CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO @CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO@) diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/arango/arango.js b/js/apps/system/_admin/aardvark/APP/frontend/js/arango/arango.js index 2ad724590b..e92b477ce2 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/arango/arango.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/arango/arango.js @@ -69,6 +69,10 @@ return shortName; }, + getDatabaseShortName: function (id) { + return this.getCoordinatorShortName(id); + }, + getDatabaseServerId: function (shortname) { var id; if (window.clusterHealth) { diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/shardsView.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/shardsView.js index fdd303cca5..1e85c5699c 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/views/shardsView.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/shardsView.js @@ -186,14 +186,15 @@ async: true, success: function (data) { if (data.id) { - arangoHelper.arangoNotification('Shard ' + shardName + ' will be moved to ' + arangoHelper.getDatabaseServerId(toServer) + '.'); + console.log(toServer); + arangoHelper.arangoNotification('Shard ' + shardName + ' will be moved to ' + arangoHelper.getDatabaseShortName(toServer) + '.'); window.setTimeout(function () { window.App.shardsView.render(); }, 3000); } }, error: function () { - arangoHelper.arangoError('Shard ' + shardName + ' could not be moved to ' + arangoHelper.getDatabaseServerId(toServer) + '.'); + arangoHelper.arangoError('Shard ' + shardName + ' could not be moved to ' + arangoHelper.getDatabaseShortName(toServer) + '.'); } }); diff --git a/js/server/modules/@arangodb/cluster.js b/js/server/modules/@arangodb/cluster.js index d0b79ac34f..15eee2fb78 100644 --- a/js/server/modules/@arangodb/cluster.js +++ b/js/server/modules/@arangodb/cluster.js @@ -120,14 +120,12 @@ function startReadLockOnLeader (endpoint, database, collName, timeout) { const id = r.id; var body = { 'id': id, 'collection': collName, 'ttl': timeout }; - r = request({ url: url + '/_api/replication/holdReadLockCollection', - body: JSON.stringify(body), - method: 'POST', headers: {'x-arango-async': true} }); - if (r.status !== 202) { - console.error('startReadLockOnLeader: Could not start read lock for shard', - collName, r); - return false; - } + request({ url: url + '/_api/replication/holdReadLockCollection', + body: JSON.stringify(body), + method: 'POST', headers: {'x-arango-async': true} }); + // Intentionally do not look at the outcome, even in case of an error + // we must make sure that the read lock on the leader is not active! + // This is done automatically below. var count = 0; while (++count < 20) { // wait for some time until read lock established: @@ -170,7 +168,10 @@ function startReadLockOnLeader (endpoint, database, collName, timeout) { // ///////////////////////////////////////////////////////////////////////////// function cancelReadLockOnLeader (endpoint, database, lockJobId) { - var url = endpointToURL(endpoint) + '/_db/' + database + + // Note that we always use the _system database here because the actual + // database might be gone already on the leader and we need to cancel + // the read lock under all circumstances. + var url = endpointToURL(endpoint) + '/_db/_system' + '/_api/replication/holdReadLockCollection'; var r; var body = {'id': lockJobId}; @@ -181,7 +182,8 @@ function cancelReadLockOnLeader (endpoint, database, lockJobId) { return false; } if (r.status !== 200) { - console.error('cancelReadLockOnLeader: error', r); + console.error('cancelReadLockOnLeader: error', lockJobId, r.status, + r.message, r.body, r.json); return false; } console.debug('cancelReadLockOnLeader: success'); @@ -453,6 +455,7 @@ function synchronizeOneShard (database, shard, planId, leader) { // synchronize this shard from the leader // this function will throw if anything goes wrong + var startTime = new Date(); var isStopping = require('internal').isStopping; var ourselves = global.ArangoServerState.id(); @@ -485,8 +488,9 @@ function synchronizeOneShard (database, shard, planId, leader) { planned[0] !== leader) { // Things have changed again, simply terminate: terminateAndStartOther(); - console.debug('synchronizeOneShard: cancelled, %s/%s, %s/%s', - database, shard, database, planId); + let endTime = new Date(); + console.debug('synchronizeOneShard: cancelled, %s/%s, %s/%s, started %s, ended %s', + database, shard, database, planId, startTime.toString(), endTime.toString()); return; } var current = []; @@ -500,8 +504,9 @@ function synchronizeOneShard (database, shard, planId, leader) { } // We are already there, this is rather strange, but never mind: terminateAndStartOther(); - console.debug('synchronizeOneShard: already done, %s/%s, %s/%s', - database, shard, database, planId); + let endTime = new Date(); + console.debug('synchronizeOneShard: already done, %s/%s, %s/%s, started %s, ended %s', + database, shard, database, planId, startTime.toString(), endTime.toString()); return; } console.debug('synchronizeOneShard: waiting for leader, %s/%s, %s/%s', @@ -522,9 +527,16 @@ function synchronizeOneShard (database, shard, planId, leader) { if (isStopping()) { throw 'server is shutting down'; } + let startTime = new Date(); sy = rep.syncCollection(shard, { endpoint: ep, incremental: true, keepBarrier: true, useCollectionId: false }); + let endTime = new Date(); + let longSync = false; + if (endTime - startTime > 5000) { + console.error('synchronizeOneShard: long call to syncCollection for shard', shard, JSON.stringify(sy), "start time: ", startTime.toString(), "end time: ", endTime.toString()); + longSync = true; + } if (sy.error) { console.error('synchronizeOneShard: could not initially synchronize', 'shard ', shard, sy); @@ -532,7 +544,15 @@ function synchronizeOneShard (database, shard, planId, leader) { } else { if (sy.collections.length === 0 || sy.collections[0].name !== shard) { + if (longSync) { + console.error('synchronizeOneShard: long sync, before cancelBarrier', + new Date().toString()); + } cancelBarrier(ep, database, sy.barrierId); + if (longSync) { + console.error('synchronizeOneShard: long sync, after cancelBarrier', + new Date().toString()); + } throw 'Shard ' + shard + ' seems to be gone from leader!'; } else { // Now start a read transaction to stop writes: @@ -592,14 +612,17 @@ function synchronizeOneShard (database, shard, planId, leader) { } else if (err2 && err2.errorNum === 1402 && err2.errorMessage.match(/HTTP 404/)) { logLevel = 'debug'; } - console[logLevel]("synchronization of local shard '%s/%s' for central '%s/%s' failed: %s", - database, shard, database, planId, JSON.stringify(err2)); + let endTime = new Date(); + console[logLevel]("synchronization of local shard '%s/%s' for central '%s/%s' failed: %s, started: %s, ended: %s", + database, shard, database, planId, JSON.stringify(err2), + startTime.toString(), endTime.toString()); } } // Tell others that we are done: terminateAndStartOther(); - console.debug('synchronizeOneShard: done, %s/%s, %s/%s', - database, shard, database, planId); + let endTime = new Date(); + console.debug('synchronizeOneShard: done, %s/%s, %s/%s, started: %s, ended: %s', + database, shard, database, planId, startTime.toString(), endTime.toString()); } // ///////////////////////////////////////////////////////////////////////////// diff --git a/js/server/tests/aql/aql-optimizer-geoindex.js b/js/server/tests/aql/aql-optimizer-geoindex.js index ec37685c5d..9b017737c8 100644 --- a/js/server/tests/aql/aql-optimizer-geoindex.js +++ b/js/server/tests/aql/aql-optimizer-geoindex.js @@ -55,6 +55,7 @@ function optimizerRuleTestSuite() { var ruleName = "geoindex"; var colName = "UnitTestsAqlOptimizer" + ruleName.replace(/-/g, "_"); + var colName2 = colName2; var geocol; var sortArray = function (l, r) { @@ -124,11 +125,21 @@ function optimizerRuleTestSuite() { internal.db._drop(colName); geocol = internal.db._create(colName); geocol.ensureIndex({type:"geo", fields:["lat","lon"]}); - for (var lat=-40; lat <=40 ; ++lat){ - for (var lon=-40; lon <= 40; ++lon){ + var lat, lon; + for (lat=-40; lat <=40 ; ++lat) { + for (lon=-40; lon <= 40; ++lon) { geocol.insert({lat,lon}); } } + + internal.db._drop(colName2); + geocol = internal.db._create(colName2); + geocol.ensureIndex({type:"geo", fields:["loca.tion.lat","loca.tion.lon"]}); + for (lat=-40; lat <=40 ; ++lat) { + for (lon=-40; lon <= 40; ++lon) { + geocol.insert({ loca : { tion : { lat , lon } } }); + } + } }, //////////////////////////////////////////////////////////////////////////////// @@ -137,6 +148,7 @@ function optimizerRuleTestSuite() { tearDown : function () { internal.db._drop(colName); + internal.db._drop(colName2); geocol = null; }, @@ -145,7 +157,13 @@ function optimizerRuleTestSuite() { geocol.ensureIndex({ type: "hash", fields: [ "y", "z" ], unique: false }); var queries = [ - { string : "FOR d IN " + colName + " SORT distance(d.lat,d.lon, 0 ,0 ) ASC LIMIT 1 RETURN d", + { string : "FOR d IN " + colName + " SORT distance(d.lat, d.lon, 0 ,0 ) ASC LIMIT 1 RETURN d", + cluster : false, + sort : false, + filter : false, + index : true + }, + { string : "FOR d IN " + colName2 + " SORT distance(d.loca.tion.lat, d.loca.tion.lon, 0 ,0 ) ASC LIMIT 1 RETURN d", cluster : false, sort : false, filter : false, @@ -213,11 +231,15 @@ function optimizerRuleTestSuite() { testRuleRemoveNodes : function () { if(enabled.removeNodes){ var queries = [ - [ "FOR d IN " + colName + " SORT distance(d.lat,d.lon, 0 ,0 ) ASC LIMIT 5 RETURN d", false, false, false ], - [ "FOR d IN " + colName + " SORT distance(0, 0, d.lat,d.lon ) ASC LIMIT 5 RETURN d", false, false, false ], - [ "FOR d IN " + colName + " FILTER distance(0, 0, d.lat,d.lon ) < 111200 RETURN d", false, false, false ], + [ "FOR d IN " + colName + " SORT distance(d.lat,d.lon, 0 ,0 ) ASC LIMIT 5 RETURN d", false, false, false ], + [ "FOR d IN " + colName + " SORT distance(0, 0, d.lat,d.lon ) ASC LIMIT 5 RETURN d", false, false, false ], + [ "FOR d IN " + colName + " FILTER distance(0, 0, d.lat,d.lon ) < 111200 RETURN d", false, false, false ], // [ "FOR i IN 1..2 FOR d IN geocol SORT distance(i,2,d.lat,d.lon) ASC LIMIT 5 RETURN d", false, false, false ], ]; + + var queries2 = [ + [ "FOR d IN " + colName2 + " SORT distance(d.loca.tion.lat,d.loca.tion.lon, 0 ,0 ) ASC LIMIT 5 RETURN d", false, false, false ] + ]; var expected = [ [[0,0], [-1,0], [0,1], [1,0], [0,-1]], @@ -234,6 +256,16 @@ function optimizerRuleTestSuite() { assertEqual(expected[qindex].sort(),pairs.sort()); //expect(expected[qindex].sort()).to.be.equal(result.json.sort()) }); + + queries2.forEach(function(query, qindex) { + var result = AQL_EXECUTE(query[0]); + expect(expected[qindex].length).to.be.equal(result.json.length); + var pairs = result.json.map(function(res){ + return [res.loca.tion.lat,res.loca.tion.lon]; + }); + assertEqual(expected[qindex].sort(),pairs.sort()); + //expect(expected[qindex].sort()).to.be.equal(result.json.sort()) + }); } }, // testRuleSort diff --git a/lib/Endpoint/Endpoint.cpp b/lib/Endpoint/Endpoint.cpp index 9c0478d9a1..ac5c54312e 100644 --- a/lib/Endpoint/Endpoint.cpp +++ b/lib/Endpoint/Endpoint.cpp @@ -280,7 +280,13 @@ Endpoint* Endpoint::factory(const Endpoint::EndpointType type, // hostname and port (e.g. [address]:port) if (found != std::string::npos && found > 2 && found + 2 < copy.size()) { - uint16_t port = (uint16_t)StringUtils::uint32(copy.substr(found + 2)); + int64_t value = StringUtils::int64(copy.substr(found + 2)); + // check port over-/underrun + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) { + LOG(ERR) << "specified port number '" << value << "' is outside the allowed range"; + return nullptr; + } + uint16_t port = static_cast(value); std::string host = copy.substr(1, found - 1); return new EndpointIpV6(type, protocol, encryption, listenBacklog, @@ -306,7 +312,13 @@ Endpoint* Endpoint::factory(const Endpoint::EndpointType type, // hostname and port if (found != std::string::npos && found + 1 < copy.size()) { - uint16_t port = (uint16_t)StringUtils::uint32(copy.substr(found + 1)); + int64_t value = StringUtils::int64(copy.substr(found + 1)); + // check port over-/underrun + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) { + LOG(ERR) << "specified port number '" << value << "' is outside the allowed range"; + return nullptr; + } + uint16_t port = static_cast(value); std::string host = copy.substr(0, found); return new EndpointIpV4(type, protocol, encryption, listenBacklog, diff --git a/lib/Logger/LoggerStream.h b/lib/Logger/LoggerStream.h index 86e5d03329..813f81fbaf 100644 --- a/lib/Logger/LoggerStream.h +++ b/lib/Logger/LoggerStream.h @@ -88,8 +88,9 @@ class LoggerStream { size_t i = 0; size_t const n = obj.size(); for (auto const& it : obj) { + _out << it; if (++i < n) { - _out << it << ", "; + _out << ", "; } } _out << ']'; @@ -102,8 +103,9 @@ class LoggerStream { size_t i = 0; size_t const n = obj.size(); for (auto const& it : obj) { + _out << it; if (++i < n) { - _out << it << ", "; + _out << ", "; } } _out << '}'; @@ -116,8 +118,9 @@ class LoggerStream { size_t i = 0; size_t n = obj.size(); for (auto const& it : obj) { + _out << it; if (++i < n) { - _out << it << ", "; + _out << ", "; } _out << it.first << " => " << it.second; } diff --git a/scripts/generate_documenation.sh b/scripts/generate_documenation.sh new file mode 100755 index 0000000000..a5add089a2 --- /dev/null +++ b/scripts/generate_documenation.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +##python3-setuptools +## +##python setup.py install +## +##node npm +## +##https://github.com/GitbookIO/gitbook +## npm install gitbook-cli -g +## +## http://calibre-ebook.com/download + +test_tools(){ + if ! type easy_install3 >> /dev/null; then + echo "you are missing setuptools" + echo "apt-get install python-setuptools" + exit 1 + fi + + if ! type node >> /dev/null; then + echo "you are missing node" + echo "apt-get install nodejs nodejs-legacy" + exit 1 + fi + + if ! type npm >> /dev/null; then + echo "you are missing node" + echo "apt-get install npm" + exit 1 + fi + + if ! type calibre >> /dev/null; then + echo "you are missing node" + echo "apt-get install calibre-bin" + exit 1 + fi +} + +install_tools(){ + ( + if ! [[ -f markdown-pp ]]; then + git clone https://github.com/arangodb-helper/markdown-pp/ + fi + cd markdown-pp + python2 setup.py install --user + ) + npm install gitbook-cli + + +} + +main(){ + #test for basic tools + test_tools + + #cd into target dir + mkdir -p "$1" + cd $1 || { echo "unable to change into $1"; exit 1; } + + install_tools + +} + +main "$@" +