Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

This commit is contained in:
Simon Grätzer 2017-02-03 21:27:08 +01:00
parent 39278f22d8 193f1213b3
commit 63dfe865ba
57 changed files with 599 additions and 342 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG
View File

@ -1,6 +1,8 @@
devel
-----
* fix potential port number over-/underruns
* added startup option `--log.shorten-filenames` for controlling whether filenames
in log message should be shortened to just the filename with the absolute path

2
CMakeLists.txt
View File

@ -490,6 +490,8 @@ if (USE_MAINTAINER_MODE)
find_program(AWK_EXECUTABLE awk)
endif ()
find_program(FILE_EXECUTABLE file)
################################################################################
## FAILURE TESTS
################################################################################

36
Documentation/Books/Manual/Indexing/Geo.mdpp
View File

@ -90,6 +90,42 @@ Create a geo index for a hash array attribute:
@END_EXAMPLE_ARANGOSH_OUTPUT
@endDocuBlock geoIndexCreateForArrayAttribute2
Use GeoIndex with AQL SORT statement:
@startDocuBlockInline geoIndexSortOptimization
@EXAMPLE_ARANGOSH_OUTPUT{geoIndexSortOptimization}
~db._create("geoSort")
db.geoSort.ensureIndex({ type: "geo", fields: [ "latitude", "longitude" ] });
| for (i = -90; i <= 90; i += 10) {
| for (j = -180; j <= 180; j += 10) {
| db.geoSort.save({ name : "Name/" + i + "/" + j, latitude : i, longitude : j });
| }
}
var query = "FOR doc in geoSort SORT distance(doc.latitude, doc.longitude, 0, 0) LIMIT 5 RETURN doc"
db._explain(query, {}, {colors: false});
db._query(query);
~db._drop("geoSort")
@END_EXAMPLE_ARANGOSH_OUTPUT
@endDocuBlock geoIndexSortOptimization
Use GeoIndex with AQL FILTER statement:
@startDocuBlockInline geoIndexFilterOptimization
@EXAMPLE_ARANGOSH_OUTPUT{geoIndexFilterOptimization}
~db._create("geoFilter")
db.geoFilter.ensureIndex({ type: "geo", fields: [ "latitude", "longitude" ] });
| for (i = -90; i <= 90; i += 10) {
| for (j = -180; j <= 180; j += 10) {
| db.geoFilter.save({ name : "Name/" + i + "/" + j, latitude : i, longitude : j });
| }
}
var query = "FOR doc in geoFilter FILTER distance(doc.latitude, doc.longitude, 0, 0) < 2000 RETURN doc"
db._explain(query, {}, {colors: false});
db._query(query);
~db._drop("geoFilter")
@END_EXAMPLE_ARANGOSH_OUTPUT
@endDocuBlock geoIndexFilterOptimization
<!-- js/common/modules/@arangodb/arango-collection-common.js-->
@startDocuBlock collectionGeo

6
Documentation/Books/Manual/Indexing/IndexBasics.mdpp
View File

@ -273,8 +273,10 @@ The geo index provides operations to find documents with coordinates nearest to
comparison coordinate, and to find documents with coordinates that are within a specifiable
radius around a comparison coordinate.
The geo index is used via dedicated functions in AQL or the simple queries functions,
but will not be used for other types of queries or conditions.
The geo index is used via dedicated functions in AQL, the simple queries
functions and it is implicitly applied when in AQL a SORT or FILTER is used with
the distance function. Otherwise it will not be used for other types of queries
or conditions.
### Fulltext Index

5
Documentation/Books/Manual/Indexing/WhichIndex.mdpp
View File

@ -71,7 +71,10 @@ different usage scenarios:
{ "coords": [ 50.9406645, 6.9599115 ] }
Geo indexes will only be invoked via special functions.
Geo indexes will be invoked via special functions or AQL optimization. The
optimization can be triggered when a collection with geo index is enumerated
and a SORT or FILTER statement is used in conjunction with the distance
function.
- fulltext index: a fulltext index can be used to index all words contained in
a specific attribute of all documents in a collection. Only words with a

31
UnitTests/Basics/EndpointTest.cpp
View File

@ -49,8 +49,6 @@ BOOST_TEST_DONT_PRINT_LOG_VALUE(arangodb::Endpoint::EndpointType)
// --SECTION-- macros
// -----------------------------------------------------------------------------
#define DELETE_ENDPOINT(e) if (e != 0) delete e;
#define FACTORY_NAME(name) name ## Factory
#define FACTORY(name, specification) arangodb::Endpoint::FACTORY_NAME(name)(specification)
@ -58,12 +56,12 @@ BOOST_TEST_DONT_PRINT_LOG_VALUE(arangodb::Endpoint::EndpointType)
#define CHECK_ENDPOINT_FEATURE(type, specification, feature, expected) \
e = FACTORY(type, specification); \
BOOST_CHECK_EQUAL((expected), (e->feature())); \
DELETE_ENDPOINT(e);
delete e;
#define CHECK_ENDPOINT_SERVER_FEATURE(type, specification, feature, expected) \
e = arangodb::Endpoint::serverFactory(specification, 1, true); \
BOOST_CHECK_EQUAL((expected), (e->feature())); \
DELETE_ENDPOINT(e);
delete e;
// -----------------------------------------------------------------------------
// --SECTION-- setup / tear-down
@ -118,6 +116,11 @@ BOOST_AUTO_TEST_CASE (EndpointInvalid) {
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("ssl@tcp://127.0.0.1:8529"));
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("https@tcp://127.0.0.1:8529"));
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("https@tcp://127.0.0.1:"));
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:65536"));
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:65537"));
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:-1"));
BOOST_CHECK_EQUAL(e, arangodb::Endpoint::clientFactory("tcp://127.0.0.1:6555555555"));
}
////////////////////////////////////////////////////////////////////////////////
@ -491,7 +494,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedServer1) {
e = arangodb::Endpoint::serverFactory("tcp://127.0.0.1", 1, true);
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
////////////////////////////////////////////////////////////////////////////////
@ -503,7 +506,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedServer2) {
e = arangodb::Endpoint::serverFactory("ssl://127.0.0.1", 1, true);
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
////////////////////////////////////////////////////////////////////////////////
@ -516,7 +519,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedServer3) {
e = arangodb::Endpoint::serverFactory("unix:///tmp/socket", 1, true);
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
#endif
@ -529,7 +532,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedClient1) {
e = arangodb::Endpoint::clientFactory("tcp://127.0.0.1");
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
////////////////////////////////////////////////////////////////////////////////
@ -541,7 +544,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedClient2) {
e = arangodb::Endpoint::clientFactory("ssl://127.0.0.1");
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
////////////////////////////////////////////////////////////////////////////////
@ -554,7 +557,7 @@ BOOST_AUTO_TEST_CASE (EndpointIsConnectedClient3) {
e = arangodb::Endpoint::clientFactory("unix:///tmp/socket");
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
#endif
@ -575,7 +578,7 @@ BOOST_AUTO_TEST_CASE (EndpointServerTcpIpv4WithPort) {
BOOST_CHECK_EQUAL(667, e->port());
BOOST_CHECK_EQUAL("127.0.0.1:667", e->hostAndPort());
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
////////////////////////////////////////////////////////////////////////////////
@ -596,7 +599,7 @@ BOOST_AUTO_TEST_CASE (EndpointServerUnix) {
BOOST_CHECK_EQUAL(0, e->port());
BOOST_CHECK_EQUAL("localhost", e->hostAndPort());
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
#endif
@ -617,7 +620,7 @@ BOOST_AUTO_TEST_CASE (EndpointClientSslIpV6WithPortHttp) {
BOOST_CHECK_EQUAL(43425, e->port());
BOOST_CHECK_EQUAL("[0001:0002:0003:0004:0005:0006:0007:0008]:43425", e->hostAndPort());
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
////////////////////////////////////////////////////////////////////////////////
@ -637,7 +640,7 @@ BOOST_AUTO_TEST_CASE (EndpointClientTcpIpv6WithoutPort) {
BOOST_CHECK_EQUAL(8529, e->port());
BOOST_CHECK_EQUAL("[::]:8529", e->hostAndPort());
BOOST_CHECK_EQUAL(false, e->isConnected());
DELETE_ENDPOINT(e);
delete e;
}
BOOST_AUTO_TEST_SUITE_END()

69
UnitTests/Basics/files-test.cpp
View File

@ -38,6 +38,7 @@
using namespace arangodb::basics;
static bool Initialized = false;
static uint64_t counter = 0;
// -----------------------------------------------------------------------------
// --SECTION-- setup / tear-down
@ -73,8 +74,6 @@ struct CFilesSetup {
}
StringBuffer* writeFile (const char* blob) {
static uint64_t counter = 0;
StringBuffer* filename = new StringBuffer(TRI_UNKNOWN_MEM_ZONE);
filename->appendText(_directory);
filename->appendText("/tmp-");
@ -108,6 +107,71 @@ struct CFilesSetup {
BOOST_FIXTURE_TEST_SUITE(CFilesTest, CFilesSetup)
BOOST_AUTO_TEST_CASE (tst_createdirectory) {
std::ostringstream out;
out << _directory.c_str() << "/tmp-" << ++counter << "-dir";
std::string filename = out.str();
long unused1;
std::string unused2;
int res = TRI_CreateDirectory(filename.c_str(), unused1, unused2);
BOOST_CHECK_EQUAL(0, res);
BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename.c_str()));
BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename.c_str()));
res = TRI_RemoveDirectory(filename.c_str());
BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename.c_str()));
BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename.c_str()));
}
BOOST_AUTO_TEST_CASE (tst_createdirectoryrecursive) {
std::ostringstream out;
out << _directory.c_str() << "/tmp-" << ++counter << "-dir";
std::string filename1 = out.str();
out << "/abc";
std::string filename2 = out.str();
long unused1;
std::string unused2;
int res = TRI_CreateRecursiveDirectory(filename2.c_str(), unused1, unused2);
BOOST_CHECK_EQUAL(0, res);
BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename1.c_str()));
BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename1.c_str()));
BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename2.c_str()));
BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename2.c_str()));
res = TRI_RemoveDirectory(filename1.c_str());
BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename1.c_str()));
BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename1.c_str()));
BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename2.c_str()));
BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename2.c_str()));
}
BOOST_AUTO_TEST_CASE (tst_removedirectorydeterministic) {
std::ostringstream out;
out << _directory.c_str() << "/tmp-" << ++counter << "-dir";
std::string filename1 = out.str();
out << "/abc";
std::string filename2 = out.str();
long unused1;
std::string unused2;
int res = TRI_CreateRecursiveDirectory(filename2.c_str(), unused1, unused2);
BOOST_CHECK_EQUAL(0, res);
BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename1.c_str()));
BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename1.c_str()));
BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename2.c_str()));
BOOST_CHECK_EQUAL(true, TRI_IsDirectory(filename2.c_str()));
res = TRI_RemoveDirectoryDeterministic(filename1.c_str());
BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename1.c_str()));
BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename1.c_str()));
BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename2.c_str()));
BOOST_CHECK_EQUAL(false, TRI_IsDirectory(filename2.c_str()));
}
////////////////////////////////////////////////////////////////////////////////
/// @brief test file exists
////////////////////////////////////////////////////////////////////////////////
@ -116,6 +180,7 @@ BOOST_AUTO_TEST_CASE (tst_existsfile) {
StringBuffer* filename = writeFile("");
BOOST_CHECK_EQUAL(true, TRI_ExistsFile(filename->c_str()));
TRI_UnlinkFile(filename->c_str());
BOOST_CHECK_EQUAL(false, TRI_ExistsFile(filename->c_str()));
delete filename;
}

4
arangod/Agency/AddFollower.cpp
View File

@ -25,11 +25,7 @@
#include "Agency/Agent.h"
#include "Agency/Job.h"
#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
AddFollower::AddFollower(Node const& snapshot, Agent* agent,
std::string const& jobId, std::string const& creator,

25
arangod/Agency/AgencyComm.cpp
View File

@ -1,4 +1,4 @@
///////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
@ -54,7 +54,6 @@
using namespace arangodb;
using namespace arangodb::application_features;
using namespace arangodb::basics;
using namespace arangodb::httpclient;
using namespace arangodb::rest;
@ -450,7 +449,7 @@ std::string AgencyCommManager::path(std::string const& p1) {
return "";
}
return MANAGER->_prefix + "/" + StringUtils::trim(p1, "/");
return MANAGER->_prefix + "/" + basics::StringUtils::trim(p1, "/");
}
std::string AgencyCommManager::path(std::string const& p1,
@ -459,8 +458,8 @@ std::string AgencyCommManager::path(std::string const& p1,
return "";
}
return MANAGER->_prefix + "/" + StringUtils::trim(p1, "/") + "/" +
StringUtils::trim(p2, "/");
return MANAGER->_prefix + "/" + basics::StringUtils::trim(p1, "/") + "/" +
basics::StringUtils::trim(p2, "/");
}
std::string AgencyCommManager::generateStamp() {
@ -674,7 +673,7 @@ void AgencyCommManager::removeEndpoint(std::string const& endpoint) {
}
std::string AgencyCommManager::endpointsString() const {
return StringUtils::join(endpoints(), ", ");
return basics::StringUtils::join(endpoints(), ", ");
}
std::vector<std::string> AgencyCommManager::endpoints() const {
@ -1280,7 +1279,7 @@ void AgencyComm::updateEndpoints(arangodb::velocypack::Slice const& current) {
for (const auto& i : VPackObjectIterator(current)) {
auto const endpoint = Endpoint::unifiedForm(i.value.copyString());
if (std::find(stored.begin(), stored.end(), endpoint) == stored.end()) {
LOG_TOPIC(INFO, Logger::CLUSTER)
LOG_TOPIC(DEBUG, Logger::CLUSTER)
<< "Adding endpoint " << endpoint << " to agent pool";
AgencyCommManager::MANAGER->addEndpoint(endpoint);
}
@ -1391,7 +1390,7 @@ AgencyCommResult AgencyComm::sendWithFailover(
b.add(VPackValue(clientId));
}
LOG_TOPIC(INFO, Logger::AGENCYCOMM) <<
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) <<
"Failed agency comm (" << result._statusCode << ")! " <<
"Inquiring about clientId " << clientId << ".";
@ -1410,25 +1409,25 @@ AgencyCommResult AgencyComm::sendWithFailover(
for (auto const& i : VPackArrayIterator(inner)) {
if (i.isUInt()) {
if (i.getUInt() == 0) {
LOG_TOPIC(INFO, Logger::AGENCYCOMM)
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< body << " failed: " << outer.toJson();
return result;
} else {
success = true;
}
} else {
LOG_TOPIC(INFO, Logger::AGENCYCOMM)
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< body << " failed with " << outer.toJson();
}
}
}
}
if (success) {
LOG_TOPIC(INFO, Logger::AGENCYCOMM)
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< body << " succeeded (" << outer.toJson() << ")";
return inq;
} else {
LOG_TOPIC(INFO, Logger::AGENCYCOMM)
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< body << " failed (" << outer.toJson() << ")";
return result;
}
@ -1437,7 +1436,7 @@ AgencyCommResult AgencyComm::sendWithFailover(
}
return inq;
} else {
LOG_TOPIC(INFO, Logger::AGENCYCOMM) <<
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM) <<
"Inquiry failed (" << inq._statusCode << "). Keep trying ...";
continue;
}

2
arangod/Agency/AgencyFeature.cpp
View File

@ -232,7 +232,7 @@ void AgencyFeature::start() {
_agent.reset(new consensus::Agent(consensus::config_t(
_size, _poolSize, _minElectionTimeout, _maxElectionTimeout, endpoint,
_agencyEndpoints, _supervision, _waitForSync, _supervisionFrequency,
_agencyEndpoints, _supervision, false, _supervisionFrequency,
_compactionStepSize, _compactionKeepSize, _supervisionGracePeriod,
_cmdLineTimings)));

48
arangod/Agency/Agent.cpp
View File

@ -257,7 +257,7 @@ bool Agent::recvAppendEntriesRPC(
term_t term, std::string const& leaderId, index_t prevIndex, term_t prevTerm,
index_t leaderCommitIndex, query_t const& queries) {
LOG_TOPIC(DEBUG, Logger::AGENCY) << "Got AppendEntriesRPC from "
LOG_TOPIC(TRACE, Logger::AGENCY) << "Got AppendEntriesRPC from "
<< leaderId << " with term " << term;
// Update commit index
@ -276,40 +276,42 @@ bool Agent::recvAppendEntriesRPC(
size_t nqs = queries->slice().length();
// State machine, _lastCommitIndex to advance atomically
MUTEX_LOCKER(mutexLocker, _ioLock);
if (nqs > 0) {
MUTEX_LOCKER(mutexLocker, _ioLock);
size_t ndups = _state.removeConflicts(queries);
if (nqs > ndups) {
LOG_TOPIC(TRACE, Logger::AGENCY)
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Appending " << nqs - ndups << " entries to state machine. ("
<< nqs << ", " << ndups << ")";
try {
_state.log(queries, ndups);
auto last = _state.log(queries, ndups);
_spearhead.apply(
_state.slices(last-ndups, last), last, _constituent.term());
_readDB.apply(
_state.slices(last-ndups, last), last, _constituent.term());
_lastCommitIndex = last;
if (_lastCommitIndex >= _nextCompationAfter) {
_state.compact(_lastCommitIndex);
_nextCompationAfter += _config.compactionStepSize();
}
} catch (std::exception const&) {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Malformed query: " << __FILE__ << __LINE__;
}
}
}
_spearhead.apply(
_state.slices(_lastCommitIndex + 1, leaderCommitIndex), _lastCommitIndex,
_constituent.term());
_readDB.apply(
_state.slices(_lastCommitIndex + 1, leaderCommitIndex), _lastCommitIndex,
_constituent.term());
_lastCommitIndex = leaderCommitIndex;
if (_lastCommitIndex >= _nextCompationAfter) {
_state.compact(_lastCommitIndex);
_nextCompationAfter += _config.compactionStepSize();
}
return true;
}
@ -348,7 +350,7 @@ void Agent::sendAppendEntriesRPC() {
duration<double> m = system_clock::now() - _lastSent[followerId];
if (highest == _lastHighest[followerId] &&
m.count() < 0.5 * _config.minPing()) {
m.count() < 0.25 * _config.minPing()) {
continue;
}

1
arangod/Agency/CleanOutServer.cpp
View File

@ -28,7 +28,6 @@
#include "Agency/MoveShard.h"
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
CleanOutServer::CleanOutServer(Node const& snapshot, Agent* agent,
std::string const& jobId,

7
arangod/Agency/Constituent.cpp
View File

@ -468,11 +468,13 @@ void Constituent::callElection() {
void Constituent::update(std::string const& leaderID, term_t t) {
MUTEX_LOCKER(guard, _castLock);
_term = t;
if (_leaderID != leaderID) {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Constituent::update: setting _leaderID to " << leaderID
<< " in term " << _term;
_leaderID = leaderID;
_role = FOLLOWER;
}
}
@ -546,6 +548,11 @@ void Constituent::run() {
LOG_TOPIC(DEBUG, Logger::AGENCY) << "Set _leaderID to " << _leaderID
<< " in term " << _term;
} else {
{
MUTEX_LOCKER(guard, _castLock);
_role = FOLLOWER;
}
while (!this->isStopping()) {
if (_role == FOLLOWER) {
static double const M = 1.0e6;

6
arangod/Agency/FailedFollower.cpp
View File

@ -75,7 +75,7 @@ bool FailedFollower::create() {
}
}
_jb = std::make_shared<velocypack::Builder>();
_jb = std::make_shared<Builder>();
_jb->openArray();
_jb->openObject();
@ -128,7 +128,7 @@ bool FailedFollower::start() {
// Copy todo to pending
velocypack::Builder todo, pending;
Builder todo, pending;
// Get todo entry
todo.openArray();
@ -254,7 +254,7 @@ JOB_STATUS FailedFollower::status() {
if (compareServerLists(planned.slice(), current.slice())) {
// Remove shard from /arango/Target/FailedServers/<server> array
velocypack::Builder del;
Builder del;
del.openArray();
del.openObject();
std::string path = _agencyPrefix + failedServersPrefix + "/" + _from;

1
arangod/Agency/FailedLeader.cpp
View File

@ -27,7 +27,6 @@
#include "Agency/Job.h"
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
FailedLeader::FailedLeader(Node const& snapshot, Agent* agent,
std::string const& jobId, std::string const& creator,

8
arangod/Agency/FailedServer.cpp
View File

@ -58,7 +58,7 @@ bool FailedServer::start() {
<< "Start FailedServer job " + _jobId + " for server " + _server;
// Copy todo to pending
velocypack::Builder todo, pending;
Builder todo, pending;
// Get todo entry
todo.openArray();
@ -210,7 +210,7 @@ bool FailedServer::create() {
std::string path = _agencyPrefix + toDoPrefix + _jobId;
_jb = std::make_shared<velocypack::Builder>();
_jb = std::make_shared<Builder>();
_jb->openArray();
_jb->openObject();
@ -271,7 +271,7 @@ JOB_STATUS FailedServer::status() {
// mop: ohhh...server is healthy again!
bool serverHealthy = serverHealth == Supervision::HEALTH_STATUS_GOOD;
std::shared_ptr<velocypack::Builder> deleteTodos;
std::shared_ptr<Builder> deleteTodos;
Node::Children const todos = _snapshot(toDoPrefix).children();
Node::Children const pends = _snapshot(pendingPrefix).children();
@ -281,7 +281,7 @@ JOB_STATUS FailedServer::status() {
if (!subJob.first.compare(0, _jobId.size() + 1, _jobId + "-")) {
if (serverHealthy) {
if (!deleteTodos) {
deleteTodos.reset(new velocypack::Builder());
deleteTodos.reset(new Builder());
deleteTodos->openArray();
deleteTodos->openObject();
}

10
arangod/Agency/Inception.cpp
View File

@ -66,7 +66,7 @@ void Inception::gossip() {
auto const version = config.version();
// Build gossip message
auto out = std::make_shared<velocypack::Builder>();
auto out = std::make_shared<Builder>();
out->openObject();
out->add("endpoint", VPackValue(config.endpoint()));
out->add("id", VPackValue(config.id()));
@ -169,7 +169,7 @@ bool Inception::restartingActiveAgent() {
auto const& clientEp = myConfig.endpoint();
auto const majority = (myConfig.size()+1)/2;
velocypack::Builder greeting;
Builder greeting;
{
VPackObjectBuilder b(&greeting);
greeting.add(clientId, VPackValue(clientEp));
@ -259,7 +259,7 @@ bool Inception::restartingActiveAgent() {
}
}
auto agency = std::make_shared<velocypack::Builder>();
auto agency = std::make_shared<Builder>();
agency->openObject();
agency->add("term", theirConfig.get("term"));
agency->add("id", VPackValue(theirLeaderId));
@ -435,7 +435,7 @@ bool Inception::estimateRAFTInterval() {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "mean(" << mean << ") stdev(" << stdev<< ")";
velocypack::Builder measurement;
Builder measurement;
measurement.openObject();
measurement.add("mean", VPackValue(mean));
measurement.add("stdev", VPackValue(stdev));
@ -541,8 +541,10 @@ void Inception::run() {
LOG_TOPIC(INFO, Logger::AGENCY) << "Activating agent.";
_agent->ready(true);
} else {
if (!this->isStopping()) {
LOG_TOPIC(FATAL, Logger::AGENCY)
<< "Unable to restart with persisted pool. Fatal exit.";
}
FATAL_ERROR_EXIT();
// FATAL ERROR
}

4
arangod/Agency/Job.cpp
View File

@ -25,7 +25,7 @@
using namespace arangodb::consensus;
bool arangodb::consensus::compareServerLists(velocypack::Slice plan, velocypack::Slice current) {
bool arangodb::consensus::compareServerLists(Slice plan, Slice current) {
if (!plan.isArray() || !current.isArray()) {
return false;
}
@ -80,7 +80,7 @@ JOB_STATUS Job::exists() const {
bool Job::finish(std::string const& type, bool success,
std::string const& reason) const {
velocypack::Builder pending, finished;
Builder pending, finished;
// Get todo entry
pending.openArray();

9
arangod/Agency/Job.h
View File

@ -28,7 +28,6 @@
#include "Node.h"
#include "Supervision.h"
#include <velocypack/Builder.h>
#include <velocypack/Iterator.h>
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>
@ -42,7 +41,7 @@ namespace consensus {
// and all others followers. Both arguments must be arrays. Returns true,
// if the first items in both slice are equal and if both arrays contain
// the same set of strings.
bool compareServerLists(velocypack::Slice plan, velocypack::Slice current);
bool compareServerLists(Slice plan, Slice current);
enum JOB_STATUS { TODO, PENDING, FINISHED, FAILED, NOTFOUND };
const std::vector<std::string> pos({"/Target/ToDo/", "/Target/Pending/",
@ -64,9 +63,9 @@ static std::string const plannedServers = "/Plan/DBServers";
static std::string const healthPrefix = "/Supervision/Health/";
inline arangodb::consensus::write_ret_t transact(Agent* _agent,
velocypack::Builder const& transaction,
Builder const& transaction,
bool waitForCommit = true) {
query_t envelope = std::make_shared<velocypack::Builder>();
query_t envelope = std::make_shared<Builder>();
try {
envelope->openArray();
@ -138,7 +137,7 @@ struct Job {
std::string _creator;
std::string _agencyPrefix;
std::shared_ptr<velocypack::Builder> _jb;
std::shared_ptr<Builder> _jb;
};

1
arangod/Agency/MoveShard.cpp
View File

@ -29,7 +29,6 @@
static std::string const DBServer = "DBServer";
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
MoveShard::MoveShard(Node const& snapshot, Agent* agent,
std::string const& jobId, std::string const& creator,

49
arangod/Agency/Node.cpp
View File

@ -33,9 +33,8 @@
#include <deque>
#include <regex>
using namespace arangodb;
using namespace arangodb::basics;
using namespace arangodb::consensus;
using namespace arangodb::basics;
struct NotEmpty {
bool operator()(const std::string& s) { return !s.empty(); }
@ -89,16 +88,16 @@ Node::Node(std::string const& name, Store* store)
Node::~Node() {}
/// Get slice to value buffer
velocypack::Slice Node::slice() const {
Slice Node::slice() const {
// Some array
if (_isArray) {
rebuildVecBuf();
return velocypack::Slice(_vecBuf.data());
return Slice(_vecBuf.data());
}
// Some value
if (!_value.empty()) {
return velocypack::Slice(_value.front().data());
return Slice(_value.front().data());
}
// Empty object
@ -107,10 +106,10 @@ velocypack::Slice Node::slice() const {
void Node::rebuildVecBuf() const {
if (_vecBufDirty) { // Dirty vector buffer
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
for (auto const& i : _value) {
tmp.add(velocypack::Slice(i.data()));
tmp.add(Slice(i.data()));
}
tmp.close();
_vecBuf = *tmp.steal();
@ -324,7 +323,7 @@ Store& Node::store() { return *(root()._store); }
Store const& Node::store() const { return *(root()._store); }
// velocypack value type of this node
velocypack::ValueType Node::valueType() const { return slice().type(); }
ValueType Node::valueType() const { return slice().type(); }
// file time to live entry for this node to now + millis
bool Node::addTimeToLive(long millis) {
@ -359,7 +358,7 @@ namespace consensus {
/// Set value
template <>
bool Node::handle<SET>(VPackSlice const& slice) {
VPackSlice val = slice.get("new");
Slice val = slice.get("new");
if (val.isObject()) {
if (val.hasKey("op")) { // No longer a keyword but a regular key "op"
@ -394,12 +393,12 @@ bool Node::handle<SET>(VPackSlice const& slice) {
/// Increment integer value or set 1
template <>
bool Node::handle<INCREMENT>(VPackSlice const& slice) {
velocypack::Builder tmp;
Builder tmp;
tmp.openObject();
try {
tmp.add("tmp", velocypack::Value(this->slice().getInt() + 1));
tmp.add("tmp", Value(this->slice().getInt() + 1));
} catch (std::exception const&) {
tmp.add("tmp", velocypack::Value(1));
tmp.add("tmp", Value(1));
}
tmp.close();
*this = tmp.slice().get("tmp");
@ -409,12 +408,12 @@ bool Node::handle<INCREMENT>(VPackSlice const& slice) {
/// Decrement integer value or set -1
template <>
bool Node::handle<DECREMENT>(VPackSlice const& slice) {
velocypack::Builder tmp;
Builder tmp;
tmp.openObject();
try {
tmp.add("tmp", velocypack::Value(this->slice().getInt() - 1));
tmp.add("tmp", Value(this->slice().getInt() - 1));
} catch (std::exception const&) {
tmp.add("tmp", velocypack::Value(-1));
tmp.add("tmp", Value(-1));
}
tmp.close();
*this = tmp.slice().get("tmp");
@ -429,7 +428,7 @@ bool Node::handle<PUSH>(VPackSlice const& slice) {
<< slice.toJson();
return false;
}
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
if (this->slice().isArray()) {
for (auto const& old : VPackArrayIterator(this->slice())) tmp.add(old);
@ -448,7 +447,7 @@ bool Node::handle<ERASE>(VPackSlice const& slice) {
<< "Operator erase without value to be erased: " << slice.toJson();
return false;
}
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
if (this->slice().isArray()) {
for (auto const& old : VPackArrayIterator(this->slice())) {
@ -475,7 +474,7 @@ bool Node::handle<REPLACE>(VPackSlice const& slice) {
<< slice.toJson();
return false;
}
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
if (this->slice().isArray()) {
for (auto const& old : VPackArrayIterator(this->slice())) {
@ -494,7 +493,7 @@ bool Node::handle<REPLACE>(VPackSlice const& slice) {
/// Remove element from end of array.
template <>
bool Node::handle<POP>(VPackSlice const& slice) {
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
if (this->slice().isArray()) {
VPackArrayIterator it(this->slice());
@ -519,7 +518,7 @@ bool Node::handle<PREPEND>(VPackSlice const& slice) {
<< slice.toJson();
return false;
}
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
tmp.add(slice.get("new"));
if (this->slice().isArray()) {
@ -533,7 +532,7 @@ bool Node::handle<PREPEND>(VPackSlice const& slice) {
/// Remove element from front of array
template <>
bool Node::handle<SHIFT>(VPackSlice const& slice) {
velocypack::Builder tmp;
Builder tmp;
tmp.openArray();
if (this->slice().isArray()) { // If a
VPackArrayIterator it(this->slice());
@ -678,7 +677,7 @@ bool Node::applies(VPackSlice const& slice) {
return true;
}
void Node::toBuilder(velocypack::Builder& builder, bool showHidden) const {
void Node::toBuilder(Builder& builder, bool showHidden) const {
try {
if (type() == NODE) {
VPackObjectBuilder guard(&builder);
@ -729,7 +728,7 @@ Node::Children& Node::children() { return _children; }
Node::Children const& Node::children() const { return _children; }
std::string Node::toJson() const {
velocypack::Builder builder;
Builder builder;
builder.openArray();
toBuilder(builder);
builder.close();
@ -796,7 +795,7 @@ std::string Node::getString() const {
return slice().copyString();
}
velocypack::Slice Node::getArray() const {
Slice Node::getArray() const {
if (type() == NODE) {
throw StoreException("Must not convert NODE type to array");
}
@ -804,6 +803,6 @@ velocypack::Slice Node::getArray() const {
throw StoreException("Not an array type");
}
rebuildVecBuf();
return velocypack::Slice(_vecBuf.data());
return Slice(_vecBuf.data());
}

14
arangod/Agency/Node.h
View File

@ -50,6 +50,8 @@ enum Operation {
REPLACE
};
using namespace arangodb::velocypack;
class StoreException : public std::exception {
public:
explicit StoreException(std::string const& message) : _message(message) {}
@ -159,7 +161,7 @@ class Node {
bool handle(arangodb::velocypack::Slice const&);
/// @brief Create Builder representing this store
void toBuilder(velocypack::Builder&, bool showHidden = false) const;
void toBuilder(Builder&, bool showHidden = false) const;
/// @brief Access children
Children& children();
@ -168,10 +170,10 @@ class Node {
Children const& children() const;
/// @brief Create slice from value
velocypack::Slice slice() const;
Slice slice() const;
/// @brief Get value type
velocypack::ValueType valueType() const;
ValueType valueType() const;
/// @brief Add observer for this node
bool addObserver(std::string const&);
@ -216,7 +218,7 @@ class Node {
std::string getString() const;
/// @brief Get array value
velocypack::Slice getArray() const;
Slice getArray() const;
protected:
/// @brief Add time to live entry
@ -232,8 +234,8 @@ class Node {
Store* _store; ///< @brief Store
Children _children; ///< @brief child nodes
TimePoint _ttl; ///< @brief my expiry
std::vector<velocypack::Buffer<uint8_t>> _value; ///< @brief my value
mutable velocypack::Buffer<uint8_t> _vecBuf;
std::vector<Buffer<uint8_t>> _value; ///< @brief my value
mutable Buffer<uint8_t> _vecBuf;
mutable bool _vecBufDirty;
bool _isArray;
};

1
arangod/Agency/RemoveServer.cpp
View File

@ -27,7 +27,6 @@
#include "Agency/Job.h"
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
RemoveServer::RemoveServer(Node const& snapshot, Agent* agent,
std::string const& jobId, std::string const& creator,

4
arangod/Agency/RestAgencyHandler.cpp
View File

@ -35,10 +35,10 @@
#include "Rest/Version.h"
using namespace arangodb;
using namespace arangodb::basics;
using namespace arangodb::consensus;
using namespace arangodb::rest;
using namespace arangodb::velocypack;
using namespace arangodb::consensus;
////////////////////////////////////////////////////////////////////////////////
/// @brief ArangoDB server

2
arangod/Agency/State.h
View File

@ -184,7 +184,7 @@ class State {
size_t _cur;
/// @brief Operation options
OperationOptions _options;
arangodb::OperationOptions _options;
/// @brief Empty log entry;
static log_t emptyLog;

3
arangod/Agency/Store.cpp
View File

@ -40,9 +40,8 @@
#include <iomanip>
#include <regex>
using namespace arangodb::basics;
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
using namespace arangodb::basics;
/// Non-Emptyness of string
struct NotEmpty {

11
arangod/Agency/Store.h
View File

@ -28,9 +28,6 @@
#include "Basics/Thread.h"
#include "Node.h"
#include <velocypack/Builder.h>
#include <velocypack/Slice.h>
namespace arangodb {
namespace consensus {
@ -61,10 +58,10 @@ class Store : public arangodb::Thread {
std::vector<bool> apply(query_t const& query, bool verbose = false);
/// @brief Apply single entry in query
bool apply(velocypack::Slice const& query, bool verbose = false);
bool apply(Slice const& query, bool verbose = false);
/// @brief Apply entry in query
std::vector<bool> apply(std::vector<velocypack::Slice> const& query,
std::vector<bool> apply(std::vector<Slice> const& query,
index_t lastCommitIndex, term_t term,
bool inform = true);
@ -82,7 +79,7 @@ class Store : public arangodb::Thread {
bool start();
/// @brief Dump everything to builder
void dumpToBuilder(velocypack::Builder&) const;
void dumpToBuilder(Builder&) const;
/// @brief Notify observers
void notifyObservers() const;
@ -93,7 +90,7 @@ class Store : public arangodb::Thread {
Store& operator=(VPackSlice const& slice);
/// @brief Create Builder representing this store
void toBuilder(velocypack::Builder&, bool showHidden = false) const;
void toBuilder(Builder&, bool showHidden = false) const;
/// @brief Copy out a node
Node get(std::string const& path) const;

4
arangod/Agency/Supervision.cpp
View File

@ -41,9 +41,9 @@
#include "Basics/MutexLocker.h"
using namespace arangodb;
using namespace arangodb::application_features;
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
using namespace arangodb::application_features;
std::string Supervision::_agencyPrefix = "/arango";

1
arangod/Agency/UnassumedLeadership.cpp
View File

@ -27,7 +27,6 @@
#include "Agency/Job.h"
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
UnassumedLeadership::UnassumedLeadership(
Node const& snapshot, Agent* agent, std::string const& jobId,

1
arangod/Agency/v8-agency.cpp
View File

@ -39,7 +39,6 @@ using namespace arangodb;
using namespace arangodb::application_features;
using namespace arangodb::basics;
using namespace arangodb::consensus;
using namespace arangodb::velocypack;
static void JS_EnabledAgent(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_TRY_CATCH_BEGIN(isolate);

1
arangod/Aql/AstNode.h
View File

@ -418,7 +418,6 @@ struct AstNode {
bool isAttributeAccessForVariable(Variable const* variable, bool allowIndexedAccess) const {
auto node = getAttributeAccessForVariable(allowIndexedAccess);
if (node == nullptr) {
return false;
}

23
arangod/Aql/ClusterBlocks.cpp
View File

@ -33,6 +33,7 @@
#include "Aql/AqlValue.h"
#include "Aql/BlockCollector.h"
#include "Aql/ExecutionEngine.h"
#include "Aql/ExecutionStats.h"
#include "Basics/Exceptions.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringBuffer.h"
@ -1328,7 +1329,7 @@ int RemoteBlock::initializeCursor(AqlItemBlock* items, size_t pos) {
responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
if (slice.hasKey("code")) {
return slice.get("code").getNumericValue<int>();
}
@ -1362,9 +1363,14 @@ int RemoteBlock::shutdown(int errorCode) {
std::shared_ptr<VPackBuilder> builder =
VPackParser::fromJson(responseBodyBuf.c_str(), responseBodyBuf.length());
VPackSlice slice = builder->slice();
// read "warnings" attribute if present and add it to our query
if (slice.isObject()) {
if (slice.hasKey("stats")) {
ExecutionStats newStats(slice.get("stats"));
_engine->_stats.add(newStats);
}
// read "warnings" attribute if present and add it to our query
VPackSlice warnings = slice.get("warnings");
if (warnings.isArray()) {
auto query = _engine->getQuery();
@ -1415,19 +1421,14 @@ AqlItemBlock* RemoteBlock::getSome(size_t atLeast, size_t atMost) {
res->result->getBodyVelocyPack();
VPackSlice responseBody = responseBodyBuilder->slice();
ExecutionStats newStats(responseBody.get("stats"));
_engine->_stats.addDelta(_deltaStats, newStats);
_deltaStats = newStats;
if (VelocyPackHelper::getBooleanValue(responseBody, "exhausted", true)) {
traceGetSomeEnd(nullptr);
return nullptr;
}
auto r = new arangodb::aql::AqlItemBlock(_engine->getQuery()->resourceMonitor(), responseBody);
traceGetSomeEnd(r);
return r;
auto r = std::make_unique<AqlItemBlock>(_engine->getQuery()->resourceMonitor(), responseBody);
traceGetSomeEnd(r.get());
return r.release();
// cppcheck-suppress style
DEBUG_END_BLOCK();

4
arangod/Aql/ClusterBlocks.h
View File

@ -28,7 +28,6 @@
#include "Aql/ClusterNodes.h"
#include "Aql/ExecutionBlock.h"
#include "Aql/ExecutionNode.h"
#include "Aql/ExecutionStats.h"
#include "Rest/GeneralRequest.h"
namespace arangodb {
@ -339,9 +338,6 @@ class RemoteBlock : public ExecutionBlock {
/// @brief the ID of the query on the server as a string
std::string _queryId;
/// @brief the ID of the query on the server as a string
ExecutionStats _deltaStats;
/// @brief whether or not this block will forward initialize,
/// initializeCursor or shutDown requests
bool const _isResponsibleForInitializeCursor;

10
arangod/Aql/ExecutionEngine.cpp
View File

@ -529,6 +529,7 @@ struct CoordinatorInstanciator : public WalkerWorker<ExecutionNode> {
VPackBuilder tmp;
query->ast()->variables()->toVelocyPack(tmp);
result.add("initialize", VPackValue(false));
result.add("variables", tmp.slice());
result.add("collections", VPackValue(VPackValueType::Array));
@ -1133,7 +1134,7 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan(
bool const isCoordinator =
arangodb::ServerState::instance()->isCoordinator(role);
bool const isDBServer = arangodb::ServerState::instance()->isDBServer(role);
TRI_ASSERT(queryRegistry != nullptr);
ExecutionEngine* engine = nullptr;
@ -1354,8 +1355,11 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan(
}
engine->_root = root;
root->initialize();
root->initializeCursor(nullptr, 0);
if (plan->isResponsibleForInitialize()) {
root->initialize();
root->initializeCursor(nullptr, 0);
}
return engine;
} catch (...) {

12
arangod/Aql/ExecutionPlan.cpp
View File

@ -177,6 +177,7 @@ ExecutionPlan::ExecutionPlan(Ast* ast)
: _ids(),
_root(nullptr),
_varUsageComputed(false),
_isResponsibleForInitialize(true),
_nextId(0),
_ast(ast),
_lastLimitNode(nullptr),
@ -280,6 +281,7 @@ ExecutionPlan* ExecutionPlan::clone() {
plan->_root = _root->clone(plan.get(), true, false);
plan->_nextId = _nextId;
plan->_appliedRules = _appliedRules;
plan->_isResponsibleForInitialize = _isResponsibleForInitialize;
CloneNodeAdder adder(plan.get());
plan->_root->walk(&adder);
@ -348,6 +350,7 @@ void ExecutionPlan::toVelocyPack(VPackBuilder& builder, Ast* ast, bool verbose)
size_t nrItems = 0;
builder.add("estimatedCost", VPackValue(_root->getCost(nrItems)));
builder.add("estimatedNrItems", VPackValue(nrItems));
builder.add("initialize", VPackValue(_isResponsibleForInitialize));
builder.close();
}
@ -1882,17 +1885,22 @@ void ExecutionPlan::insertDependency(ExecutionNode* oldNode,
/// @brief create a plan from VPack
ExecutionNode* ExecutionPlan::fromSlice(VPackSlice const& slice) {
ExecutionNode* ret = nullptr;
if (!slice.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "plan slice is not an object");
}
if (slice.hasKey("initialize")) {
// whether or not this plan (or fragment) is responsible for calling initialize
_isResponsibleForInitialize = slice.get("initialize").getBoolean();
}
VPackSlice nodes = slice.get("nodes");
if (!nodes.isArray()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "plan \"nodes\" attribute is not an array");
}
ExecutionNode* ret = nullptr;
// first, re-create all nodes from the Slice, using the node ids
// no dependency links will be set up in this step

4
arangod/Aql/ExecutionPlan.h
View File

@ -75,6 +75,8 @@ class ExecutionPlan {
/// @brief check if the plan is empty
inline bool empty() const { return (_root == nullptr); }
bool isResponsibleForInitialize() const { return _isResponsibleForInitialize; }
/// @brief note that an optimizer rule was applied
inline void addAppliedRule(int level) { _appliedRules.emplace_back(level); }
@ -299,6 +301,8 @@ class ExecutionPlan {
/// @brief flag to indicate whether the variable usage is computed
bool _varUsageComputed;
bool _isResponsibleForInitialize;
/// @brief auto-increment sequence for node ids
size_t _nextId;

5
arangod/Aql/ExecutionStats.cpp
View File

@ -74,9 +74,7 @@ ExecutionStats::ExecutionStats()
executionTime(0.0) {}
ExecutionStats::ExecutionStats(VPackSlice const& slice)
: httpRequests(0),
fullCount(-1),
executionTime(0.0) {
: ExecutionStats() {
if (!slice.isObject()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"stats is not an object");
@ -88,7 +86,6 @@ ExecutionStats::ExecutionStats(VPackSlice const& slice)
scannedIndex = slice.get("scannedIndex").getNumber<int64_t>();
filtered = slice.get("filtered").getNumber<int64_t>();
if (slice.hasKey("httpRequests")) {
httpRequests = slice.get("httpRequests").getNumber<int64_t>();
}

25
arangod/Aql/ExecutionStats.h
View File

@ -58,21 +58,22 @@ struct ExecutionStats {
scannedIndex += summand.scannedIndex;
filtered += summand.filtered;
httpRequests += summand.httpRequests;
fullCount += summand.fullCount;
if (summand.fullCount > 0) {
// fullCount may be negative, don't add it then
fullCount += summand.fullCount;
}
// intentionally no modification of executionTime
}
/// @brief sumarize the delta of two other sets of ExecutionStats to us
void addDelta(ExecutionStats const& lastStats,
ExecutionStats const& newStats) {
writesExecuted += newStats.writesExecuted - lastStats.writesExecuted;
writesIgnored += newStats.writesIgnored - lastStats.writesIgnored;
scannedFull += newStats.scannedFull - lastStats.scannedFull;
scannedIndex += newStats.scannedIndex - lastStats.scannedIndex;
filtered += newStats.filtered - lastStats.filtered;
httpRequests += newStats.httpRequests - lastStats.httpRequests;
fullCount += newStats.fullCount - lastStats.fullCount;
// intentionally no modification of executionTime
void clear() {
writesExecuted = 0;
writesIgnored = 0;
scannedFull = 0;
scannedIndex = 0;
filtered = 0;
httpRequests = 0;
fullCount = -1;
executionTime = 0.0;
}
/// @brief number of successfully executed write operations

83
arangod/Aql/OptimizerRules.cpp
View File

@ -2350,13 +2350,15 @@ void arangodb::aql::scatterInClusterRule(Optimizer* opt, ExecutionPlan* plan,
// Using Index for sort only works if all indexes are equal.
auto first = allIndexes[0].getIndex();
for (auto const& path : first->fieldNames()) {
elements.emplace_back(sortVariable, !isSortReverse, path);
}
for (auto const& it : allIndexes) {
if (first != it.getIndex()) {
elements.clear();
break;
if (first->isSorted()) {
for (auto const& path : first->fieldNames()) {
elements.emplace_back(sortVariable, !isSortReverse, path);
}
for (auto const& it : allIndexes) {
if (first != it.getIndex()) {
elements.clear();
break;
}
}
}
} else if (nodeType == ExecutionNode::INSERT ||
@ -4098,47 +4100,54 @@ MMFilesGeoIndexInfo iterativePreorderWithCondition(EN::NodeType type, AstNode* r
return MMFilesGeoIndexInfo{};
}
MMFilesGeoIndexInfo geoDistanceFunctionArgCheck(std::pair<AstNode*,AstNode*> const& pair, ExecutionPlan* plan, MMFilesGeoIndexInfo info){
using SV = std::vector<std::string>;
MMFilesGeoIndexInfo geoDistanceFunctionArgCheck(std::pair<AstNode const*, AstNode const*> const& pair,
ExecutionPlan* plan, MMFilesGeoIndexInfo info){
std::pair<Variable const*, std::vector<arangodb::basics::AttributeName>> attributeAccess1;
std::pair<Variable const*, std::vector<arangodb::basics::AttributeName>> attributeAccess2;
// first and second should be based on the same document - need to provide the document
// in order to see which collection is bound to it and if that collections supports geo-index
if( !pair.first->isAttributeAccessForVariable() || !pair.second->isAttributeAccessForVariable()){
if (!pair.first->isAttributeAccessForVariable(attributeAccess1) ||
!pair.second->isAttributeAccessForVariable(attributeAccess2)) {
info.invalidate();
return info;
}
TRI_ASSERT(attributeAccess1.first != nullptr);
TRI_ASSERT(attributeAccess2.first != nullptr);
// expect access of the for doc.attribute
// TODO: more complex access path have to be added: loop until REFERENCE TYPE IS FOUND
auto setter1 = plan->getVarSetBy(static_cast<Variable const*>(pair.first->getMember(0)->getData())->id);
auto setter2 = plan->getVarSetBy(static_cast<Variable const*>(pair.second->getMember(0)->getData())->id);
SV accessPath1{pair.first->getString()};
SV accessPath2{pair.second->getString()};
auto setter1 = plan->getVarSetBy(attributeAccess1.first->id);
auto setter2 = plan->getVarSetBy(attributeAccess2.first->id);
if(setter1 == setter2){
if(setter1->getType() == EN::ENUMERATE_COLLECTION){
auto collNode = reinterpret_cast<EnumerateCollectionNode*>(setter1);
if (setter1 != nullptr &&
setter2 != nullptr &&
setter1 == setter2 &&
setter1->getType() == EN::ENUMERATE_COLLECTION) {
auto collNode = reinterpret_cast<EnumerateCollectionNode*>(setter1);
auto coll = collNode->collection(); //what kind of indexes does it have on what attributes
auto lcoll = coll->getCollection();
// TODO - check collection for suitable geo-indexes
for(auto indexShardPtr : lcoll->getIndexes()){
// get real index
arangodb::Index& index = *indexShardPtr.get();
auto coll = collNode->collection(); //what kind of indexes does it have on what attributes
auto lcoll = coll->getCollection();
// TODO - check collection for suitable geo-indexes
for(auto indexShardPtr : lcoll->getIndexes()){
// get real index
arangodb::Index& index = *indexShardPtr.get();
// check if current index is a geo-index
if( index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX
&& index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO2_INDEX) {
continue;
}
// check if current index is a geo-index
if( index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX
&& index.type() != arangodb::Index::IndexType::TRI_IDX_TYPE_GEO2_INDEX){
continue;
}
TRI_ASSERT(index.fields().size() == 2);
//check access paths of attributes in ast and those in index match
if( index.fieldNames()[0] == accessPath1 && index.fieldNames()[1] == accessPath2 ){
info.collectionNode = collNode;
info.index = indexShardPtr;
info.longitude = std::move(accessPath1);
info.latitude = std::move(accessPath2);
return info;
}
//check access paths of attributes in ast and those in index match
if (index.fields()[0] == attributeAccess1.second &&
index.fields()[1] == attributeAccess2.second) {
info.collectionNode = collNode;
info.index = indexShardPtr;
TRI_AttributeNamesJoinNested(attributeAccess1.second, info.longitude, true);
TRI_AttributeNamesJoinNested(attributeAccess2.second, info.latitude, true);
return info;
}
}
}

24
arangod/Aql/Query.cpp
View File

@ -731,14 +731,12 @@ QueryResult Query::execute(QueryRegistry* registry) {
}
_trx->commit();
result.context = _trx->transactionContext();
_engine->_stats.setExecutionTime(TRI_microtime() - _startTime);
auto stats = std::make_shared<VPackBuilder>();
_engine->_stats.toVelocyPack(*(stats.get()));
result.context = _trx->transactionContext();
cleanupPlanAndEngine(TRI_ERROR_NO_ERROR);
cleanupPlanAndEngine(TRI_ERROR_NO_ERROR, stats.get());
enterState(FINALIZATION);
@ -913,18 +911,15 @@ QueryResultV8 Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry) {
_trx->commit();
_engine->_stats.setExecutionTime(TRI_microtime() - _startTime);
auto stats = std::make_shared<VPackBuilder>();
_engine->_stats.toVelocyPack(*(stats.get()));
result.context = _trx->transactionContext();
LOG_TOPIC(DEBUG, Logger::QUERIES)
<< TRI_microtime() - _startTime << " "
<< "Query::executeV8: before cleanupPlanAndEngine"
<< " this: " << (uintptr_t) this;
cleanupPlanAndEngine(TRI_ERROR_NO_ERROR);
result.context = _trx->transactionContext();
_engine->_stats.setExecutionTime(TRI_microtime() - _startTime);
auto stats = std::make_shared<VPackBuilder>();
cleanupPlanAndEngine(TRI_ERROR_NO_ERROR, stats.get());
enterState(FINALIZATION);
@ -1387,10 +1382,13 @@ std::string Query::getStateString() const {
}
/// @brief cleanup plan and engine for current query
void Query::cleanupPlanAndEngine(int errorCode) {
void Query::cleanupPlanAndEngine(int errorCode, VPackBuilder* statsBuilder) {
if (_engine != nullptr) {
try {
_engine->shutdown(errorCode);
if (statsBuilder != nullptr) {
_engine->_stats.toVelocyPack(*statsBuilder);
}
} catch (...) {
// shutdown may fail but we must not throw here
// (we're also called from the destructor)

2
arangod/Aql/Query.h
View File

@ -378,7 +378,7 @@ class Query {
void enterState(ExecutionState);
/// @brief cleanup plan and engine for current query
void cleanupPlanAndEngine(int);
void cleanupPlanAndEngine(int, VPackBuilder* statsBuilder = nullptr);
/// @brief create a TransactionContext
std::shared_ptr<arangodb::TransactionContext> createTransactionContext();

19
arangod/Aql/RestAqlHandler.cpp
View File

@ -697,7 +697,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
try {
res = query->trx()->lockCollections();
} catch (...) {
LOG(ERR) << "lock lead to an exception";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"lock lead to an exception");
@ -726,15 +725,10 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
if (items.get() == nullptr) {
answerBuilder.add("exhausted", VPackValue(true));
answerBuilder.add("error", VPackValue(false));
answerBuilder.add(VPackValue("stats"));
query->getStats(answerBuilder);
} else {
try {
items->toVelocyPack(query->trx(), answerBuilder);
answerBuilder.add(VPackValue("stats"));
query->getStats(answerBuilder);
} catch (...) {
LOG(ERR) << "cannot transform AqlItemBlock to VelocyPack";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"cannot transform AqlItemBlock to VelocyPack");
@ -760,7 +754,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
skipped = block->skipSomeForShard(atLeast, atMost, shardId);
}
} catch (...) {
LOG(ERR) << "skipSome lead to an exception";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"skipSome lead to an exception");
@ -768,8 +761,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
}
answerBuilder.add("skipped", VPackValue(static_cast<double>(skipped)));
answerBuilder.add("error", VPackValue(false));
answerBuilder.add(VPackValue("stats"));
query->getStats(answerBuilder);
} else if (operation == "skip") {
auto number =
VelocyPackHelper::getNumericValue<size_t>(querySlice, "number", 1);
@ -789,10 +780,7 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
}
answerBuilder.add("exhausted", VPackValue(exhausted));
answerBuilder.add("error", VPackValue(false));
answerBuilder.add(VPackValue("stats"));
query->getStats(answerBuilder);
} catch (...) {
LOG(ERR) << "skip lead to an exception";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"skip lead to an exception");
@ -803,7 +791,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
try {
res = query->engine()->initialize();
} catch (...) {
LOG(ERR) << "initialize lead to an exception";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"initialize lead to an exception");
@ -825,7 +812,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
res = query->engine()->initializeCursor(items.get(), pos);
}
} catch (...) {
LOG(ERR) << "initializeCursor lead to an exception";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"initializeCursor lead to an exception");
@ -833,8 +819,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
}
answerBuilder.add("error", VPackValue(res != TRI_ERROR_NO_ERROR));
answerBuilder.add("code", VPackValue(static_cast<double>(res)));
answerBuilder.add(VPackValue("stats"));
query->getStats(answerBuilder);
} else if (operation == "shutdown") {
int res = TRI_ERROR_INTERNAL;
int errorCode = VelocyPackHelper::getNumericValue<int>(
@ -854,7 +838,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
_queryRegistry->destroy(_vocbase, _qId, errorCode);
_qId = 0;
} catch (...) {
LOG(ERR) << "shutdown lead to an exception";
generateError(rest::ResponseCode::SERVER_ERROR,
TRI_ERROR_HTTP_SERVER_ERROR,
"shutdown lead to an exception");
@ -863,7 +846,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
answerBuilder.add("error", VPackValue(res != TRI_ERROR_NO_ERROR));
answerBuilder.add("code", VPackValue(res));
} else {
LOG(ERR) << "Unknown operation!";
generateError(rest::ResponseCode::NOT_FOUND,
TRI_ERROR_HTTP_NOT_FOUND);
return;
@ -875,7 +857,6 @@ void RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
generateError(rest::ResponseCode::BAD, e.code());
return;
} catch (...) {
LOG(ERR) << "OUT OF MEMORY when handling query.";
generateError(rest::ResponseCode::BAD, TRI_ERROR_OUT_OF_MEMORY);
return;
}

1
arangod/Cluster/ClusterMethods.cpp
View File

@ -2033,6 +2033,7 @@ int flushWalOnAllDBServers(bool waitForSync, bool waitForCollector) {
}
if (nrok != (int)DBservers.size()) {
LOG(WARN) << "could not flush WAL on all servers. confirmed: " << nrok << ", expected: " << DBservers.size();
return TRI_ERROR_INTERNAL;
}

7
arangod/Cluster/DBServerAgencySync.cpp
View File

@ -54,6 +54,8 @@ void DBServerAgencySync::work() {
DBServerAgencySyncResult DBServerAgencySync::execute() {
// default to system database
double startTime = TRI_microtime();
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "DBServerAgencySync::execute starting";
DatabaseFeature* database =
ApplicationServer::getFeature<DatabaseFeature>("Database");
@ -80,6 +82,11 @@ DBServerAgencySyncResult DBServerAgencySync::execute() {
return result;
}
double now = TRI_microtime();
if (now - startTime > 5.0) {
LOG(INFO) << "DBServerAgencySync::execute took more than 5s to get free V8 context, starting handle-plan-change now";
}
TRI_DEFER(V8DealerFeature::DEALER->exitContext(context));
auto isolate = context->_isolate;

2
arangod/MMFiles/MMFilesGeoIndex.h
View File

@ -118,7 +118,7 @@ friend class MMFilesGeoIndexIterator;
bool canBeDropped() const override { return true; }
bool isSorted() const override { return false; }
bool isSorted() const override { return true; }
bool hasSelectivityEstimate() const override { return false; }

21
arangod/dbg.cmake
View File

@ -1,17 +1,12 @@
# -*- mode: CMAKE; -*-
# these are the install targets for the client package.
# we can't use RUNTIME DESTINATION here.
# include(/tmp/dump_vars.cmake)
message( "CMAKE_PROJECT_NAME ${CMAKE_PROJECT_NAME}/ CMAKE_INSTALL_SBINDIR ${CMAKE_INSTALL_SBINDIR}")
set(STRIP_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip")
execute_process(COMMAND mkdir -p ${STRIP_DIR})
set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX})
set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_SBINDIR})
install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_SBINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX}"
"${STRIP_DIR}/${BIN_ARANGOD}${CMAKE_EXECUTABLE_SUFFIX}"
)

83
arangosh/dbg.cmake
View File

@ -2,63 +2,30 @@
# these are the install targets for the client package.
# we can't use RUNTIME DESTINATION here.
set(STRIP_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip")
execute_process(COMMAND mkdir -p ${STRIP_DIR})
set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX})
set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR})
set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX})
set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR})
set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX})
set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR})
set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX})
set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR})
set(FILE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX})
set(STRIP_FILE ${STRIP_DIR}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${CMAKE_INSTALL_BINDIR})
install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX}"
"${STRIP_DIR}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX}")
install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX}"
"${STRIP_DIR}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX}")
install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX}"
"${STRIP_DIR}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX}")
install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX}"
"${STRIP_DIR}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX}")
install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX}"
"${STRIP_DIR}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX}")

4
cmake/ArangoDBInstall.cmake
View File

@ -19,9 +19,9 @@ endif()
# debug info directory:
if (${CMAKE_INSTALL_LIBDIR} STREQUAL "usr/lib64")
# some systems have weird places for usr/lib:
set(CMAKE_INSTALL_DEBINFO_DIR "usr/lib/debug/${CMAKE_PROJECT_NAME}")
set(CMAKE_INSTALL_DEBINFO_DIR "usr/lib/debug/")
else ()
set(CMAKE_INSTALL_DEBINFO_DIR "${CMAKE_INSTALL_LIBDIR}/debug/${CMAKE_PROJECT_NAME}")
set(CMAKE_INSTALL_DEBINFO_DIR "${CMAKE_INSTALL_LIBDIR}/debug/")
endif ()
set(CMAKE_INSTALL_SYSCONFDIR_ARANGO "${CMAKE_INSTALL_SYSCONFDIR}/${CMAKE_PROJECT_NAME}")

36
cmake/InstallMacros.cmake
View File

@ -157,3 +157,39 @@ macro(to_native_path sourceVarName)
endif()
set("INC_${sourceVarName}" ${myVar})
endmacro()
macro(install_debinfo
STRIP_DIR
USER_SUB_DEBINFO_DIR
USER_FILE
USER_STRIP_FILE)
set(SUB_DEBINFO_DIR ${USER_SUB_DEBINFO_DIR})
set(FILE ${USER_FILE})
set(STRIP_FILE ${USER_STRIP_FILE})
execute_process(COMMAND mkdir -p ${STRIP_DIR})
if (NOT MSVC AND CMAKE_STRIP)
execute_process(COMMAND "rm" -f ${STRIP_FILE})
execute_process(
COMMAND ${FILE_EXECUTABLE} ${FILE}
OUTPUT_VARIABLE FILE_RESULT)
string(REGEX
REPLACE ".*=([a-z0-9]*),.*" "\\1"
FILE_CHECKSUM
${FILE_RESULT}
)
if (NOT ${FILE_CHECKSUM} STREQUAL "")
string(SUBSTRING ${FILE_CHECKSUM} 0 2 SUB_DIR)
string(SUBSTRING ${FILE_CHECKSUM} 2 -1 STRIP_FILE)
set(SUB_DEBINFO_DIR .build-id/${SUB_DIR})
endif()
execute_process(COMMAND ${CMAKE_OBJCOPY} --only-keep-debug ${FILE} ${STRIP_FILE})
set(FILE ${STRIP_FILE})
endif()
install(
PROGRAMS ${FILE}
DESTINATION ${CMAKE_INSTALL_DEBINFO_DIR}/${SUB_DEBINFO_DIR})
endmacro()

5
cmake/packages/dbg/deb.txt
View File

@ -1,7 +1,7 @@
################################################################################
# the client package is a complete cmake sub package.
################################################################################
project(PACKAGE-DBG)
project(@CMAKE_PROJECT_NAME@)
cmake_minimum_required(VERSION 2.8)
################################################################################
@ -15,6 +15,9 @@ set(CROSS_COMPILING @CROSS_COMPILING@)
set(CMAKE_INSTALL_BINDIR @CMAKE_INSTALL_BINDIR@)
set(CMAKE_INSTALL_FULL_BINDIR @CMAKE_INSTALL_FULL_BINDIR@)
set(CMAKE_INSTALL_SBINDIR @CMAKE_INSTALL_SBINDIR@)
set(CMAKE_INSTALL_FULL_SBINDIR @CMAKE_INSTALL_FULL_SBINDIR@)
set(CMAKE_INSTALL_DATAROOTDIR @CMAKE_INSTALL_DATAROOTDIR@)
set(CMAKE_INSTALL_DATAROOTDIR_ARANGO @CMAKE_INSTALL_DATAROOTDIR_ARANGO@)
set(CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO @CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO@)

4
js/apps/system/_admin/aardvark/APP/frontend/js/arango/arango.js
View File

@ -69,6 +69,10 @@
return shortName;
},
getDatabaseShortName: function (id) {
return this.getCoordinatorShortName(id);
},
getDatabaseServerId: function (shortname) {
var id;
if (window.clusterHealth) {

5
js/apps/system/_admin/aardvark/APP/frontend/js/views/shardsView.js
View File

@ -186,14 +186,15 @@
async: true,
success: function (data) {
if (data.id) {
arangoHelper.arangoNotification('Shard ' + shardName + ' will be moved to ' + arangoHelper.getDatabaseServerId(toServer) + '.');
console.log(toServer);
arangoHelper.arangoNotification('Shard ' + shardName + ' will be moved to ' + arangoHelper.getDatabaseShortName(toServer) + '.');
window.setTimeout(function () {
window.App.shardsView.render();
}, 3000);
}
},
error: function () {
arangoHelper.arangoError('Shard ' + shardName + ' could not be moved to ' + arangoHelper.getDatabaseServerId(toServer) + '.');
arangoHelper.arangoError('Shard ' + shardName + ' could not be moved to ' + arangoHelper.getDatabaseShortName(toServer) + '.');
}
});

59
js/server/modules/@arangodb/cluster.js
View File

@ -120,14 +120,12 @@ function startReadLockOnLeader (endpoint, database, collName, timeout) {
const id = r.id;
var body = { 'id': id, 'collection': collName, 'ttl': timeout };
r = request({ url: url + '/_api/replication/holdReadLockCollection',
body: JSON.stringify(body),
method: 'POST', headers: {'x-arango-async': true} });
if (r.status !== 202) {
console.error('startReadLockOnLeader: Could not start read lock for shard',
collName, r);
return false;
}
request({ url: url + '/_api/replication/holdReadLockCollection',
body: JSON.stringify(body),
method: 'POST', headers: {'x-arango-async': true} });
// Intentionally do not look at the outcome, even in case of an error
// we must make sure that the read lock on the leader is not active!
// This is done automatically below.
var count = 0;
while (++count < 20) { // wait for some time until read lock established:
@ -170,7 +168,10 @@ function startReadLockOnLeader (endpoint, database, collName, timeout) {
// /////////////////////////////////////////////////////////////////////////////
function cancelReadLockOnLeader (endpoint, database, lockJobId) {
var url = endpointToURL(endpoint) + '/_db/' + database +
// Note that we always use the _system database here because the actual
// database might be gone already on the leader and we need to cancel
// the read lock under all circumstances.
var url = endpointToURL(endpoint) + '/_db/_system' +
'/_api/replication/holdReadLockCollection';
var r;
var body = {'id': lockJobId};
@ -181,7 +182,8 @@ function cancelReadLockOnLeader (endpoint, database, lockJobId) {
return false;
}
if (r.status !== 200) {
console.error('cancelReadLockOnLeader: error', r);
console.error('cancelReadLockOnLeader: error', lockJobId, r.status,
r.message, r.body, r.json);
return false;
}
console.debug('cancelReadLockOnLeader: success');
@ -453,6 +455,7 @@ function synchronizeOneShard (database, shard, planId, leader) {
// synchronize this shard from the leader
// this function will throw if anything goes wrong
var startTime = new Date();
var isStopping = require('internal').isStopping;
var ourselves = global.ArangoServerState.id();
@ -485,8 +488,9 @@ function synchronizeOneShard (database, shard, planId, leader) {
planned[0] !== leader) {
// Things have changed again, simply terminate:
terminateAndStartOther();
console.debug('synchronizeOneShard: cancelled, %s/%s, %s/%s',
database, shard, database, planId);
let endTime = new Date();
console.debug('synchronizeOneShard: cancelled, %s/%s, %s/%s, started %s, ended %s',
database, shard, database, planId, startTime.toString(), endTime.toString());
return;
}
var current = [];
@ -500,8 +504,9 @@ function synchronizeOneShard (database, shard, planId, leader) {
}
// We are already there, this is rather strange, but never mind:
terminateAndStartOther();
console.debug('synchronizeOneShard: already done, %s/%s, %s/%s',
database, shard, database, planId);
let endTime = new Date();
console.debug('synchronizeOneShard: already done, %s/%s, %s/%s, started %s, ended %s',
database, shard, database, planId, startTime.toString(), endTime.toString());
return;
}
console.debug('synchronizeOneShard: waiting for leader, %s/%s, %s/%s',
@ -522,9 +527,16 @@ function synchronizeOneShard (database, shard, planId, leader) {
if (isStopping()) {
throw 'server is shutting down';
}
let startTime = new Date();
sy = rep.syncCollection(shard,
{ endpoint: ep, incremental: true,
keepBarrier: true, useCollectionId: false });
let endTime = new Date();
let longSync = false;
if (endTime - startTime > 5000) {
console.error('synchronizeOneShard: long call to syncCollection for shard', shard, JSON.stringify(sy), "start time: ", startTime.toString(), "end time: ", endTime.toString());
longSync = true;
}
if (sy.error) {
console.error('synchronizeOneShard: could not initially synchronize',
'shard ', shard, sy);
@ -532,7 +544,15 @@ function synchronizeOneShard (database, shard, planId, leader) {
} else {
if (sy.collections.length === 0 ||
sy.collections[0].name !== shard) {
if (longSync) {
console.error('synchronizeOneShard: long sync, before cancelBarrier',
new Date().toString());
}
cancelBarrier(ep, database, sy.barrierId);
if (longSync) {
console.error('synchronizeOneShard: long sync, after cancelBarrier',
new Date().toString());
}
throw 'Shard ' + shard + ' seems to be gone from leader!';
} else {
// Now start a read transaction to stop writes:
@ -592,14 +612,17 @@ function synchronizeOneShard (database, shard, planId, leader) {
} else if (err2 && err2.errorNum === 1402 && err2.errorMessage.match(/HTTP 404/)) {
logLevel = 'debug';
}
console[logLevel]("synchronization of local shard '%s/%s' for central '%s/%s' failed: %s",
database, shard, database, planId, JSON.stringify(err2));
let endTime = new Date();
console[logLevel]("synchronization of local shard '%s/%s' for central '%s/%s' failed: %s, started: %s, ended: %s",
database, shard, database, planId, JSON.stringify(err2),
startTime.toString(), endTime.toString());
}
}
// Tell others that we are done:
terminateAndStartOther();
console.debug('synchronizeOneShard: done, %s/%s, %s/%s',
database, shard, database, planId);
let endTime = new Date();
console.debug('synchronizeOneShard: done, %s/%s, %s/%s, started: %s, ended: %s',
database, shard, database, planId, startTime.toString(), endTime.toString());
}
// /////////////////////////////////////////////////////////////////////////////

44
js/server/tests/aql/aql-optimizer-geoindex.js
View File

@ -55,6 +55,7 @@ function optimizerRuleTestSuite() {
var ruleName = "geoindex";
var colName = "UnitTestsAqlOptimizer" + ruleName.replace(/-/g, "_");
var colName2 = colName2;
var geocol;
var sortArray = function (l, r) {
@ -124,11 +125,21 @@ function optimizerRuleTestSuite() {
internal.db._drop(colName);
geocol = internal.db._create(colName);
geocol.ensureIndex({type:"geo", fields:["lat","lon"]});
for (var lat=-40; lat <=40 ; ++lat){
for (var lon=-40; lon <= 40; ++lon){
var lat, lon;
for (lat=-40; lat <=40 ; ++lat) {
for (lon=-40; lon <= 40; ++lon) {
geocol.insert({lat,lon});
}
}
internal.db._drop(colName2);
geocol = internal.db._create(colName2);
geocol.ensureIndex({type:"geo", fields:["loca.tion.lat","loca.tion.lon"]});
for (lat=-40; lat <=40 ; ++lat) {
for (lon=-40; lon <= 40; ++lon) {
geocol.insert({ loca : { tion : { lat , lon } } });
}
}
},
////////////////////////////////////////////////////////////////////////////////
@ -137,6 +148,7 @@ function optimizerRuleTestSuite() {
tearDown : function () {
internal.db._drop(colName);
internal.db._drop(colName2);
geocol = null;
},
@ -145,7 +157,13 @@ function optimizerRuleTestSuite() {
geocol.ensureIndex({ type: "hash", fields: [ "y", "z" ], unique: false });
var queries = [
{ string : "FOR d IN " + colName + " SORT distance(d.lat,d.lon, 0 ,0 ) ASC LIMIT 1 RETURN d",
{ string : "FOR d IN " + colName + " SORT distance(d.lat, d.lon, 0 ,0 ) ASC LIMIT 1 RETURN d",
cluster : false,
sort : false,
filter : false,
index : true
},
{ string : "FOR d IN " + colName2 + " SORT distance(d.loca.tion.lat, d.loca.tion.lon, 0 ,0 ) ASC LIMIT 1 RETURN d",
cluster : false,
sort : false,
filter : false,
@ -213,11 +231,15 @@ function optimizerRuleTestSuite() {
testRuleRemoveNodes : function () {
if(enabled.removeNodes){
var queries = [
[ "FOR d IN " + colName + " SORT distance(d.lat,d.lon, 0 ,0 ) ASC LIMIT 5 RETURN d", false, false, false ],
[ "FOR d IN " + colName + " SORT distance(0, 0, d.lat,d.lon ) ASC LIMIT 5 RETURN d", false, false, false ],
[ "FOR d IN " + colName + " FILTER distance(0, 0, d.lat,d.lon ) < 111200 RETURN d", false, false, false ],
[ "FOR d IN " + colName + " SORT distance(d.lat,d.lon, 0 ,0 ) ASC LIMIT 5 RETURN d", false, false, false ],
[ "FOR d IN " + colName + " SORT distance(0, 0, d.lat,d.lon ) ASC LIMIT 5 RETURN d", false, false, false ],
[ "FOR d IN " + colName + " FILTER distance(0, 0, d.lat,d.lon ) < 111200 RETURN d", false, false, false ],
// [ "FOR i IN 1..2 FOR d IN geocol SORT distance(i,2,d.lat,d.lon) ASC LIMIT 5 RETURN d", false, false, false ],
];
var queries2 = [
[ "FOR d IN " + colName2 + " SORT distance(d.loca.tion.lat,d.loca.tion.lon, 0 ,0 ) ASC LIMIT 5 RETURN d", false, false, false ]
];
var expected = [
[[0,0], [-1,0], [0,1], [1,0], [0,-1]],
@ -234,6 +256,16 @@ function optimizerRuleTestSuite() {
assertEqual(expected[qindex].sort(),pairs.sort());
//expect(expected[qindex].sort()).to.be.equal(result.json.sort())
});
queries2.forEach(function(query, qindex) {
var result = AQL_EXECUTE(query[0]);
expect(expected[qindex].length).to.be.equal(result.json.length);
var pairs = result.json.map(function(res){
return [res.loca.tion.lat,res.loca.tion.lon];
});
assertEqual(expected[qindex].sort(),pairs.sort());
//expect(expected[qindex].sort()).to.be.equal(result.json.sort())
});
}
}, // testRuleSort

16
lib/Endpoint/Endpoint.cpp
View File

@ -280,7 +280,13 @@ Endpoint* Endpoint::factory(const Endpoint::EndpointType type,
// hostname and port (e.g. [address]:port)
if (found != std::string::npos && found > 2 && found + 2 < copy.size()) {
uint16_t port = (uint16_t)StringUtils::uint32(copy.substr(found + 2));
int64_t value = StringUtils::int64(copy.substr(found + 2));
// check port over-/underrun
if (value < (std::numeric_limits<uint16_t>::min)() || value > (std::numeric_limits<uint16_t>::max)()) {
LOG(ERR) << "specified port number '" << value << "' is outside the allowed range";
return nullptr;
}
uint16_t port = static_cast<uint16_t>(value);
std::string host = copy.substr(1, found - 1);
return new EndpointIpV6(type, protocol, encryption, listenBacklog,
@ -306,7 +312,13 @@ Endpoint* Endpoint::factory(const Endpoint::EndpointType type,
// hostname and port
if (found != std::string::npos && found + 1 < copy.size()) {
uint16_t port = (uint16_t)StringUtils::uint32(copy.substr(found + 1));
int64_t value = StringUtils::int64(copy.substr(found + 1));
// check port over-/underrun
if (value < (std::numeric_limits<uint16_t>::min)() || value > (std::numeric_limits<uint16_t>::max)()) {
LOG(ERR) << "specified port number '" << value << "' is outside the allowed range";
return nullptr;
}
uint16_t port = static_cast<uint16_t>(value);
std::string host = copy.substr(0, found);
return new EndpointIpV4(type, protocol, encryption, listenBacklog,

9
lib/Logger/LoggerStream.h
View File

@ -88,8 +88,9 @@ class LoggerStream {
size_t i = 0;
size_t const n = obj.size();
for (auto const& it : obj) {
_out << it;
if (++i < n) {
_out << it << ", ";
_out << ", ";
}
}
_out << ']';
@ -102,8 +103,9 @@ class LoggerStream {
size_t i = 0;
size_t const n = obj.size();
for (auto const& it : obj) {
_out << it;
if (++i < n) {
_out << it << ", ";
_out << ", ";
}
}
_out << '}';
@ -116,8 +118,9 @@ class LoggerStream {
size_t i = 0;
size_t n = obj.size();
for (auto const& it : obj) {
_out << it;
if (++i < n) {
_out << it << ", ";
_out << ", ";
}
_out << it.first << " => " << it.second;
}

66
scripts/generate_documenation.sh Executable file
View File

@ -0,0 +1,66 @@
#!/bin/bash
##python3-setuptools
##
##python setup.py install
##
##node npm
##
##https://github.com/GitbookIO/gitbook
## npm install gitbook-cli -g
##
## http://calibre-ebook.com/download
test_tools(){
if ! type easy_install3 >> /dev/null; then
echo "you are missing setuptools"
echo "apt-get install python-setuptools"
exit 1
fi
if ! type node >> /dev/null; then
echo "you are missing node"
echo "apt-get install nodejs nodejs-legacy"
exit 1
fi
if ! type npm >> /dev/null; then
echo "you are missing node"
echo "apt-get install npm"
exit 1
fi
if ! type calibre >> /dev/null; then
echo "you are missing node"
echo "apt-get install calibre-bin"
exit 1
fi
}
install_tools(){
(
if ! [[ -f markdown-pp ]]; then
git clone https://github.com/arangodb-helper/markdown-pp/
fi
cd markdown-pp
python2 setup.py install --user
)
npm install gitbook-cli
}
main(){
#test for basic tools
test_tools
#cd into target dir
mkdir -p "$1"
cd $1 || { echo "unable to change into $1"; exit 1; }
install_tools
}
main "$@"