Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Performance tuning for many shards. (#8577)

This commit is contained in:
Max Neunhöffer 2019-03-29 21:34:45 +01:00 committed by Frank Celler
parent 3631d55146
commit 54f84cab92
39 changed files with 658 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG
View File

@ -157,6 +157,12 @@ v3.4.5 (2019-03-27)
* coordinator route for full agency dumps contains compactions and
time stamps
* lots of agency performance improvements, mostly avoiding copying
* priority queue for maintenance jobs
* do not wait for replication after each job execution in Supervision
v3.4.4 (2019-03-12)
-------------------

4
arangod/Agency/ActiveFailoverJob.cpp
View File

@ -119,7 +119,7 @@ bool ActiveFailoverJob::create(std::shared_ptr<VPackBuilder> envelope) {
return true;
}
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
return true;
}
@ -211,7 +211,7 @@ bool ActiveFailoverJob::start(bool&) {
} // array for transaction done
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, pending);
write_ret_t res = singleWriteTransaction(_agent, pending, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
_status = FINISHED;

6
arangod/Agency/AddFollower.cpp
View File

@ -107,7 +107,7 @@ bool AddFollower::create(std::shared_ptr<VPackBuilder> envelope) {
_jb->close(); // transaction object
_jb->close(); // close array
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
return true;
@ -128,7 +128,7 @@ bool AddFollower::start(bool&) {
finish("", "", true, "collection has been dropped in the meantime");
return false;
}
Node collection =
Node const& collection =
_snapshot.hasAsNode(planColPrefix + _database + "/" + _collection).first;
if (collection.has("distributeShardsLike")) {
finish("", "", false,
@ -298,7 +298,7 @@ bool AddFollower::start(bool&) {
} // array for transaction done
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, trx);
write_ret_t res = singleWriteTransaction(_agent, trx, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
_status = FINISHED;

16
arangod/Agency/AgencyComm.cpp
View File

@ -1345,6 +1345,17 @@ AgencyCommResult AgencyComm::sendWithFailover(arangodb::rest::RequestType method
} else if (waitInterval.count() < 5.0) {
waitInterval *= 1.0749292929292;
}
// Check again for shutdown, since some time has passed:
if (!application_features::ApplicationServer::isRetryOK()) {
LOG_TOPIC(INFO, Logger::AGENCYCOMM)
<< "Unsuccessful AgencyComm: Timeout because of shutdown "
<< "errorCode: " << result.errorCode()
<< " errorMessage: " << result.errorMessage()
<< " errorDetails: " << result.errorDetails();
return true;
}
return false;
};
@ -1461,6 +1472,7 @@ AgencyCommResult AgencyComm::sendWithFailover(arangodb::rest::RequestType method
if (isWriteTrans && !clientIds.empty() && result._sent &&
(result._statusCode == 0 || result._statusCode == 503)) {
isInquiry = true;
conTimeout = 16.0;
}
// This leaves the redirect, timeout and 503 cases, which are handled
@ -1535,7 +1547,7 @@ AgencyCommResult AgencyComm::sendWithFailover(arangodb::rest::RequestType method
// In case of a timeout, we increase the patience:
if (result._statusCode == 0) {
if (conTimeout < 15.0) { // double until we have 16s
if (conTimeout < 33.0) { // double until we have 64s
conTimeout *= 2;
}
}
@ -1745,6 +1757,8 @@ bool AgencyComm::tryInitializeStructure() {
builder.add("NumberOfDBServers", VPackSlice::nullSlice());
builder.add(VPackValue("CleanedServers"));
{ VPackArrayBuilder dd(&builder); }
builder.add(VPackValue("ToBeCleanedServers"));
{ VPackArrayBuilder dd(&builder); }
builder.add(VPackValue("FailedServers"));
{ VPackObjectBuilder dd(&builder); }
builder.add("Lock", VPackValue("UNLOCKED"));

5
arangod/Agency/Agent.cpp
View File

@ -738,7 +738,7 @@ void Agent::advanceCommitIndex() {
WRITE_LOCKER(oLocker, _outputLock);
if (index > _commitIndex) {
CONDITION_LOCKER(guard, _waitForCV);
LOG_TOPIC(TRACE, Logger::AGENCY)
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Critical mass for commiting " << _commitIndex + 1 << " through "
<< index << " to read db";
// Change _readDB and _commitIndex atomically together:
@ -747,6 +747,9 @@ void Agent::advanceCommitIndex() {
_commitIndex, t, true);
_commitIndex = index;
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Critical mass for commiting " << _commitIndex + 1 << " through "
<< index << " to read db, done";
// Wake up rest handlers:
_waitForCV.broadcast();

16
arangod/Agency/CleanOutServer.cpp
View File

@ -65,8 +65,8 @@ JOB_STATUS CleanOutServer::status() {
return _status;
}
Node::Children const todos = _snapshot.hasAsChildren(toDoPrefix).first;
Node::Children const pends = _snapshot.hasAsChildren(pendingPrefix).first;
Node::Children const& todos = _snapshot.hasAsChildren(toDoPrefix).first;
Node::Children const& pends = _snapshot.hasAsChildren(pendingPrefix).first;
size_t found = 0;
for (auto const& subJob : todos) {
@ -94,7 +94,7 @@ JOB_STATUS CleanOutServer::status() {
return PENDING;
}
Node::Children const failed = _snapshot.hasAsChildren(failedPrefix).first;
Node::Children const& failed = _snapshot.hasAsChildren(failedPrefix).first;
size_t failedFound = 0;
for (auto const& subJob : failed) {
if (!subJob.first.compare(0, _jobId.size() + 1, _jobId + "-")) {
@ -136,7 +136,7 @@ JOB_STATUS CleanOutServer::status() {
}
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, reportTrx);
write_ret_t res = singleWriteTransaction(_agent, reportTrx, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0] != 0) {
LOG_TOPIC(DEBUG, Logger::SUPERVISION)
@ -184,7 +184,7 @@ bool CleanOutServer::create(std::shared_ptr<VPackBuilder> envelope) {
return true;
}
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
return true;
@ -225,7 +225,7 @@ bool CleanOutServer::start(bool& aborts) {
// Check that _to is not in `Target/CleanedServers`:
VPackBuilder cleanedServersBuilder;
auto cleanedServersNode = _snapshot.hasAsNode(cleanedPrefix);
auto const& cleanedServersNode = _snapshot.hasAsNode(cleanedPrefix);
if (cleanedServersNode.second) {
cleanedServersNode.first.toBuilder(cleanedServersBuilder);
} else {
@ -248,7 +248,7 @@ bool CleanOutServer::start(bool& aborts) {
// so that hasAsNode does not generate a warning log message)
VPackBuilder failedServersBuilder;
if (_snapshot.has(failedServersPrefix)) {
auto failedServersNode = _snapshot.hasAsNode(failedServersPrefix);
auto const& failedServersNode = _snapshot.hasAsNode(failedServersPrefix);
if (failedServersNode.second) {
failedServersNode.first.toBuilder(failedServersBuilder);
} else {
@ -344,7 +344,7 @@ bool CleanOutServer::start(bool& aborts) {
} // array for transaction done
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, *pending);
write_ret_t res = singleWriteTransaction(_agent, *pending, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(DEBUG, Logger::SUPERVISION) << "Pending: Clean out server " + _server;

4
arangod/Agency/FailedFollower.cpp
View File

@ -112,7 +112,7 @@ bool FailedFollower::create(std::shared_ptr<VPackBuilder> envelope) {
if (envelope == nullptr) {
_jb->close(); // object
_jb->close(); // array
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
return (res.accepted && res.indices.size() == 1 && res.indices[0]);
}
@ -186,7 +186,7 @@ bool FailedFollower::start(bool& aborts) {
{
VPackArrayBuilder a(&todo);
if (_jb == nullptr) {
auto jobIdNode = _snapshot.hasAsNode(toDoPrefix + _jobId);
auto const& jobIdNode = _snapshot.hasAsNode(toDoPrefix + _jobId);
if (jobIdNode.second) {
jobIdNode.first.toBuilder(todo);
} else {

6
arangod/Agency/FailedLeader.cpp
View File

@ -156,7 +156,7 @@ bool FailedLeader::create(std::shared_ptr<VPackBuilder> b) {
if (b == nullptr) {
_jb->close(); // object
_jb->close(); // array
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
return (res.accepted && res.indices.size() == 1 && res.indices[0]);
}
@ -209,7 +209,7 @@ bool FailedLeader::start(bool& aborts) {
{
VPackArrayBuilder t(&todo);
if (_jb == nullptr) {
auto jobIdNode = _snapshot.hasAsNode(toDoPrefix + _jobId);
auto const& jobIdNode = _snapshot.hasAsNode(toDoPrefix + _jobId);
if (jobIdNode.second) {
jobIdNode.first.toBuilder(todo);
} else {
@ -401,7 +401,7 @@ JOB_STATUS FailedLeader::status() {
}
std::string database, shard;
auto job = _snapshot.hasAsNode(pendingPrefix + _jobId);
auto const& job = _snapshot.hasAsNode(pendingPrefix + _jobId);
if (job.second) {
auto tmp_database = job.first.hasAsString("database");
auto tmp_shard = job.first.hasAsString("shard");

8
arangod/Agency/FailedServer.cpp
View File

@ -94,7 +94,7 @@ bool FailedServer::start(bool& aborts) {
{
VPackArrayBuilder t(&todo);
if (_jb == nullptr) {
auto toDoJob = _snapshot.hasAsNode(toDoPrefix + _jobId);
auto const& toDoJob = _snapshot.hasAsNode(toDoPrefix + _jobId);
if (toDoJob.second) {
toDoJob.first.toBuilder(todo);
} else {
@ -214,7 +214,7 @@ bool FailedServer::start(bool& aborts) {
}
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, *transactions);
write_ret_t res = singleWriteTransaction(_agent, *transactions, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(DEBUG, Logger::SUPERVISION)
@ -285,7 +285,7 @@ bool FailedServer::create(std::shared_ptr<VPackBuilder> envelope) {
}
if (selfCreate) {
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
if (!res.accepted || res.indices.size() != 1 || res.indices[0] == 0) {
LOG_TOPIC(INFO, Logger::SUPERVISION) << "Failed to insert job " + _jobId;
return false;
@ -345,7 +345,7 @@ JOB_STATUS FailedServer::status() {
deleteTodos->close();
deleteTodos->close();
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, *deleteTodos);
write_ret_t res = singleWriteTransaction(_agent, *deleteTodos, false);
if (!res.accepted || res.indices.size() != 1 || !res.indices[0]) {
LOG_TOPIC(WARN, Logger::SUPERVISION)

171
arangod/Agency/Job.cpp
View File

@ -72,102 +72,109 @@ std::string Job::agencyPrefix = "arango";
bool Job::finish(std::string const& server, std::string const& shard,
bool success, std::string const& reason, query_t const payload) {
Builder pending, finished;
try { // protect everything, just in case
Builder pending, finished;
// Get todo entry
bool started = false;
{
VPackArrayBuilder guard(&pending);
if (_snapshot.exists(pendingPrefix + _jobId).size() == 3) {
_snapshot.hasAsBuilder(pendingPrefix + _jobId, pending);
started = true;
} else if (_snapshot.exists(toDoPrefix + _jobId).size() == 3) {
_snapshot.hasAsBuilder(toDoPrefix + _jobId, pending);
} else {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Nothing in pending to finish up for job " << _jobId;
return false;
// Get todo entry
bool started = false;
{
VPackArrayBuilder guard(&pending);
if (_snapshot.exists(pendingPrefix + _jobId).size() == 3) {
_snapshot.hasAsBuilder(pendingPrefix + _jobId, pending);
started = true;
} else if (_snapshot.exists(toDoPrefix + _jobId).size() == 3) {
_snapshot.hasAsBuilder(toDoPrefix + _jobId, pending);
} else {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Nothing in pending to finish up for job " << _jobId;
return false;
}
}
}
std::string jobType;
try {
jobType = pending.slice()[0].get("type").copyString();
} catch (std::exception const&) {
LOG_TOPIC(WARN, Logger::AGENCY) << "Failed to obtain type of job " << _jobId;
}
std::string jobType;
try {
jobType = pending.slice()[0].get("type").copyString();
} catch (std::exception const&) {
LOG_TOPIC(WARN, Logger::AGENCY) << "Failed to obtain type of job " << _jobId;
}
// Additional payload, which is to be executed in the finish transaction
Slice operations = Slice::emptyObjectSlice();
Slice preconditions = Slice::emptyObjectSlice();
if (payload != nullptr) {
Slice slice = payload->slice();
TRI_ASSERT(slice.isObject() || slice.isArray());
if (slice.isObject()) { // opers only
operations = slice;
TRI_ASSERT(operations.isObject());
} else {
TRI_ASSERT(slice.length() < 3); // opers + precs only
if (slice.length() > 0) {
operations = slice[0];
// Additional payload, which is to be executed in the finish transaction
Slice operations = Slice::emptyObjectSlice();
Slice preconditions = Slice::emptyObjectSlice();
if (payload != nullptr) {
Slice slice = payload->slice();
TRI_ASSERT(slice.isObject() || slice.isArray());
if (slice.isObject()) { // opers only
operations = slice;
TRI_ASSERT(operations.isObject());
if (slice.length() > 1) {
preconditions = slice[1];
TRI_ASSERT(preconditions.isObject());
} else {
TRI_ASSERT(slice.length() < 3); // opers + precs only
if (slice.length() > 0) {
operations = slice[0];
TRI_ASSERT(operations.isObject());
if (slice.length() > 1) {
preconditions = slice[1];
TRI_ASSERT(preconditions.isObject());
}
}
}
}
}
// Prepare pending entry, block toserver
{
VPackArrayBuilder guard(&finished);
// Prepare pending entry, block toserver
{
VPackArrayBuilder guard(&finished);
{ // operations --
VPackObjectBuilder operguard(&finished);
{ // operations --
VPackObjectBuilder operguard(&finished);
addPutJobIntoSomewhere(finished, success ? "Finished" : "Failed",
pending.slice()[0], reason);
addPutJobIntoSomewhere(finished, success ? "Finished" : "Failed",
pending.slice()[0], reason);
addRemoveJobFromSomewhere(finished, "ToDo", _jobId);
addRemoveJobFromSomewhere(finished, "Pending", _jobId);
addRemoveJobFromSomewhere(finished, "ToDo", _jobId);
addRemoveJobFromSomewhere(finished, "Pending", _jobId);
if (operations.length() > 0) {
for (auto const& oper : VPackObjectIterator(operations)) {
finished.add(oper.key.copyString(), oper.value);
if (operations.length() > 0) {
for (auto const& oper : VPackObjectIterator(operations)) {
finished.add(oper.key.copyString(), oper.value);
}
}
}
// --- Remove blocks if specified:
if (started && !server.empty()) {
addReleaseServer(finished, server);
}
if (started && !shard.empty()) {
addReleaseShard(finished, shard);
}
} // -- operations
if (preconditions != Slice::emptyObjectSlice()) { // preconditions --
VPackObjectBuilder precguard(&finished);
if (preconditions.length() > 0) {
for (auto const& prec : VPackObjectIterator(preconditions)) {
finished.add(prec.key.copyString(), prec.value);
// --- Remove blocks if specified:
if (started && !server.empty()) {
addReleaseServer(finished, server);
}
if (started && !shard.empty()) {
addReleaseShard(finished, shard);
}
}
} // -- preconditions
} // -- operations
if (preconditions != Slice::emptyObjectSlice()) { // preconditions --
VPackObjectBuilder precguard(&finished);
if (preconditions.length() > 0) {
for (auto const& prec : VPackObjectIterator(preconditions)) {
finished.add(prec.key.copyString(), prec.value);
}
}
} // -- preconditions
}
write_ret_t res = singleWriteTransaction(_agent, finished, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Successfully finished job " << jobType << "(" << _jobId << ")";
_status = (success ? FINISHED : FAILED);
return true;
}
} catch (std::exception const& e) {
LOG_TOPIC(WARN, Logger::AGENCY)
<< "Caught exception in finish, message: " << e.what();
} catch (...) {
LOG_TOPIC(WARN, Logger::AGENCY)
<< "Caught unspecified exception in finish.";
}
write_ret_t res = singleWriteTransaction(_agent, finished);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(DEBUG, Logger::AGENCY)
<< "Successfully finished job " << jobType << "(" << _jobId << ")";
_status = (success ? FINISHED : FAILED);
return true;
}
return false;
}
@ -410,16 +417,16 @@ std::vector<Job::shard_t> Job::clones(Node const& snapshot, std::string const& d
for (const auto& colptr : snapshot.hasAsChildren(databasePath).first) { // collections
auto const col = *colptr.second;
auto const otherCollection = colptr.first;
auto const &col = *colptr.second;
auto const &otherCollection = colptr.first;
if (otherCollection != collection && col.has("distributeShardsLike") && // use .has() form to prevent logging of missing
col.hasAsSlice("distributeShardsLike").first.copyString() == collection) {
auto const theirshards = sortedShardList(col("shards"));
auto const& theirshards = sortedShardList(col.hasAsNode("shards").first);
if (theirshards.size() > 0) { // do not care about virtual collections
if (theirshards.size() == myshards.size()) {
ret.emplace_back(otherCollection,
sortedShardList(col.hasAsNode("shards").first)[steps]);
theirshards[steps]);
} else {
LOG_TOPIC(ERR, Logger::SUPERVISION)
<< "Shard distribution of clone(" << otherCollection

8
arangod/Agency/Job.h
View File

@ -258,9 +258,11 @@ inline arangodb::consensus::trans_ret_t generalTransaction(AgentInterface* _agen
auto ret = _agent->transact(envelope);
if (ret.maxind > 0) {
_agent->waitFor(ret.maxind);
}
// This is for now disabled to speed up things. We wait after a full
// Supervision run, which is good enough.
//if (ret.maxind > 0) {
// _agent->waitFor(ret.maxind);
//
return ret;
}

12
arangod/Agency/MoveShard.cpp
View File

@ -157,7 +157,7 @@ bool MoveShard::create(std::shared_ptr<VPackBuilder> envelope) {
_jb->close(); // transaction object
_jb->close(); // close array
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
return true;
@ -190,7 +190,7 @@ bool MoveShard::start(bool&) {
finish("", "", true, "collection has been dropped in the meantime");
return false;
}
auto collection = _snapshot.hasAsNode(planColPrefix + _database + "/" + _collection);
auto const& collection = _snapshot.hasAsNode(planColPrefix + _database + "/" + _collection);
if (collection.second && collection.first.has("distributeShardsLike")) {
finish("", "", false,
"collection must not have 'distributeShardsLike' attribute");
@ -389,7 +389,7 @@ bool MoveShard::start(bool&) {
} // array for transaction done
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, pending);
write_ret_t res = singleWriteTransaction(_agent, pending, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(DEBUG, Logger::SUPERVISION)
@ -649,7 +649,7 @@ JOB_STATUS MoveShard::pendingLeader() {
}
// Transact to agency:
write_ret_t res = singleWriteTransaction(_agent, trx);
write_ret_t res = singleWriteTransaction(_agent, trx, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
LOG_TOPIC(DEBUG, Logger::SUPERVISION)
@ -735,7 +735,7 @@ JOB_STATUS MoveShard::pendingFollower() {
trx.add(precondition.slice());
}
write_ret_t res = singleWriteTransaction(_agent, trx);
write_ret_t res = singleWriteTransaction(_agent, trx, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
return FINISHED;
@ -859,7 +859,7 @@ arangodb::Result MoveShard::abort() {
});
}
}
write_ret_t res = singleWriteTransaction(_agent, trx);
write_ret_t res = singleWriteTransaction(_agent, trx, false);
if (!res.accepted) {
result = Result(TRI_ERROR_SUPERVISION_GENERAL_FAILURE,

16
arangod/Agency/Node.cpp
View File

@ -37,14 +37,6 @@
using namespace arangodb::consensus;
using namespace arangodb::basics;
struct NotEmpty {
bool operator()(const std::string& s) { return !s.empty(); }
};
struct Empty {
bool operator()(const std::string& s) { return s.empty(); }
};
const Node::Children Node::dummyChildren = Node::Children();
const Node Node::_dummyNode = Node("dumm-di-dumm");
@ -71,7 +63,10 @@ inline static std::vector<std::string> split(const std::string& str, char separa
p = q + 1;
}
result.emplace_back(key, p);
result.erase(std::find_if(result.rbegin(), result.rend(), NotEmpty()).base(),
result.erase(std::find_if(result.rbegin(), result.rend(),
[](std::string const& s) -> bool {
return !s.empty();
}).base(),
result.end());
return result;
}
@ -716,6 +711,7 @@ bool Node::applieOp(VPackSlice const& slice) {
if (_parent == nullptr) { // root node
_children.clear();
_value.clear();
_vecBufDirty = true; // just in case there was an array
return true;
} else {
return _parent->removeChild(_nodeName);
@ -995,7 +991,7 @@ std::pair<Slice, bool> Node::hasAsSlice(std::string const& url) const {
ret_pair.second = true;
} catch (...) {
// do nothing, ret_pair second already false
LOG_TOPIC(DEBUG, Logger::SUPERVISION)
LOG_TOPIC(TRACE, Logger::SUPERVISION)
<< "hasAsSlice had exception processing " << url;
} // catch

6
arangod/Agency/RemoveFollower.cpp
View File

@ -110,7 +110,7 @@ bool RemoveFollower::create(std::shared_ptr<VPackBuilder> envelope) {
_jb->close(); // transaction object
_jb->close(); // close array
write_ret_t res = singleWriteTransaction(_agent, *_jb);
write_ret_t res = singleWriteTransaction(_agent, *_jb, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
return true;
@ -131,7 +131,7 @@ bool RemoveFollower::start(bool&) {
finish("", "", true, "collection has been dropped in the meantime");
return false;
}
Node collection =
Node const& collection =
_snapshot.hasAsNode(planColPrefix + _database + "/" + _collection).first;
if (collection.has("distributeShardsLike")) {
finish("", "", false,
@ -410,7 +410,7 @@ bool RemoveFollower::start(bool&) {
} // array for transaction done
// Transact to agency
write_ret_t res = singleWriteTransaction(_agent, trx);
write_ret_t res = singleWriteTransaction(_agent, trx, false);
if (res.accepted && res.indices.size() == 1 && res.indices[0]) {
_status = FINISHED;

5
arangod/Agency/RestAgencyHandler.cpp
View File

@ -530,6 +530,7 @@ RestStatus RestAgencyHandler::handleRead() {
}
RestStatus RestAgencyHandler::handleConfig() {
LOG_TOPIC(DEBUG, Logger::AGENCY) << "handleConfig start";
// Update endpoint of peer
if (_request->requestType() == rest::RequestType::POST) {
try {
@ -542,6 +543,7 @@ RestStatus RestAgencyHandler::handleConfig() {
// Respond with configuration
auto last = _agent->lastCommitted();
LOG_TOPIC(DEBUG, Logger::AGENCY) << "handleConfig after lastCommitted";
Builder body;
{
VPackObjectBuilder b(&body);
@ -549,13 +551,16 @@ RestStatus RestAgencyHandler::handleConfig() {
body.add("leaderId", Value(_agent->leaderID()));
body.add("commitIndex", Value(last));
_agent->lastAckedAgo(body);
LOG_TOPIC(DEBUG, Logger::AGENCY) << "handleConfig after lastAckedAgo";
body.add("configuration", _agent->config().toBuilder()->slice());
body.add("engine", VPackValue(EngineSelectorFeature::engineName()));
body.add("version", VPackValue(ARANGODB_VERSION));
}
LOG_TOPIC(DEBUG, Logger::AGENCY) << "handleConfig after before generateResult";
generateResult(rest::ResponseCode::OK, body.slice());
LOG_TOPIC(DEBUG, Logger::AGENCY) << "handleConfig after before done";
return RestStatus::DONE;
}

65
arangod/Agency/Store.cpp
View File

@ -49,11 +49,6 @@ struct NotEmpty {
bool operator()(const std::string& s) { return !s.empty(); }
};
/// Emptyness of string
struct Empty {
bool operator()(const std::string& s) { return s.empty(); }
};
/// @brief Split strings by separator
inline static std::vector<std::string> split(const std::string& str, char separator) {
std::vector<std::string> result;
@ -532,7 +527,7 @@ bool Store::read(VPackSlice const& query, Builder& ret) const {
bool showHidden = false;
// Collect all paths
std::list<std::string> query_strs;
std::vector<std::string> query_strs;
if (query.isArray()) {
for (auto const& sub_query : VPackArrayIterator(query)) {
std::string subqstr = sub_query.copyString();
@ -544,7 +539,7 @@ bool Store::read(VPackSlice const& query, Builder& ret) const {
}
// Remove double ranges (inclusion / identity)
query_strs.sort(); // sort paths
std::sort(query_strs.begin(), query_strs.end()); // sort paths
for (auto i = query_strs.begin(), j = i; i != query_strs.end(); ++i) {
if (i != j && i->compare(0, i->size(), *j) == 0) {
*i = "";
@ -552,29 +547,55 @@ bool Store::read(VPackSlice const& query, Builder& ret) const {
j = i;
}
}
auto cut = std::remove_if(query_strs.begin(), query_strs.end(), Empty());
auto cut = std::remove_if(query_strs.begin(), query_strs.end(),
[](std::string const& s) -> bool { return s.empty(); });
query_strs.erase(cut, query_strs.end());
// Create response tree
Node copy("copy");
// Distinguish two cases:
// a fast path for exactly one path, in which we do not have to copy all
// a slow path for more than one path
MUTEX_LOCKER(storeLocker, _storeLock); // Freeze KV-Store for read
for (auto const path : query_strs) {
if (query_strs.size() == 1) {
auto const& path = query_strs[0];
std::vector<std::string> pv = split(path, '/');
size_t e = _node.exists(pv).size();
// Build surrounding object structure:
size_t e = _node.exists(pv).size(); // note: e <= pv.size()!
size_t i = 0;
for (auto it = pv.begin(); i < e; ++i, ++it) {
ret.openObject();
ret.add(VPackValue(*it));
}
if (e == pv.size()) { // existing
copy(pv) = _node(pv);
} else { // non-existing
for (size_t i = 0; i < pv.size() - e + 1; ++i) {
pv.pop_back();
}
if (copy(pv).type() == LEAF && copy(pv).slice().isNone()) {
copy(pv) = arangodb::velocypack::Slice::emptyObjectSlice();
_node(pv).toBuilder(ret, showHidden);
} else {
VPackObjectBuilder guard(&ret);
}
// And close surrounding object structures:
for (i = 0; i < e; ++i) {
ret.close();
}
} else { // slow path for 0 or more than 2 paths:
// Create response tree
Node copy("copy");
for (auto const& path : query_strs) {
std::vector<std::string> pv = split(path, '/');
size_t e = _node.exists(pv).size();
if (e == pv.size()) { // existing
copy(pv) = _node(pv);
} else { // non-existing
for (size_t i = 0; i < pv.size() - e + 1; ++i) {
pv.pop_back();
}
if (copy(pv).type() == LEAF && copy(pv).slice().isNone()) {
copy(pv) = arangodb::velocypack::Slice::emptyObjectSlice();
}
}
}
}
// Into result builder
copy.toBuilder(ret, showHidden);
// Into result builder
copy.toBuilder(ret, showHidden);
}
return success;
}

206
arangod/Agency/Supervision.cpp
View File

@ -38,6 +38,7 @@
#include "Basics/ConditionLocker.h"
#include "Basics/MutexLocker.h"
#include "Cluster/ServerState.h"
#include "Random/RandomGenerator.h"
using namespace arangodb;
using namespace arangodb::consensus;
@ -334,7 +335,7 @@ void handleOnStatusDBServer(Agent* agent, Node const& snapshot,
void handleOnStatusCoordinator(Agent* agent, Node const& snapshot, HealthRecord& persisted,
HealthRecord& transisted, std::string const& serverID) {
if (transisted.status == Supervision::HEALTH_STATUS_FAILED) {
// if the current foxxmaster server failed => reset the value to ""
if (snapshot.hasAsString(foxxmaster).first == serverID) {
@ -346,7 +347,7 @@ void handleOnStatusCoordinator(Agent* agent, Node const& snapshot, HealthRecord&
create.add(foxxmaster, VPackValue(""));
}
}
singleWriteTransaction(agent, create);
singleWriteTransaction(agent, create, false);
}
}
}
@ -607,7 +608,7 @@ std::vector<check_t> Supervision::check(std::string const& type) {
if (!this->isStopping()) {
// Replicate special event and only then transient store
if (changed) {
write_ret_t res = singleWriteTransaction(_agent, *pReport);
write_ret_t res = singleWriteTransaction(_agent, *pReport, false);
if (res.accepted && res.indices.front() != 0) {
++_jobId; // Job was booked
transient(_agent, *tReport);
@ -744,7 +745,7 @@ void Supervision::reportStatus(std::string const& status) {
}
if (persist) {
write_ret_t res = singleWriteTransaction(_agent, *report);
write_ret_t res = singleWriteTransaction(_agent, *report, false);
}
}
@ -793,7 +794,7 @@ void Supervision::run() {
while (!this->isStopping()) {
auto lapStart = std::chrono::steady_clock::now();
{
MUTEX_LOCKER(locker, _lock);
@ -812,8 +813,9 @@ void Supervision::run() {
if (_jobId == 0 || _jobId == _jobIdMax) {
getUniqueIds(); // cannot fail but only hang
}
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin updateSnapshot";
updateSnapshot();
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Finished updateSnapshot";
if (!_snapshot.has("Supervision/Maintenance")) {
reportStatus("Normal");
@ -828,12 +830,14 @@ void Supervision::run() {
// 55 seconds is less than a minute, which fits to the
// 60 seconds timeout in /_admin/cluster/health
try {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin doChecks";
doChecks();
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Finished doChecks";
} catch (std::exception const& e) {
LOG_TOPIC(ERR, Logger::SUPERVISION)
LOG_TOPIC(WARN, Logger::SUPERVISION)
<< e.what() << " " << __FILE__ << " " << __LINE__;
} catch (...) {
LOG_TOPIC(ERR, Logger::SUPERVISION)
LOG_TOPIC(WARN, Logger::SUPERVISION)
<< "Supervision::doChecks() generated an uncaught "
"exception.";
}
@ -843,9 +847,15 @@ void Supervision::run() {
"heartbeats: "
<< _agent->leaderFor();
}
handleJobs();
try {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin handleJobs";
handleJobs();
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Finished handleJobs";
} catch(std::exception const& e) {
LOG_TOPIC(WARN, Logger::SUPERVISION)
<< "Caught exception in handleJobs(), error message: "
<< e.what();
}
} else {
reportStatus("Maintenance");
}
@ -875,7 +885,7 @@ void Supervision::run() {
auto lapTime = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - lapStart).count();
if (lapTime < 1000000) {
_cv.wait(static_cast<uint64_t>((1000000 - lapTime) * _frequency));
}
@ -961,7 +971,7 @@ void Supervision::handleShutdown() {
}
void Supervision::cleanupLostCollections(Node const& snapshot, AgentInterface* agent,
std::string const& jobId) {
uint64_t& jobId) {
std::unordered_set<std::string> failedServers;
// Search for failed server
@ -1011,21 +1021,23 @@ void Supervision::cleanupLostCollections(Node const& snapshot, AgentInterface* a
auto const& servername = servers[0].copyString();
if (failedServers.find(servername) != failedServers.end()) {
// lost shard
LOG_TOPIC(TRACE, Logger::SUPERVISION)
<< "Found a lost shard: " << shard.first;
// potentially lost shard
auto const& shardname = shard.first;
auto const& planurl = "/arango/Plan/Collections/" + dbname + "/" +
colname + "/shards/" + shardname;
auto const& planurlinsnapshot = "/Plan/Collections/" + dbname
+ "/" + colname + "/shards/" + shardname;
auto const& planurl = "/arango" + planurlinsnapshot;
auto const& currenturl = "/arango/Current/Collections/" + dbname +
"/" + colname + "/" + shardname;
auto const& healthurl =
"/arango/Supervision/Health/" + servername + "/Status";
// check if it exists in Plan
if (snapshot.has(planurl)) {
if (snapshot.has(planurlinsnapshot)) {
continue;
}
LOG_TOPIC(TRACE, Logger::SUPERVISION)
<< "Found a lost shard: " << shard.first;
// Now remove that shard
{
VPackArrayBuilder trx(builder.get());
@ -1038,16 +1050,17 @@ void Supervision::cleanupLostCollections(Node const& snapshot, AgentInterface* a
builder->add("op", VPackValue("delete"));
}
// add a job done entry to "Target/Finished"
builder->add(VPackValue("/arango/Target/Finished"));
std::string jobIdStr = std::to_string(jobId++);
builder->add(VPackValue("/arango/Target/Finished/" + jobIdStr));
{
VPackObjectBuilder op(builder.get());
builder->add("op", VPackValue("push"));
builder->add("op", VPackValue("set"));
builder->add(VPackValue("new"));
{
VPackObjectBuilder job(builder.get());
builder->add("type", VPackValue("cleanUpLostCollection"));
builder->add("server", VPackValue(shardname));
builder->add("jobId", VPackValue(jobId));
builder->add("jobId", VPackValue(jobIdStr));
builder->add("creator", VPackValue("supervision"));
builder->add("timeCreated", VPackValue(timepointToString(
std::chrono::system_clock::now())));
@ -1102,51 +1115,162 @@ void Supervision::cleanupLostCollections(Node const& snapshot, AgentInterface* a
bool Supervision::handleJobs() {
_lock.assertLockedByCurrentThread();
// Do supervision
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin shrinkCluster";
shrinkCluster();
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin enforceReplication";
enforceReplication();
cleanupLostCollections(_snapshot, _agent, std::to_string(_jobId++));
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin cleanupLostCollections";
cleanupLostCollections(_snapshot, _agent, _jobId);
// Note that this function consumes job IDs, potentially many, so the member
// is incremented inside the function. Furthermore, `cleanupLostCollections`
// is static for catch testing purposes.
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin readyOrphanedIndexCreations";
readyOrphanedIndexCreations();
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin workJobs";
workJobs();
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin cleanupFinishedAndFailedJobs";
cleanupFinishedAndFailedJobs();
return true;
}
// Guarded by caller
void Supervision::cleanupFinishedAndFailedJobs() {
// This deletes old Supervision jobs in /Target/Finished and
// /Target/Failed. We can be rather generous here since old
// snapshots and log entries are kept for much longer.
// We only keep up to 500 finished jobs and 1000 failed jobs.
_lock.assertLockedByCurrentThread();
constexpr size_t maximalFinishedJobs = 500;
constexpr size_t maximalFailedJobs = 1000;
auto cleanup = [&](std::string prefix, size_t limit) {
auto const& jobs = _snapshot.hasAsChildren(prefix).first;
if (jobs.size() <= 2 * limit) {
return;
}
typedef std::pair<std::string, std::string> keyDate;
std::vector<keyDate> v;
v.reserve(jobs.size());
for (auto const& p: jobs) {
auto created = p.second->hasAsString("timeCreated");
if (created.second) {
v.emplace_back(p.first, created.first);
} else {
v.emplace_back(p.first, "1970"); // will be sorted very early
}
}
std::sort(v.begin(), v.end(), [](keyDate const& a, keyDate const& b) -> bool {
return a.second < b.second;
});
size_t toBeDeleted = v.size() - limit; // known to be positive
LOG_TOPIC(INFO, Logger::AGENCY) << "Deleting " << toBeDeleted << " old jobs"
" in " << prefix;
VPackBuilder trx; // We build a transaction here
{ // Pair for operation, no precondition here
VPackArrayBuilder guard1(&trx);
{
VPackObjectBuilder guard2(&trx);
for (auto it = v.begin(); toBeDeleted-- > 0 && it != v.end(); ++it) {
trx.add(VPackValue(prefix + it->first));
{
VPackObjectBuilder guard2(&trx);
trx.add("op", VPackValue("delete"));
}
}
}
}
singleWriteTransaction(_agent, trx, false); // do not care about the result
};
cleanup(finishedPrefix, maximalFinishedJobs);
cleanup(failedPrefix, maximalFailedJobs);
}
// Guarded by caller
void Supervision::workJobs() {
_lock.assertLockedByCurrentThread();
bool dummy = false;
// ATTENTION: It is necessary to copy the todos here, since we modify
// below!
auto todos = _snapshot.hasAsChildren(toDoPrefix).first;
auto it = todos.begin();
static std::string const FAILED = "failed";
// In the case that there are a lot of jobs in ToDo or in Pending we cannot
// afford to run through all of them before we do another Supervision round.
// This is because only in a new round we discover things like a server
// being good again. Currently, we manage to work through approx. 200 jobs
// per second. Therefore, we have - for now - chosen to limit the number of
// jobs actually worked on to 1000 in ToDo and 1000 in Pending. However,
// since some jobs are just waiting, we cannot work on the same 1000
// jobs in each round. This is where the randomization comes in. We work
// on up to 1000 *random* jobs. This will eventually cover everything with
// very high probability. Note that the snapshot does not change, so
// `todos.size()` is constant for the loop, even though we do agency
// transactions to remove ToDo jobs.
size_t const maximalJobsPerRound = 1000;
bool selectRandom = todos.size() > maximalJobsPerRound;
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin ToDos of type Failed*";
while (it != todos.end()) {
auto jobNode = *(it->second);
if (selectRandom && RandomGenerator::interval(static_cast<uint64_t>(todos.size())) > maximalJobsPerRound) {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Skipped ToDo Job";
++it;
continue;
}
auto const& jobNode = *(it->second);
if (jobNode.hasAsString("type").first.compare(0, FAILED.length(), FAILED) == 0) {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin JobContext::run()";
JobContext(TODO, jobNode.hasAsString("jobId").first, _snapshot, _agent)
.run(_haveAborts);
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Finish JobContext::run()";
it = todos.erase(it);
} else {
++it;
}
}
// Do not start other jobs, if above resilience jobs aborted stuff
// Do not start other jobs, if above resilience jobs aborted stuff
if (!_haveAborts) {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin ToDos";
for (auto const& todoEnt : todos) {
auto jobNode = *(todoEnt.second);
if (selectRandom && RandomGenerator::interval(static_cast<uint64_t>(todos.size())) > maximalJobsPerRound) {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Skipped ToDo Job";
continue;
}
auto const& jobNode = *(todoEnt.second);
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin JobContext::run()";
JobContext(TODO, jobNode.hasAsString("jobId").first, _snapshot, _agent)
.run(dummy);
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Finish JobContext::run()";
}
}
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin Pendings";
auto const& pends = _snapshot.hasAsChildren(pendingPrefix).first;
selectRandom = pends.size() > maximalJobsPerRound;
for (auto const& pendEnt : pends) {
auto jobNode = *(pendEnt.second);
if (selectRandom && RandomGenerator::interval(static_cast<uint64_t>(pends.size())) > maximalJobsPerRound) {
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Skipped Pending Job";
continue;
}
auto const& jobNode = *(pendEnt.second);
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Begin JobContext::run()";
JobContext(PENDING, jobNode.hasAsString("jobId").first, _snapshot, _agent)
.run(dummy);
LOG_TOPIC(TRACE, Logger::SUPERVISION) << "Finish JobContext::run()";
}
}
@ -1266,9 +1390,9 @@ void Supervision::enforceReplication() {
// there is no overload on the Agency job system. Therefore, if this
// number is at least maxNrAddRemoveJobsInTodo, we skip the rest of
// the function:
int const maxNrAddRemoveJobsInTodo = 15;
int const maxNrAddRemoveJobsInTodo = 50;
auto todos = _snapshot.hasAsChildren(toDoPrefix).first;
auto const& todos = _snapshot.hasAsChildren(toDoPrefix).first;
int nrAddRemoveJobsInTodo = 0;
for (auto it = todos.begin(); it != todos.end(); ++it) {
auto jobNode = *(it->second);
@ -1280,7 +1404,10 @@ void Supervision::enforceReplication() {
}
}
// We will loop over plannedDBs, so we use hasAsChildren
auto const& plannedDBs = _snapshot.hasAsChildren(planColPrefix).first;
// We will lookup in currentDBs, so we use hasAsNode
auto const& currentDBs = _snapshot.hasAsNode(curColPrefix).first;
for (const auto& db_ : plannedDBs) { // Planned databases
auto const& db = *(db_.second);
@ -1327,6 +1454,17 @@ void Supervision::enforceReplication() {
if (actualReplicationFactor != replicationFactor ||
apparentReplicationFactor != replicationFactor) {
// First check the case that not all are in sync:
std::string curPath = db_.first + "/" + col_.first + "/"
+ shard_.first + "/servers";
auto const& currentServers = currentDBs.hasAsArray(curPath);
size_t inSyncReplicationFactor = actualReplicationFactor;
if (currentServers.second) {
if (currentServers.first.length() < actualReplicationFactor) {
inSyncReplicationFactor = currentServers.first.length();
}
}
// Check that there is not yet an addFollower or removeFollower
// or moveShard job in ToDo for this shard:
auto const& todo = _snapshot.hasAsChildren(toDoPrefix).first;
@ -1361,7 +1499,7 @@ void Supervision::enforceReplication() {
return;
}
} else if (apparentReplicationFactor > replicationFactor &&
actualReplicationFactor >= replicationFactor) {
inSyncReplicationFactor >= replicationFactor) {
RemoveFollower(_snapshot, _agent, std::to_string(_jobId++),
"supervision", db_.first, col_.first, shard_.first)
.create();
@ -1385,7 +1523,7 @@ void Supervision::fixPrototypeChain(Builder& migrate) {
std::function<std::string(std::string const&, std::string const&)> resolve;
resolve = [&](std::string const& db, std::string const& col) {
std::string s;
auto tmp_n = snap.hasAsNode(planColPrefix + db + "/" + col);
auto const& tmp_n = snap.hasAsNode(planColPrefix + db + "/" + col);
if (tmp_n.second) {
Node const& n = tmp_n.first;
s = n.hasAsString("distributeShardsLike").first;

5
arangod/Agency/Supervision.h
View File

@ -154,6 +154,9 @@ class Supervision : public arangodb::CriticalThread {
// @brief Check shards in agency
std::vector<check_t> checkShards();
// @brief
void cleanupFinishedAndFailedJobs();
void workJobs();
/// @brief Get unique ids from agency
@ -169,7 +172,7 @@ class Supervision : public arangodb::CriticalThread {
public:
static void cleanupLostCollections(Node const& snapshot, AgentInterface* agent,
std::string const& jobId);
uint64_t& jobId);
private:
/**

23
arangod/Cluster/Action.cpp
View File

@ -171,6 +171,29 @@ void Action::toVelocyPack(arangodb::velocypack::Builder& builder) const {
_action->toVelocyPack(builder);
}
bool Action::operator<(Action const& other) const {
// This is to sort actions in a priority queue, therefore, the higher, the
// higher the priority. FastTrack is always higher, priority counts then,
// and finally creation time (earlier is higher):
if (!fastTrack() && other.fastTrack()) {
return true;
}
if (fastTrack() && !other.fastTrack()) {
return false;
}
if (priority() < other.priority()) {
return true;
}
if (priority() > other.priority()) {
return false;
}
if (getCreateTime() > other.getCreateTime()) {
// Intentional inversion! smaller time is higher priority.
return true;
}
return false;
}
namespace std {
ostream& operator<<(ostream& out, arangodb::maintenance::Action const& d) {
out << d.toVelocyPack().toJson();

16
arangod/Cluster/Action.h
View File

@ -54,6 +54,9 @@ class Action {
Action() = delete;
Action& operator=(Action const&) = delete;
/// @brief for sorting in a priority queue:
bool operator<(Action const& other) const;
/**
* @brief construct with concrete action base
* @param feature Maintenance feature
@ -73,9 +76,6 @@ class Action {
/// @brief run for some time and tell, if need more time or done
bool first();
/// @brief run for some time and tell, if need more time or done
ActionState state() const;
/// @brief is object in a usable condition
bool ok() const { return (nullptr != _action.get() && _action->ok()); };
@ -180,6 +180,16 @@ class Action {
return _action->getDoneTime();
}
/// @brief fastTrack
bool fastTrack() const {
return _action->fastTrack();
}
/// @brief priority
int priority() const {
return _action->priority();
}
private:
/// @brief actually create the concrete action
void create(MaintenanceFeature&, ActionDescription const&);

5
arangod/Cluster/ActionBase.cpp
View File

@ -40,12 +40,13 @@ inline static std::chrono::system_clock::duration secs_since_epoch() {
}
ActionBase::ActionBase(MaintenanceFeature& feature, ActionDescription const& desc)
: _feature(feature), _description(desc), _state(READY), _progress(0) {
: _feature(feature), _description(desc), _state(READY), _progress(0),
_priority(desc.priority()) {
init();
}
ActionBase::ActionBase(MaintenanceFeature& feature, ActionDescription&& desc)
: _feature(feature), _description(std::move(desc)), _state(READY), _progress(0) {
: _feature(feature), _description(std::move(desc)), _state(READY), _progress(0), _priority(desc.priority()) {
init();
}

6
arangod/Cluster/ActionBase.h
View File

@ -182,6 +182,11 @@ class ActionBase {
std::string const static FAST_TRACK;
/// @brief return priority, inherited from ActionDescription
int priority() const {
return _priority;
}
protected:
/// @brief common initialization for all constructors
void init();
@ -215,6 +220,7 @@ class ActionBase {
Result _result;
int _priority;
}; // class ActionBase
} // namespace maintenance

5
arangod/Cluster/ActionDescription.cpp
View File

@ -31,8 +31,9 @@ using namespace arangodb::maintenance;
/// @brief ctor
ActionDescription::ActionDescription(std::map<std::string, std::string> const& d,
int priority,
std::shared_ptr<VPackBuilder> const p)
: _description(d), _properties(p) {
: _description(d), _properties(p), _priority(priority) {
TRI_ASSERT(d.find(NAME) != d.end());
TRI_ASSERT(p == nullptr || p->isEmpty() || p->slice().isObject());
}
@ -134,7 +135,7 @@ std::size_t hash<ActionDescription>::operator()(ActionDescription const& a) cons
}
ostream& operator<<(ostream& out, arangodb::maintenance::ActionDescription const& d) {
out << d.toJson();
out << d.toJson() << " Priority: " << d.priority();
return out;
}
} // namespace std

12
arangod/Cluster/ActionDescription.h
View File

@ -67,6 +67,7 @@ struct ActionDescription {
*/
explicit ActionDescription(
std::map<std::string, std::string> const& desc,
int priority,
std::shared_ptr<VPackBuilder> const suppl = std::make_shared<VPackBuilder>());
/**
@ -155,12 +156,23 @@ struct ActionDescription {
*/
std::shared_ptr<VPackBuilder> const properties() const;
/**
* @brief Get priority, the higher the more priority, 1 is default
* @return int
*/
int priority() const {
return _priority;
}
private:
/** @brief discriminatory properties */
std::map<std::string, std::string> const _description;
/** @brief non-discriminatory properties */
std::shared_ptr<VPackBuilder> const _properties;
/** @brief priority */
int _priority;
};
} // namespace maintenance

16
arangod/Cluster/ClusterInfo.cpp
View File

@ -450,7 +450,7 @@ void ClusterInfo::loadPlan() {
uint64_t storedVersion = _planProt.wantedVersion; // this is the version
// we will set in the end
LOG_TOPIC(TRACE, Logger::CLUSTER) << "loadPlan: wantedVersion=" << storedVersion
LOG_TOPIC(DEBUG, Logger::CLUSTER) << "loadPlan: wantedVersion=" << storedVersion
<< ", doneVersion=" << _planProt.doneVersion;
if (_planProt.doneVersion == storedVersion) {
// Somebody else did, what we intended to do, so just return
@ -877,6 +877,9 @@ void ClusterInfo::loadPlan() {
} else {
LOG_TOPIC(ERR, Logger::CLUSTER) << "\"Plan\" is not an object in agency";
}
LOG_TOPIC(DEBUG, Logger::CLUSTER)
<< "loadPlan done: wantedVersion=" << storedVersion
<< ", doneVersion=" << _planProt.doneVersion;
return;
}
@ -904,6 +907,9 @@ void ClusterInfo::loadCurrent() {
return;
}
LOG_TOPIC(DEBUG, Logger::CLUSTER) << "loadCurrent: wantedVersion: "
<< _currentProt.wantedVersion;
// Now contact the agency:
AgencyCommResult result = _agency.getValues(prefixCurrent);
@ -1022,6 +1028,8 @@ void ClusterInfo::loadCurrent() {
LOG_TOPIC(ERR, Logger::CLUSTER) << "Current is not an object!";
}
LOG_TOPIC(DEBUG, Logger::CLUSTER) << "loadCurrent done.";
return;
}
@ -1511,6 +1519,8 @@ int ClusterInfo::createCollectionCoordinator(
return TRI_ERROR_BAD_PARAMETER; // must not be empty
}
LOG_TOPIC(DEBUG, Logger::CLUSTER) << "createCollectionCoordinator, loading Plan from agency...";
{
// check if a collection with the same name is already planned
loadPlan();
@ -1542,6 +1552,8 @@ int ClusterInfo::createCollectionCoordinator(
}
}
LOG_TOPIC(DEBUG, Logger::CLUSTER) << "createCollectionCoordinator, checking things...";
// mop: why do these ask the agency instead of checking cluster info?
if (!ac.exists("Plan/Databases/" + databaseName)) {
events::CreateCollection(name, TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
@ -1758,6 +1770,8 @@ int ClusterInfo::createCollectionCoordinator(
break; // Leave loop, since we are done
}
LOG_TOPIC(DEBUG, Logger::CLUSTER) << "createCollectionCoordinator, Plan changed, waiting for success...";
bool isSmart = false;
VPackSlice smartSlice = json.get(StaticStrings::IsSmart);
if (smartSlice.isBool() && smartSlice.getBool()) {

3
arangod/Cluster/CreateCollection.cpp
View File

@ -185,6 +185,9 @@ bool CreateCollection::first() {
_description.get(SERVER_ID), _result);
}
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "Create collection done, notifying Maintenance";
notify();
return false;

45
arangod/Cluster/DBServerAgencySync.cpp
View File

@ -58,6 +58,7 @@ void DBServerAgencySync::work() {
}
Result DBServerAgencySync::getLocalCollections(VPackBuilder& collections) {
using namespace arangodb::basics;
Result result;
DatabaseFeature* dbfeature = nullptr;
@ -197,7 +198,23 @@ DBServerAgencySyncResult DBServerAgencySync::execute() {
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "DBServerAgencySync::phaseOne done";
LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo";
// Give some asynchronous jobs created in phaseOne a chance to complete
// before we collect data for phaseTwo:
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "DBServerAgencySync::hesitating between phases 1 and 2 for 0.1s...";
std::this_thread::sleep_for(std::chrono::milliseconds(100));
auto current = clusterInfo->getCurrent();
if (current == nullptr) {
// TODO increase log level, except during shutdown?
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "DBServerAgencySync::execute no current";
result.errorMessage = "DBServerAgencySync::execute no current";
return result;
}
LOG_TOPIC(TRACE, Logger::MAINTENANCE)
<< "DBServerAgencySync::phaseTwo - current state: " << current->toJson();
local.clear();
glc = getLocalCollections(local);
// We intentionally refetch local collections here, such that phase 2
@ -210,16 +227,7 @@ DBServerAgencySyncResult DBServerAgencySync::execute() {
return result;
}
auto current = clusterInfo->getCurrent();
if (current == nullptr) {
// TODO increase log level, except during shutdown?
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "DBServerAgencySync::execute no current";
result.errorMessage = "DBServerAgencySync::execute no current";
return result;
}
LOG_TOPIC(TRACE, Logger::MAINTENANCE)
<< "DBServerAgencySync::phaseTwo - current state: " << current->toJson();
LOG_TOPIC(DEBUG, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo";
tmp = arangodb::maintenance::phaseTwo(plan->slice(), current->slice(),
local.slice(), serverId, *mfeature, rb);
@ -243,21 +251,30 @@ DBServerAgencySyncResult DBServerAgencySync::execute() {
// Report to current
if (!agency.isEmptyObject()) {
std::vector<AgencyOperation> operations;
std::vector<AgencyPrecondition> preconditions;
for (auto const& ao : VPackObjectIterator(agency)) {
auto const key = ao.key.copyString();
auto const op = ao.value.get("op").copyString();
if (ao.value.hasKey("precondition")) {
auto const precondition = ao.value.get("precondition");
preconditions.push_back(
AgencyPrecondition(
precondition.keyAt(0).copyString(), AgencyPrecondition::Type::VALUE, precondition.valueAt(0)));
}
if (op == "set") {
auto const value = ao.value.get("payload");
operations.push_back(AgencyOperation(key, AgencyValueOperationType::SET, value));
} else if (op == "delete") {
operations.push_back(AgencyOperation(key, AgencySimpleOperationType::DELETE_OP));
}
}
operations.push_back(AgencyOperation("Current/Version",
AgencySimpleOperationType::INCREMENT_OP));
AgencyPrecondition precondition("Plan/Version",
AgencyPrecondition::Type::VALUE, plan->slice().get("Version"));
AgencyWriteTransaction currentTransaction(operations, precondition);
AgencyWriteTransaction currentTransaction(operations, preconditions);
AgencyCommResult r = comm.sendTransactionWithFailover(currentTransaction);
if (!r.successful()) {
LOG_TOPIC(INFO, Logger::MAINTENANCE)

44
arangod/Cluster/Maintenance.cpp
View File

@ -246,7 +246,7 @@ void handlePlanShard(VPackSlice const& cprops, VPackSlice const& ldb,
{SERVER_ID, serverId},
{LOCAL_LEADER, lcol.get(THE_LEADER).copyString()},
{FOLLOWERS_TO_DROP, followersToDropString}},
properties));
HIGHER_PRIORITY, properties));
} else {
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "Previous failure exists for local shard " << dbname << "/" << shname
@ -274,7 +274,7 @@ void handlePlanShard(VPackSlice const& cprops, VPackSlice const& ldb,
index.get(StaticStrings::IndexType).copyString()},
{FIELDS, index.get(FIELDS).toJson()},
{ID, index.get(ID).copyString()}},
std::make_shared<VPackBuilder>(index)));
INDEX_PRIORITY, std::make_shared<VPackBuilder>(index)));
}
}
}
@ -288,7 +288,7 @@ void handlePlanShard(VPackSlice const& cprops, VPackSlice const& ldb,
{DATABASE, dbname},
{SERVER_ID, serverId},
{THE_LEADER, shouldBeLeading ? std::string() : leaderId}},
props));
shouldBeLeading ? LEADER_PRIORITY : FOLLOWER_PRIORITY, props));
} else {
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "Previous failure exists for creating local shard " << dbname << "/"
@ -311,7 +311,8 @@ void handleLocalShard(std::string const& dbname, std::string const& colname,
bool localLeader = cprops.get(THE_LEADER).copyString().empty();
if (plannedLeader == UNDERSCORE + serverId && localLeader) {
actions.emplace_back(ActionDescription(
{{NAME, "ResignShardLeadership"}, {DATABASE, dbname}, {SHARD, colname}}));
{{NAME, "ResignShardLeadership"}, {DATABASE, dbname}, {SHARD, colname}},
RESIGN_PRIORITY));
} else {
bool drop = false;
// check if shard is in plan, if not drop it
@ -326,7 +327,8 @@ void handleLocalShard(std::string const& dbname, std::string const& colname,
if (drop) {
actions.emplace_back(ActionDescription(
{{NAME, DROP_COLLECTION}, {DATABASE, dbname}, {COLLECTION, colname}}));
{{NAME, DROP_COLLECTION}, {DATABASE, dbname}, {COLLECTION, colname}},
localLeader ? LEADER_PRIORITY : FOLLOWER_PRIORITY));
} else {
// The shard exists in both Plan and Local
commonShrds.erase(it); // it not a common shard?
@ -345,7 +347,7 @@ void handleLocalShard(std::string const& dbname, std::string const& colname,
indis.erase(id);
} else {
actions.emplace_back(ActionDescription(
{{NAME, "DropIndex"}, {DATABASE, dbname}, {COLLECTION, colname}, {"index", id}}));
{{NAME, "DropIndex"}, {DATABASE, dbname}, {COLLECTION, colname}, {"index", id}}, INDEX_PRIORITY));
}
}
}
@ -394,7 +396,7 @@ arangodb::Result arangodb::maintenance::diffPlanLocal(
if (errors.databases.find(dbname) == errors.databases.end()) {
actions.emplace_back(
ActionDescription({{std::string(NAME), std::string(CREATE_DATABASE)},
{std::string(DATABASE), std::string(dbname)}}));
{std::string(DATABASE), std::string(dbname)}}, HIGHER_PRIORITY));
} else {
LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
<< "Previous failure exists for creating database " << dbname << "skipping";
@ -408,7 +410,7 @@ arangodb::Result arangodb::maintenance::diffPlanLocal(
if (!plan.hasKey(std::vector<std::string>{DATABASES, dbname})) {
actions.emplace_back(
ActionDescription({{std::string(NAME), std::string(DROP_DATABASE)},
{std::string(DATABASE), std::string(dbname)}}));
{std::string(DATABASE), std::string(dbname)}}, HIGHER_PRIORITY));
}
}
@ -619,7 +621,9 @@ arangodb::Result arangodb::maintenance::phaseOne(VPackSlice const& plan,
std::string const& serverId,
MaintenanceFeature& feature,
VPackBuilder& report) {
arangodb::Result result;
report.add(VPackValue(PHASE_ONE));
{
@ -847,7 +851,28 @@ arangodb::Result arangodb::maintenance::reportInCurrent(
{
VPackObjectBuilder o(&report);
report.add(OP, VP_SET);
// Report new current entry ...
report.add("payload", localCollectionInfo.slice());
// ... if and only if plan for this shard has changed in the meantime
try {
// Try to add a precondition, just in case we catch the exception.
// Even if the Plan entry is gone by now, it is still OK to
// report the Current value, it will be deleted in due course.
// It is the case that the Plan value is there but has changed
// that we want to protect against.
VPackSlice oldValue = pdbs.get(std::vector<std::string>{dbName, colName, "shards", shName});
if (!oldValue.isNone()) {
report.add(VPackValue("precondition"));
{
VPackObjectBuilder p(&report);
report.add(
PLAN_COLLECTIONS + dbName + "/" + colName + "/shards/" + shName,
oldValue);
}
}
} catch(...) {
}
}
}
} else { // Follower
@ -1080,7 +1105,7 @@ arangodb::Result arangodb::maintenance::syncReplicatedShardsWithLeaders(
{COLLECTION, colname},
{SHARD, shname},
{THE_LEADER, leader},
{SHARD_VERSION, std::to_string(feature.shardVersion(shname))}}));
{SHARD_VERSION, std::to_string(feature.shardVersion(shname))}}, SYNCHRONIZE_PRIORITY));
}
}
}
@ -1097,6 +1122,7 @@ arangodb::Result arangodb::maintenance::phaseTwo(VPackSlice const& plan,
std::string const& serverId,
MaintenanceFeature& feature,
VPackBuilder& report) {
MaintenanceFeature::errors_t allErrors;
feature.copyAllErrors(allErrors);

13
arangod/Cluster/Maintenance.h
View File

@ -36,6 +36,19 @@ class LogicalCollection;
namespace maintenance {
// A few constants for priorities used in Maintenance for ActionDescriptions:
// For fast track:
constexpr int NORMAL_PRIORITY = 1;
constexpr int FOLLOWER_PRIORITY = 1;
constexpr int LEADER_PRIORITY = 2;
constexpr int HIGHER_PRIORITY = 2;
constexpr int RESIGN_PRIORITY = 3;
// For non fast track:
constexpr int INDEX_PRIORITY = 2;
constexpr int SYNCHRONIZE_PRIORITY = 1;
using Transactions = std::vector<std::pair<VPackBuilder, VPackBuilder>>;
arangodb::Result diffPlanLocalForDatabases(VPackSlice const&,

67
arangod/Cluster/MaintenanceFeature.cpp
View File

@ -33,6 +33,7 @@
#include "Cluster/CreateDatabase.h"
#include "Cluster/MaintenanceWorker.h"
#include "Cluster/ServerState.h"
#include "Random/RandomGenerator.h"
using namespace arangodb;
using namespace arangodb::application_features;
@ -152,7 +153,7 @@ void MaintenanceFeature::stop() {
// loop on each worker, retesting at 10ms just in case
while (itWorker->isRunning()) {
_workerCompletion.wait(10000);
_workerCompletion.wait(std::chrono::microseconds(10000));
} // if
} // for
@ -299,7 +300,9 @@ void MaintenanceFeature::registerAction(std::shared_ptr<Action> action, bool exe
// mark as executing so no other workers accidentally grab it
if (executeNow) {
action->setState(maintenance::EXECUTING);
} // if
} else if (action->getState() == maintenance::READY) {
_prioQueue.push(action);
}
// WARNING: holding write lock to _actionRegistry and about to
// lock condition variable
@ -308,7 +311,13 @@ void MaintenanceFeature::registerAction(std::shared_ptr<Action> action, bool exe
if (!executeNow) {
CONDITION_LOCKER(cLock, _actionRegistryCond);
_actionRegistryCond.signal();
_actionRegistryCond.broadcast();
// Note that we do a broadcast here for the following reason: if we did
// signal here, we cannot control which of the sleepers is woken up.
// If the new action is not fast track, then we could wake up the
// fast track worker, which would leave the action as it is. This would
// cause a delay of up to 0.1 seconds. With the broadcast, the worst
// case is that we wake up sleeping workers unnecessarily.
} // if
} // lock
}
@ -398,28 +407,48 @@ std::shared_ptr<Action> MaintenanceFeature::findActionIdNoLock(uint64_t id) {
std::shared_ptr<Action> MaintenanceFeature::findReadyAction(std::unordered_set<std::string> const& labels) {
std::shared_ptr<Action> ret_ptr;
while (!_isShuttingDown && !ret_ptr) {
// scan for ready action (and purge any that are done waiting)
while (!_isShuttingDown) {
// use priority queue for ready action (and purge any that are done waiting)
{
WRITE_LOCKER(wLock, _actionRegistryLock);
for (auto loop = _actionRegistry.begin(); _actionRegistry.end() != loop && !ret_ptr;) {
auto state = (*loop)->getState();
if (state == maintenance::READY && (*loop)->matches(labels)) {
ret_ptr = *loop;
ret_ptr->setState(maintenance::EXECUTING);
} else if ((*loop)->done()) {
loop = _actionRegistry.erase(loop);
} else {
++loop;
} // else
} // for
while (!_prioQueue.empty()) {
// If _prioQueue is empty, we have no ready job and simply loop in the
// outer loop.
auto const& top = _prioQueue.top();
if (top->getState() != maintenance::READY) { // in case it is deleted
_prioQueue.pop();
continue;
}
if (top->matches(labels)) {
ret_ptr = top;
_prioQueue.pop();
return ret_ptr;
}
// We are not interested, this can only mean that we are fast track
// and the top action is not. Therefore, the whole queue does not
// contain any fast track, so we can idle.
break;
}
// When we get here, there is currently nothing to do, so we might
// as well clean up those jobs in the _actionRegistry, which are
// in state DONE:
if (RandomGenerator::interval(uint32_t(10)) == 0) {
for (auto loop = _actionRegistry.begin(); _actionRegistry.end() != loop;) {
if ((*loop)->done()) {
loop = _actionRegistry.erase(loop);
} else {
++loop;
} // else
} // for
}
} // WRITE
// no pointer ... wait 5 second
if (!_isShuttingDown && !ret_ptr) {
// no pointer ... wait 0.1 seconds unless woken up
if (!_isShuttingDown) {
CONDITION_LOCKER(cLock, _actionRegistryCond);
_actionRegistryCond.wait(100000);
_actionRegistryCond.wait(std::chrono::milliseconds(100));
} // if
} // while

28
arangod/Cluster/MaintenanceFeature.h
View File

@ -32,8 +32,20 @@
#include "Cluster/MaintenanceWorker.h"
#include "ProgramOptions/ProgramOptions.h"
#include <queue>
namespace arangodb {
template<typename T>
struct SharedPtrComparer {
bool operator()(std::shared_ptr<T> const& a, std::shared_ptr<T> const& b) {
if (a == nullptr || b == nullptr) {
return false;
}
return *a < *b;
}
};
class MaintenanceFeature : public application_features::ApplicationFeature {
public:
explicit MaintenanceFeature(application_features::ApplicationServer&);
@ -350,6 +362,22 @@ class MaintenanceFeature : public application_features::ApplicationFeature {
/// @brief all actions executing, waiting, and done
std::deque<std::shared_ptr<maintenance::Action>> _actionRegistry;
// The following is protected with the _actionRegistryLock exactly as
// the _actionRegistry. This priority queue is used to find the highest
// priority action that is ready. Therefore, _prioQueue contains all the
// actions in state READY. The sorting is done such that all fast track
// actions come before all non-fast track actions. Therefore, a fast track
// thread can just look at the top action and if this is not fast track,
// it does not have to pop anything. If a worker picks an action and starts
// work on it, the action leaves state READY and is popped from the priority
// queue.
// We also need to be able to delete actions which are READY. In that case
// we need to leave the action in _prioQueue (since we cannot remove anything
// but the top from it), and simply put it into a different state.
std::priority_queue<std::shared_ptr<maintenance::Action>,
std::vector<std::shared_ptr<maintenance::Action>>,
SharedPtrComparer<maintenance::Action>> _prioQueue;
/// @brief lock to protect _actionRegistry and state changes to MaintenanceActions within
mutable arangodb::basics::ReadWriteLock _actionRegistryLock;

5
arangod/Cluster/MaintenanceRestHandler.cpp
View File

@ -121,6 +121,7 @@ bool MaintenanceRestHandler::parsePutBody(VPackSlice const& parameters) {
std::map<std::string, std::string> desc;
auto prop = std::make_shared<VPackBuilder>();
int priority = 1;
VPackObjectIterator it(parameters, true);
for (; it.valid() && good; ++it) {
@ -135,12 +136,14 @@ bool MaintenanceRestHandler::parsePutBody(VPackSlice const& parameters) {
} else if (key.isString() && (key.copyString() == "properties") && value.isObject()) {
// code here
prop.reset(new VPackBuilder(value));
} else if (key.isString() && (key.copyString() == "priority") && value.isInteger()) {
priority = static_cast<int>(value.getInt());
} else {
good = false;
} // else
} // for
_actionDesc = std::make_shared<maintenance::ActionDescription>(desc, prop);
_actionDesc = std::make_shared<maintenance::ActionDescription>(desc, priority, prop);
return good;

2
arangod/VocBase/Methods/UpgradeTasks.cpp
View File

@ -90,7 +90,7 @@ bool createSystemCollection(TRI_vocbase_t* vocbase, std::string const& name) {
bb.close();
res = Collections::create(vocbase, name, TRI_COL_TYPE_DOCUMENT, bb.slice(),
/*waitsForSyncReplication*/ true,
/*waitsForSyncReplication*/ false,
/*enforceReplicationFactor*/ true,
[](std::shared_ptr<LogicalCollection> const&) -> void {});
}

3
js/server/modules/@arangodb/cluster.js
View File

@ -274,7 +274,8 @@ function moveShard (info) {
'jobId': id,
'timeCreated': (new Date()).toISOString(),
'creator': ArangoServerState.id(),
'isLeader': isLeader };
'isLeader': isLeader,
'remainsFollower': isLeader};
global.ArangoAgency.set('Target/ToDo/' + id, todo);
} catch (e1) {
return {error: true, errorMessage: 'Cannot write to agency.'};

8
scripts/startLocalCluster.sh
View File

@ -157,7 +157,7 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do
--server.endpoint $TRANSPORT://$ENDPOINT:$PORT \
--server.statistics false \
--log.file cluster/$PORT.log \
--log.force-direct true \
--log.force-direct false \
--log.level $LOG_LEVEL_AGENCY \
--javascript.allow-admin-execute true \
$STORAGE_ENGINE \
@ -184,7 +184,7 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do
--server.endpoint $TRANSPORT://$ENDPOINT:$PORT \
--server.statistics false \
--log.file cluster/$PORT.log \
--log.force-direct true \
--log.force-direct false \
--log.level $LOG_LEVEL_AGENCY \
--javascript.allow-admin-execute true \
$STORAGE_ENGINE \
@ -235,7 +235,7 @@ start() {
--javascript.startup-directory $SRC_DIR/js \
--javascript.module-directory $SRC_DIR/enterprise/js \
--javascript.app-path cluster/apps$PORT \
--log.force-direct true \
--log.force-direct false \
--log.level $LOG_LEVEL_CLUSTER \
--javascript.allow-admin-execute true \
$STORAGE_ENGINE \
@ -257,7 +257,7 @@ start() {
--javascript.startup-directory $SRC_DIR/js \
--javascript.module-directory $SRC_DIR/enterprise/js \
--javascript.app-path cluster/apps$PORT \
--log.force-direct true \
--log.force-direct false \
--log.thread true \
--log.level $LOG_LEVEL_CLUSTER \
--javascript.allow-admin-execute true \

19
tests/Agency/CleanOutServerTest.cpp
View File

@ -210,7 +210,6 @@ SECTION("cleanout server should fail if the server does not exist") {
checkFailed(JOB_STATUS::TODO, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -224,7 +223,6 @@ SECTION("cleanout server should fail if the server does not exist") {
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("cleanout server should wait if the server is currently blocked") {
@ -347,7 +345,6 @@ SECTION("cleanout server should fail if the server is already cleaned") {
checkFailed(JOB_STATUS::TODO, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -361,7 +358,6 @@ SECTION("cleanout server should fail if the server is already cleaned") {
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("cleanout server should fail if the server is failed") {
@ -394,7 +390,6 @@ SECTION("cleanout server should fail if the server is failed") {
checkFailed(JOB_STATUS::TODO, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -408,7 +403,6 @@ SECTION("cleanout server should fail if the server is failed") {
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("cleanout server should fail if the replicationFactor is too big for any shard after counting in failedservers") {
@ -443,7 +437,6 @@ SECTION("cleanout server should fail if the replicationFactor is too big for any
checkFailed(JOB_STATUS::TODO, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -457,7 +450,6 @@ SECTION("cleanout server should fail if the replicationFactor is too big for any
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("cleanout server should fail if the replicationFactor is too big for any shard after counting in cleanedoutservers") {
@ -493,7 +485,6 @@ SECTION("cleanout server should fail if the replicationFactor is too big for any
checkFailed(JOB_STATUS::TODO, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -507,7 +498,6 @@ SECTION("cleanout server should fail if the replicationFactor is too big for any
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("cleanout server should fail if the replicationFactor is too big for any shard after counting in tobecleanedoutservers") {
@ -543,7 +533,6 @@ SECTION("cleanout server should fail if the replicationFactor is too big for any
checkFailed(JOB_STATUS::TODO, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -557,7 +546,6 @@ SECTION("cleanout server should fail if the replicationFactor is too big for any
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("a cleanout server job should move into pending when everything is ok") {
@ -612,7 +600,6 @@ SECTION("a cleanout server job should move into pending when everything is ok")
REQUIRE(preconditions.get("/arango/Target/FailedServers").get("old").toJson() == "{}");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -626,7 +613,6 @@ SECTION("a cleanout server job should move into pending when everything is ok")
);
cleanOutServer.start(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("a cleanout server job should abort after a long timeout") {
@ -692,7 +678,6 @@ SECTION("a cleanout server job should abort after a long timeout") {
}
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
@ -706,7 +691,6 @@ SECTION("a cleanout server job should abort after a long timeout") {
);
cleanOutServer.run(aborts);
Verify(Method(mockAgent, write));
Verify(Method(mockAgent, waitFor));
}
SECTION("when there are still subjobs to be done it should wait") {
@ -793,7 +777,6 @@ SECTION("once all subjobs were successful then the job should be finished") {
CHECK(std::string(writes.get("/arango/Target/Finished/1").typeName()) == "object");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
INFO("AGENCY: " << agency.toJson());
@ -838,7 +821,6 @@ SECTION("if there was a failed subjob then the job should also fail") {
checkFailed(JOB_STATUS::PENDING, q);
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
INFO("AGENCY: " << agency.toJson());
@ -904,7 +886,6 @@ SECTION("when the cleanout server job is aborted all subjobs should be aborted t
}
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
Node agency = createAgency(createTestStructure);
INFO("AGENCY: " << agency.toJson());

10
tests/Agency/CleanUpLostCollectionTest.cpp
View File

@ -97,7 +97,7 @@ VPackBuilder createJob(std::string const& collection, std::string const& from, s
TEST_CASE("CleanUpLostCollectionTest", "[agency][supervision]") {
auto baseStructure = createRootNode();
write_ret_t fakeWriteResult {true, "", std::vector<apply_ret_t> {APPLIED}, std::vector<index_t> {1}};
std::string const jobId = "1";
uint64_t jobId = 1;
SECTION("clean up a lost collection when the leader is failed") {
@ -112,7 +112,7 @@ SECTION("clean up a lost collection when the leader is failed") {
// 1. Transaction:
// - Operation:
// delete /arango/Current/Collections/database/collection/s99
// push {
// set /arango/Target/Finished/1 {
// "creator": "supervision",
// "jobId": "1",
// "server": "s99",
@ -139,8 +139,8 @@ SECTION("clean up a lost collection when the leader is failed") {
REQUIRE(op1delete.hasKey("op"));
REQUIRE(op1delete.get("op").isEqualString("delete"));
REQUIRE(op1.hasKey("/arango/Target/Finished"));
auto const& op1push = op1.get("/arango/Target/Finished");
REQUIRE(op1.hasKey("/arango/Target/Finished/1"));
auto const& op1push = op1.get("/arango/Target/Finished/1");
REQUIRE(op1push.hasKey("new"));
auto const& op1new = op1push.get("new");
REQUIRE(op1new.get("creator").isEqualString("supervision"));
@ -150,7 +150,7 @@ SECTION("clean up a lost collection when the leader is failed") {
REQUIRE(op1new.get("type").isEqualString("cleanUpLostCollection"));
REQUIRE(op1push.hasKey("op"));
REQUIRE(op1push.get("op").isEqualString("push"));
REQUIRE(op1push.get("op").isEqualString("set"));
REQUIRE(pre1.hasKey("/arango/Current/Collections/database/collection/s99"));
REQUIRE(pre1.hasKey("/arango/Plan/Collections/database/collection/shards/s99"));

33
tests/Maintenance/MaintenanceFeatureTest.cpp
View File

@ -37,6 +37,7 @@
#include "Basics/Result.h"
#include "Cluster/Action.h"
#include "Cluster/MaintenanceFeature.h"
#include "Cluster/Maintenance.h"
#include "MaintenanceFeatureMock.h"
@ -76,7 +77,7 @@ public:
if (gres.ok()) {
pred.insert({"iterate_count",iterate_count});
}
_preAction = std::make_shared<ActionDescription>(pred);
_preAction = std::make_shared<ActionDescription>(pred, arangodb::maintenance::NORMAL_PRIORITY);
} // if
@ -86,7 +87,7 @@ public:
if (gres.ok()) {
postd.insert({"iterate_count",iterate_count});
}
_postAction = std::make_shared<ActionDescription>(postd);
_postAction = std::make_shared<ActionDescription>(postd, arangodb::maintenance::NORMAL_PRIORITY);
} // if
};
@ -175,7 +176,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
action_base_ptr.reset(
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","0"}})));
{"name","TestActionBasic"},{"iterate_count","0"}}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -208,7 +209,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","0"},{"result_code","1"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -239,7 +240,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
action_base_ptr.reset(
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","1"}})));
{"name","TestActionBasic"},{"iterate_count","1"}}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -271,7 +272,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","1"},{"result_code","1"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -304,7 +305,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","2"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -336,7 +337,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
action_base_ptr.reset(
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"}})));
{"name","TestActionBasic"},{"iterate_count","100"}}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -369,7 +370,7 @@ TEST_CASE("MaintenanceFeatureUnthreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},{"result_code","1"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf.addAction(
std::make_shared<Action>(std::move(action_base_ptr)), true);
@ -413,7 +414,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},{"result_code","1"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);
@ -426,7 +427,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","2"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);
@ -440,7 +441,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},{"result_code","1"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);
@ -496,7 +497,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},{"preaction_result_code","0"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);
@ -550,7 +551,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},{"postaction_result_code","0"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);
@ -607,7 +608,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},
{TestActionBasic::FAST_TRACK, ""}})));
{TestActionBasic::FAST_TRACK, ""}}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);
@ -650,7 +651,7 @@ TEST_CASE("MaintenanceFeatureThreaded", "[cluster][maintenance][devel]") {
(ActionBase*) new TestActionBasic(
*tf, ActionDescription(std::map<std::string,std::string>{
{"name","TestActionBasic"},{"iterate_count","100"},{"postaction_result_code","0"}
})));
}, arangodb::maintenance::NORMAL_PRIORITY)));
arangodb::Result result = tf->addAction(
std::make_shared<Action>(std::move(action_base_ptr)), false);

24
tests/Maintenance/MaintenanceTest.cpp
View File

@ -289,24 +289,24 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
dbsIds = matchShortLongIds(supervision);
SECTION("Construct minimal ActionDescription") {
ActionDescription desc(std::map<std::string,std::string>{{"name", "SomeAction"}});
ActionDescription desc(std::map<std::string,std::string>{{"name", "SomeAction"}}, NORMAL_PRIORITY);
REQUIRE(desc.get("name") == "SomeAction");
}
SECTION("Construct minimal ActionDescription with nullptr props") {
std::shared_ptr<VPackBuilder> props;
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
}
SECTION("Construct minimal ActionDescription with empty props") {
std::shared_ptr<VPackBuilder> props;
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
}
SECTION("Retrieve non-assigned key from ActionDescription") {
std::shared_ptr<VPackBuilder> props;
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
try {
auto bogus = desc.get("bogus");
@ -320,7 +320,7 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
SECTION("Retrieve non-assigned key from ActionDescription") {
std::shared_ptr<VPackBuilder> props;
ActionDescription desc({{"name", "SomeAction"}, {"bogus", "bogus"}}, props);
ActionDescription desc({{"name", "SomeAction"}, {"bogus", "bogus"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
try {
auto bogus = desc.get("bogus");
@ -334,14 +334,14 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
SECTION("Retrieve non-assigned properties from ActionDescription") {
std::shared_ptr<VPackBuilder> props;
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties() == nullptr);
}
SECTION("Retrieve empty properties from ActionDescription") {
auto props = std::make_shared<VPackBuilder>();
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties()->isEmpty());
}
@ -349,7 +349,7 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
SECTION("Retrieve empty object properties from ActionDescription") {
auto props = std::make_shared<VPackBuilder>();
{ VPackObjectBuilder empty(props.get()); }
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties()->slice().isEmptyObject());
}
@ -358,7 +358,7 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
auto props = std::make_shared<VPackBuilder>();
{ VPackObjectBuilder obj(props.get());
props->add("hello", VPackValue("world")); }
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties()->slice().hasKey("hello"));
REQUIRE(desc.properties()->slice().get("hello").copyString() == "world");
@ -369,7 +369,7 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
auto props = std::make_shared<VPackBuilder>();
{ VPackObjectBuilder obj(props.get());
props->add("pi", VPackValue(3.14159265359)); }
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties()->slice().hasKey("pi"));
REQUIRE(
@ -381,7 +381,7 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
auto props = std::make_shared<VPackBuilder>();
{ VPackObjectBuilder obj(props.get());
props->add("one", VPackValue(one)); }
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties()->slice().hasKey("one"));
REQUIRE(
@ -399,7 +399,7 @@ TEST_CASE("ActionDescription", "[cluster][maintenance]") {
props->add(VPackValue(pi));
props->add(VPackValue(one));
props->add(VPackValue(hello)); }}
ActionDescription desc({{"name", "SomeAction"}}, props);
ActionDescription desc({{"name", "SomeAction"}}, NORMAL_PRIORITY, props);
REQUIRE(desc.get("name") == "SomeAction");
REQUIRE(desc.properties()->slice().hasKey("array"));
REQUIRE(desc.properties()->slice().get("array").isArray());