From 75474ad665b2fbd357544bf15d0e59f372167000 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 7 Dec 2016 15:28:46 +0100 Subject: [PATCH 1/3] fixed docu PUT /_api/simple/by-example --- .../Rest/Simple Queries/JSA_put_api_simple_by_example.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md b/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md index fe27be1bf3..b2f37ab09e 100644 --- a/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md +++ b/Documentation/DocuBlocks/Rest/Simple Queries/JSA_put_api_simple_by_example.md @@ -17,6 +17,12 @@ The number of documents to skip in the query (optional). The maximal amount of documents to return. The *skip* is applied before the *limit* restriction. (optional) +@RESTBODYPARAM{batchSize,integer,optional,int64} +maximum number of result documents to be transferred from +the server to the client in one roundtrip. If this attribute is +not set, a server-controlled default value will be used. A *batchSize* value of +*0* is disallowed. + @RESTDESCRIPTION This will find all documents matching a given example. From b930b23fc249cf7514c96d68c351882768fd9c47 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Wed, 7 Dec 2016 16:20:47 +0100 Subject: [PATCH 2/3] AddFollower jobs for newly arrived db server to satisfy replication factors --- arangod/Agency/AddFollower.cpp | 24 ++++++++++++++++++++++++ arangod/Agency/CleanOutServer.cpp | 2 +- arangod/Agency/FailedFollower.cpp | 3 ++- arangod/Agency/FailedServer.cpp | 2 +- arangod/Agency/Job.cpp | 8 ++++---- arangod/Agency/Job.h | 3 ++- arangod/Agency/RemoveServer.cpp | 2 +- arangod/Agency/Supervision.cpp | 31 +++++++++++++++++++++++++------ 8 files changed, 60 insertions(+), 15 deletions(-) diff --git a/arangod/Agency/AddFollower.cpp b/arangod/Agency/AddFollower.cpp index 21e1d4c9a6..c7fa57165e 100644 --- a/arangod/Agency/AddFollower.cpp +++ b/arangod/Agency/AddFollower.cpp @@ -73,6 +73,30 @@ bool AddFollower::create() { TRI_ASSERT(current[0].isString()); #endif + std::string planPath = + planColPrefix + _database + "/" + _collection + "/shards"; + + auto const& myClones = clones(_snapshot, _database, _collection); + if (!myClones.empty()) { + + size_t sub = 0; + auto myshards = _snapshot(planPath).children(); + auto mpos = std::distance(myshards.begin(), myshards.find(_shard)); + + // Deal with my clones + for (auto const& collection : myClones) { + auto othershards = _snapshot( + planColPrefix + _database + "/" + collection + "/shards").children(); + auto opos = othershards.begin(); + std::advance(opos, mpos); + auto const& shard = opos->first; + + AddFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++), + _jobId, _agencyPrefix, _database, collection, shard, + _newFollower); + } + } + _jb = std::make_shared(); _jb->openArray(); _jb->openObject(); diff --git a/arangod/Agency/CleanOutServer.cpp b/arangod/Agency/CleanOutServer.cpp index bce1fb58c1..c8043511b2 100644 --- a/arangod/Agency/CleanOutServer.cpp +++ b/arangod/Agency/CleanOutServer.cpp @@ -243,7 +243,7 @@ bool CleanOutServer::start() { bool CleanOutServer::scheduleMoveShards() { - std::vector servers = availableServers(); + std::vector servers = availableServers(_snapshot); // Minimum 1 DB server must remain if (servers.size() == 1) { diff --git a/arangod/Agency/FailedFollower.cpp b/arangod/Agency/FailedFollower.cpp index 67fe1f7788..92c0de7629 100644 --- a/arangod/Agency/FailedFollower.cpp +++ b/arangod/Agency/FailedFollower.cpp @@ -78,7 +78,8 @@ bool FailedFollower::create() { // Deal with my clones for (auto const& collection : myClones) { - auto othershards = _snapshot(planPath).children(); + auto othershards = _snapshot( + planColPrefix + _database + "/" + collection + "/shards").children(); auto opos = othershards.begin(); std::advance(opos, mpos); auto const& shard = opos->first; diff --git a/arangod/Agency/FailedServer.cpp b/arangod/Agency/FailedServer.cpp index 6f750a5e2b..2324c0333c 100644 --- a/arangod/Agency/FailedServer.cpp +++ b/arangod/Agency/FailedServer.cpp @@ -146,7 +146,7 @@ bool FailedServer::start() { } } catch (...) {} // Not clone - auto available = availableServers(); + auto available = availableServers(_snapshot); for (auto const& shard : collection("shards").children()) { diff --git a/arangod/Agency/Job.cpp b/arangod/Agency/Job.cpp index 3270b45a91..09de7dac85 100644 --- a/arangod/Agency/Job.cpp +++ b/arangod/Agency/Job.cpp @@ -143,12 +143,12 @@ bool Job::finish(std::string const& type, bool success, } -std::vector Job::availableServers() const { +std::vector Job::availableServers(Node const& snapshot) { std::vector ret; // Get servers from plan - Node::Children const& dbservers = _snapshot(plannedServers).children(); + Node::Children const& dbservers = snapshot(plannedServers).children(); for (auto const& srv : dbservers) { ret.push_back(srv.first); } @@ -156,7 +156,7 @@ std::vector Job::availableServers() const { // Remove cleaned servers from ist try { for (auto const& srv : - VPackArrayIterator(_snapshot(cleanedPrefix).slice())) { + VPackArrayIterator(snapshot(cleanedPrefix).slice())) { ret.erase( std::remove(ret.begin(), ret.end(), srv.copyString()), ret.end()); @@ -167,7 +167,7 @@ std::vector Job::availableServers() const { // Remove failed servers from list try { for (auto const& srv : - VPackArrayIterator(_snapshot(failedServersPrefix).slice())) { + VPackArrayIterator(snapshot(failedServersPrefix).slice())) { ret.erase( std::remove(ret.begin(), ret.end(), srv.copyString()), ret.end()); diff --git a/arangod/Agency/Job.h b/arangod/Agency/Job.h index 5c9fef1ecf..e1406414e3 100644 --- a/arangod/Agency/Job.h +++ b/arangod/Agency/Job.h @@ -109,7 +109,8 @@ struct Job { virtual bool start() = 0; - virtual std::vector availableServers() const; + static std::vector availableServers( + const arangodb::consensus::Node&); static std::vector clones( Node const& snapshot, std::string const& database, diff --git a/arangod/Agency/RemoveServer.cpp b/arangod/Agency/RemoveServer.cpp index 5744ad1e3c..c3d241a80b 100644 --- a/arangod/Agency/RemoveServer.cpp +++ b/arangod/Agency/RemoveServer.cpp @@ -286,7 +286,7 @@ bool RemoveServer::start() { bool RemoveServer::scheduleAddFollowers() { - std::vector servers = availableServers(); + std::vector servers = availableServers(_snapshot); // Minimum 1 DB server must remain if (servers.size() == 1) { diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 4ce39a087d..a3d5187939 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -548,6 +548,7 @@ bool Supervision::handleJobs() { } // Do supervision + shrinkCluster(); workJobs(); enforceReplication(); @@ -614,18 +615,36 @@ void Supervision::workJobs() { void Supervision::enforceReplication() { auto const& plannedDBs = _snapshot(planColPrefix).children(); + auto available = Job::availableServers(_snapshot); for (const auto& db_ : plannedDBs) { // Planned databases auto const& db = *(db_.second); for (const auto& col_ : db.children()) { // Planned collections auto const& col = *(col_.second); auto const& replicationFactor = col("replicationFactor").slice().getUInt(); - for (auto const& shard_ : col("shards").children()) { // Pl shards - auto const& shard = *(shard_.second); - if (replicationFactor != shard.slice().length()) { - LOG(WARN) << shard.slice().typeName() - << " target repl(" << replicationFactor - << ") actual repl(" << shard.slice().length() << ")"; + + bool clone = false; + try { + clone = !col("distributeShardsLike").slice().copyString().empty(); + } catch (...) {} + + if (!clone) { + for (auto const& shard_ : col("shards").children()) { // Pl shards + auto const& shard = *(shard_.second); + + // Enough DBServer to + if (replicationFactor > shard.slice().length() && + available.size() >= replicationFactor) { + for (auto const& i : VPackArrayIterator(shard.slice())) { + available.erase( + std::remove( + available.begin(), available.end(), i.copyString()), + available.end()); + } + AddFollower( + _snapshot, _agent, std::to_string(_jobId++), "supervision", + _agencyPrefix, db_.first, col_.first, shard_.first, available.back()); + } } } } From 9746f6c7add8609847be1df56f476c5556f007b8 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 7 Dec 2016 16:23:54 +0100 Subject: [PATCH 3/3] Finally fix condition for coordinator bootstrap. --- arangod/Cluster/ClusterFeature.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arangod/Cluster/ClusterFeature.cpp b/arangod/Cluster/ClusterFeature.cpp index 944bfe1b59..ee987e50c5 100644 --- a/arangod/Cluster/ClusterFeature.cpp +++ b/arangod/Cluster/ClusterFeature.cpp @@ -308,7 +308,8 @@ void ClusterFeature::prepare() { LOG(INFO) << "Waiting for DBservers to show up..."; ci->loadCurrentDBServers(); std::vector DBServers = ci->getCurrentDBServers(); - if (DBServers.size() > 1 || TRI_microtime() - start > 30.0) { + if (DBServers.size() >= 1 && + (DBServers.size() > 1 || TRI_microtime() - start > 15.0)) { LOG(INFO) << "Found " << DBServers.size() << " DBservers."; break; }