Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bugfix / Supervision: removeFollower should remove the last follower(s) first (#4923) 

* Added a test asserting the last followers are removed first as required by moveShard
* Remove the last followers first
* Removed unused includes
* Updated CHANGELOG
This commit is contained in:
Tobias Gödderz 2018-03-23 09:34:04 +01:00 committed by Max Neunhöffer
parent 5df7a744a8
commit 7e53d3ed75
4 changed files with 1140 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG
View File

@ -1,6 +1,8 @@
devel devel
----- -----
* fix issue #4924: removeFollower now prefers to remove the last follower(s)
* Improvement: The AQL query planner in cluster is now a bit more clever and * Improvement: The AQL query planner in cluster is now a bit more clever and
can prepare AQL queries with less network overhead. can prepare AQL queries with less network overhead.
This should speed up simple queries in cluster mode, on complex queries it This should speed up simple queries in cluster mode, on complex queries it

22
arangod/Agency/RemoveFollower.cpp
View File

@ -24,6 +24,7 @@
#include "Agency/AgentInterface.h" #include "Agency/AgentInterface.h"
#include "Agency/Job.h" #include "Agency/Job.h"
#include "Cluster/ClusterInfo.h"
#include "Random/RandomGenerator.h" #include "Random/RandomGenerator.h"
using namespace arangodb::consensus; using namespace arangodb::consensus;
@ -234,7 +235,20 @@ bool RemoveFollower::start() {
size_t currentReplFactor = actualReplFactor; // will be counted down size_t currentReplFactor = actualReplFactor; // will be counted down
if (currentReplFactor > desiredReplFactor) { if (currentReplFactor > desiredReplFactor) {
// First choose BAD servers: // First choose BAD servers:
for (auto const& pair : overview) {
// Iterate the list of planned servers in reverse order,
// because the last must be removed first
std::vector<ServerID> reversedPlannedServers { planned.length() };
{
auto rDbIt = reversedPlannedServers.rbegin();
for (auto const &vPackIt : VPackArrayIterator(planned)) {
*rDbIt = vPackIt.copyString();
rDbIt++;
}
}
for (auto const& it : reversedPlannedServers) {
auto const pair = *overview.find(it);
if (pair.second == -1) { if (pair.second == -1) {
chosenToRemove.insert(pair.first); chosenToRemove.insert(pair.first);
--currentReplFactor; --currentReplFactor;
@ -245,7 +259,8 @@ bool RemoveFollower::start() {
} }
if (currentReplFactor > desiredReplFactor) { if (currentReplFactor > desiredReplFactor) {
// Now choose servers that are not in sync for all shards: // Now choose servers that are not in sync for all shards:
for (auto const& pair : overview) { for (auto const& it : reversedPlannedServers) {
auto const pair = *overview.find(it);
if (pair.second >= 0 && if (pair.second >= 0 &&
static_cast<size_t>(pair.second) < shardsLikeMe.size()) { static_cast<size_t>(pair.second) < shardsLikeMe.size()) {
chosenToRemove.insert(pair.first); chosenToRemove.insert(pair.first);
@ -257,7 +272,8 @@ bool RemoveFollower::start() {
} }
if (currentReplFactor > desiredReplFactor) { if (currentReplFactor > desiredReplFactor) {
// Finally choose servers that are in sync, but are no leader: // Finally choose servers that are in sync, but are no leader:
for (auto const& pair : overview) { for (auto const& it : reversedPlannedServers) {
auto const pair = *overview.find(it);
if (pair.second >= 0 && if (pair.second >= 0 &&
static_cast<size_t>(pair.second) >= shardsLikeMe.size() && static_cast<size_t>(pair.second) >= shardsLikeMe.size() &&
pair.first != planned[0].copyString()) { pair.first != planned[0].copyString()) {

776
tests/Agency/RemoveFollowerTest.cpp
View File

@ -62,6 +62,9 @@ const std::string FREE_SERVER2 = "free2";
const char *agency = const char *agency =
#include "RemoveFollowerTest.json" #include "RemoveFollowerTest.json"
; ;
const char *agencyLarge =
#include "RemoveFollowerTestLarge.json"
;
const char *todo = const char *todo =
#include "RemoveFollowerTestToDo.json" #include "RemoveFollowerTestToDo.json"
; ;
@ -107,354 +110,509 @@ inline static std::string typeName (Slice const& slice) {
} }
TEST_CASE("RemoveFollower", "[agency][supervision]") { TEST_CASE("RemoveFollower", "[agency][supervision]") {
auto baseStructure = createRootNode();
arangodb::RandomGenerator::initialize(arangodb::RandomGenerator::RandomType::MERSENNE);
Builder builder;
baseStructure.toBuilder(builder);
std::string jobId = "1"; GIVEN("An agency with 5 DBservers") {
write_ret_t fakeWriteResult {true, "", std::vector<bool> {true}, std::vector<index_t> {1}}; auto baseStructure = createRootNode();
trans_ret_t fakeTransResult {true, "", 1, 0, std::make_shared<Builder>()}; arangodb::RandomGenerator::initialize(arangodb::RandomGenerator::RandomType::MERSENNE);
SECTION("creating a job should create a job in todo") {
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t { Builder builder;
INFO(q->slice().toJson()); baseStructure.toBuilder(builder);
auto expectedJobKey = "/arango/Target/ToDo/" + jobId;
REQUIRE(typeName(q->slice()) == "array" );
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
REQUIRE(q->slice()[0][0].length() == 1); // should ONLY do an entry in todo
REQUIRE(typeName(q->slice()[0][0].get(expectedJobKey)) == "object");
auto job = q->slice()[0][0].get(expectedJobKey); std::string jobId = "1";
REQUIRE(typeName(job.get("creator")) == "string");
REQUIRE(typeName(job.get("type")) == "string");
CHECK(job.get("type").copyString() == "removeFollower");
REQUIRE(typeName(job.get("database")) == "string");
CHECK(job.get("database").copyString() == DATABASE);
REQUIRE(typeName(job.get("collection")) == "string");
CHECK(job.get("collection").copyString() == COLLECTION);
REQUIRE(typeName(job.get("shard")) == "string");
CHECK(job.get("shard").copyString() == SHARD);
CHECK(typeName(job.get("jobId")) == "string");
CHECK(typeName(job.get("timeCreated")) == "string");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK); write_ret_t fakeWriteResult{true, "", std::vector<bool> {true}, std::vector<index_t> {1}};
auto& agent = mockAgent.get(); trans_ret_t fakeTransResult{true, "", 1, 0, std::make_shared<Builder>()};
auto removeFollower = RemoveFollower(
baseStructure, &agent, jobId, "unittest", DATABASE, COLLECTION, SHARD);
removeFollower.create();
}
SECTION("<collection> still exists, if missing, job is finished, move to " SECTION("creating a job should create a job in todo") {
"Target/Finished") { Mock<AgentInterface> mockAgent;
TestStructType createTestStructure = [&]( When(Method(mockAgent, write)).AlwaysDo(
Slice const& s, std::string const& path) { [&](query_t const &q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
auto expectedJobKey = "/arango/Target/ToDo/" + jobId;
REQUIRE(typeName(q->slice()) == "array");
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
REQUIRE(q->slice()[0][0].length() == 1); // should ONLY do an entry in todo
REQUIRE(typeName(q->slice()[0][0].get(expectedJobKey)) == "object");
std::unique_ptr<Builder> builder; auto job = q->slice()[0][0].get(expectedJobKey);
if (path == "/arango/Plan/Collections/" + DATABASE + "/" + COLLECTION) { REQUIRE(typeName(job.get("creator")) == "string");
return builder; REQUIRE(typeName(job.get("type")) == "string");
} CHECK(job.get("type").copyString() == "removeFollower");
builder = std::make_unique<Builder>(); REQUIRE(typeName(job.get("database")) == "string");
CHECK(job.get("database").copyString() == DATABASE);
if (s.isObject()) { REQUIRE(typeName(job.get("collection")) == "string");
VPackObjectBuilder b(builder.get()); CHECK(job.get("collection").copyString() == COLLECTION);
for (auto const& it: VPackObjectIterator(s)) { REQUIRE(typeName(job.get("shard")) == "string");
auto childBuilder = createTestStructure(it.value, path + "/" + it.key.copyString()); CHECK(job.get("shard").copyString() == SHARD);
if (childBuilder) { CHECK(typeName(job.get("jobId")) == "string");
builder->add(it.key.copyString(), childBuilder->slice()); CHECK(typeName(job.get("timeCreated")) == "string");
}
return fakeWriteResult;
} }
if (path == "/arango/Target/ToDo") { );
builder->add(jobId, createBuilder(todo).slice());
}
builder->close();
} else {
builder->add(s);
}
return builder;
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
Node agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array" );
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
// we always simply override! no preconditions...
REQUIRE(q->slice()[0].length() == 1);
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto& agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
}
SECTION("if <collection> has a non-empty distributeShardsLike attribute, the "
"job immediately fails and is moved to Target/Failed") {
TestStructType createTestStructure = [&]( When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
Slice const& s, std::string const& path) { auto &agent = mockAgent.get();
auto removeFollower = RemoveFollower(
baseStructure, &agent, jobId, "unittest", DATABASE, COLLECTION, SHARD
);
std::unique_ptr<Builder> builder = std::make_unique<Builder>(); removeFollower.create();
if (s.isObject()) {
VPackObjectBuilder b(builder.get()); }
for (auto const& it: VPackObjectIterator(s)) {
auto childBuilder = SECTION("<collection> still exists, if missing, job is finished, move to "
createTestStructure(it.value, path + "/" + it.key.copyString()); "Target/Finished") {
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice()); TestStructType createTestStructure = [&](
} Slice const &s, std::string const &path
} ) {
std::unique_ptr<Builder> builder;
if (path == "/arango/Plan/Collections/" + DATABASE + "/" + COLLECTION) { if (path == "/arango/Plan/Collections/" + DATABASE + "/" + COLLECTION) {
builder->add("distributeShardsLike", VPackValue("PENG")); return builder;
} }
if (path == "/arango/Target/ToDo") { builder = std::make_unique<Builder>();
builder->add(jobId, createBuilder(todo).slice());
}
} else {
builder->add(s);
}
return builder;
}; if (s.isObject()) {
VPackObjectBuilder b(builder.get());
auto builder = createTestStructure(baseStructure.toBuilder().slice(), ""); for (auto const &it: VPackObjectIterator(s)) {
REQUIRE(builder); auto childBuilder = createTestStructure(it.value, path + "/" + it.key.copyString());
auto agency = createNodeFromBuilder(*builder); if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array" );
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Failed/1")) == "object");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto& agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
}
SECTION("condition (*) still holds for the mentioned collections, if not, "
"job is finished, move to Target/Finished.") {
TestStructType createTestStructure = [&](
Slice const& s, std::string const& path) {
std::unique_ptr<Builder> builder = std::make_unique<Builder>();
if (s.isObject()) {
VPackObjectBuilder b(builder.get());
for (auto const& it: VPackObjectIterator(s)) {
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString());
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
}
}
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
} else {
if (path == "/arango/Plan/Collections/" + DATABASE + "/" +
COLLECTION + "/shards/" + SHARD) {
VPackArrayBuilder a(builder.get());
for (auto const& serv : VPackArrayIterator(s)) {
if (serv.copyString() != SHARD_FOLLOWER1
&& serv.copyString() != SHARD_FOLLOWER2) {
builder->add(serv);
} }
} }
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
builder->close();
} else {
builder->add(s);
}
return builder;
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
Node agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo(
[&](query_t const &q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array");
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
// we always simply override! no preconditions...
REQUIRE(q->slice()[0].length() == 1);
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object");
return fakeWriteResult;
}
);
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto &agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
}
SECTION("if <collection> has a non-empty distributeShardsLike attribute, the "
"job immediately fails and is moved to Target/Failed") {
TestStructType createTestStructure = [&](
Slice const &s, std::string const &path
) {
std::unique_ptr<Builder> builder = std::make_unique<Builder>();
if (s.isObject()) {
VPackObjectBuilder b(builder.get());
for (auto const &it: VPackObjectIterator(s)) {
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString());
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
}
}
if (path == "/arango/Plan/Collections/" + DATABASE + "/" + COLLECTION) {
builder->add("distributeShardsLike", VPackValue("PENG"));
}
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
} else { } else {
builder->add(s); builder->add(s);
} }
} return builder;
return builder;
}; };
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
auto agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent; auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t { REQUIRE(builder);
INFO(q->slice().toJson()); auto agency = createNodeFromBuilder(*builder);
REQUIRE(typeName(q->slice()) == "array" );
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0]; Mock<AgentInterface> mockAgent;
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object"); When(Method(mockAgent, write)).AlwaysDo(
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string"); [&](query_t const &q, bool d) -> write_ret_t {
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete"); INFO(q->slice().toJson());
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object"); REQUIRE(typeName(q->slice()) == "array");
return fakeWriteResult; REQUIRE(q->slice().length() == 1);
}); REQUIRE(typeName(q->slice()[0]) == "array");
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK); REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
auto& agent = mockAgent.get(); REQUIRE(typeName(q->slice()[0][0]) == "object");
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
}
SECTION("compute the list allShards of collection/shard pairs that " auto writes = q->slice()[0][0];
"correspond to / because of distributeShardsLike attributes with " REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
"value , include / as the first pair") { REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Failed/1")) == "object");
return fakeWriteResult;
}
);
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto &agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
TestStructType createTestStructure = [&]( }
Slice const& s, std::string const& path) {
SECTION("condition (*) still holds for the mentioned collections, if not, "
std::unique_ptr<Builder> builder = std::make_unique<Builder>(); "job is finished, move to Target/Finished.") {
if (s.isObject()) {
TestStructType createTestStructure = [&](
VPackObjectBuilder b(builder.get()); Slice const &s, std::string const &path
for (auto const& it: VPackObjectIterator(s)) { ) {
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString()); std::unique_ptr<Builder> builder = std::make_unique<Builder>();
if (childBuilder) { if (s.isObject()) {
builder->add(it.key.copyString(), childBuilder->slice()); VPackObjectBuilder b(builder.get());
for (auto const &it: VPackObjectIterator(s)) {
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString());
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
}
}
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
} else {
if (path == "/arango/Plan/Collections/" + DATABASE + "/" +
COLLECTION + "/shards/" + SHARD) {
VPackArrayBuilder a(builder.get());
for (auto const &serv : VPackArrayIterator(s)) {
if (serv.copyString() != SHARD_FOLLOWER1
&& serv.copyString() != SHARD_FOLLOWER2) {
builder->add(serv);
}
}
} else {
builder->add(s);
} }
} }
if (path == "/arango/Target/ToDo") { return builder;
builder->add(jobId, createBuilder(todo).slice());
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
auto agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo(
[&](query_t const &q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array");
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object");
return fakeWriteResult;
} }
if (path == "/arango/Plan/Collections/" + DATABASE + "/" + CLONE) { );
builder->add("distributeShardsLike", VPackValue(COLLECTION)); When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
} auto &agent = mockAgent.get();
} else { RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
builder->add(s);
}
return builder;
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
std::cout << builder->toJson() << std::endl;
Node agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array" );
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(writes.get("/arango/Target/Finished/1").get("collection").copyString() == COLLECTION);
CHECK(writes.get("/arango/Target/Pending/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Failed/1")) == "none");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
AgentInterface &agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
} }
SECTION("the leader has acknowledged its leadership in Current for all " SECTION("compute the list allShards of collection/shard pairs that "
"these shards and at least replicationFactor - 1 servers are in sync " "correspond to / because of distributeShardsLike attributes with "
"with the leader according to Current for all shards") { "value , include / as the first pair") {
}
SECTION("if there is no job under Supervision/Shards/<shard> (if so, do " TestStructType createTestStructure = [&](
"nothing, leave job in ToDo") { Slice const &s, std::string const &path
) {
}
SECTION("All good should remove follower") { std::unique_ptr<Builder> builder = std::make_unique<Builder>();
if (s.isObject()) {
TestStructType createTestStructure = [&]( VPackObjectBuilder b(builder.get());
Slice const& s, std::string const& path) { for (auto const &it: VPackObjectIterator(s)) {
auto childBuilder =
std::unique_ptr<Builder> builder = std::make_unique<Builder>(); createTestStructure(it.value, path + "/" + it.key.copyString());
if (s.isObject()) { if (childBuilder) {
VPackObjectBuilder b(builder.get()); builder->add(it.key.copyString(), childBuilder->slice());
for (auto const& it: VPackObjectIterator(s)) { }
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString());
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
} }
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
if (path == "/arango/Plan/Collections/" + DATABASE + "/" + CLONE) {
builder->add("distributeShardsLike", VPackValue(COLLECTION));
}
} else {
builder->add(s);
} }
if (path == "/arango/Target/ToDo") { return builder;
builder->add(jobId, createBuilder(todo).slice());
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
std::cout << builder->toJson() << std::endl;
Node agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo(
[&](query_t const &q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array");
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 1); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(writes.get("/arango/Target/Finished/1").get("collection").copyString() == COLLECTION);
CHECK(writes.get("/arango/Target/Pending/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Failed/1")) == "none");
return fakeWriteResult;
} }
} else { );
builder->add(s); When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
} AgentInterface &agent = mockAgent.get();
return builder; RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
}; }
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
auto agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent; SECTION("the leader has acknowledged its leadership in Current for all "
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t { "these shards and at least replicationFactor - 1 servers are in sync "
INFO(q->slice().toJson()); "with the leader according to Current for all shards") {
REQUIRE(typeName(q->slice()) == "array" ); }
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 2); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0]; SECTION("if there is no job under Supervision/Shards/<shard> (if so, do "
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object"); "nothing, leave job in ToDo") {
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto& agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
}
SECTION("All good should remove follower") {
TestStructType createTestStructure = [&](
Slice const& s, std::string const& path) {
std::unique_ptr<Builder> builder = std::make_unique<Builder>();
if (s.isObject()) {
VPackObjectBuilder b(builder.get());
for (auto const& it: VPackObjectIterator(s)) {
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString());
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
}
}
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
} else {
builder->add(s);
}
return builder;
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
auto agency = createNodeFromBuilder(*builder);
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo([&](query_t const& q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array" );
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 2); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
auto writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object");
return fakeWriteResult;
});
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto& agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
REQUIRE_NOTHROW(Verify(Method(mockAgent, write)));
}
}
GIVEN("An agency with 12 DBServers") {
auto baseStructure = createNode(agencyLarge);
arangodb::RandomGenerator::initialize(arangodb::RandomGenerator::RandomType::MERSENNE);
Builder builder;
baseStructure.toBuilder(builder);
std::string jobId = "1";
write_ret_t fakeWriteResult{true, "", std::vector<bool> {true}, std::vector<index_t> {1}};
trans_ret_t fakeTransResult{true, "", 1, 0, std::make_shared<Builder>()};
SECTION("Should remove the last followers") {
TestStructType createTestStructure = [&](
Slice const &s, std::string const &path
) {
std::unique_ptr<Builder> builder = std::make_unique<Builder>();
if (s.isObject()) {
VPackObjectBuilder b(builder.get());
for (auto const &it: VPackObjectIterator(s)) {
auto childBuilder =
createTestStructure(it.value, path + "/" + it.key.copyString());
if (childBuilder) {
builder->add(it.key.copyString(), childBuilder->slice());
}
}
if (path == "/arango/Target/ToDo") {
builder->add(jobId, createBuilder(todo).slice());
}
} else {
builder->add(s);
}
return builder;
};
auto builder = createTestStructure(baseStructure.toBuilder().slice(), "");
REQUIRE(builder);
auto agency = createNodeFromBuilder(*builder);
// The reason for using so much DBServers is to make it nearly impossible
// for the test to pass by accident. Trying with lower numbers
// (say remove 5 out of 10) seemed to indicate that the implementation
// of unordered_map makes it, at least for small numbers, likely
// that the last added elements appear first when iterating over it.
// When choosing different names for the 10 DBServers, I expected the
// order to change (and it did), but in 4 out of 5 tries made the correct
// DBServers were removed, while I expected the chance for that to be
// 1 / (10 choose 5) = 1/252.
// Thus I increased the agency to 100 DBServers with a replicationFactor
// of 50.
Mock<AgentInterface> mockAgent;
When(Method(mockAgent, write)).AlwaysDo(
[&](query_t const &q, bool d) -> write_ret_t {
INFO(q->slice().toJson());
REQUIRE(typeName(q->slice()) == "array");
REQUIRE(q->slice().length() == 1);
REQUIRE(typeName(q->slice()[0]) == "array");
REQUIRE(q->slice()[0].length() == 2); // we always simply override! no preconditions...
REQUIRE(typeName(q->slice()[0][0]) == "object");
Slice writes = q->slice()[0][0];
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1")) == "object");
REQUIRE(typeName(writes.get("/arango/Target/ToDo/1").get("op")) == "string");
CHECK(writes.get("/arango/Target/ToDo/1").get("op").copyString() == "delete");
CHECK(typeName(writes.get("/arango/Target/Finished/1")) == "object");
Slice servers = writes.get("/arango/Plan/Collections/database/collection1/shards/s1");
REQUIRE(typeName(servers) == "array");
INFO(servers.toJson());
REQUIRE(servers.length() == 50);
REQUIRE(servers[0].copyString() == "leader");
REQUIRE(servers[1].copyString() == "follower-1-48a5e486-10c9-4953-8630-9a3de12a6169");
REQUIRE(servers[2].copyString() == "follower-2-34e18222-8ce3-4016-9558-7092e41eb22c");
REQUIRE(servers[3].copyString() == "follower-3-27452c0b-efc8-4d9a-b5b1-d557997c4337");
REQUIRE(servers[4].copyString() == "follower-4-cc56c772-58cc-4c16-b571-56283ad813c8");
REQUIRE(servers[5].copyString() == "follower-5-aec34c5c-9939-42af-bf5d-afc15f960c50");
REQUIRE(servers[6].copyString() == "follower-6-8477db61-d46f-46f7-a816-8176e1514494");
REQUIRE(servers[7].copyString() == "follower-7-58b689ae-e7e8-45e7-83cb-6006b2375f61");
REQUIRE(servers[8].copyString() == "follower-8-d5e9c550-4a68-4dca-a50d-c84d4b690945");
REQUIRE(servers[9].copyString() == "follower-9-349e7296-b4fc-4fd3-b8a8-02befbb0380e");
REQUIRE(servers[10].copyString() == "follower-10-1ad1aa18-00b3-430a-8144-ba973bfed5fe");
REQUIRE(servers[11].copyString() == "follower-11-110920e0-d079-4f06-99ed-c482d19b5112");
REQUIRE(servers[12].copyString() == "follower-12-b7d64986-c458-4332-934a-ecb2caf19259");
REQUIRE(servers[13].copyString() == "follower-13-2e7ba82c-b837-4126-8cda-0db6ac98e30b");
REQUIRE(servers[14].copyString() == "follower-14-6b9d6b95-420f-44d8-a714-d17ae95eecdd");
REQUIRE(servers[15].copyString() == "follower-15-08c7dc8d-bb31-4cc3-a7ca-4b8bfba19b70");
REQUIRE(servers[16].copyString() == "follower-16-5a301b07-d1d8-4c86-8e8a-bd7957a2cafb");
REQUIRE(servers[17].copyString() == "follower-17-fd04d0f0-821e-401e-8f8e-0b2837ddc41d");
REQUIRE(servers[18].copyString() == "follower-18-0a2bfdf4-c277-45ea-8af3-4d60eba67910");
REQUIRE(servers[19].copyString() == "follower-19-69c98e93-b1c2-416f-b5e4-84cc50b65efe");
REQUIRE(servers[20].copyString() == "follower-20-e9eca0f2-530b-4496-950e-341b71086f8b");
REQUIRE(servers[21].copyString() == "follower-21-1042a97a-aa82-48ee-8388-8480a6e57249");
REQUIRE(servers[22].copyString() == "follower-22-c3922c1e-53df-42d6-9bcd-476d01e581fd");
REQUIRE(servers[23].copyString() == "follower-23-cec0e2ed-3a5b-4b9a-a615-39a1f24179c2");
REQUIRE(servers[24].copyString() == "follower-24-1753643f-2d1e-4014-8cc6-4f063c0f143e");
REQUIRE(servers[25].copyString() == "follower-25-1a4edf05-e6ed-47bc-8765-0b8292fc3175");
REQUIRE(servers[26].copyString() == "follower-26-fcc5fb9e-b4a0-4986-ae14-8b330350fa67");
REQUIRE(servers[27].copyString() == "follower-27-aa738702-aeb8-4306-86cd-a77516eef44d");
REQUIRE(servers[28].copyString() == "follower-28-4f6cd6dc-9e12-4fcc-9083-23900ffad0d1");
REQUIRE(servers[29].copyString() == "follower-29-884e050b-0d33-440b-88bf-13cd41e00c10");
REQUIRE(servers[30].copyString() == "follower-30-bac109ba-a0ba-4235-b665-743fec5e2ea1");
REQUIRE(servers[31].copyString() == "follower-31-62a74a8e-f141-44bb-a818-57259c7d6323");
REQUIRE(servers[32].copyString() == "follower-32-7a0e8f27-04a4-4094-a00c-830dfe3e937c");
REQUIRE(servers[33].copyString() == "follower-33-83c9df58-91b1-4703-bce7-1d47c633a2c4");
REQUIRE(servers[34].copyString() == "follower-34-d8f1aa6e-fbd0-49c0-9560-b447417d0284");
REQUIRE(servers[35].copyString() == "follower-35-77b8626e-30d8-4b04-8ac9-42dd788a4c46");
REQUIRE(servers[36].copyString() == "follower-36-8239c391-86fe-462d-9036-c129983103f2");
REQUIRE(servers[37].copyString() == "follower-37-41b1fe2a-2826-43a6-8222-fc9480b4f211");
REQUIRE(servers[38].copyString() == "follower-38-4a4b54db-17ff-4f5f-882b-973907d9dc27");
REQUIRE(servers[39].copyString() == "follower-39-e6e0cb50-a609-4f5f-b376-4ec72fefb938");
REQUIRE(servers[40].copyString() == "follower-40-2c6f13c1-46dc-4d54-992f-4f923169e5e2");
REQUIRE(servers[41].copyString() == "follower-41-b4c3d57c-ec01-4162-8107-823a09176fc4");
REQUIRE(servers[42].copyString() == "follower-42-e65dfaf4-cdbd-485a-a4d3-f56848e58d28");
REQUIRE(servers[43].copyString() == "follower-43-a248deeb-817f-4f0d-9813-c08a40e9027a");
REQUIRE(servers[44].copyString() == "follower-44-c8f4e52e-7a12-4a3b-8a93-cd543f512a55");
REQUIRE(servers[45].copyString() == "follower-45-d2a70a84-2a12-4fa3-b0e9-68945fd34cfc");
REQUIRE(servers[46].copyString() == "follower-46-bf70b49c-ff50-4255-a704-70a5a4d7a4b3");
REQUIRE(servers[47].copyString() == "follower-47-ca6aaf76-0bf8-4289-9033-605883e514ca");
REQUIRE(servers[48].copyString() == "follower-48-30442bc5-2dc0-434c-b21f-989610a199e7");
REQUIRE(servers[49].copyString() == "follower-49-788d2a9b-6d56-42a7-bacb-1dafff7d58a9");
return fakeWriteResult;
}
);
When(Method(mockAgent, waitFor)).AlwaysReturn(AgentInterface::raft_commit_t::OK);
auto &agent = mockAgent.get();
RemoveFollower(agency("arango"), &agent, JOB_STATUS::TODO, jobId).start();
REQUIRE_NOTHROW(Verify(Method(mockAgent, write)));
}
} }
}; };

652
tests/Agency/RemoveFollowerTestLarge.json Normal file
View File

@ -0,0 +1,652 @@
R"=(
{
"arango": {
"Current": {
"Collections": {
"database": {
"collection1": {
"s1": {
"servers": [
"leader",
"follower-1-48a5e486-10c9-4953-8630-9a3de12a6169",
"follower-2-34e18222-8ce3-4016-9558-7092e41eb22c",
"follower-3-27452c0b-efc8-4d9a-b5b1-d557997c4337",
"follower-4-cc56c772-58cc-4c16-b571-56283ad813c8",
"follower-5-aec34c5c-9939-42af-bf5d-afc15f960c50",
"follower-6-8477db61-d46f-46f7-a816-8176e1514494",
"follower-7-58b689ae-e7e8-45e7-83cb-6006b2375f61",
"follower-8-d5e9c550-4a68-4dca-a50d-c84d4b690945",
"follower-9-349e7296-b4fc-4fd3-b8a8-02befbb0380e",
"follower-10-1ad1aa18-00b3-430a-8144-ba973bfed5fe",
"follower-11-110920e0-d079-4f06-99ed-c482d19b5112",
"follower-12-b7d64986-c458-4332-934a-ecb2caf19259",
"follower-13-2e7ba82c-b837-4126-8cda-0db6ac98e30b",
"follower-14-6b9d6b95-420f-44d8-a714-d17ae95eecdd",
"follower-15-08c7dc8d-bb31-4cc3-a7ca-4b8bfba19b70",
"follower-16-5a301b07-d1d8-4c86-8e8a-bd7957a2cafb",
"follower-17-fd04d0f0-821e-401e-8f8e-0b2837ddc41d",
"follower-18-0a2bfdf4-c277-45ea-8af3-4d60eba67910",
"follower-19-69c98e93-b1c2-416f-b5e4-84cc50b65efe",
"follower-20-e9eca0f2-530b-4496-950e-341b71086f8b",
"follower-21-1042a97a-aa82-48ee-8388-8480a6e57249",
"follower-22-c3922c1e-53df-42d6-9bcd-476d01e581fd",
"follower-23-cec0e2ed-3a5b-4b9a-a615-39a1f24179c2",
"follower-24-1753643f-2d1e-4014-8cc6-4f063c0f143e",
"follower-25-1a4edf05-e6ed-47bc-8765-0b8292fc3175",
"follower-26-fcc5fb9e-b4a0-4986-ae14-8b330350fa67",
"follower-27-aa738702-aeb8-4306-86cd-a77516eef44d",
"follower-28-4f6cd6dc-9e12-4fcc-9083-23900ffad0d1",
"follower-29-884e050b-0d33-440b-88bf-13cd41e00c10",
"follower-30-bac109ba-a0ba-4235-b665-743fec5e2ea1",
"follower-31-62a74a8e-f141-44bb-a818-57259c7d6323",
"follower-32-7a0e8f27-04a4-4094-a00c-830dfe3e937c",
"follower-33-83c9df58-91b1-4703-bce7-1d47c633a2c4",
"follower-34-d8f1aa6e-fbd0-49c0-9560-b447417d0284",
"follower-35-77b8626e-30d8-4b04-8ac9-42dd788a4c46",
"follower-36-8239c391-86fe-462d-9036-c129983103f2",
"follower-37-41b1fe2a-2826-43a6-8222-fc9480b4f211",
"follower-38-4a4b54db-17ff-4f5f-882b-973907d9dc27",
"follower-39-e6e0cb50-a609-4f5f-b376-4ec72fefb938",
"follower-40-2c6f13c1-46dc-4d54-992f-4f923169e5e2",
"follower-41-b4c3d57c-ec01-4162-8107-823a09176fc4",
"follower-42-e65dfaf4-cdbd-485a-a4d3-f56848e58d28",
"follower-43-a248deeb-817f-4f0d-9813-c08a40e9027a",
"follower-44-c8f4e52e-7a12-4a3b-8a93-cd543f512a55",
"follower-45-d2a70a84-2a12-4fa3-b0e9-68945fd34cfc",
"follower-46-bf70b49c-ff50-4255-a704-70a5a4d7a4b3",
"follower-47-ca6aaf76-0bf8-4289-9033-605883e514ca",
"follower-48-30442bc5-2dc0-434c-b21f-989610a199e7",
"follower-49-788d2a9b-6d56-42a7-bacb-1dafff7d58a9",
"follower-50-fc63b181-9848-4f7b-8e03-84e39bc37cd3",
"follower-51-9659f049-9f0c-4ae3-8bc4-98d07e042f44",
"follower-52-05475c47-994b-4360-b90d-c874df3b7268",
"follower-53-0d8b6c5c-63c2-4f2d-bc0e-5ecfd5df518c",
"follower-54-3e264e2c-ab6e-4aae-a17e-28c4926b587f",
"follower-55-ebbc5038-de11-46aa-ba3b-447fc6e84ecb",
"follower-56-a18414b2-d3d3-4310-af85-3986a2c03ce0",
"follower-57-9a9c258b-7a7d-4afe-a39b-63ff865a1e14",
"follower-58-65bb2bb2-b371-4ef7-bbe6-77775223d505",
"follower-59-ec854bb2-5f6c-4ec1-8cb7-0ddb751c8afe",
"follower-60-33091fa1-893a-426c-ae6e-ec4db28cca92",
"follower-61-bca22a37-b48a-4e33-86a6-8f1a9c4d05ac",
"follower-62-d3282d65-dc64-4022-9604-2563af4373cf",
"follower-63-cdacaed8-cb57-4b7d-818c-435f37665bf1",
"follower-64-94c6ac3a-3611-48df-9dbd-4d034091b775",
"follower-65-b7570937-9ea5-4c72-9a94-67fccbbdbaa8",
"follower-66-beb74106-ce6e-415d-b170-664217c88124",
"follower-67-7b2086f2-b321-4a38-8557-768e0c64dd7d",
"follower-68-d4adf1d9-9091-4391-8d0b-739297fead93",
"follower-69-ab287171-4f08-4162-bdf7-d4ea14f7fc4c",
"follower-70-b8eb7ae3-270a-4c26-bd01-312173bca03a",
"follower-71-cb71b675-c801-41c3-9fe1-f0fb38e36c3c",
"follower-72-d0c90536-685c-4cae-90be-aa2ee048569e",
"follower-73-8ec7fdd0-9c62-423b-be49-1895489f4d2f",
"follower-74-15735f4f-cdd3-44c1-b87c-9d56495ceefd",
"follower-75-93283be9-2341-472d-a2d0-5ab834a94669",
"follower-76-71b140b4-251b-46b9-bf87-39251292edbc",
"follower-77-4bfb6d9b-7144-4d50-92af-45e33205f8c0",
"follower-78-f37ec045-4abb-4c1a-974e-e19e3a3a396a",
"follower-79-ce8c1b0f-fc48-4d62-99c8-f6e3391f16b7",
"follower-80-096f7f42-77b6-4959-843b-9c0157b0c422",
"follower-81-fe55e212-615f-4735-beea-239ddc93c702",
"follower-82-71234abb-fe81-40b5-b9c4-bbaef36e2a55",
"follower-83-c255faed-96f9-4a4d-b060-ee6d39448906",
"follower-84-85d2cdc9-65d0-4def-8773-ce741da76136",
"follower-85-a83536c7-6cea-44b3-903e-44c0fb1ced6c",
"follower-86-094742b0-5604-43d3-b667-8da02ed0689f",
"follower-87-176673c3-2095-4313-87df-407390f71684",
"follower-88-62924ed5-0efc-493d-bc9e-695f627acbca",
"follower-89-cadb3768-eabc-46b5-a2f8-2ab2c704f2e6",
"follower-90-de533425-9713-4068-aaa3-697fe9fc8af2",
"follower-91-d5904d2d-af0c-411f-ac29-20519091eb62",
"follower-92-a47f583f-63dd-416a-942d-8230b84a4e25",
"follower-93-b5596822-5153-4e62-8669-27c86f56a50f",
"follower-94-14dbfeed-05ce-4467-878b-8841540b139a",
"follower-95-3d2bc5c8-4fba-4be5-8a56-867dc358cf48",
"follower-96-9b09e6a0-bffb-4b96-984f-06a77656d0e7",
"follower-97-858dc032-8e3c-4f10-b2c8-63e42979cf54",
"follower-98-46db1df2-4754-46b2-beda-b982e97a9e86",
"follower-99-fb36b0f8-61f4-4d5d-bfbe-61aff7cb39ce"
]
}
}
}
}
},
"Plan": {
"Collections": {
"database": {
"collection1": {
"replicationFactor" : 50,
"shards": {
"s1": [
"leader",
"follower-1-48a5e486-10c9-4953-8630-9a3de12a6169",
"follower-2-34e18222-8ce3-4016-9558-7092e41eb22c",
"follower-3-27452c0b-efc8-4d9a-b5b1-d557997c4337",
"follower-4-cc56c772-58cc-4c16-b571-56283ad813c8",
"follower-5-aec34c5c-9939-42af-bf5d-afc15f960c50",
"follower-6-8477db61-d46f-46f7-a816-8176e1514494",
"follower-7-58b689ae-e7e8-45e7-83cb-6006b2375f61",
"follower-8-d5e9c550-4a68-4dca-a50d-c84d4b690945",
"follower-9-349e7296-b4fc-4fd3-b8a8-02befbb0380e",
"follower-10-1ad1aa18-00b3-430a-8144-ba973bfed5fe",
"follower-11-110920e0-d079-4f06-99ed-c482d19b5112",
"follower-12-b7d64986-c458-4332-934a-ecb2caf19259",
"follower-13-2e7ba82c-b837-4126-8cda-0db6ac98e30b",
"follower-14-6b9d6b95-420f-44d8-a714-d17ae95eecdd",
"follower-15-08c7dc8d-bb31-4cc3-a7ca-4b8bfba19b70",
"follower-16-5a301b07-d1d8-4c86-8e8a-bd7957a2cafb",
"follower-17-fd04d0f0-821e-401e-8f8e-0b2837ddc41d",
"follower-18-0a2bfdf4-c277-45ea-8af3-4d60eba67910",
"follower-19-69c98e93-b1c2-416f-b5e4-84cc50b65efe",
"follower-20-e9eca0f2-530b-4496-950e-341b71086f8b",
"follower-21-1042a97a-aa82-48ee-8388-8480a6e57249",
"follower-22-c3922c1e-53df-42d6-9bcd-476d01e581fd",
"follower-23-cec0e2ed-3a5b-4b9a-a615-39a1f24179c2",
"follower-24-1753643f-2d1e-4014-8cc6-4f063c0f143e",
"follower-25-1a4edf05-e6ed-47bc-8765-0b8292fc3175",
"follower-26-fcc5fb9e-b4a0-4986-ae14-8b330350fa67",
"follower-27-aa738702-aeb8-4306-86cd-a77516eef44d",
"follower-28-4f6cd6dc-9e12-4fcc-9083-23900ffad0d1",
"follower-29-884e050b-0d33-440b-88bf-13cd41e00c10",
"follower-30-bac109ba-a0ba-4235-b665-743fec5e2ea1",
"follower-31-62a74a8e-f141-44bb-a818-57259c7d6323",
"follower-32-7a0e8f27-04a4-4094-a00c-830dfe3e937c",
"follower-33-83c9df58-91b1-4703-bce7-1d47c633a2c4",
"follower-34-d8f1aa6e-fbd0-49c0-9560-b447417d0284",
"follower-35-77b8626e-30d8-4b04-8ac9-42dd788a4c46",
"follower-36-8239c391-86fe-462d-9036-c129983103f2",
"follower-37-41b1fe2a-2826-43a6-8222-fc9480b4f211",
"follower-38-4a4b54db-17ff-4f5f-882b-973907d9dc27",
"follower-39-e6e0cb50-a609-4f5f-b376-4ec72fefb938",
"follower-40-2c6f13c1-46dc-4d54-992f-4f923169e5e2",
"follower-41-b4c3d57c-ec01-4162-8107-823a09176fc4",
"follower-42-e65dfaf4-cdbd-485a-a4d3-f56848e58d28",
"follower-43-a248deeb-817f-4f0d-9813-c08a40e9027a",
"follower-44-c8f4e52e-7a12-4a3b-8a93-cd543f512a55",
"follower-45-d2a70a84-2a12-4fa3-b0e9-68945fd34cfc",
"follower-46-bf70b49c-ff50-4255-a704-70a5a4d7a4b3",
"follower-47-ca6aaf76-0bf8-4289-9033-605883e514ca",
"follower-48-30442bc5-2dc0-434c-b21f-989610a199e7",
"follower-49-788d2a9b-6d56-42a7-bacb-1dafff7d58a9",
"follower-50-fc63b181-9848-4f7b-8e03-84e39bc37cd3",
"follower-51-9659f049-9f0c-4ae3-8bc4-98d07e042f44",
"follower-52-05475c47-994b-4360-b90d-c874df3b7268",
"follower-53-0d8b6c5c-63c2-4f2d-bc0e-5ecfd5df518c",
"follower-54-3e264e2c-ab6e-4aae-a17e-28c4926b587f",
"follower-55-ebbc5038-de11-46aa-ba3b-447fc6e84ecb",
"follower-56-a18414b2-d3d3-4310-af85-3986a2c03ce0",
"follower-57-9a9c258b-7a7d-4afe-a39b-63ff865a1e14",
"follower-58-65bb2bb2-b371-4ef7-bbe6-77775223d505",
"follower-59-ec854bb2-5f6c-4ec1-8cb7-0ddb751c8afe",
"follower-60-33091fa1-893a-426c-ae6e-ec4db28cca92",
"follower-61-bca22a37-b48a-4e33-86a6-8f1a9c4d05ac",
"follower-62-d3282d65-dc64-4022-9604-2563af4373cf",
"follower-63-cdacaed8-cb57-4b7d-818c-435f37665bf1",
"follower-64-94c6ac3a-3611-48df-9dbd-4d034091b775",
"follower-65-b7570937-9ea5-4c72-9a94-67fccbbdbaa8",
"follower-66-beb74106-ce6e-415d-b170-664217c88124",
"follower-67-7b2086f2-b321-4a38-8557-768e0c64dd7d",
"follower-68-d4adf1d9-9091-4391-8d0b-739297fead93",
"follower-69-ab287171-4f08-4162-bdf7-d4ea14f7fc4c",
"follower-70-b8eb7ae3-270a-4c26-bd01-312173bca03a",
"follower-71-cb71b675-c801-41c3-9fe1-f0fb38e36c3c",
"follower-72-d0c90536-685c-4cae-90be-aa2ee048569e",
"follower-73-8ec7fdd0-9c62-423b-be49-1895489f4d2f",
"follower-74-15735f4f-cdd3-44c1-b87c-9d56495ceefd",
"follower-75-93283be9-2341-472d-a2d0-5ab834a94669",
"follower-76-71b140b4-251b-46b9-bf87-39251292edbc",
"follower-77-4bfb6d9b-7144-4d50-92af-45e33205f8c0",
"follower-78-f37ec045-4abb-4c1a-974e-e19e3a3a396a",
"follower-79-ce8c1b0f-fc48-4d62-99c8-f6e3391f16b7",
"follower-80-096f7f42-77b6-4959-843b-9c0157b0c422",
"follower-81-fe55e212-615f-4735-beea-239ddc93c702",
"follower-82-71234abb-fe81-40b5-b9c4-bbaef36e2a55",
"follower-83-c255faed-96f9-4a4d-b060-ee6d39448906",
"follower-84-85d2cdc9-65d0-4def-8773-ce741da76136",
"follower-85-a83536c7-6cea-44b3-903e-44c0fb1ced6c",
"follower-86-094742b0-5604-43d3-b667-8da02ed0689f",
"follower-87-176673c3-2095-4313-87df-407390f71684",
"follower-88-62924ed5-0efc-493d-bc9e-695f627acbca",
"follower-89-cadb3768-eabc-46b5-a2f8-2ab2c704f2e6",
"follower-90-de533425-9713-4068-aaa3-697fe9fc8af2",
"follower-91-d5904d2d-af0c-411f-ac29-20519091eb62",
"follower-92-a47f583f-63dd-416a-942d-8230b84a4e25",
"follower-93-b5596822-5153-4e62-8669-27c86f56a50f",
"follower-94-14dbfeed-05ce-4467-878b-8841540b139a",
"follower-95-3d2bc5c8-4fba-4be5-8a56-867dc358cf48",
"follower-96-9b09e6a0-bffb-4b96-984f-06a77656d0e7",
"follower-97-858dc032-8e3c-4f10-b2c8-63e42979cf54",
"follower-98-46db1df2-4754-46b2-beda-b982e97a9e86",
"follower-99-fb36b0f8-61f4-4d5d-bfbe-61aff7cb39ce"
]
}
}
}
},
"DBServers": {
"follower-1-48a5e486-10c9-4953-8630-9a3de12a6169": "none",
"follower-2-34e18222-8ce3-4016-9558-7092e41eb22c": "none",
"follower-3-27452c0b-efc8-4d9a-b5b1-d557997c4337": "none",
"follower-4-cc56c772-58cc-4c16-b571-56283ad813c8": "none",
"follower-5-aec34c5c-9939-42af-bf5d-afc15f960c50": "none",
"follower-6-8477db61-d46f-46f7-a816-8176e1514494": "none",
"follower-7-58b689ae-e7e8-45e7-83cb-6006b2375f61": "none",
"follower-8-d5e9c550-4a68-4dca-a50d-c84d4b690945": "none",
"follower-9-349e7296-b4fc-4fd3-b8a8-02befbb0380e": "none",
"follower-10-1ad1aa18-00b3-430a-8144-ba973bfed5fe": "none",
"follower-11-110920e0-d079-4f06-99ed-c482d19b5112": "none",
"follower-12-b7d64986-c458-4332-934a-ecb2caf19259": "none",
"follower-13-2e7ba82c-b837-4126-8cda-0db6ac98e30b": "none",
"follower-14-6b9d6b95-420f-44d8-a714-d17ae95eecdd": "none",
"follower-15-08c7dc8d-bb31-4cc3-a7ca-4b8bfba19b70": "none",
"follower-16-5a301b07-d1d8-4c86-8e8a-bd7957a2cafb": "none",
"follower-17-fd04d0f0-821e-401e-8f8e-0b2837ddc41d": "none",
"follower-18-0a2bfdf4-c277-45ea-8af3-4d60eba67910": "none",
"follower-19-69c98e93-b1c2-416f-b5e4-84cc50b65efe": "none",
"follower-20-e9eca0f2-530b-4496-950e-341b71086f8b": "none",
"follower-21-1042a97a-aa82-48ee-8388-8480a6e57249": "none",
"follower-22-c3922c1e-53df-42d6-9bcd-476d01e581fd": "none",
"follower-23-cec0e2ed-3a5b-4b9a-a615-39a1f24179c2": "none",
"follower-24-1753643f-2d1e-4014-8cc6-4f063c0f143e": "none",
"follower-25-1a4edf05-e6ed-47bc-8765-0b8292fc3175": "none",
"follower-26-fcc5fb9e-b4a0-4986-ae14-8b330350fa67": "none",
"follower-27-aa738702-aeb8-4306-86cd-a77516eef44d": "none",
"follower-28-4f6cd6dc-9e12-4fcc-9083-23900ffad0d1": "none",
"follower-29-884e050b-0d33-440b-88bf-13cd41e00c10": "none",
"follower-30-bac109ba-a0ba-4235-b665-743fec5e2ea1": "none",
"follower-31-62a74a8e-f141-44bb-a818-57259c7d6323": "none",
"follower-32-7a0e8f27-04a4-4094-a00c-830dfe3e937c": "none",
"follower-33-83c9df58-91b1-4703-bce7-1d47c633a2c4": "none",
"follower-34-d8f1aa6e-fbd0-49c0-9560-b447417d0284": "none",
"follower-35-77b8626e-30d8-4b04-8ac9-42dd788a4c46": "none",
"follower-36-8239c391-86fe-462d-9036-c129983103f2": "none",
"follower-37-41b1fe2a-2826-43a6-8222-fc9480b4f211": "none",
"follower-38-4a4b54db-17ff-4f5f-882b-973907d9dc27": "none",
"follower-39-e6e0cb50-a609-4f5f-b376-4ec72fefb938": "none",
"follower-40-2c6f13c1-46dc-4d54-992f-4f923169e5e2": "none",
"follower-41-b4c3d57c-ec01-4162-8107-823a09176fc4": "none",
"follower-42-e65dfaf4-cdbd-485a-a4d3-f56848e58d28": "none",
"follower-43-a248deeb-817f-4f0d-9813-c08a40e9027a": "none",
"follower-44-c8f4e52e-7a12-4a3b-8a93-cd543f512a55": "none",
"follower-45-d2a70a84-2a12-4fa3-b0e9-68945fd34cfc": "none",
"follower-46-bf70b49c-ff50-4255-a704-70a5a4d7a4b3": "none",
"follower-47-ca6aaf76-0bf8-4289-9033-605883e514ca": "none",
"follower-48-30442bc5-2dc0-434c-b21f-989610a199e7": "none",
"follower-49-788d2a9b-6d56-42a7-bacb-1dafff7d58a9": "none",
"follower-50-fc63b181-9848-4f7b-8e03-84e39bc37cd3": "none",
"follower-51-9659f049-9f0c-4ae3-8bc4-98d07e042f44": "none",
"follower-52-05475c47-994b-4360-b90d-c874df3b7268": "none",
"follower-53-0d8b6c5c-63c2-4f2d-bc0e-5ecfd5df518c": "none",
"follower-54-3e264e2c-ab6e-4aae-a17e-28c4926b587f": "none",
"follower-55-ebbc5038-de11-46aa-ba3b-447fc6e84ecb": "none",
"follower-56-a18414b2-d3d3-4310-af85-3986a2c03ce0": "none",
"follower-57-9a9c258b-7a7d-4afe-a39b-63ff865a1e14": "none",
"follower-58-65bb2bb2-b371-4ef7-bbe6-77775223d505": "none",
"follower-59-ec854bb2-5f6c-4ec1-8cb7-0ddb751c8afe": "none",
"follower-60-33091fa1-893a-426c-ae6e-ec4db28cca92": "none",
"follower-61-bca22a37-b48a-4e33-86a6-8f1a9c4d05ac": "none",
"follower-62-d3282d65-dc64-4022-9604-2563af4373cf": "none",
"follower-63-cdacaed8-cb57-4b7d-818c-435f37665bf1": "none",
"follower-64-94c6ac3a-3611-48df-9dbd-4d034091b775": "none",
"follower-65-b7570937-9ea5-4c72-9a94-67fccbbdbaa8": "none",
"follower-66-beb74106-ce6e-415d-b170-664217c88124": "none",
"follower-67-7b2086f2-b321-4a38-8557-768e0c64dd7d": "none",
"follower-68-d4adf1d9-9091-4391-8d0b-739297fead93": "none",
"follower-69-ab287171-4f08-4162-bdf7-d4ea14f7fc4c": "none",
"follower-70-b8eb7ae3-270a-4c26-bd01-312173bca03a": "none",
"follower-71-cb71b675-c801-41c3-9fe1-f0fb38e36c3c": "none",
"follower-72-d0c90536-685c-4cae-90be-aa2ee048569e": "none",
"follower-73-8ec7fdd0-9c62-423b-be49-1895489f4d2f": "none",
"follower-74-15735f4f-cdd3-44c1-b87c-9d56495ceefd": "none",
"follower-75-93283be9-2341-472d-a2d0-5ab834a94669": "none",
"follower-76-71b140b4-251b-46b9-bf87-39251292edbc": "none",
"follower-77-4bfb6d9b-7144-4d50-92af-45e33205f8c0": "none",
"follower-78-f37ec045-4abb-4c1a-974e-e19e3a3a396a": "none",
"follower-79-ce8c1b0f-fc48-4d62-99c8-f6e3391f16b7": "none",
"follower-80-096f7f42-77b6-4959-843b-9c0157b0c422": "none",
"follower-81-fe55e212-615f-4735-beea-239ddc93c702": "none",
"follower-82-71234abb-fe81-40b5-b9c4-bbaef36e2a55": "none",
"follower-83-c255faed-96f9-4a4d-b060-ee6d39448906": "none",
"follower-84-85d2cdc9-65d0-4def-8773-ce741da76136": "none",
"follower-85-a83536c7-6cea-44b3-903e-44c0fb1ced6c": "none",
"follower-86-094742b0-5604-43d3-b667-8da02ed0689f": "none",
"follower-87-176673c3-2095-4313-87df-407390f71684": "none",
"follower-88-62924ed5-0efc-493d-bc9e-695f627acbca": "none",
"follower-89-cadb3768-eabc-46b5-a2f8-2ab2c704f2e6": "none",
"follower-90-de533425-9713-4068-aaa3-697fe9fc8af2": "none",
"follower-91-d5904d2d-af0c-411f-ac29-20519091eb62": "none",
"follower-92-a47f583f-63dd-416a-942d-8230b84a4e25": "none",
"follower-93-b5596822-5153-4e62-8669-27c86f56a50f": "none",
"follower-94-14dbfeed-05ce-4467-878b-8841540b139a": "none",
"follower-95-3d2bc5c8-4fba-4be5-8a56-867dc358cf48": "none",
"follower-96-9b09e6a0-bffb-4b96-984f-06a77656d0e7": "none",
"follower-97-858dc032-8e3c-4f10-b2c8-63e42979cf54": "none",
"follower-98-46db1df2-4754-46b2-beda-b982e97a9e86": "none",
"follower-99-fb36b0f8-61f4-4d5d-bfbe-61aff7cb39ce": "none",
"free": "none",
"free2": "none",
"leader": "none"
}
},
"Supervision": {
"DBServers": {},
"Health": {
"leader": {
"Status": "GOOD"
},
"follower-1-48a5e486-10c9-4953-8630-9a3de12a6169": {
"Status": "GOOD"
},
"follower-2-34e18222-8ce3-4016-9558-7092e41eb22c": {
"Status": "GOOD"
},
"follower-3-27452c0b-efc8-4d9a-b5b1-d557997c4337": {
"Status": "GOOD"
},
"follower-4-cc56c772-58cc-4c16-b571-56283ad813c8": {
"Status": "GOOD"
},
"follower-5-aec34c5c-9939-42af-bf5d-afc15f960c50": {
"Status": "GOOD"
},
"follower-6-8477db61-d46f-46f7-a816-8176e1514494": {
"Status": "GOOD"
},
"follower-7-58b689ae-e7e8-45e7-83cb-6006b2375f61": {
"Status": "GOOD"
},
"follower-8-d5e9c550-4a68-4dca-a50d-c84d4b690945": {
"Status": "GOOD"
},
"follower-9-349e7296-b4fc-4fd3-b8a8-02befbb0380e": {
"Status": "GOOD"
},
"follower-10-1ad1aa18-00b3-430a-8144-ba973bfed5fe": {
"Status": "GOOD"
},
"follower-11-110920e0-d079-4f06-99ed-c482d19b5112": {
"Status": "GOOD"
},
"follower-12-b7d64986-c458-4332-934a-ecb2caf19259": {
"Status": "GOOD"
},
"follower-13-2e7ba82c-b837-4126-8cda-0db6ac98e30b": {
"Status": "GOOD"
},
"follower-14-6b9d6b95-420f-44d8-a714-d17ae95eecdd": {
"Status": "GOOD"
},
"follower-15-08c7dc8d-bb31-4cc3-a7ca-4b8bfba19b70": {
"Status": "GOOD"
},
"follower-16-5a301b07-d1d8-4c86-8e8a-bd7957a2cafb": {
"Status": "GOOD"
},
"follower-17-fd04d0f0-821e-401e-8f8e-0b2837ddc41d": {
"Status": "GOOD"
},
"follower-18-0a2bfdf4-c277-45ea-8af3-4d60eba67910": {
"Status": "GOOD"
},
"follower-19-69c98e93-b1c2-416f-b5e4-84cc50b65efe": {
"Status": "GOOD"
},
"follower-20-e9eca0f2-530b-4496-950e-341b71086f8b": {
"Status": "GOOD"
},
"follower-21-1042a97a-aa82-48ee-8388-8480a6e57249": {
"Status": "GOOD"
},
"follower-22-c3922c1e-53df-42d6-9bcd-476d01e581fd": {
"Status": "GOOD"
},
"follower-23-cec0e2ed-3a5b-4b9a-a615-39a1f24179c2": {
"Status": "GOOD"
},
"follower-24-1753643f-2d1e-4014-8cc6-4f063c0f143e": {
"Status": "GOOD"
},
"follower-25-1a4edf05-e6ed-47bc-8765-0b8292fc3175": {
"Status": "GOOD"
},
"follower-26-fcc5fb9e-b4a0-4986-ae14-8b330350fa67": {
"Status": "GOOD"
},
"follower-27-aa738702-aeb8-4306-86cd-a77516eef44d": {
"Status": "GOOD"
},
"follower-28-4f6cd6dc-9e12-4fcc-9083-23900ffad0d1": {
"Status": "GOOD"
},
"follower-29-884e050b-0d33-440b-88bf-13cd41e00c10": {
"Status": "GOOD"
},
"follower-30-bac109ba-a0ba-4235-b665-743fec5e2ea1": {
"Status": "GOOD"
},
"follower-31-62a74a8e-f141-44bb-a818-57259c7d6323": {
"Status": "GOOD"
},
"follower-32-7a0e8f27-04a4-4094-a00c-830dfe3e937c": {
"Status": "GOOD"
},
"follower-33-83c9df58-91b1-4703-bce7-1d47c633a2c4": {
"Status": "GOOD"
},
"follower-34-d8f1aa6e-fbd0-49c0-9560-b447417d0284": {
"Status": "GOOD"
},
"follower-35-77b8626e-30d8-4b04-8ac9-42dd788a4c46": {
"Status": "GOOD"
},
"follower-36-8239c391-86fe-462d-9036-c129983103f2": {
"Status": "GOOD"
},
"follower-37-41b1fe2a-2826-43a6-8222-fc9480b4f211": {
"Status": "GOOD"
},
"follower-38-4a4b54db-17ff-4f5f-882b-973907d9dc27": {
"Status": "GOOD"
},
"follower-39-e6e0cb50-a609-4f5f-b376-4ec72fefb938": {
"Status": "GOOD"
},
"follower-40-2c6f13c1-46dc-4d54-992f-4f923169e5e2": {
"Status": "GOOD"
},
"follower-41-b4c3d57c-ec01-4162-8107-823a09176fc4": {
"Status": "GOOD"
},
"follower-42-e65dfaf4-cdbd-485a-a4d3-f56848e58d28": {
"Status": "GOOD"
},
"follower-43-a248deeb-817f-4f0d-9813-c08a40e9027a": {
"Status": "GOOD"
},
"follower-44-c8f4e52e-7a12-4a3b-8a93-cd543f512a55": {
"Status": "GOOD"
},
"follower-45-d2a70a84-2a12-4fa3-b0e9-68945fd34cfc": {
"Status": "GOOD"
},
"follower-46-bf70b49c-ff50-4255-a704-70a5a4d7a4b3": {
"Status": "GOOD"
},
"follower-47-ca6aaf76-0bf8-4289-9033-605883e514ca": {
"Status": "GOOD"
},
"follower-48-30442bc5-2dc0-434c-b21f-989610a199e7": {
"Status": "GOOD"
},
"follower-49-788d2a9b-6d56-42a7-bacb-1dafff7d58a9": {
"Status": "GOOD"
},
"follower-50-fc63b181-9848-4f7b-8e03-84e39bc37cd3": {
"Status": "GOOD"
},
"follower-51-9659f049-9f0c-4ae3-8bc4-98d07e042f44": {
"Status": "GOOD"
},
"follower-52-05475c47-994b-4360-b90d-c874df3b7268": {
"Status": "GOOD"
},
"follower-53-0d8b6c5c-63c2-4f2d-bc0e-5ecfd5df518c": {
"Status": "GOOD"
},
"follower-54-3e264e2c-ab6e-4aae-a17e-28c4926b587f": {
"Status": "GOOD"
},
"follower-55-ebbc5038-de11-46aa-ba3b-447fc6e84ecb": {
"Status": "GOOD"
},
"follower-56-a18414b2-d3d3-4310-af85-3986a2c03ce0": {
"Status": "GOOD"
},
"follower-57-9a9c258b-7a7d-4afe-a39b-63ff865a1e14": {
"Status": "GOOD"
},
"follower-58-65bb2bb2-b371-4ef7-bbe6-77775223d505": {
"Status": "GOOD"
},
"follower-59-ec854bb2-5f6c-4ec1-8cb7-0ddb751c8afe": {
"Status": "GOOD"
},
"follower-60-33091fa1-893a-426c-ae6e-ec4db28cca92": {
"Status": "GOOD"
},
"follower-61-bca22a37-b48a-4e33-86a6-8f1a9c4d05ac": {
"Status": "GOOD"
},
"follower-62-d3282d65-dc64-4022-9604-2563af4373cf": {
"Status": "GOOD"
},
"follower-63-cdacaed8-cb57-4b7d-818c-435f37665bf1": {
"Status": "GOOD"
},
"follower-64-94c6ac3a-3611-48df-9dbd-4d034091b775": {
"Status": "GOOD"
},
"follower-65-b7570937-9ea5-4c72-9a94-67fccbbdbaa8": {
"Status": "GOOD"
},
"follower-66-beb74106-ce6e-415d-b170-664217c88124": {
"Status": "GOOD"
},
"follower-67-7b2086f2-b321-4a38-8557-768e0c64dd7d": {
"Status": "GOOD"
},
"follower-68-d4adf1d9-9091-4391-8d0b-739297fead93": {
"Status": "GOOD"
},
"follower-69-ab287171-4f08-4162-bdf7-d4ea14f7fc4c": {
"Status": "GOOD"
},
"follower-70-b8eb7ae3-270a-4c26-bd01-312173bca03a": {
"Status": "GOOD"
},
"follower-71-cb71b675-c801-41c3-9fe1-f0fb38e36c3c": {
"Status": "GOOD"
},
"follower-72-d0c90536-685c-4cae-90be-aa2ee048569e": {
"Status": "GOOD"
},
"follower-73-8ec7fdd0-9c62-423b-be49-1895489f4d2f": {
"Status": "GOOD"
},
"follower-74-15735f4f-cdd3-44c1-b87c-9d56495ceefd": {
"Status": "GOOD"
},
"follower-75-93283be9-2341-472d-a2d0-5ab834a94669": {
"Status": "GOOD"
},
"follower-76-71b140b4-251b-46b9-bf87-39251292edbc": {
"Status": "GOOD"
},
"follower-77-4bfb6d9b-7144-4d50-92af-45e33205f8c0": {
"Status": "GOOD"
},
"follower-78-f37ec045-4abb-4c1a-974e-e19e3a3a396a": {
"Status": "GOOD"
},
"follower-79-ce8c1b0f-fc48-4d62-99c8-f6e3391f16b7": {
"Status": "GOOD"
},
"follower-80-096f7f42-77b6-4959-843b-9c0157b0c422": {
"Status": "GOOD"
},
"follower-81-fe55e212-615f-4735-beea-239ddc93c702": {
"Status": "GOOD"
},
"follower-82-71234abb-fe81-40b5-b9c4-bbaef36e2a55": {
"Status": "GOOD"
},
"follower-83-c255faed-96f9-4a4d-b060-ee6d39448906": {
"Status": "GOOD"
},
"follower-84-85d2cdc9-65d0-4def-8773-ce741da76136": {
"Status": "GOOD"
},
"follower-85-a83536c7-6cea-44b3-903e-44c0fb1ced6c": {
"Status": "GOOD"
},
"follower-86-094742b0-5604-43d3-b667-8da02ed0689f": {
"Status": "GOOD"
},
"follower-87-176673c3-2095-4313-87df-407390f71684": {
"Status": "GOOD"
},
"follower-88-62924ed5-0efc-493d-bc9e-695f627acbca": {
"Status": "GOOD"
},
"follower-89-cadb3768-eabc-46b5-a2f8-2ab2c704f2e6": {
"Status": "GOOD"
},
"follower-90-de533425-9713-4068-aaa3-697fe9fc8af2": {
"Status": "GOOD"
},
"follower-91-d5904d2d-af0c-411f-ac29-20519091eb62": {
"Status": "GOOD"
},
"follower-92-a47f583f-63dd-416a-942d-8230b84a4e25": {
"Status": "GOOD"
},
"follower-93-b5596822-5153-4e62-8669-27c86f56a50f": {
"Status": "GOOD"
},
"follower-94-14dbfeed-05ce-4467-878b-8841540b139a": {
"Status": "GOOD"
},
"follower-95-3d2bc5c8-4fba-4be5-8a56-867dc358cf48": {
"Status": "GOOD"
},
"follower-96-9b09e6a0-bffb-4b96-984f-06a77656d0e7": {
"Status": "GOOD"
},
"follower-97-858dc032-8e3c-4f10-b2c8-63e42979cf54": {
"Status": "GOOD"
},
"follower-98-46db1df2-4754-46b2-beda-b982e97a9e86": {
"Status": "GOOD"
},
"follower-99-fb36b0f8-61f4-4d5d-bfbe-61aff7cb39ce": {
"Status": "GOOD"
},
"free": {
"Status": "FAILED"
},
"free2": {
"Status": "GOOD"
}
},
"Shards": {}
},
"Target": {
"Failed": {},
"Finished": {},
"ToDo": {}
}
}
}
)="