Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cluster supervision moves broken leaders

This commit is contained in:
Kaveh Vahedipour 2016-05-11 14:59:10 +02:00
parent 6694179e57
commit 5f8f8a628a
7 changed files with 107 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
arangod/Agency/Node.cpp
View File

@ -143,6 +143,11 @@ bool Node::operator== (VPackSlice const& rhs) const {
}
}
// Comparison with slice
bool Node::operator!= (VPackSlice const& rhs) const {
return !(*this == rhs);
}
// Remove this node from store
bool Node::remove () {
Node& parent = *_parent;

3
arangod/Agency/Node.h
View File

@ -103,6 +103,9 @@ public:
/// @brief Check equality with slice
bool operator== (arangodb::velocypack::Slice const&) const;
/// @brief Check equality with slice
bool operator!= (arangodb::velocypack::Slice const&) const;
/// @brief Type of this node (LEAF / NODE)
NodeType type() const;

28
arangod/Agency/Store.cpp
View File

@ -232,12 +232,14 @@ std::vector<bool> Store::apply (
// Check precondition
bool Store::check (VPackSlice const& slice) const {
if (!slice.isObject()) {
if (!slice.isObject()) { // Must be object
LOG_TOPIC(WARN, Logger::AGENCY)
<< "Cannot check precondition: " << slice.toJson();
return false;
}
for (auto const& precond : VPackObjectIterator(slice)) {
for (auto const& precond : VPackObjectIterator(slice)) { // Preconditions
std::string path = precond.key.copyString();
bool found = false;
Node node ("precond");
@ -251,21 +253,29 @@ bool Store::check (VPackSlice const& slice) const {
for (auto const& op : VPackObjectIterator(precond.value)) {
std::string const& oper = op.key.copyString();
if (oper == "old") { // old
return (node == op.value);
if (node != op.value) {
return false;
}
} else if (oper == "isArray") { // isArray
if (!op.value.isBoolean()) {
LOG (FATAL) << "Non boolsh expression for 'isArray' precondition";
LOG_TOPIC(ERR, Logger::AGENCY)
<< "Non boolsh expression for 'isArray' precondition";
return false;
}
bool isArray =
(node.type() == LEAF && node.slice().isArray());
return op.value.getBool() ? isArray : !isArray;
} else if (oper == "oldEmpty") { // isEmpty
if (!op.value.isBoolean()) {
LOG (FATAL) << "Non boolsh expression for 'oldEmpty' precondition";
if(op.value.getBool() ? !isArray : isArray) {
return false;
}
} else if (oper == "oldEmpty") { // isEmpty
if (!op.value.isBoolean()) {
LOG_TOPIC(ERR, Logger::AGENCY)
<< "Non boolsh expression for 'oldEmpty' precondition";
return false;
}
if (op.value.getBool() ? found : !found) {
return false;
}
return op.value.getBool() ? !found : found;
}
}
} else {

89
arangod/Agency/Supervision.cpp
View File

@ -43,7 +43,7 @@ inline void makeReport (query_t& envelope, Builder const& report) {
envelope->close();
}
template<> struct Job<FAILED_LEADER> {
template<> struct Job<FAILED_DBSERVER> {
Job (Node const& snapshot, Agent* agent, uint64_t jobId,
std::string const& failed) {
@ -59,32 +59,68 @@ template<> struct Job<FAILED_LEADER> {
Node const& replicationFactor = collection("replicationFactor");
if (replicationFactor.slice().getUInt() > 1) {
for (auto const& shard : collection("shards").children()) {
for (auto const& dbserver :
VPackArrayIterator(shard.second->slice())) {
std::vector<std::string> bad, reordered;
VPackArrayIterator dbsit (shard.second->slice());
if ((*dbsit.begin()).copyString() != failed) { // Cannot do much
continue;
}
for (auto const& dbserver : dbsit) {
std::string serverID = dbserver.copyString();
if (dbserver.copyString() == failed) {
std::string path ("/arango/Supervision/Jobs/Pending/");
path += arangodb::basics::StringUtils::itoa(jobId);
LOG(WARN) << path;
query_t envelope = std::make_shared<Builder>();
Builder report;
report.openObject();
report.add(path, VPackValue(VPackValueType::Object));
report.add("shard", VPackValue(shard.first));
report.add("dbservers", VPackValue(failed));
report.close();
report.close();
makeReport(envelope, report);
envelope->clear();
report.clear();
report.openObject();
path = std::string("/arango/Plan/Collections")
+ database.first + "/" + collptr.first;
report.close();
agent->write(envelope);
bad.push_back(serverID);
} else {
reordered.push_back(serverID);
}
}
// Put into supervision
std::string path ("/arango/Supervision/Jobs/Pending/");
path += arangodb::basics::StringUtils::itoa(jobId);
LOG(WARN) << path;
query_t envelope = std::make_shared<Builder>();
Builder report;
report.openObject();
report.add(path, VPackValue(VPackValueType::Object));
report.add("shard", VPackValue(shard.first));
report.add("dbservers", VPackValue(failed));
report.add("old", shard.second->slice());
report.add("new", VPackValue(VPackValueType::Array));
for (auto const& server : reordered) {
report.add(VPackValue(server));
}
for (auto const& server : bad) {
report.add(VPackValue(server));
}
report.close();
report.close();
report.close();
LOG(WARN) << report.toJson();
makeReport(envelope, report);
envelope->clear();
report.clear();
// Put into plan
path = std::string("/arango/Plan/Collections/")
+ database.first + "/" + collptr.first + "/shards/" + shard.first;
LOG(WARN) << path;
/*report.openObject();
path = std::string("/arango/Plan/Collections")
+ database.first + "/" + collptr.first + "/" + shard.first;
report.close();
LOG(WARN) << report.toJson();
makeReport(envelope, report);
LOG(WARN) << envelope->toJson();*/
//agent->write(envelope);
// }
//}
}
}
}
}
}
@ -99,7 +135,7 @@ using namespace arangodb::consensus;
Supervision::Supervision() : arangodb::Thread("Supervision"), _agent(nullptr),
_snapshot("Supervision"), _frequency(5),
_gracePeriod(60) {
_gracePeriod(10) {
}
@ -150,7 +186,7 @@ std::vector<check_t> Supervision::checkDBServers () {
VPackValue(VPackValueType::Object));
report->add("LastHearbeatReceived",
VPackValue(printTimestamp(it->second->myTimestamp)));
report->add("LastHearbeatSent", VPackValue(lastHeartbeatTime));
report->add("LastHearbeatSent", VPackValue(it->second->serverTimestamp));
report->add("LastHearbeatStatus", VPackValue(lastHeartbeatStatus));
if (it->second->serverTimestamp == lastHeartbeatTime) {
@ -161,10 +197,9 @@ std::vector<check_t> Supervision::checkDBServers () {
if (t.count() > _gracePeriod) { // Failure
if (it->second->maintenance() == 0) {
it->second->maintenance(TRI_NewTickServer());
Job<FAILED_LEADER> jfl(_snapshot, _agent, it->second->maintenance(),
Job<FAILED_DBSERVER> jfl(_snapshot, _agent, it->second->maintenance(),
serverID);
}
report->add("Alert", VPackValue(true));
}
} else {

2
arangod/Agency/Supervision.h
View File

@ -46,7 +46,7 @@ struct JobCallback {
virtual bool operator()(JobResult*) = 0;
};
enum JOB_CASE {FAILED_LEADER};
enum JOB_CASE {FAILED_DBSERVER};
template<enum JOB_CASE> struct Job {};
struct check_t {

9
js/client/tests/agency/agency-test.js
View File

@ -157,6 +157,15 @@ function agencyTestSuite () {
assertEqual(res.statusCode, 412);
res = writeAgency([[{"/a/b/c":{"op":"set","new":14}},{"/a":{"old":{"b":{"c":13}}}}]]);
assertEqual(res.statusCode, 200);
// multiple preconditions
res = writeAgency([[{"/a":12},{"/a":{"oldEmpty":false}}]]);
assertEqual(readAndCheck([["/a"]]), [{a:12}]);
res = writeAgency([[{"/a":13},{"/a":{"oldEmpty":false},"/a":{"oldEmpty":true}}]]);
assertEqual(res.statusCode, 412);
assertEqual(readAndCheck([["/a"]]), [{a:12}]);
//res = writeAgency([[{"/a":13},{"/a":{"oldEmpty":true},"/a":{"oldEmpty":false}}]]);
//assertEqual(res.statusCode, 412);
//assertEqual(readAndCheck([["/a"]]), [{a:12}]);
},

15
lib/Basics/crc1.s
View File

@ -31,9 +31,9 @@
.text
.globl TRI_BlockCrc32_SSE42
.globl _TRI_BlockCrc32_SSE42
#ifndef __APPLE__
.type TRI_BlockCrc32_SSE42, @function
.type _TRI_BlockCrc32_SSE42, @function
#ifndef __APPLE__
// .type TRI_BlockCrc32_SSE42, @function
// .type _TRI_BlockCrc32_SSE42, @function
#endif
TRI_BlockCrc32_SSE42:
_TRI_BlockCrc32_SSE42:
@ -55,9 +55,10 @@ crca4:
addq $1,%rsi
jmp crca4
crca9:
ret
#ifndef __APPLE__
.size TRI_BlockCrc32_SSE42, .-TRI_BlockCrc32_SSE42
.size _TRI_BlockCrc32_SSE42, .-_TRI_BlockCrc32_SSE42
ret
#ifndef __APPLE__
// .size TRI_BlockCrc32_SSE42, .-TRI_BlockCrc32_SSE42
// .size _TRI_BlockCrc32_SSE42, .-_TRI_BlockCrc32_SSE42
#endif
/* end of TRI_BlockCrc32_SSE42 */