cluster supervision moves broken leaders

arangod/Agency/Node.cpp

@@ -143,6 +143,11 @@ bool Node::operator== (VPackSlice const& rhs) const {
   }
 }
 
+// Comparison with slice
+bool Node::operator!= (VPackSlice const& rhs) const {
+  return !(*this == rhs);
+}
+
 // Remove this node from store
 bool Node::remove () {
   Node& parent = *_parent;

arangod/Agency/Node.h

@@ -103,6 +103,9 @@ public:
   /// @brief Check equality with slice
   bool operator== (arangodb::velocypack::Slice const&) const;
 
+  /// @brief Check equality with slice
+  bool operator!= (arangodb::velocypack::Slice const&) const;
+
   /// @brief Type of this node (LEAF / NODE)
   NodeType type() const;
 

arangod/Agency/Store.cpp

@@ -232,12 +232,14 @@ std::vector<bool> Store::apply (
 
 // Check precondition
 bool Store::check (VPackSlice const& slice) const {
-  if (!slice.isObject()) {
+
+  if (!slice.isObject()) { // Must be object
     LOG_TOPIC(WARN, Logger::AGENCY)
       << "Cannot check precondition: " << slice.toJson();
     return false;
   }
-  for (auto const& precond : VPackObjectIterator(slice)) {
+
+  for (auto const& precond : VPackObjectIterator(slice)) { // Preconditions
     std::string path = precond.key.copyString();
     bool found = false;
     Node node ("precond");
@@ -251,21 +253,29 @@ bool Store::check (VPackSlice const& slice) const {
       for (auto const& op : VPackObjectIterator(precond.value)) {
         std::string const& oper = op.key.copyString();
         if (oper == "old") {                           // old
-          return (node == op.value);
+          if (node != op.value) {
+            return false;
+          }
         } else if (oper == "isArray") {                // isArray
           if (!op.value.isBoolean()) {
-            LOG (FATAL) << "Non boolsh expression for 'isArray' precondition";
+            LOG_TOPIC(ERR, Logger::AGENCY)
+              << "Non boolsh expression for 'isArray' precondition";
             return false;
           }
           bool isArray =
             (node.type() == LEAF && node.slice().isArray());
-          return op.value.getBool() ? isArray : !isArray;
+          if(op.value.getBool() ? !isArray : isArray) {
-        } else if (oper == "oldEmpty") {              // isEmpty
+            return false;
-          if (!op.value.isBoolean()) {
+          }
-            LOG (FATAL) << "Non boolsh expression for 'oldEmpty' precondition";
+        } else if (oper == "oldEmpty") {              // isEmpty
+          if (!op.value.isBoolean()) {
+            LOG_TOPIC(ERR, Logger::AGENCY)
+              << "Non boolsh expression for 'oldEmpty' precondition";
+            return false;
+          }
+          if (op.value.getBool() ? found : !found) {
             return false;
           }
-          return op.value.getBool() ? !found : found;
         }
       }
     } else {

arangod/Agency/Supervision.cpp

@@ -43,7 +43,7 @@ inline void makeReport (query_t& envelope, Builder const& report) {
   envelope->close();
 }
 
-template<> struct Job<FAILED_LEADER> {
+template<> struct Job<FAILED_DBSERVER> {
 
   Job (Node const& snapshot, Agent* agent, uint64_t jobId,
        std::string const& failed) {
@@ -59,32 +59,68 @@ template<> struct Job<FAILED_LEADER> {
         Node const& replicationFactor = collection("replicationFactor");
         if (replicationFactor.slice().getUInt() > 1) {
           for (auto const& shard : collection("shards").children()) {
-            for (auto const& dbserver :
+
-                   VPackArrayIterator(shard.second->slice())) {
+            std::vector<std::string> bad, reordered;
+            VPackArrayIterator dbsit (shard.second->slice());
+
+            if ((*dbsit.begin()).copyString() != failed) { // Cannot do much
+              continue;
+            }
+            
+            for (auto const& dbserver : dbsit) {
+              std::string serverID = dbserver.copyString();
               if (dbserver.copyString() == failed) {
-                std::string path ("/arango/Supervision/Jobs/Pending/");
+                bad.push_back(serverID);
-                path += arangodb::basics::StringUtils::itoa(jobId);
+              } else {
-                LOG(WARN) << path;
+                reordered.push_back(serverID);
-                query_t envelope = std::make_shared<Builder>();
-                Builder report;
-                report.openObject();
-                report.add(path, VPackValue(VPackValueType::Object));
-                report.add("shard", VPackValue(shard.first));
-                report.add("dbservers", VPackValue(failed));
-                report.close();
-                report.close();
-                makeReport(envelope, report);
-                envelope->clear();
-                report.clear();
-                report.openObject();
-                path = std::string("/arango/Plan/Collections")
-                  + database.first + "/" + collptr.first;
-                report.close();
-                agent->write(envelope);
               }
             }
+
+            // Put into supervision
+            std::string path ("/arango/Supervision/Jobs/Pending/");
+            path += arangodb::basics::StringUtils::itoa(jobId);
+            LOG(WARN) << path;
+            query_t envelope = std::make_shared<Builder>();
+            Builder report;
+            report.openObject();
+            report.add(path, VPackValue(VPackValueType::Object));
+            report.add("shard", VPackValue(shard.first));
+            report.add("dbservers", VPackValue(failed));
+            report.add("old", shard.second->slice());
+            report.add("new", VPackValue(VPackValueType::Array));
+            for (auto const& server : reordered) {
+              report.add(VPackValue(server));
+            }
+            for (auto const& server : bad) {
+              report.add(VPackValue(server));
+            }
+            report.close();
+            report.close();
+            report.close();
+            LOG(WARN) << report.toJson();
+            makeReport(envelope, report);
+
+            envelope->clear();
+            report.clear();
+
+            // Put into plan
+            path = std::string("/arango/Plan/Collections/")
+              + database.first + "/" + collptr.first + "/shards/" + shard.first;
+
+            LOG(WARN) << path;
+              
+            /*report.openObject();
+              path = std::string("/arango/Plan/Collections")
+              + database.first + "/" + collptr.first + "/" + shard.first;
+              report.close();
+              LOG(WARN) << report.toJson();
+              makeReport(envelope, report);
+              LOG(WARN) << envelope->toJson();*/
+            //agent->write(envelope);
+            // }
+            //}
           }
-        }
+        } 
       }
     }
     
@@ -99,7 +135,7 @@ using namespace arangodb::consensus;
 
 Supervision::Supervision() : arangodb::Thread("Supervision"), _agent(nullptr),
                              _snapshot("Supervision"), _frequency(5),
-                             _gracePeriod(60) {
+                             _gracePeriod(10) {
   
 }
 
@@ -150,7 +186,7 @@ std::vector<check_t> Supervision::checkDBServers () {
                   VPackValue(VPackValueType::Object));
       report->add("LastHearbeatReceived",
                   VPackValue(printTimestamp(it->second->myTimestamp)));
-      report->add("LastHearbeatSent", VPackValue(lastHeartbeatTime));
+      report->add("LastHearbeatSent", VPackValue(it->second->serverTimestamp));
       report->add("LastHearbeatStatus", VPackValue(lastHeartbeatStatus));
 
       if (it->second->serverTimestamp == lastHeartbeatTime) {
@@ -161,10 +197,9 @@ std::vector<check_t> Supervision::checkDBServers () {
         if (t.count() > _gracePeriod) {               // Failure
           if (it->second->maintenance() == 0) {
             it->second->maintenance(TRI_NewTickServer());
-            Job<FAILED_LEADER> jfl(_snapshot, _agent, it->second->maintenance(),
+            Job<FAILED_DBSERVER> jfl(_snapshot, _agent, it->second->maintenance(),
                                    serverID);
           }
-          report->add("Alert", VPackValue(true));
         }
 
       } else {

arangod/Agency/Supervision.h

@@ -46,7 +46,7 @@ struct JobCallback {
   virtual bool operator()(JobResult*) = 0;
 };
 
-enum JOB_CASE {FAILED_LEADER};
+enum JOB_CASE {FAILED_DBSERVER};
 template<enum JOB_CASE> struct Job {};
 
 struct check_t {

js/client/tests/agency/agency-test.js

@@ -157,6 +157,15 @@ function agencyTestSuite () {
       assertEqual(res.statusCode, 412);
       res = writeAgency([[{"/a/b/c":{"op":"set","new":14}},{"/a":{"old":{"b":{"c":13}}}}]]);
       assertEqual(res.statusCode, 200);
+      // multiple preconditions
+      res = writeAgency([[{"/a":12},{"/a":{"oldEmpty":false}}]]);
+      assertEqual(readAndCheck([["/a"]]), [{a:12}]);
+      res = writeAgency([[{"/a":13},{"/a":{"oldEmpty":false},"/a":{"oldEmpty":true}}]]);
+      assertEqual(res.statusCode, 412);
+      assertEqual(readAndCheck([["/a"]]), [{a:12}]);
+      //res = writeAgency([[{"/a":13},{"/a":{"oldEmpty":true},"/a":{"oldEmpty":false}}]]);
+      //assertEqual(res.statusCode, 412);
+      //assertEqual(readAndCheck([["/a"]]), [{a:12}]);
     },
 
 

lib/Basics/crc1.s

@@ -31,9 +31,9 @@
 	.text
 	.globl	TRI_BlockCrc32_SSE42
 	.globl	_TRI_BlockCrc32_SSE42
-#ifndef  __APPLE__
+#ifndef __APPLE__
-	.type	TRI_BlockCrc32_SSE42, @function
+//	.type	TRI_BlockCrc32_SSE42, @function
-	.type	_TRI_BlockCrc32_SSE42, @function
+//	.type	_TRI_BlockCrc32_SSE42, @function
 #endif
 TRI_BlockCrc32_SSE42:
 _TRI_BlockCrc32_SSE42:
@@ -55,9 +55,10 @@ crca4:
         addq    $1,%rsi
         jmp     crca4
 crca9:
-        ret
+  ret
-#ifndef  __APPLE__
+
-	.size	TRI_BlockCrc32_SSE42, .-TRI_BlockCrc32_SSE42
+#ifndef __APPLE__
-	.size	_TRI_BlockCrc32_SSE42, .-_TRI_BlockCrc32_SSE42
+//	.size	TRI_BlockCrc32_SSE42, .-TRI_BlockCrc32_SSE42
+//	.size	_TRI_BlockCrc32_SSE42, .-_TRI_BlockCrc32_SSE42
 #endif
 /* end of TRI_BlockCrc32_SSE42  */