correct handling of distributeShardsLike in FailedFollower

arangod/Agency/FailedFollower.cpp

@@ -66,7 +66,32 @@ bool FailedFollower::create() {
       << "Todo: failed Follower for " + _shard + " from " + _from + " to " + _to;
 
   std::string path = _agencyPrefix + toDoPrefix + _jobId;
+  std::string planPath =
+    planColPrefix + _database + "/" + _collection + "/shards";
+  
+  auto const& myClones = clones(_snapshot, _database, _collection);
+  if (!myClones.empty()) {
 
+    size_t sub = 0;
+    auto myshards = _snapshot(
+      planColPrefix + _database + "/" + _collection + "/shards").children();
+    auto mpos = std::distance(myshards.begin(), myshards.find(_shard));
+
+    // Deal with my clones
+    for (auto const& collection : myClones) {
+      auto othershards =
+        _snapshot(planColPrefix + _database + "/" + collection + "/shards")
+        .children();
+      auto opos = othershards.begin();
+      std::advance(opos, mpos);
+      auto const& shard = opos->first;
+
+      FailedFollower(_snapshot, _agent, _jobId + "-" + std::to_string(sub++),
+                     _jobId, _agencyPrefix, _database, collection, shard,
+                     _from, _to);
+    }
+  }
+   
   _jb = std::make_shared<Builder>();
   _jb->openArray();
   _jb->openObject();
@@ -182,16 +207,10 @@ bool FailedFollower::start() {
 
   pending.close();
 
-  // Precondition
+  // Preconditions
-  // --- Check that Current servers are as we expect
- 
   pending.openObject();
-  /* pending.add(_agencyPrefix + curPath, VPackValue(VPackValueType::Object));
-  pending.add("old", current.slice());
-  pending.close();
-  */
   
-  // --- Check if shard is not blocked
+  // --- Check if shard is not blocked by other job
   pending.add(_agencyPrefix + blockedShardsPrefix + _shard,
               VPackValue(VPackValueType::Object));
   pending.add("oldEmpty", VPackValue(true));

arangod/Agency/FailedServer.cpp

@@ -130,14 +130,23 @@ bool FailedServer::start() {
       auto cdatabase = current.at(database.first)->children();
 
       for (auto const& collptr : database.second->children()) {
-        Node const& collection = *(collptr.second);
+        auto const& collection = *(collptr.second);
         
         if (!cdatabase.find(collptr.first)->second->children().empty()) {
-          Node const& collection = *(collptr.second);
+
-          Node const& replicationFactor = collection("replicationFactor");
+          auto const& collection = *(collptr.second);
+          auto const& replicationFactor = collection("replicationFactor");
+
           if (replicationFactor.slice().getUInt() > 1) {
-            auto available = availableServers();
+
+            bool isClone = false;
+            try { // Clone
+              if(!collection("distributeShardsLike").slice().copyString().empty()) {
+                isClone = true;
+              }
+            } catch (...) {} // Not clone
             
+            auto available = availableServers();
               
             for (auto const& shard : collection("shards").children()) {
 
@@ -167,7 +176,7 @@ bool FailedServer::start() {
                 ++pos;
               }
 
-              if (found && available.size() > 0) {
+              if (found && !available.empty() > 0 && !isClone) {
                 auto randIt = available.begin();
                 std::advance(randIt, std::rand() % available.size());
                 FailedFollower(

arangod/Agency/Inception.cpp

@@ -368,7 +368,7 @@ bool Inception::estimateRAFTInterval() {
   auto config = _agent->config();
 
   auto myid = _agent->id();
-  double to = 0.25;
+  auto to = std::chrono::duration<double,std::milli>(1.0); // 
 
   for (size_t i = 0; i < nrep; ++i) {
     for (auto const& peer : config.pool()) {
@@ -383,7 +383,7 @@ bool Inception::estimateRAFTInterval() {
           2.0, true);
       }
     }
-    std::this_thread::sleep_for(std::chrono::duration<double,std::milli>(to));
+    std::this_thread::sleep_for(to);
     to *= 1.01;
   }
 

arangod/Agency/Job.cpp

@@ -177,3 +177,23 @@ std::vector<std::string> Job::availableServers() const {
   return ret;
   
 }
+
+std::vector<std::string> Job::clones(Node const& snapshot,
+                                     std::string const& database,
+                                     std::string const& collection) {
+
+  std::vector<std::string> ret;
+  std::string databasePath = planColPrefix + database;
+  try {
+    for (const auto& colptr : snapshot(databasePath).children()) { // databases
+      try {
+        auto const col = *colptr.second;
+        if (col("distributeShardsLike").slice().copyString() == collection) {
+          ret.push_back(colptr.first);
+        }
+      } catch(...) {}
+    }
+  } catch (...) {}
+  return ret;
+  
+}

arangod/Agency/Job.h

@@ -111,6 +111,10 @@ struct Job {
 
   virtual std::vector<std::string> availableServers() const;
 
+  static std::vector<std::string> clones(
+    Node const& snapshot, std::string const& database,
+    std::string const& collection);
+
   Node const _snapshot;
   Agent* _agent;
   std::string _jobId;

arangod/Agency/Node.cpp

@@ -781,3 +781,15 @@ std::string Node::getString() const {
   }
   return slice().copyString();
 }
+
+Slice Node::getArray() const {
+  if (type() == NODE) {
+    throw StoreException("Must not convert NODE type to array");
+  }
+  if (!_isArray) {
+    throw StoreException("Not an array type");
+  }
+  rebuildVecBuf();
+  return Slice(_vecBuf.data());
+}
+

arangod/Agency/Node.h

@@ -217,6 +217,9 @@ class Node {
   /// @brief Get string value (throws if type NODE or if conversion fails)
   std::string getString() const;
 
+  /// @brief Get array value
+  Slice getArray() const;
+
  protected:
   /// @brief Add time to live entry
   virtual bool addTimeToLive(long millis);
@@ -231,8 +234,7 @@ class Node {
   Store* _store;           ///< @brief Store
   Children _children;      ///< @brief child nodes
   TimePoint _ttl;          ///< @brief my expiry
-  // Buffer<uint8_t> _value; ///< @brief my value
+  std::vector<Buffer<uint8_t>> _value; ///< @brief my value
-  std::vector<Buffer<uint8_t>> _value;  ///< @brief my value
   mutable Buffer<uint8_t> _vecBuf;
   mutable bool _vecBufDirty;
   bool _isArray;