agent can now join with disaster-recovery-id and changed endpoint (#5828)

CHANGELOG

@@ -1,7 +1,7 @@
 v3.3.12 (XXXX-XX-XX)
 --------------------
 
-* fixed issue #5727: Edge document with user provided key is inserted as many 
+* fixed issue #5727: Edge document with user provided key is inserted as many
   times as the number of shards, violating the primary index
 
 * fixed internal issue #2658: AQL modification queries did not allow `_rev`
@@ -9,7 +9,7 @@ v3.3.12 (XXXX-XX-XX)
   in order to force AQL modification queries to match revision ids before
   doing any modifications
 
-* fixed issue #5679: Replication applier restrictions will crash synchronisation 
+* fixed issue #5679: Replication applier restrictions will crash synchronisation
   after initial sync
 
 * fixed potential issue in RETURN DISTINCT CollectBlock implementation
@@ -19,6 +19,9 @@ v3.3.12 (XXXX-XX-XX)
   this fixes a race condition with parallel VST communication over
   SSL
 
+* fixed for agent coming back to agency with changed endpoint and
+  total data loss
+
 
 v3.3.11 (2018-06-26)
 --------------------

arangod/Agency/Agent.cpp

@@ -1675,14 +1675,11 @@ query_t Agent::gossip(query_t const& in, bool isCallback, size_t version) {
 
 
   LOG_TOPIC(TRACE, Logger::AGENCY) << "Received gossip " << slice.toJson();
-
-  std::unordered_map<std::string, std::string> incoming;
   for (auto const& pair : VPackObjectIterator(pslice)) {
     if (!pair.value.isString()) {
       THROW_ARANGO_EXCEPTION_MESSAGE(
           20004, "Gossip message pool must contain string parameters");
     }
-    incoming[pair.key.copyString()] = pair.value.copyString();
   }
 
   query_t out = std::make_shared<Builder>();
@@ -1700,14 +1697,10 @@ query_t Agent::gossip(query_t const& in, bool isCallback, size_t version) {
       }
     }
 
-    for (auto const& i : incoming) {
-
-      /// disagreement over pool membership: fatal!
-      if (!_config.addToPool(i)) {
-        LOG_TOPIC(FATAL, Logger::AGENCY) << "Discrepancy in agent pool!";
-        FATAL_ERROR_EXIT();
-      }
-
+    /// disagreement over pool membership: fatal!
+    if (!_config.upsertPool(pslice, id)) {
+      LOG_TOPIC(FATAL, Logger::AGENCY) << "Discrepancy in agent pool!";
+      FATAL_ERROR_EXIT();
     }
 
     if (!isCallback) { // no gain in callback to a callback.

arangod/Agency/AgentConfiguration.cpp

@@ -265,16 +265,25 @@ void config_t::eraseFromGossipPeers(std::string const& endpoint) {
   }
 }
 
-bool config_t::addToPool(std::pair<std::string, std::string> const& i) {
+bool config_t::upsertPool(
+  VPackSlice const& otherPool, std::string const& otherId) {
   WRITE_LOCKER(readLocker, _lock);
-  if (_pool.find(i.first) == _pool.end()) {
-    LOG_TOPIC(INFO, Logger::AGENCY)
-      << "Adding " << i.first << "(" << i.second << ") to agent pool";
-    _pool[i.first] = i.second;
-    ++_version;
-  } else {
-    if (_pool.at(i.first) != i.second) {  /// discrepancy!
-      return false;
+  for (auto const& entry : VPackObjectIterator(otherPool)) {
+    auto const id = entry.key.copyString();
+    auto const endpoint = entry.value.copyString();
+    if (_pool.find(id) == _pool.end()) {
+      LOG_TOPIC(INFO, Logger::AGENCY)
+        << "Adding " << id << "(" << endpoint << ") to agent pool";
+      _pool[id] = endpoint;
+      ++_version;
+    } else {
+      if (_pool.at(id) != endpoint) {   
+        if (id != otherId) {          /// discrepancy!
+          return false;
+        } else {                      /// we trust the other guy on his own endpoint
+          _pool.at(id) = endpoint;
+        }
+      }
     }
   }
   return true;

arangod/Agency/AgentConfiguration.h

@@ -127,8 +127,20 @@ struct config_t {
   /// @brief wait for sync requested
   bool waitForSync() const;
 
-  /// @brief add pool member
-  bool addToPool(std::pair<std::string, std::string> const& i);
+  /**
+   * @brief           Verify other agent's pool against our own:
+   *                  - We only get here, if our pool is not complete yet or the
+   *                    id is member of this agency
+   *                  - We match their pool to ours and allow only for an update
+   *                    of it's own endpoint
+   * 
+   * @param otherPool Other agent's pool
+   * @param otherId   Other agent's id
+   *
+   * @return          Success
+   */
+  bool upsertPool(
+    VPackSlice const& otherPool, std::string const& otherId);
 
   /// @brief active agency size
   void activate();