update gossip loop to be more responsive to other agents (#5390)

arangod/Agency/Agent.cpp

@@ -1,7 +1,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 /// DISCLAIMER
 ///
-/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany
 /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
@@ -1727,6 +1727,11 @@ query_t Agent::gossip(query_t const& in, bool isCallback, size_t version) {
                                      << out->slice().toJson();
   }
 
+  // let gossip loop know that it has new data
+  if ( _inception != nullptr && isCallback) {
+    _inception->signalConditionVar();
+  }
+
   return out;
 }
 

arangod/Agency/AgentConfiguration.cpp

@@ -1,7 +1,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 /// DISCLAIMER
 ///
-/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany
 /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
@@ -176,7 +176,7 @@ void config_t::setTimeoutMult(int64_t m) {
   WRITE_LOCKER(writeLocker, _lock);
   if (_timeoutMult != m) {
     _timeoutMult = m;
-    ++_version;
+    // this is called during election, do NOT update ++_version
   }
 }
 
@@ -637,5 +637,3 @@ bool config_t::merge(VPackSlice const& conf) {
   ++_version;
   return true;
 }
-
-

arangod/Agency/Inception.cpp

@@ -1,7 +1,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 /// DISCLAIMER
 ///
-/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany
 /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
@@ -103,8 +103,8 @@ void Inception::gossip() {
           }
         }
         std::string clientid = config.id() + std::to_string(j++);
-        LOG_TOPIC(DEBUG, Logger::AGENCY) << "Sending gossip message: "
-            << out->toJson() << " to peer " << clientid;
+        LOG_TOPIC(DEBUG, Logger::AGENCY) << "Sending gossip message 1: "
+            << out->toJson() << " to peer " << p;
         if (this->isStopping() || _agent->isStopping() || cc == nullptr) {
           return;
         }
@@ -128,8 +128,8 @@ void Inception::gossip() {
         }
         complete = false;
         auto const clientid = config.id() + std::to_string(j++);
-        LOG_TOPIC(DEBUG, Logger::AGENCY) << "Sending gossip message: "
-            << out->toJson() << " to pool member " << clientid;
+        LOG_TOPIC(DEBUG, Logger::AGENCY) << "Sending gossip message 2: "
+            << out->toJson() << " to pool member " << pair.second;
         if (this->isStopping() || _agent->isStopping() || cc == nullptr) {
           return;
         }
@@ -163,10 +163,14 @@ void Inception::gossip() {
     }
 
     // don't panic just yet
-    _cv.wait(waitInterval);
+    //  wait() is true on signal, false on timeout
+    if (_cv.wait(waitInterval)) {
+        waitInterval = 250000;
+    } else {
       if (waitInterval < 2500000) { // 2.5s
         waitInterval *= 2;
       }
+    }
 
   }
 
@@ -297,11 +301,9 @@ bool Inception::restartingActiveAgent() {
                   continue;
                 }
               }
-              
               auto const  theirConfigL = comres->result->getBodyVelocyPack();
               auto const& lcc           =
                 theirConfigL->slice().get("configuration");
-              
               auto agency = std::make_shared<Builder>();
               { VPackObjectBuilder b(agency.get());
                 agency->add("term", theirConfigL->slice().get("term"));
@@ -450,3 +452,9 @@ void Inception::beginShutdown() {
   CONDITION_LOCKER(guard, _cv);
   guard.broadcast();
 }
+
+// @brief Let external routines, like Agent::gossip(), signal our condition
+void Inception::signalConditionVar() {
+  CONDITION_LOCKER(guard, _cv);
+  guard.broadcast();
+}

arangod/Agency/Inception.h

@@ -1,7 +1,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 /// DISCLAIMER
 ///
-/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2014-2018 ArangoDB GmbH, Cologne, Germany
 /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
 ///
 /// Licensed under the Apache License, Version 2.0 (the "License");
@@ -61,6 +61,8 @@ public:
   void beginShutdown() override;
   void run() override;
 
+  void signalConditionVar();
+
  private:
 
   /// @brief We are a restarting active RAFT agent