Fix agency election lockstep bug. (#9348)

* Fix agency election lockstep bug.

Reset the base point for the random election timeout to now whenever we have
cast a vote, be it for us or for some other server.

* CHANGELOG.

CHANGELOG

@@ -1,9 +1,6 @@
 v3.4.8 (XXXX-XX-XX)
 -------------------
 
-* reduce resident memory usage when creating non-unique indexes with the RocksDB storage engine
-
-
 v3.4.7 (2019-07-02)
 -------------------
 
@@ -81,6 +78,11 @@ v3.4.7 (2019-07-02)
 
 * Fix agency's handling of object assignments with TTL
 
+* reduce resident memory usage when creating non-unique indexes with the RocksDB storage engine
+
+* Fix agency election bug.
+
+
 v3.4.6.1 (2019-06-05)
 ---------------------
 

arangod/Agency/Constituent.cpp

@@ -393,6 +393,10 @@ bool Constituent::vote(term_t termOfPeer, std::string const& id,
   if (_votedFor != NO_LEADER) {  // already voted in this term
     if (_votedFor == id) {
       LOG_TOPIC(DEBUG, Logger::AGENCY) << "repeating vote for " << id;
+      // Set the last heart beat seen to now, to grant the other guy some time
+      // to establish itself as a leader, before we call for another election:
+      _lastHeartbeatSeen = TRI_microtime();
+      LOG_TOPIC(TRACE, Logger::AGENCY) << "setting last heartbeat time to now, since we repeated a vote grant: " << _lastHeartbeatSeen;
       return true;
     }
     LOG_TOPIC(DEBUG, Logger::AGENCY)
@@ -408,6 +412,10 @@ bool Constituent::vote(term_t termOfPeer, std::string const& id,
   if (prevLogTerm > myLastLogEntry.term ||
       (prevLogTerm == myLastLogEntry.term && prevLogIndex >= myLastLogEntry.index)) {
     LOG_TOPIC(DEBUG, Logger::AGENCY) << "voting for " << id << " in term " << _term;
+    // Set the last heart beat seen to now, to grant the other guy some time
+    // to establish itself as a leader, before we call for another election:
+    _lastHeartbeatSeen = TRI_microtime();
+    LOG_TOPIC(TRACE, Logger::AGENCY) << "setting last heartbeat time to now, since we granted a vote: " << _lastHeartbeatSeen;
     termNoLock(_term, id);
     return true;
   }
@@ -700,6 +708,12 @@ void Constituent::run() {
 
       } else if (role == CANDIDATE) {
         callElection();  // Run for office
+        // Now we take this point of time as the next base point for a
+        // potential next random timeout, since we have just cast a vote for
+        // ourselves:
+        _lastHeartbeatSeen = TRI_microtime();
+        LOG_TOPIC(TRACE, Logger::AGENCY) << "setting last heartbeat because we voted for us: " << _lastHeartbeatSeen;
+
       } else {
         double interval =
             0.25 * _agent->config().minPing() * _agent->config().timeoutMult();

arangod/Agency/Constituent.h

@@ -156,7 +156,10 @@ class Constituent : public Thread {
   std::string _id;        // My own id
 
   // Last time an AppendEntriesRPC message has arrived, this is used to
-  // organize out-of-patience in the follower:
+  // organize out-of-patience in the follower. Note that this variable is
+  // also set to the current time when a vote is cast, either for ourselves
+  // or for somebody else. The constituent calls for an election if and only
+  // if the time since _lastHeartbeatSeen is greater than a random timeout:
   std::atomic<double> _lastHeartbeatSeen;
 
   role_t _role;           // My role