Bug fix/agency prepare leading bug (#2752)

arangod/Agency/Agent.cpp

@@ -170,7 +170,12 @@ std::string Agent::leaderID() const {
 
 /// Are we leading?
 bool Agent::leading() const {
-  return _preparing || _constituent.leading();
+  // When we become leader, we first are officially still a follower, but
+  // prepare for the leading. This is indicated by the _preparing flag in the
+  // Agent, the Constituent stays with role FOLLOWER for now. The agent has
+  // to send out AppendEntriesRPC calls immediately, but only when we are
+  // properly leading (with initialized stores etc.) can we execute requests.
+  return (_preparing && _constituent.following()) || _constituent.leading();
 }
 
 /// Start constituent personality
@@ -555,6 +560,7 @@ void Agent::sendAppendEntriesRPC() {
       // Really leading?
       if (challengeLeadership()) {
         _constituent.candidate();
+        _preparing = false;
         return;
       }
       
@@ -815,6 +821,7 @@ trans_ret_t Agent::transact(query_t const& queries) {
     // Only leader else redirect
     if (challengeLeadership()) {
       _constituent.candidate();
+      _preparing = false;
       return trans_ret_t(false, NO_LEADER);
     }
     
@@ -872,6 +879,7 @@ trans_ret_t Agent::transient(query_t const& queries) {
     // Only leader else redirect
     if (challengeLeadership()) {
       _constituent.candidate();
+      _preparing = false;
       return trans_ret_t(false, NO_LEADER);
     }
 
@@ -982,6 +990,7 @@ write_ret_t Agent::write(query_t const& query, bool discardStartup) {
     // Only leader else redirect
     if (multihost && challengeLeadership()) {
       _constituent.candidate();
+      _preparing = false;
       return write_ret_t(false, NO_LEADER);
     }
     
@@ -1027,6 +1036,7 @@ read_ret_t Agent::read(query_t const& query) {
   // Only leader else redirect
   if (challengeLeadership()) {
     _constituent.candidate();
+    _preparing = false;
     return read_ret_t(false, NO_LEADER);
   }
 
@@ -1382,13 +1392,13 @@ void Agent::notify(query_t const& message) {
     THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL);
   }
   if (!slice.hasKey("min ping") || !slice.get("min ping").isNumber()) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL);
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_MIN_PING);
   }
   if (!slice.hasKey("max ping") || !slice.get("max ping").isNumber()) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL);
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_MAX_PING);
   }
   if (!slice.hasKey("timeoutMult") || !slice.get("timeoutMult").isInteger()) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL);
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT);
   }
 
   _config.update(message);

arangod/Agency/Agent.h

@@ -97,6 +97,11 @@ class Agent : public arangodb::Thread,
   /// @brief Prepare leadership
   bool prepareLead();
 
+  /// @brief Unprepare for leadership, needed when we resign during preparation
+  void unprepareLead() {
+    _preparing = false;
+  }
+
   /// @brief Load persistent state
   void load();
 

arangod/Agency/Constituent.cpp

@@ -705,6 +705,7 @@ void Constituent::run() {
         if (isTimeout) {
           LOG_TOPIC(TRACE, Logger::AGENCY) << "timeout, calling an election";
           candidate();
+          _agent->unprepareLead();
         }
       } else if (_role == CANDIDATE) {
         callElection();  // Run for office

arangod/Agency/Inception.cpp

@@ -308,6 +308,7 @@ bool Inception::restartingActiveAgent() {
               agency->add("pool",        tcc.get("pool"));
               agency->add("min ping",    tcc.get("min ping"));
               agency->add("max ping",    tcc.get("max ping"));
+              agency->add("timeoutMult", tcc.get("timeoutMult"));
               agency->close();
               _agent->notify(agency);
               return true;

arangod/Agency/RestAgencyPrivHandler.h

@@ -56,7 +56,7 @@ class RestAgencyPrivHandler : public arangodb::RestBaseHandler {
       return false;
     } else {
       try {
-        val = std::stoul(val_str);
+        val = std::stoull(val_str);
       } catch (std::invalid_argument const&) {
         LOG_TOPIC(WARN, Logger::AGENCY)
             << "Value for query string " << name

js/common/bootstrap/errors.js

@@ -307,6 +307,9 @@
     "ERROR_AGENCY_INFORM_MUST_CONTAIN_ID" : { "code" : 20013, "message" : "Inform message must contain string parameter 'id'" },
     "ERROR_AGENCY_INFORM_MUST_CONTAIN_ACTIVE" : { "code" : 20014, "message" : "Inform message must contain array 'active'" },
     "ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL" : { "code" : 20015, "message" : "Inform message must contain object 'pool'" },
+    "ERROR_AGENCY_INFORM_MUST_CONTAIN_MIN_PING" : { "code" : 20016, "message" : "Inform message must contain object 'min ping'" },
+    "ERROR_AGENCY_INFORM_MUST_CONTAIN_MAX_PING" : { "code" : 20017, "message" : "Inform message must contain object 'max ping'" },
+    "ERROR_AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT" : { "code" : 20018, "message" : "Inform message must contain object 'timeoutMult'" },
     "ERROR_AGENCY_INQUIRE_CLIENT_ID_MUST_BE_STRING" : { "code" : 20020, "message" : "Inquiry failed" },
     "ERROR_AGENCY_CANNOT_REBUILD_DBS" : { "code" : 20021, "message" : "Cannot rebuild readDB and spearHead" },
     "ERROR_SUPERVISION_GENERAL_FAILURE" : { "code" : 20501, "message" : "general supervision failure" },

lib/Basics/errors.dat

@@ -429,6 +429,9 @@ ERROR_AGENCY_INFORM_MUST_CONTAIN_TERM,20012,"Inform message must contain uint pa
 ERROR_AGENCY_INFORM_MUST_CONTAIN_ID,20013,"Inform message must contain string parameter 'id'","The inform message in the agency must contain a string parameter 'id'."
 ERROR_AGENCY_INFORM_MUST_CONTAIN_ACTIVE,20014,"Inform message must contain array 'active'","The inform message in the agency must contain an array 'active'."
 ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL,20015,"Inform message must contain object 'pool'","The inform message in the agency must contain an object 'pool'."
+ERROR_AGENCY_INFORM_MUST_CONTAIN_MIN_PING,20016,"Inform message must contain object 'min ping'","The inform message in the agency must contain an object 'min ping'."
+ERROR_AGENCY_INFORM_MUST_CONTAIN_MAX_PING,20017,"Inform message must contain object 'max ping'","The inform message in the agency must contain an object 'max ping'."
+ERROR_AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT,20018,"Inform message must contain object 'timeoutMult'","The inform message in the agency must contain an object 'timeoutMult'."
 ERROR_AGENCY_INQUIRE_CLIENT_ID_MUST_BE_STRING,20020,"Inquiry failed","Inquiry by clientId failed"
 ERROR_AGENCY_CANNOT_REBUILD_DBS,20021,"Cannot rebuild readDB and spearHead","Will be raised if the readDB or the spearHead cannot be rebuilt from the replicated log."
 

lib/Basics/voc-errors.cpp

@@ -303,6 +303,9 @@ void TRI_InitializeErrorMessages () {
   REG_ERROR(ERROR_AGENCY_INFORM_MUST_CONTAIN_ID, "Inform message must contain string parameter 'id'");
   REG_ERROR(ERROR_AGENCY_INFORM_MUST_CONTAIN_ACTIVE, "Inform message must contain array 'active'");
   REG_ERROR(ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL, "Inform message must contain object 'pool'");
+  REG_ERROR(ERROR_AGENCY_INFORM_MUST_CONTAIN_MIN_PING, "Inform message must contain object 'min ping'");
+  REG_ERROR(ERROR_AGENCY_INFORM_MUST_CONTAIN_MAX_PING, "Inform message must contain object 'max ping'");
+  REG_ERROR(ERROR_AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT, "Inform message must contain object 'timeoutMult'");
   REG_ERROR(ERROR_AGENCY_INQUIRE_CLIENT_ID_MUST_BE_STRING, "Inquiry failed");
   REG_ERROR(ERROR_AGENCY_CANNOT_REBUILD_DBS, "Cannot rebuild readDB and spearHead");
   REG_ERROR(ERROR_SUPERVISION_GENERAL_FAILURE, "general supervision failure");

lib/Basics/voc-errors.h

@@ -720,6 +720,12 @@
 ///   The inform message in the agency must contain an array 'active'.
 /// - 20015: @LIT{Inform message must contain object 'pool'}
 ///   The inform message in the agency must contain an object 'pool'.
+/// - 20016: @LIT{Inform message must contain object 'min ping'}
+///   The inform message in the agency must contain an object 'min ping'.
+/// - 20017: @LIT{Inform message must contain object 'max ping'}
+///   The inform message in the agency must contain an object 'max ping'.
+/// - 20018: @LIT{Inform message must contain object 'timeoutMult'}
+///   The inform message in the agency must contain an object 'timeoutMult'.
 /// - 20020: @LIT{Inquiry failed}
 ///   Inquiry by clientId failed
 /// - 20021: @LIT{Cannot rebuild readDB and spearHead}
@@ -3829,6 +3835,36 @@ void TRI_InitializeErrorMessages ();
 
 #define TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_POOL                         (20015)
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief 20016: ERROR_AGENCY_INFORM_MUST_CONTAIN_MIN_PING
+///
+/// Inform message must contain object 'min ping'
+///
+/// The inform message in the agency must contain an object 'min ping'.
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_MIN_PING                     (20016)
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief 20017: ERROR_AGENCY_INFORM_MUST_CONTAIN_MAX_PING
+///
+/// Inform message must contain object 'max ping'
+///
+/// The inform message in the agency must contain an object 'max ping'.
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_MAX_PING                     (20017)
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief 20018: ERROR_AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT
+///
+/// Inform message must contain object 'timeoutMult'
+///
+/// The inform message in the agency must contain an object 'timeoutMult'.
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_ERROR_AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT                 (20018)
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief 20020: ERROR_AGENCY_INQUIRE_CLIENT_ID_MUST_BE_STRING
 ///