Agent send logs to followers in packs of 250 max for now

arangod/Agency/AgencyFeature.cpp

@@ -50,6 +50,7 @@ AgencyFeature::AgencyFeature(application_features::ApplicationServer* server)
       _supervisionFrequency(1.0),
       _compactionStepSize(20000),
       _compactionKeepSize(10000),
+      _maxAppendSize(250),
       _supervisionGracePeriod(10.0),
       _cmdLineTimings(false)
 {
@@ -121,6 +122,11 @@ void AgencyFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
                            "wait for hard disk syncs on every persistence call "
                            "(required in production)",
                            new BooleanParameter(&_waitForSync));
+
+  options->addHiddenOption("--agency.max-append-size",
+                           "maximum size of appendEntries document (# log entries)",
+                           new UInt64Parameter(&_maxAppendSize));
+
 }
 
 void AgencyFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
@@ -253,7 +259,7 @@ void AgencyFeature::start() {
       _size, _poolSize, _minElectionTimeout, _maxElectionTimeout, endpoint,
       _agencyEndpoints, _supervision, _waitForSync, _supervisionFrequency,
       _compactionStepSize, _compactionKeepSize, _supervisionGracePeriod,
-      _cmdLineTimings)));
+      _cmdLineTimings, _maxAppendSize)));
 
   LOG_TOPIC(DEBUG, Logger::AGENCY) << "Starting agency personality";
   _agent->start();

arangod/Agency/AgencyFeature.h

@@ -55,6 +55,7 @@ class AgencyFeature : virtual public application_features::ApplicationFeature {
   double _supervisionFrequency;
   uint64_t _compactionStepSize;
   uint64_t _compactionKeepSize;
+  uint64_t _maxAppendSize;
   double _supervisionGracePeriod;
   std::string _agencyMyAddress;
   std::vector<std::string> _agencyEndpoints;

arangod/Agency/Agent.cpp

@@ -420,7 +420,8 @@ void Agent::sendAppendEntriesRPC() {
         commitIndex = _commitIndex;
       }
 
-      std::vector<log_t> unconfirmed = _state.get(lastConfirmed);
+      std::vector<log_t> unconfirmed =
+        _state.get(lastConfirmed, lastConfirmed + _config.maxAppendSize());
 
       // Note that dispite compaction this vector can never be empty, since
       // any compaction keeps at least one active log entry!

arangod/Agency/AgentConfiguration.cpp

@@ -43,13 +43,14 @@ config_t::config_t()
       _cmdLineTimings(false),
       _version(0),
       _startup("origin"),
+      _maxAppendSize(250),
       _lock()
       {}
 
 config_t::config_t(size_t as, size_t ps, double minp, double maxp,
                    std::string const& e, std::vector<std::string> const& g,
                    bool s, bool w, double f, uint64_t c, uint64_t k, double p,
-                   bool t)
+                   bool t, size_t a)
     : _agencySize(as),
       _poolSize(ps),
       _minPing(minp),
@@ -64,8 +65,10 @@ config_t::config_t(size_t as, size_t ps, double minp, double maxp,
       _supervisionGracePeriod(p),
       _cmdLineTimings(t),
       _version(0),
-      _startup("origin"),     
-      _lock() {}
+      _startup("origin"),
+      _maxAppendSize(a),
+      _lock()
+      {}
 
 config_t::config_t(config_t const& other) { *this = other; }
 
@@ -87,7 +90,8 @@ config_t::config_t(config_t&& other)
       _supervisionGracePeriod(std::move(other._supervisionGracePeriod)),
       _cmdLineTimings(std::move(other._cmdLineTimings)),
       _version(std::move(other._version)),
-      _startup(std::move(other._startup)){}
+      _startup(std::move(other._startup)),
+      _maxAppendSize(std::move(other._maxAppendSize)){}
 
 config_t& config_t::operator=(config_t const& other) {
   // must hold the lock of other to copy _pool, _minPing, _maxPing etc.
@@ -111,6 +115,7 @@ config_t& config_t::operator=(config_t const& other) {
   _cmdLineTimings = other._cmdLineTimings;
   _version = other._version;
   _startup = other._startup;
+  _maxAppendSize = other._maxAppendSize;
   return *this;
 }
 
@@ -133,6 +138,7 @@ config_t& config_t::operator=(config_t&& other) {
   _cmdLineTimings = std::move(other._cmdLineTimings);
   _version = std::move(other._version);
   _startup = std::move(other._startup);
+  _maxAppendSize = std::move(other._maxAppendSize);
   return *this;
 }
 
@@ -288,6 +294,11 @@ std::string config_t::nextAgentInLine() const {
   return ""; // No one left
 }
 
+size_t config_t::maxAppendSize() const {
+  READ_LOCKER(readLocker, _lock);
+  return _maxAppendSize;
+}
+
 size_t config_t::compactionStepSize() const {
   READ_LOCKER(readLocker, _lock);
   return _compactionStepSize;

arangod/Agency/AgentConfiguration.h

@@ -75,6 +75,7 @@ struct config_t {
   bool _cmdLineTimings;
   size_t _version;
   std::string _startup;
+  size_t _maxAppendSize;
 
   mutable arangodb::basics::ReadWriteLock _lock; // guard member variables
 
@@ -84,7 +85,7 @@ struct config_t {
   /// @brief ctor
   config_t(size_t as, size_t ps, double minp, double maxp, std::string const& e,
            std::vector<std::string> const& g, bool s, bool w, double f,
-           uint64_t c, uint64_t k, double p, bool t);
+           uint64_t c, uint64_t k, double p, bool t, size_t a);
 
   /// @brief copy constructor
   config_t(config_t const&);
@@ -122,6 +123,9 @@ struct config_t {
   /// @brief active agency size
   size_t size() const;
 
+  /// @brief maximum appendEntries #logs
+  size_t maxAppendSize() const;
+
   /// @brief active empty?
   bool activeEmpty() const;