[3.5] Minimize unnecessary dropping of followers due to poorly set synchronous replication timeouts (#9810)

* Minimize unnecessary dropping of followers due to poorly set synchronous replication timeouts.

* Add changelog entry.

* Add setIntroducedIn versions.

CHANGELOG

@@ -1,6 +1,10 @@
 v3.5.1 (XXXX-XX-XX)
 -------------------
 
+* Make minimum timeout for synchronous replication configurable via parameter
+  (--cluster.synchronous-replication-timeout-minimum) and increase default value
+  to prevent dropping followers unnecessarily.
+
 * Added support for TLS 1.3 for the arangod server and the client tools.
 
   The default TLS protocol for the arangod server is still TLS 1.2 however, in order

arangod/Cluster/ReplicationTimeoutFeature.cpp

@@ -31,7 +31,7 @@ namespace arangodb {
 
 double ReplicationTimeoutFeature::timeoutFactor = 1.0;
 double ReplicationTimeoutFeature::timeoutPer4k = 0.1;
-double ReplicationTimeoutFeature::lowerLimit = 0.5;
+double ReplicationTimeoutFeature::lowerLimit = 30.0;
 
 ReplicationTimeoutFeature::ReplicationTimeoutFeature(application_features::ApplicationServer& server)
     : ApplicationFeature(server, "ReplicationTimeout") {
@@ -42,6 +42,14 @@ ReplicationTimeoutFeature::ReplicationTimeoutFeature(application_features::Appli
 void ReplicationTimeoutFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
   options->addSection("cluster", "Configure the cluster");
 
+  options
+      ->addOption("--cluster.synchronous-replication-timeout-minimum",
+                  "all synchronous replication timeouts will be at least "
+                  "this value (in seconds)",
+                  new DoubleParameter(&lowerLimit))
+      .setIntroducedIn(30501)
+      .setIntroducedIn(30408);
+
   options->addOption(
       "--cluster.synchronous-replication-timeout-factor",
       "all synchronous replication timeouts are multiplied by this factor",
@@ -55,10 +63,4 @@ void ReplicationTimeoutFeature::collectOptions(std::shared_ptr<ProgramOptions> o
       arangodb::options::makeFlags(arangodb::options::Flags::Hidden));
 }
 
-void ReplicationTimeoutFeature::prepare() {
-  // set minimum timeout. this depends on the selected storage engine
-  TRI_ASSERT(EngineSelectorFeature::ENGINE != nullptr);
-  lowerLimit = EngineSelectorFeature::ENGINE->minimumSyncReplicationTimeout();
-}
-
 }  // namespace arangodb

arangod/Cluster/ReplicationTimeoutFeature.h

@@ -35,7 +35,6 @@ class ReplicationTimeoutFeature : public application_features::ApplicationFeatur
   explicit ReplicationTimeoutFeature(application_features::ApplicationServer& server);
 
   void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;
-  void prepare() override final;
 
   static double timeoutFactor;
   static double timeoutPer4k;

arangod/ClusterEngine/ClusterEngine.h

@@ -63,9 +63,6 @@ class ClusterEngine final : public StorageEngine {
   void prepare() override;
   void start() override;
 
-  // minimum timeout for the synchronous replication
-  double minimumSyncReplicationTimeout() const override { return 1.0; }
-
   bool supportsDfdb() const override { return false; }
   bool useRawDocumentPointers() override { return false; }
 

arangod/MMFiles/MMFilesEngine.h

@@ -80,9 +80,6 @@ class MMFilesEngine final : public StorageEngine {
   // flush wal wait for collector
   void stop() override;
 
-  // minimum timeout for the synchronous replication
-  double minimumSyncReplicationTimeout() const override { return 0.5; }
-
   bool supportsDfdb() const override { return true; }
 
   bool useRawDocumentPointers() override { return true; }

arangod/RocksDBEngine/RocksDBEngine.h

@@ -143,9 +143,6 @@ class RocksDBEngine final : public StorageEngine {
   void stop() override;
   void unprepare() override;
 
-  // minimum timeout for the synchronous replication
-  double minimumSyncReplicationTimeout() const override { return 1.0; }
-
   bool supportsDfdb() const override { return false; }
   bool useRawDocumentPointers() override { return false; }
 

arangod/StorageEngine/StorageEngine.h

@@ -114,9 +114,6 @@ class StorageEngine : public application_features::ApplicationFeature {
   virtual std::unique_ptr<PhysicalCollection> createPhysicalCollection(
       LogicalCollection& collection, velocypack::Slice const& info) = 0;
 
-  // minimum timeout for the synchronous replication
-  virtual double minimumSyncReplicationTimeout() const = 0;
-
   // status functionality
   // --------------------
 

tests/Mocks/StorageEngineMock.h

@@ -195,7 +195,6 @@ class StorageEngineMock: public arangodb::StorageEngine {
   virtual arangodb::RecoveryState recoveryState() override;
   virtual TRI_voc_tick_t recoveryTick() override;
   virtual arangodb::Result lastLogger(TRI_vocbase_t& vocbase, std::shared_ptr<arangodb::transaction::Context> transactionContext, uint64_t tickStart, uint64_t tickEnd, std::shared_ptr<VPackBuilder>& builderSPtr) override;
-  virtual double minimumSyncReplicationTimeout() const override { return 1.0; }
   virtual std::unique_ptr<TRI_vocbase_t> openDatabase(arangodb::velocypack::Slice const& args, bool isUpgrade, bool isVersionCheck, int& status) override;
   virtual arangodb::Result persistCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection) override;
   virtual void prepareDropDatabase(TRI_vocbase_t& vocbase, bool useWriteMarker, int& status) override;