back port PR 9874 to 3.5. Add write amplification stat and option to disable statistics history. (#9947)

CHANGELOG

@@ -1,6 +1,11 @@
 v3.5.1 (XXXX-XX-XX)
 -------------------
 
+* Add --server.statistics-history flag to allow disabling of only the historical
+  statistics.  Also added rocksdbengine.write.amplification.x100 statistics
+  for measurement of compaction option impact.  Enabled non-historical
+  statistics for agents.
+
 * Fixed AQL constrained-heap sort in conjunction with fullCount.
 
 * Added support for AQL expressions such as `a NOT LIKE b`, `a NOT =~ b` and

arangod/Agency/AgencyFeature.cpp

@@ -234,12 +234,11 @@ void AgencyFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
   // - ArangoSearch: not needed by agency even if MMFiles is the selected
   //   storage engine
   // - IResearchAnalyzer: analyzers are not needed by agency
-  // - Statistics: turn off statistics gathering for agency
   // - Action/Script/FoxxQueues/Frontend: Foxx and JavaScript APIs
 
   std::vector<std::string> disabledFeatures({
     "MMFilesPersistentIndex", "ArangoSearch", "ArangoSearchAnalyzer",
-    "Statistics", "Action", "Script", "FoxxQueues", "Frontend"});
+    "Action", "Script", "FoxxQueues", "Frontend"});
 
   if (!result.touched("console") || !*(options->get<BooleanParameter>("console")->ptr)) {
     // specifiying --console requires JavaScript, so we can only turn it off

arangod/RocksDBEngine/RocksDBEngine.cpp

@@ -2235,6 +2235,14 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
     for (auto const& stat : rocksdb::TickersNameMap) {
       builder.add(stat.second, VPackValue(_options.statistics->getTickerCount(stat.first)));
     }
+
+    uint64_t walWrite, flushWrite, compactionWrite, userWrite;
+    walWrite = _options.statistics->getTickerCount(rocksdb::WAL_FILE_BYTES);
+    flushWrite = _options.statistics->getTickerCount(rocksdb::FLUSH_WRITE_BYTES);
+    compactionWrite = _options.statistics->getTickerCount(rocksdb::COMPACT_WRITE_BYTES);
+    userWrite = _options.statistics->getTickerCount(rocksdb::BYTES_WRITTEN);
+    builder.add("rocksdbengine.write.amplification.x100",
+                VPackValue( (0 != userWrite) ? ((walWrite+flushWrite+compactionWrite)*100)/userWrite : 100));
   }
 
   cache::Manager* manager = CacheManagerFeature::MANAGER;

arangod/Statistics/StatisticsFeature.cpp

@@ -21,6 +21,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "StatisticsFeature.h"
+#include "Cluster/ServerState.h"
 #include "Logger/Logger.h"
 #include "ProgramOptions/ProgramOptions.h"
 #include "ProgramOptions/Section.h"
@@ -127,6 +128,8 @@ StatisticsFeature* StatisticsFeature::STATISTICS = nullptr;
 StatisticsFeature::StatisticsFeature(application_features::ApplicationServer& server)
     : ApplicationFeature(server, "Statistics"),
       _statistics(true),
+      _statisticsHistory(true),
+      _statisticsHistoryTouched(false),
       _descriptions(new stats::Descriptions()) {
   startsAfter("AQLPhase");
   setOptional(true);
@@ -141,13 +144,22 @@ void StatisticsFeature::collectOptions(std::shared_ptr<ProgramOptions> options)
                      "turn statistics gathering on or off",
                      new BooleanParameter(&_statistics),
                      arangodb::options::makeFlags(arangodb::options::Flags::Hidden));
+  options->addOption("--server.statistics-history",
+                     "turn storing statistics in database on or off",
+                     new BooleanParameter(&_statisticsHistory),
+                     arangodb::options::makeFlags(arangodb::options::Flags::Hidden))
+    .setIntroducedIn(30409)
+    .setIntroducedIn(30501);
 }
 
-void StatisticsFeature::validateOptions(std::shared_ptr<ProgramOptions>) {
+void StatisticsFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
   if (!_statistics) {
     // turn ourselves off
     disable();
   }
+
+  _statisticsHistoryTouched = options->processingResult().touched("--server.statistics-history");
+
 }
 
 void StatisticsFeature::prepare() {
@@ -181,7 +193,6 @@ void StatisticsFeature::start() {
   }
 
   _statisticsThread.reset(new StatisticsThread);
-  _statisticsWorker.reset(new StatisticsWorker(*vocbase));
 
   if (!_statisticsThread->start()) {
     LOG_TOPIC("46b0c", FATAL, arangodb::Logger::STATISTICS)
@@ -189,11 +200,20 @@ void StatisticsFeature::start() {
     FATAL_ERROR_EXIT();
   }
 
+  // force history disable on Agents
+  if (arangodb::ServerState::instance()->isAgent() && !_statisticsHistoryTouched) {
+    _statisticsHistory = false;
+  } // if
+
+  if (_statisticsHistory) {
+    _statisticsWorker.reset(new StatisticsWorker(*vocbase));
+
     if (!_statisticsWorker->start()) {
       LOG_TOPIC("6ecdc", FATAL, arangodb::Logger::STATISTICS)
         << "could not start statistics worker";
       FATAL_ERROR_EXIT();
     }
+  } // if
 }
 
 void StatisticsFeature::stop() {

arangod/Statistics/StatisticsFeature.h

@@ -93,6 +93,8 @@ class StatisticsFeature final : public application_features::ApplicationFeature
 
  private:
   bool _statistics;
+  bool _statisticsHistory;
+  bool _statisticsHistoryTouched;
 
   std::unique_ptr<stats::Descriptions> _descriptions;
   std::unique_ptr<StatisticsThread> _statisticsThread;