From eb5f4241a5c64d5850ebca46abcee502ff7fab48 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 19 Aug 2019 22:18:57 +0200 Subject: [PATCH] issue #9654 (#9752) --- CHANGELOG | 7 +++++++ arangod/RocksDBEngine/RocksDBEngine.cpp | 20 ++++++++++++++----- lib/ApplicationFeatures/ApplicationServer.h | 3 +++ .../RocksDBOptionFeature.cpp | 2 +- .../RocksDBOptionFeature.h | 1 + 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 62d1f01e8d..91be494aa4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,13 @@ v3.4.8 (XXXX-XX-XX) ------------------- +* Fixed issue #9654: honor value of `--rocksdb.max-write-buffer-number` if it + is set to at least 9 (which is the recommended value). Ignore it if it is + set to a lower value than 9, and warn the end user about it. + + Previous versions of ArangoDB always silently ignored the value of this setting + and effectively hard-coded the value to 9. + * Fixed cut'n'pasting code from the documentation into arangosh. * Refactor maintenance to use a TakeoverShardLeadership job. This fixes a bug diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 301fe1d1f7..5698a0be1e 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -557,9 +557,18 @@ void RocksDBEngine::start() { _options.db_write_buffer_size = opts->_totalWriteBufferSize; } - // this is cfFamilies.size() + 2 ... but _option needs to be set before - // building cfFamilies - _options.max_write_buffer_number = 7 + 2; + if (!application_features::ApplicationServer::server->options()->processingResult().touched("rocksdb.max-write-buffer-number")) { + // user hasn't explicitly set the number of write buffers, so we use a default value based + // on the number of column families + // this is cfFamilies.size() + 2 ... but _option needs to be set before + // building cfFamilies + // Update max_write_buffer_number above if you change number of families used + _options.max_write_buffer_number = 7 + 2; + } else if (_options.max_write_buffer_number < 7 + 2) { + // user set the value explicitly, and it is lower than recommended + LOG_TOPIC(WARN, Logger::ENGINES) << "ignoring value for option `--rocksdb.max-write-buffer-number` because it is lower than recommended"; + _options.max_write_buffer_number = 7 + 2; + } // cf options for definitons (dbs, collections, views, ...) rocksdb::ColumnFamilyOptions definitionsCF(_options); @@ -597,8 +606,9 @@ void RocksDBEngine::start() { cfFamilies.emplace_back("VPackIndex", vpackFixedPrefCF); // 4 cfFamilies.emplace_back("GeoIndex", fixedPrefCF); // 5 cfFamilies.emplace_back("FulltextIndex", fixedPrefCF); // 6 - // DO NOT FORGET TO DESTROY THE CFs ON CLOSE - // Update max_write_buffer_number above if you change number of families used + + TRI_ASSERT(static_cast(_options.max_write_buffer_number) >= static_cast(cfFamilies.size())); + // Update max_write_buffer_number above if you change number of families used // validate sizes of existing RocksDB journal files // RocksDB may just crash when opening a logfile with an unexpected size diff --git a/lib/ApplicationFeatures/ApplicationServer.h b/lib/ApplicationFeatures/ApplicationServer.h index 479fb6ab38..420f247611 100644 --- a/lib/ApplicationFeatures/ApplicationServer.h +++ b/lib/ApplicationFeatures/ApplicationServer.h @@ -229,6 +229,9 @@ class ApplicationServer { // return VPack options VPackBuilder options(std::unordered_set const& excludes) const; + + // return the program options object + std::shared_ptr options() const { return _options; } // return the server state ServerState state() const { return _state; } diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp index 158a814e9a..ec5eb18464 100644 --- a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp +++ b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp @@ -46,7 +46,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(application_features::ApplicationServ _transactionLockTimeout(rocksDBTrxDefaults.transaction_lock_timeout), _totalWriteBufferSize(rocksDBDefaults.db_write_buffer_size), _writeBufferSize(rocksDBDefaults.write_buffer_size), - _maxWriteBufferNumber(rocksDBDefaults.max_write_buffer_number), + _maxWriteBufferNumber(7 + 2), // number of column families plus 2 _maxTotalWalSize(80 << 20), _delayedWriteRate(rocksDBDefaults.delayed_write_rate), _minWriteBufferNumberToMerge(rocksDBDefaults.min_write_buffer_number_to_merge), diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.h b/lib/ApplicationFeatures/RocksDBOptionFeature.h index 9615f50d08..bab1929cbe 100644 --- a/lib/ApplicationFeatures/RocksDBOptionFeature.h +++ b/lib/ApplicationFeatures/RocksDBOptionFeature.h @@ -49,6 +49,7 @@ class RocksDBOptionFeature final : public application_features::ApplicationFeatu std::string _walDirectory; uint64_t _totalWriteBufferSize; uint64_t _writeBufferSize; + // Update max_write_buffer_number above if you change number of families used uint64_t _maxWriteBufferNumber; uint64_t _maxTotalWalSize; uint64_t _delayedWriteRate;