1
0
Fork 0

Bugfix 3.4: Calculate Write Amplification / Flag to disable StatisticsWorker (#9874)

* release version 3.4.8

* [3.4] fix agency lockup when removing 404-ed callbacks (#9839)

* add write amplification statistic

* add --server.statistics-history flag that blocks StatisticsWorker

* no longer disable statistics for Agency

* update for statistics changes

* correct grammar

* correct grammar and only force statistical history off on Agent when option not explicitly used.

* correct grammar and only force statistical history off on Agent when option not explicitly used.

* correct variable naming
This commit is contained in:
Matthew Von-Maszewski 2019-09-11 05:54:33 -04:00 committed by KVS85
parent f9fdc39340
commit d1d2edba57
5 changed files with 43 additions and 9 deletions

View File

@ -1,6 +1,11 @@
v3.4.9 (XXXX-XX-XX)
-------------------
* Add --server.statistics-history flag to allow disabling of only the historical
statistics. Also added rocksdbengine.write.amplification.x100 statistics
for measurement of compaction option impact. Enabled non-historical
statistics for agents.
* Fixed output of `_listObjects()` function of "general-graph" and "smart-graph"
modules, which did not print the `_id` values of the return graphs correctly.

View File

@ -233,11 +233,10 @@ void AgencyFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
// the selected storage engine
// - ArangoSearch: not needed by agency even if MMFiles is the selected
// storage engine
// - Statistics: turn off statistics gathering for agency
// - Action/Script/FoxxQueues/Frontend: Foxx and JavaScript APIs
std::vector<std::string> disabledFeatures(
{"MMFilesPersistentIndex", "ArangoSearch", "Statistics", "Action",
{"MMFilesPersistentIndex", "ArangoSearch", "Action",
"Script", "FoxxQueues", "Frontend"});
if (!result.touched("console") || !*(options->get<BooleanParameter>("console")->ptr)) {
// specifiying --console requires JavaScript, so we can only turn it off

View File

@ -2229,6 +2229,14 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
for (auto const& stat : rocksdb::TickersNameMap) {
builder.add(stat.second, VPackValue(_options.statistics->getTickerCount(stat.first)));
}
uint64_t walWrite, flushWrite, compactionWrite, userWrite;
walWrite = _options.statistics->getTickerCount(rocksdb::WAL_FILE_BYTES);
flushWrite = _options.statistics->getTickerCount(rocksdb::FLUSH_WRITE_BYTES);
compactionWrite = _options.statistics->getTickerCount(rocksdb::COMPACT_WRITE_BYTES);
userWrite = _options.statistics->getTickerCount(rocksdb::BYTES_WRITTEN);
builder.add("rocksdbengine.write.amplification.x100",
VPackValue( (0 != userWrite) ? ((walWrite+flushWrite+compactionWrite)*100)/userWrite : 100));
}
cache::Manager* manager = CacheManagerFeature::MANAGER;

View File

@ -21,6 +21,7 @@
////////////////////////////////////////////////////////////////////////////////
#include "StatisticsFeature.h"
#include "Cluster/ServerState.h"
#include "Logger/Logger.h"
#include "ProgramOptions/ProgramOptions.h"
#include "ProgramOptions/Section.h"
@ -124,6 +125,8 @@ StatisticsFeature* StatisticsFeature::STATISTICS = nullptr;
StatisticsFeature::StatisticsFeature(application_features::ApplicationServer& server)
: ApplicationFeature(server, "Statistics"),
_statistics(true),
_statisticsHistory(true),
_statisticsHistoryTouched(false),
_descriptions(new stats::Descriptions()) {
startsAfter("AQLPhase");
setOptional(true);
@ -138,13 +141,22 @@ void StatisticsFeature::collectOptions(std::shared_ptr<ProgramOptions> options)
"turn statistics gathering on or off",
new BooleanParameter(&_statistics),
arangodb::options::makeFlags(arangodb::options::Flags::Hidden));
options->addOption("--server.statistics-history",
"turn storing statistics in database on or off",
new BooleanParameter(&_statisticsHistory),
arangodb::options::makeFlags(arangodb::options::Flags::Hidden))
.setIntroducedIn(30409)
.setIntroducedIn(30501);
}
void StatisticsFeature::validateOptions(std::shared_ptr<ProgramOptions>) {
void StatisticsFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
if (!_statistics) {
// turn ourselves off
disable();
}
_statisticsHistoryTouched = options->processingResult().touched("--server.statistics-history");
}
void StatisticsFeature::prepare() {
@ -186,7 +198,6 @@ void StatisticsFeature::start() {
}
_statisticsThread.reset(new StatisticsThread);
_statisticsWorker.reset(new StatisticsWorker(*vocbase));
if (!_statisticsThread->start()) {
LOG_TOPIC(FATAL, arangodb::Logger::STATISTICS)
@ -194,11 +205,20 @@ void StatisticsFeature::start() {
FATAL_ERROR_EXIT();
}
if (!_statisticsWorker->start()) {
LOG_TOPIC(FATAL, arangodb::Logger::STATISTICS)
// force history disable on Agents
if (arangodb::ServerState::instance()->isAgent() && !_statisticsHistoryTouched) {
_statisticsHistory = false;
} // if
if (_statisticsHistory) {
_statisticsWorker.reset(new StatisticsWorker(*vocbase));
if (!_statisticsWorker->start()) {
LOG_TOPIC(FATAL, arangodb::Logger::STATISTICS)
<< "could not start statistics worker";
FATAL_ERROR_EXIT();
}
FATAL_ERROR_EXIT();
}
} // if
}
void StatisticsFeature::unprepare() {

View File

@ -85,6 +85,8 @@ class StatisticsFeature final : public application_features::ApplicationFeature
private:
bool _statistics;
bool _statisticsHistory;
bool _statisticsHistoryTouched;
std::unique_ptr<stats::Descriptions> _descriptions;
std::unique_ptr<StatisticsThread> _statisticsThread;
@ -93,4 +95,4 @@ class StatisticsFeature final : public application_features::ApplicationFeature
} // namespace arangodb
#endif
#endif