1
0
Fork 0

Improved RocksDB engine statistics with docs and rest endpoint (#2598)

* Improved statistics reporting for RocksDB engine (including cache info).

* Added documentation about db._engineStats().

* Added rest endpoint for engine stats.
This commit is contained in:
Dan Larkin 2017-06-18 16:55:25 -04:00 committed by Frank Celler
parent 9b53d3cd45
commit a0c4e3d9df
6 changed files with 120 additions and 32 deletions

View File

@ -150,7 +150,7 @@ If no initial users are specified, a default user *root* will be created
with an empty string password. This ensures that the new database will be
accessible via HTTP after it is created.
You can create users in a database if no initial user is specified. Switch
You can create users in a database if no initial user is specified. Switch
into the new database (username and password must be identical to the current
session) and add or modify users with the following commands.
@ -184,3 +184,12 @@ database. The *_system* database itself cannot be dropped.
Databases are dropped asynchronously, and will be physically removed if
all clients have disconnected and references have been garbage-collected.
### Engine statistics
retrieve statistics related to the storage engine-rocksdb
`db._engineStats()`
Returns some statistics related to storage engine activity, including figures
about data size, cache usage, etc.
**Note**: Currently this only produces useful output for the RocksDB engine.

View File

@ -257,8 +257,8 @@ std::pair<double, double> Manager::globalHitRates() {
uint64_t currentHits = _findHits.load();
uint64_t currentMisses = _findMisses.load();
if (currentHits + currentMisses > 0) {
lifetimeRate = 100 * (static_cast<double>(currentHits) /
static_cast<double>(currentHits + currentMisses));
lifetimeRate = 100.0 * (static_cast<double>(currentHits) /
static_cast<double>(currentHits + currentMisses));
}
if (_enableWindowedStats && _findStats.get() != nullptr) {
@ -278,8 +278,9 @@ std::pair<double, double> Manager::globalHitRates() {
currentMisses = (*stats)[0].second;
}
if (currentHits + currentMisses > 0) {
windowedRate = 100 * (static_cast<double>(currentHits) /
static_cast<double>(currentHits + currentMisses));
windowedRate =
100.0 * (static_cast<double>(currentHits) /
static_cast<double>(currentHits + currentMisses));
}
}
}
@ -750,8 +751,8 @@ std::shared_ptr<Manager::PriorityList> Manager::priorityList() {
double baseWeight = std::max(minimumWeight, uniformMarginalWeight);
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
if (1.0 < (baseWeight * static_cast<double>(_caches.size()))) {
LOG_TOPIC(FATAL, Logger::FIXME) << "weight: " << baseWeight
<< ", count: " << _caches.size();
LOG_TOPIC(FATAL, Logger::FIXME)
<< "weight: " << baseWeight << ", count: " << _caches.size();
TRI_ASSERT(1.0 >= (baseWeight * static_cast<double>(_caches.size())));
}
#endif

View File

@ -443,11 +443,11 @@ void GeneralServerFeature::defineHandlers() {
AsyncJobManager*>,
_jobManager.get());
_handlerFactory->addHandler(
"/_api/version", RestHandlerCreator<RestVersionHandler>::createNoData);
_handlerFactory->addPrefixHandler(
"/_api/engine", RestHandlerCreator<RestEngineHandler>::createNoData);
_handlerFactory->addHandler(
"/_api/engine", RestHandlerCreator<RestEngineHandler>::createNoData);
"/_api/version", RestHandlerCreator<RestVersionHandler>::createNoData);
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
_handlerFactory->addHandler(

View File

@ -38,10 +38,51 @@ RestEngineHandler::RestEngineHandler(GeneralRequest* request,
: RestBaseHandler(request, response) {}
RestStatus RestEngineHandler::execute() {
// extract the sub-request type
auto const type = _request->requestType();
if (type == rest::RequestType::GET) {
handleGet();
return RestStatus::DONE;
}
generateError(rest::ResponseCode::METHOD_NOT_ALLOWED,
TRI_ERROR_HTTP_METHOD_NOT_ALLOWED);
return RestStatus::DONE;
}
void RestEngineHandler::handleGet() {
std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() > 1 || (suffixes.size() == 1 && suffixes[0] != "stats")) {
generateError(rest::ResponseCode::BAD, TRI_ERROR_BAD_PARAMETER,
"expecting GET /_api/engine[/stats]");
return;
}
if (suffixes.size() == 0) {
getCapabilities();
return;
}
getStats();
return;
}
void RestEngineHandler::getCapabilities() {
VPackBuilder result;
StorageEngine* engine = EngineSelectorFeature::ENGINE;
engine->getCapabilities(result);
generateResult(rest::ResponseCode::OK, result.slice());
return RestStatus::DONE;
return;
}
void RestEngineHandler::getStats() {
VPackBuilder result;
StorageEngine* engine = EngineSelectorFeature::ENGINE;
engine->getStatistics(result);
generateResult(rest::ResponseCode::OK, result.slice());
return;
}

View File

@ -35,6 +35,11 @@ class RestEngineHandler : public arangodb::RestBaseHandler {
char const* name() const override final { return "RestEngineHandler"; }
bool isDirect() const override { return true; }
RestStatus execute() override;
protected:
void handleGet();
void getCapabilities();
void getStats();
};
}

View File

@ -34,6 +34,8 @@
#include "Basics/VelocyPackHelper.h"
#include "Basics/WriteLocker.h"
#include "Basics/build.h"
#include "Cache/CacheManagerFeature.h"
#include "Cache/Manager.h"
#include "GeneralServer/RestHandlerFactory.h"
#include "Logger/Logger.h"
#include "ProgramOptions/ProgramOptions.h"
@ -150,7 +152,7 @@ void RocksDBEngine::collectOptions(
"timeout after which unused WAL files are deleted",
new DoubleParameter(&_pruneWaitTime));
#ifdef USE_ENTERPRISE
#ifdef USE_ENTERPRISE
collectEnterpriseOptions(options);
#endif
}
@ -158,7 +160,7 @@ void RocksDBEngine::collectOptions(
// validate the storage engine's specific options
void RocksDBEngine::validateOptions(std::shared_ptr<options::ProgramOptions> options) {
transaction::Options::setLimits(_maxTransactionSize, _intermediateCommitSize, _intermediateCommitCount);
#ifdef USE_ENTERPRISE
#ifdef USE_ENTERPRISE
validateEnterpriseOptions(options);
#endif
}
@ -1414,29 +1416,52 @@ RocksDBReplicationManager* RocksDBEngine::replicationManager() const {
void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
builder.openObject();
// add int properties
auto c1 = [&](std::string const& s) {
auto addInt = [&](std::string const& s) {
std::string v;
if (_db->GetProperty(s, &v)) {
int64_t i = basics::StringUtils::int64(v);
builder.add(s, VPackValue(i));
}
};
auto addStr = [&](std::string const& s) {
std::string v;
if (_db->GetProperty(s, &v)) {
builder.add(s, VPackValue(v));
}
};
c1(rocksdb::DB::Properties::kNumImmutableMemTable);
c1(rocksdb::DB::Properties::kMemTableFlushPending);
c1(rocksdb::DB::Properties::kCompactionPending);
c1(rocksdb::DB::Properties::kBackgroundErrors);
c1(rocksdb::DB::Properties::kCurSizeActiveMemTable);
c1(rocksdb::DB::Properties::kCurSizeAllMemTables);
c1(rocksdb::DB::Properties::kSizeAllMemTables);
c1(rocksdb::DB::Properties::kNumEntriesImmMemTables);
c1(rocksdb::DB::Properties::kNumSnapshots);
c1(rocksdb::DB::Properties::kDBStats);
c1(rocksdb::DB::Properties::kCFStats);
c1(rocksdb::DB::Properties::kSSTables);
c1(rocksdb::DB::Properties::kNumRunningCompactions);
c1(rocksdb::DB::Properties::kNumRunningFlushes);
c1(rocksdb::DB::Properties::kIsFileDeletionsEnabled);
c1(rocksdb::DB::Properties::kBaseLevel);
c1(rocksdb::DB::Properties::kTotalSstFilesSize);
auto addCf = [&](std::string const& name, rocksdb::ColumnFamilyHandle* c) {
std::string v;
if (_db->GetProperty(c, rocksdb::DB::Properties::kCFStats, &v)) {
builder.add(name, VPackValue(v));
}
};
addInt(rocksdb::DB::Properties::kNumImmutableMemTable);
addInt(rocksdb::DB::Properties::kMemTableFlushPending);
addInt(rocksdb::DB::Properties::kCompactionPending);
addInt(rocksdb::DB::Properties::kBackgroundErrors);
addInt(rocksdb::DB::Properties::kCurSizeActiveMemTable);
addInt(rocksdb::DB::Properties::kCurSizeAllMemTables);
addInt(rocksdb::DB::Properties::kSizeAllMemTables);
addInt(rocksdb::DB::Properties::kNumEntriesImmMemTables);
addInt(rocksdb::DB::Properties::kNumSnapshots);
addStr(rocksdb::DB::Properties::kDBStats);
addStr(rocksdb::DB::Properties::kSSTables);
addInt(rocksdb::DB::Properties::kNumRunningCompactions);
addInt(rocksdb::DB::Properties::kNumRunningFlushes);
addInt(rocksdb::DB::Properties::kIsFileDeletionsEnabled);
addInt(rocksdb::DB::Properties::kBaseLevel);
addInt(rocksdb::DB::Properties::kTotalSstFilesSize);
builder.add("rocksdb.cfstats", VPackValue(VPackValueType::Object));
addCf("other", RocksDBColumnFamily::other());
addCf("documents", RocksDBColumnFamily::documents());
addCf("primary", RocksDBColumnFamily::primary());
addCf("edge", RocksDBColumnFamily::edge());
addCf("geo", RocksDBColumnFamily::geo());
addCf("fulltext", RocksDBColumnFamily::fulltext());
addCf("index", RocksDBColumnFamily::index());
addCf("uniqueIndex", RocksDBColumnFamily::uniqueIndex());
addCf("views", RocksDBColumnFamily::views());
builder.close();
if (_options.table_factory) {
void* options = _options.table_factory->GetOptions();
@ -1445,6 +1470,13 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
}
}
cache::Manager* manager = CacheManagerFeature::MANAGER;
auto rates = manager->globalHitRates();
builder.add("cache.size", VPackValue(manager->globalLimit()));
builder.add("cache.used", VPackValue(manager->globalAllocation()));
builder.add("cache.hit-rate-lifetime", VPackValue(rates.first));
builder.add("cache.hit-rate-recent", VPackValue(rates.second));
builder.close();
}