mirror of https://gitee.com/bigwinds/arangodb
Improved RocksDB engine statistics with docs and rest endpoint (#2598)
* Improved statistics reporting for RocksDB engine (including cache info). * Added documentation about db._engineStats(). * Added rest endpoint for engine stats.
This commit is contained in:
parent
9b53d3cd45
commit
a0c4e3d9df
|
@ -150,7 +150,7 @@ If no initial users are specified, a default user *root* will be created
|
|||
with an empty string password. This ensures that the new database will be
|
||||
accessible via HTTP after it is created.
|
||||
|
||||
You can create users in a database if no initial user is specified. Switch
|
||||
You can create users in a database if no initial user is specified. Switch
|
||||
into the new database (username and password must be identical to the current
|
||||
session) and add or modify users with the following commands.
|
||||
|
||||
|
@ -184,3 +184,12 @@ database. The *_system* database itself cannot be dropped.
|
|||
Databases are dropped asynchronously, and will be physically removed if
|
||||
all clients have disconnected and references have been garbage-collected.
|
||||
|
||||
### Engine statistics
|
||||
|
||||
retrieve statistics related to the storage engine-rocksdb
|
||||
`db._engineStats()`
|
||||
|
||||
Returns some statistics related to storage engine activity, including figures
|
||||
about data size, cache usage, etc.
|
||||
|
||||
**Note**: Currently this only produces useful output for the RocksDB engine.
|
||||
|
|
|
@ -257,8 +257,8 @@ std::pair<double, double> Manager::globalHitRates() {
|
|||
uint64_t currentHits = _findHits.load();
|
||||
uint64_t currentMisses = _findMisses.load();
|
||||
if (currentHits + currentMisses > 0) {
|
||||
lifetimeRate = 100 * (static_cast<double>(currentHits) /
|
||||
static_cast<double>(currentHits + currentMisses));
|
||||
lifetimeRate = 100.0 * (static_cast<double>(currentHits) /
|
||||
static_cast<double>(currentHits + currentMisses));
|
||||
}
|
||||
|
||||
if (_enableWindowedStats && _findStats.get() != nullptr) {
|
||||
|
@ -278,8 +278,9 @@ std::pair<double, double> Manager::globalHitRates() {
|
|||
currentMisses = (*stats)[0].second;
|
||||
}
|
||||
if (currentHits + currentMisses > 0) {
|
||||
windowedRate = 100 * (static_cast<double>(currentHits) /
|
||||
static_cast<double>(currentHits + currentMisses));
|
||||
windowedRate =
|
||||
100.0 * (static_cast<double>(currentHits) /
|
||||
static_cast<double>(currentHits + currentMisses));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -750,8 +751,8 @@ std::shared_ptr<Manager::PriorityList> Manager::priorityList() {
|
|||
double baseWeight = std::max(minimumWeight, uniformMarginalWeight);
|
||||
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
|
||||
if (1.0 < (baseWeight * static_cast<double>(_caches.size()))) {
|
||||
LOG_TOPIC(FATAL, Logger::FIXME) << "weight: " << baseWeight
|
||||
<< ", count: " << _caches.size();
|
||||
LOG_TOPIC(FATAL, Logger::FIXME)
|
||||
<< "weight: " << baseWeight << ", count: " << _caches.size();
|
||||
TRI_ASSERT(1.0 >= (baseWeight * static_cast<double>(_caches.size())));
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -443,11 +443,11 @@ void GeneralServerFeature::defineHandlers() {
|
|||
AsyncJobManager*>,
|
||||
_jobManager.get());
|
||||
|
||||
_handlerFactory->addHandler(
|
||||
"/_api/version", RestHandlerCreator<RestVersionHandler>::createNoData);
|
||||
_handlerFactory->addPrefixHandler(
|
||||
"/_api/engine", RestHandlerCreator<RestEngineHandler>::createNoData);
|
||||
|
||||
_handlerFactory->addHandler(
|
||||
"/_api/engine", RestHandlerCreator<RestEngineHandler>::createNoData);
|
||||
"/_api/version", RestHandlerCreator<RestVersionHandler>::createNoData);
|
||||
|
||||
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
|
||||
_handlerFactory->addHandler(
|
||||
|
|
|
@ -38,10 +38,51 @@ RestEngineHandler::RestEngineHandler(GeneralRequest* request,
|
|||
: RestBaseHandler(request, response) {}
|
||||
|
||||
RestStatus RestEngineHandler::execute() {
|
||||
// extract the sub-request type
|
||||
auto const type = _request->requestType();
|
||||
|
||||
if (type == rest::RequestType::GET) {
|
||||
handleGet();
|
||||
return RestStatus::DONE;
|
||||
}
|
||||
|
||||
generateError(rest::ResponseCode::METHOD_NOT_ALLOWED,
|
||||
TRI_ERROR_HTTP_METHOD_NOT_ALLOWED);
|
||||
return RestStatus::DONE;
|
||||
}
|
||||
|
||||
void RestEngineHandler::handleGet() {
|
||||
std::vector<std::string> const& suffixes = _request->suffixes();
|
||||
|
||||
if (suffixes.size() > 1 || (suffixes.size() == 1 && suffixes[0] != "stats")) {
|
||||
generateError(rest::ResponseCode::BAD, TRI_ERROR_BAD_PARAMETER,
|
||||
"expecting GET /_api/engine[/stats]");
|
||||
return;
|
||||
}
|
||||
|
||||
if (suffixes.size() == 0) {
|
||||
getCapabilities();
|
||||
return;
|
||||
}
|
||||
|
||||
getStats();
|
||||
return;
|
||||
}
|
||||
|
||||
void RestEngineHandler::getCapabilities() {
|
||||
VPackBuilder result;
|
||||
StorageEngine* engine = EngineSelectorFeature::ENGINE;
|
||||
engine->getCapabilities(result);
|
||||
|
||||
|
||||
generateResult(rest::ResponseCode::OK, result.slice());
|
||||
return RestStatus::DONE;
|
||||
return;
|
||||
}
|
||||
|
||||
void RestEngineHandler::getStats() {
|
||||
VPackBuilder result;
|
||||
StorageEngine* engine = EngineSelectorFeature::ENGINE;
|
||||
engine->getStatistics(result);
|
||||
|
||||
generateResult(rest::ResponseCode::OK, result.slice());
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -35,6 +35,11 @@ class RestEngineHandler : public arangodb::RestBaseHandler {
|
|||
char const* name() const override final { return "RestEngineHandler"; }
|
||||
bool isDirect() const override { return true; }
|
||||
RestStatus execute() override;
|
||||
|
||||
protected:
|
||||
void handleGet();
|
||||
void getCapabilities();
|
||||
void getStats();
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
@ -34,6 +34,8 @@
|
|||
#include "Basics/VelocyPackHelper.h"
|
||||
#include "Basics/WriteLocker.h"
|
||||
#include "Basics/build.h"
|
||||
#include "Cache/CacheManagerFeature.h"
|
||||
#include "Cache/Manager.h"
|
||||
#include "GeneralServer/RestHandlerFactory.h"
|
||||
#include "Logger/Logger.h"
|
||||
#include "ProgramOptions/ProgramOptions.h"
|
||||
|
@ -150,7 +152,7 @@ void RocksDBEngine::collectOptions(
|
|||
"timeout after which unused WAL files are deleted",
|
||||
new DoubleParameter(&_pruneWaitTime));
|
||||
|
||||
#ifdef USE_ENTERPRISE
|
||||
#ifdef USE_ENTERPRISE
|
||||
collectEnterpriseOptions(options);
|
||||
#endif
|
||||
}
|
||||
|
@ -158,7 +160,7 @@ void RocksDBEngine::collectOptions(
|
|||
// validate the storage engine's specific options
|
||||
void RocksDBEngine::validateOptions(std::shared_ptr<options::ProgramOptions> options) {
|
||||
transaction::Options::setLimits(_maxTransactionSize, _intermediateCommitSize, _intermediateCommitCount);
|
||||
#ifdef USE_ENTERPRISE
|
||||
#ifdef USE_ENTERPRISE
|
||||
validateEnterpriseOptions(options);
|
||||
#endif
|
||||
}
|
||||
|
@ -1414,29 +1416,52 @@ RocksDBReplicationManager* RocksDBEngine::replicationManager() const {
|
|||
void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
|
||||
builder.openObject();
|
||||
// add int properties
|
||||
auto c1 = [&](std::string const& s) {
|
||||
auto addInt = [&](std::string const& s) {
|
||||
std::string v;
|
||||
if (_db->GetProperty(s, &v)) {
|
||||
int64_t i = basics::StringUtils::int64(v);
|
||||
builder.add(s, VPackValue(i));
|
||||
}
|
||||
};
|
||||
auto addStr = [&](std::string const& s) {
|
||||
std::string v;
|
||||
if (_db->GetProperty(s, &v)) {
|
||||
builder.add(s, VPackValue(v));
|
||||
}
|
||||
};
|
||||
c1(rocksdb::DB::Properties::kNumImmutableMemTable);
|
||||
c1(rocksdb::DB::Properties::kMemTableFlushPending);
|
||||
c1(rocksdb::DB::Properties::kCompactionPending);
|
||||
c1(rocksdb::DB::Properties::kBackgroundErrors);
|
||||
c1(rocksdb::DB::Properties::kCurSizeActiveMemTable);
|
||||
c1(rocksdb::DB::Properties::kCurSizeAllMemTables);
|
||||
c1(rocksdb::DB::Properties::kSizeAllMemTables);
|
||||
c1(rocksdb::DB::Properties::kNumEntriesImmMemTables);
|
||||
c1(rocksdb::DB::Properties::kNumSnapshots);
|
||||
c1(rocksdb::DB::Properties::kDBStats);
|
||||
c1(rocksdb::DB::Properties::kCFStats);
|
||||
c1(rocksdb::DB::Properties::kSSTables);
|
||||
c1(rocksdb::DB::Properties::kNumRunningCompactions);
|
||||
c1(rocksdb::DB::Properties::kNumRunningFlushes);
|
||||
c1(rocksdb::DB::Properties::kIsFileDeletionsEnabled);
|
||||
c1(rocksdb::DB::Properties::kBaseLevel);
|
||||
c1(rocksdb::DB::Properties::kTotalSstFilesSize);
|
||||
auto addCf = [&](std::string const& name, rocksdb::ColumnFamilyHandle* c) {
|
||||
std::string v;
|
||||
if (_db->GetProperty(c, rocksdb::DB::Properties::kCFStats, &v)) {
|
||||
builder.add(name, VPackValue(v));
|
||||
}
|
||||
};
|
||||
addInt(rocksdb::DB::Properties::kNumImmutableMemTable);
|
||||
addInt(rocksdb::DB::Properties::kMemTableFlushPending);
|
||||
addInt(rocksdb::DB::Properties::kCompactionPending);
|
||||
addInt(rocksdb::DB::Properties::kBackgroundErrors);
|
||||
addInt(rocksdb::DB::Properties::kCurSizeActiveMemTable);
|
||||
addInt(rocksdb::DB::Properties::kCurSizeAllMemTables);
|
||||
addInt(rocksdb::DB::Properties::kSizeAllMemTables);
|
||||
addInt(rocksdb::DB::Properties::kNumEntriesImmMemTables);
|
||||
addInt(rocksdb::DB::Properties::kNumSnapshots);
|
||||
addStr(rocksdb::DB::Properties::kDBStats);
|
||||
addStr(rocksdb::DB::Properties::kSSTables);
|
||||
addInt(rocksdb::DB::Properties::kNumRunningCompactions);
|
||||
addInt(rocksdb::DB::Properties::kNumRunningFlushes);
|
||||
addInt(rocksdb::DB::Properties::kIsFileDeletionsEnabled);
|
||||
addInt(rocksdb::DB::Properties::kBaseLevel);
|
||||
addInt(rocksdb::DB::Properties::kTotalSstFilesSize);
|
||||
builder.add("rocksdb.cfstats", VPackValue(VPackValueType::Object));
|
||||
addCf("other", RocksDBColumnFamily::other());
|
||||
addCf("documents", RocksDBColumnFamily::documents());
|
||||
addCf("primary", RocksDBColumnFamily::primary());
|
||||
addCf("edge", RocksDBColumnFamily::edge());
|
||||
addCf("geo", RocksDBColumnFamily::geo());
|
||||
addCf("fulltext", RocksDBColumnFamily::fulltext());
|
||||
addCf("index", RocksDBColumnFamily::index());
|
||||
addCf("uniqueIndex", RocksDBColumnFamily::uniqueIndex());
|
||||
addCf("views", RocksDBColumnFamily::views());
|
||||
builder.close();
|
||||
|
||||
if (_options.table_factory) {
|
||||
void* options = _options.table_factory->GetOptions();
|
||||
|
@ -1445,6 +1470,13 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
|
|||
}
|
||||
}
|
||||
|
||||
cache::Manager* manager = CacheManagerFeature::MANAGER;
|
||||
auto rates = manager->globalHitRates();
|
||||
builder.add("cache.size", VPackValue(manager->globalLimit()));
|
||||
builder.add("cache.used", VPackValue(manager->globalAllocation()));
|
||||
builder.add("cache.hit-rate-lifetime", VPackValue(rates.first));
|
||||
builder.add("cache.hit-rate-recent", VPackValue(rates.second));
|
||||
|
||||
builder.close();
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue