Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bugfix 3.4: Clean up rocksdb getStatistics() (#9703) 

* clean up basic rocksdb statistics.  More clean-up needed.

* add rocksdb throttle value to statistics

* update comments about rocksdb properties

* rename function per PR comments
This commit is contained in:
Matthew Von-Maszewski 2019-08-13 13:10:45 -04:00 committed by GitHub
parent 5c8aee1b67
commit 38b9ec80b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 48 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
arangod/RocksDBEngine/RocksDBEngine.cpp
View File

@ -2130,6 +2130,23 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
} }
}; };
// get string property from each column family and return sum;
auto addIntAllCf = [&](std::string const& s) {
int64_t sum = 0;
for (auto cfh : RocksDBColumnFamily::_allHandles) {
std::string v;
if (_db->GetProperty(cfh, s, &v)) {
int64_t temp=basics::StringUtils::int64(v);
// -1 returned for somethings that are valid property but no value
if (0 < temp) {
sum += temp;
} // if
} // if
} // for
builder.add(s, VPackValue(sum));
};
// add column family properties // add column family properties
auto addCf = [&](std::string const& name, rocksdb::ColumnFamilyHandle* c) { auto addCf = [&](std::string const& name, rocksdb::ColumnFamilyHandle* c) {
std::string v; std::string v;
@ -2157,40 +2174,43 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
builder.openObject(); builder.openObject();
for (int i = 0; i < _options.num_levels; ++i) { for (int i = 0; i < _options.num_levels; ++i) {
addStr(rocksdb::DB::Properties::kNumFilesAtLevelPrefix + std::to_string(i)); addIntAllCf(rocksdb::DB::Properties::kNumFilesAtLevelPrefix + std::to_string(i));
addStr(rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix + std::to_string(i)); // ratio needs new calculation with all cf, not a simple add operation
addIntAllCf(rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix + std::to_string(i));
} }
addInt(rocksdb::DB::Properties::kNumImmutableMemTable); // caution: you must read rocksdb/db/interal_stats.cc carefully to
addInt(rocksdb::DB::Properties::kNumImmutableMemTableFlushed); // determine if a property is for whole database or one column family
addInt(rocksdb::DB::Properties::kMemTableFlushPending); addIntAllCf(rocksdb::DB::Properties::kNumImmutableMemTable);
addInt(rocksdb::DB::Properties::kCompactionPending); addIntAllCf(rocksdb::DB::Properties::kNumImmutableMemTableFlushed);
addIntAllCf(rocksdb::DB::Properties::kMemTableFlushPending);
addIntAllCf(rocksdb::DB::Properties::kCompactionPending);
addInt(rocksdb::DB::Properties::kBackgroundErrors); addInt(rocksdb::DB::Properties::kBackgroundErrors);
addInt(rocksdb::DB::Properties::kCurSizeActiveMemTable); addIntAllCf(rocksdb::DB::Properties::kCurSizeActiveMemTable);
addInt(rocksdb::DB::Properties::kCurSizeAllMemTables); addIntAllCf(rocksdb::DB::Properties::kCurSizeAllMemTables);
addInt(rocksdb::DB::Properties::kSizeAllMemTables); addIntAllCf(rocksdb::DB::Properties::kSizeAllMemTables);
addInt(rocksdb::DB::Properties::kNumEntriesActiveMemTable); addIntAllCf(rocksdb::DB::Properties::kNumEntriesActiveMemTable);
addInt(rocksdb::DB::Properties::kNumEntriesImmMemTables); addIntAllCf(rocksdb::DB::Properties::kNumEntriesImmMemTables);
addInt(rocksdb::DB::Properties::kNumDeletesActiveMemTable); addIntAllCf(rocksdb::DB::Properties::kNumDeletesActiveMemTable);
addInt(rocksdb::DB::Properties::kNumDeletesImmMemTables); addIntAllCf(rocksdb::DB::Properties::kNumDeletesImmMemTables);
addInt(rocksdb::DB::Properties::kEstimateNumKeys); addIntAllCf(rocksdb::DB::Properties::kEstimateNumKeys);
addInt(rocksdb::DB::Properties::kEstimateTableReadersMem); addIntAllCf(rocksdb::DB::Properties::kEstimateTableReadersMem);
addInt(rocksdb::DB::Properties::kNumSnapshots); addInt(rocksdb::DB::Properties::kNumSnapshots);
addInt(rocksdb::DB::Properties::kOldestSnapshotTime); addInt(rocksdb::DB::Properties::kOldestSnapshotTime);
addInt(rocksdb::DB::Properties::kNumLiveVersions); addIntAllCf(rocksdb::DB::Properties::kNumLiveVersions);
addInt(rocksdb::DB::Properties::kMinLogNumberToKeep); addInt(rocksdb::DB::Properties::kMinLogNumberToKeep);
addInt(rocksdb::DB::Properties::kEstimateLiveDataSize); addIntAllCf(rocksdb::DB::Properties::kEstimateLiveDataSize);
addInt(rocksdb::DB::Properties::kLiveSstFilesSize); addIntAllCf(rocksdb::DB::Properties::kLiveSstFilesSize);
addStr(rocksdb::DB::Properties::kDBStats); addStr(rocksdb::DB::Properties::kDBStats);
addStr(rocksdb::DB::Properties::kSSTables); addStr(rocksdb::DB::Properties::kSSTables);
addInt(rocksdb::DB::Properties::kNumRunningCompactions); addInt(rocksdb::DB::Properties::kNumRunningCompactions);
addInt(rocksdb::DB::Properties::kNumRunningFlushes); addInt(rocksdb::DB::Properties::kNumRunningFlushes);
addInt(rocksdb::DB::Properties::kIsFileDeletionsEnabled); addInt(rocksdb::DB::Properties::kIsFileDeletionsEnabled);
addInt(rocksdb::DB::Properties::kEstimatePendingCompactionBytes); addIntAllCf(rocksdb::DB::Properties::kEstimatePendingCompactionBytes);
addInt(rocksdb::DB::Properties::kBaseLevel); addInt(rocksdb::DB::Properties::kBaseLevel);
addInt(rocksdb::DB::Properties::kBlockCacheCapacity); addInt(rocksdb::DB::Properties::kBlockCacheCapacity);
addInt(rocksdb::DB::Properties::kBlockCacheUsage); addInt(rocksdb::DB::Properties::kBlockCacheUsage);
addInt(rocksdb::DB::Properties::kBlockCachePinnedUsage); addInt(rocksdb::DB::Properties::kBlockCachePinnedUsage);
addInt(rocksdb::DB::Properties::kTotalSstFilesSize); addIntAllCf(rocksdb::DB::Properties::kTotalSstFilesSize);
addInt(rocksdb::DB::Properties::kActualDelayedWriteRate); addInt(rocksdb::DB::Properties::kActualDelayedWriteRate);
addInt(rocksdb::DB::Properties::kIsWriteStopped); addInt(rocksdb::DB::Properties::kIsWriteStopped);
@ -2209,6 +2229,7 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
builder.add("cache.hit-rate-recent", VPackValue(rates.second >= 0.0 ? rates.second : 0.0)); builder.add("cache.hit-rate-recent", VPackValue(rates.second >= 0.0 ? rates.second : 0.0));
// print column family statistics // print column family statistics
// warning: output format limits numbers to 3 digits of precision or less.
builder.add("columnFamilies", VPackValue(VPackValueType::Object)); builder.add("columnFamilies", VPackValue(VPackValueType::Object));
addCf("definitions", RocksDBColumnFamily::definitions()); addCf("definitions", RocksDBColumnFamily::definitions());
addCf("documents", RocksDBColumnFamily::documents()); addCf("documents", RocksDBColumnFamily::documents());
@ -2219,6 +2240,10 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
addCf("fulltext", RocksDBColumnFamily::fulltext()); addCf("fulltext", RocksDBColumnFamily::fulltext());
builder.close(); builder.close();
if (_listener) {
builder.add("rocksdbengine.throttle.bps", VPackValue(_listener->GetThrottle()));
} // if
builder.close(); builder.close();
} }

4
arangod/RocksDBEngine/RocksDBThrottle.h
View File

@ -85,6 +85,8 @@ class RocksDBThrottle : public rocksdb::EventListener {
void StopThread(); void StopThread();
uint64_t GetThrottle() const {return _throttleBps;};
protected: protected:
void Startup(rocksdb::DB* db); void Startup(rocksdb::DB* db);
@ -137,7 +139,7 @@ class RocksDBThrottle : public rocksdb::EventListener {
ThrottleData_t _throttleData[THROTTLE_INTERVALS]; ThrottleData_t _throttleData[THROTTLE_INTERVALS];
size_t _replaceIdx; size_t _replaceIdx;
uint64_t _throttleBps; std::atomic<uint64_t> _throttleBps;
bool _firstThrottle; bool _firstThrottle;
std::unique_ptr<WriteControllerToken> _delayToken; std::unique_ptr<WriteControllerToken> _delayToken;