Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bugfix 3.4: Clean up rocksdb getStatistics() (#9703) 

* clean up basic rocksdb statistics.  More clean-up needed.

* add rocksdb throttle value to statistics

* update comments about rocksdb properties

* rename function per PR comments
This commit is contained in:
Matthew Von-Maszewski 2019-08-13 13:10:45 -04:00 committed by GitHub
parent 5c8aee1b67
commit 38b9ec80b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 48 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
arangod/RocksDBEngine/RocksDBEngine.cpp
View File

@ -2130,6 +2130,23 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
}
};
// get string property from each column family and return sum;
auto addIntAllCf = [&](std::string const& s) {
int64_t sum = 0;
for (auto cfh : RocksDBColumnFamily::_allHandles) {
std::string v;
if (_db->GetProperty(cfh, s, &v)) {
int64_t temp=basics::StringUtils::int64(v);
// -1 returned for somethings that are valid property but no value
if (0 < temp) {
sum += temp;
} // if
} // if
} // for
builder.add(s, VPackValue(sum));
};
// add column family properties
auto addCf = [&](std::string const& name, rocksdb::ColumnFamilyHandle* c) {
std::string v;
@ -2157,40 +2174,43 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
builder.openObject();
for (int i = 0; i < _options.num_levels; ++i) {
addStr(rocksdb::DB::Properties::kNumFilesAtLevelPrefix + std::to_string(i));
addStr(rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix + std::to_string(i));
addIntAllCf(rocksdb::DB::Properties::kNumFilesAtLevelPrefix + std::to_string(i));
// ratio needs new calculation with all cf, not a simple add operation
addIntAllCf(rocksdb::DB::Properties::kCompressionRatioAtLevelPrefix + std::to_string(i));
}
addInt(rocksdb::DB::Properties::kNumImmutableMemTable);
addInt(rocksdb::DB::Properties::kNumImmutableMemTableFlushed);
addInt(rocksdb::DB::Properties::kMemTableFlushPending);
addInt(rocksdb::DB::Properties::kCompactionPending);
// caution: you must read rocksdb/db/interal_stats.cc carefully to
// determine if a property is for whole database or one column family
addIntAllCf(rocksdb::DB::Properties::kNumImmutableMemTable);
addIntAllCf(rocksdb::DB::Properties::kNumImmutableMemTableFlushed);
addIntAllCf(rocksdb::DB::Properties::kMemTableFlushPending);
addIntAllCf(rocksdb::DB::Properties::kCompactionPending);
addInt(rocksdb::DB::Properties::kBackgroundErrors);
addInt(rocksdb::DB::Properties::kCurSizeActiveMemTable);
addInt(rocksdb::DB::Properties::kCurSizeAllMemTables);
addInt(rocksdb::DB::Properties::kSizeAllMemTables);
addInt(rocksdb::DB::Properties::kNumEntriesActiveMemTable);
addInt(rocksdb::DB::Properties::kNumEntriesImmMemTables);
addInt(rocksdb::DB::Properties::kNumDeletesActiveMemTable);
addInt(rocksdb::DB::Properties::kNumDeletesImmMemTables);
addInt(rocksdb::DB::Properties::kEstimateNumKeys);
addInt(rocksdb::DB::Properties::kEstimateTableReadersMem);
addIntAllCf(rocksdb::DB::Properties::kCurSizeActiveMemTable);
addIntAllCf(rocksdb::DB::Properties::kCurSizeAllMemTables);
addIntAllCf(rocksdb::DB::Properties::kSizeAllMemTables);
addIntAllCf(rocksdb::DB::Properties::kNumEntriesActiveMemTable);
addIntAllCf(rocksdb::DB::Properties::kNumEntriesImmMemTables);
addIntAllCf(rocksdb::DB::Properties::kNumDeletesActiveMemTable);
addIntAllCf(rocksdb::DB::Properties::kNumDeletesImmMemTables);
addIntAllCf(rocksdb::DB::Properties::kEstimateNumKeys);
addIntAllCf(rocksdb::DB::Properties::kEstimateTableReadersMem);
addInt(rocksdb::DB::Properties::kNumSnapshots);
addInt(rocksdb::DB::Properties::kOldestSnapshotTime);
addInt(rocksdb::DB::Properties::kNumLiveVersions);
addIntAllCf(rocksdb::DB::Properties::kNumLiveVersions);
addInt(rocksdb::DB::Properties::kMinLogNumberToKeep);
addInt(rocksdb::DB::Properties::kEstimateLiveDataSize);
addInt(rocksdb::DB::Properties::kLiveSstFilesSize);
addIntAllCf(rocksdb::DB::Properties::kEstimateLiveDataSize);
addIntAllCf(rocksdb::DB::Properties::kLiveSstFilesSize);
addStr(rocksdb::DB::Properties::kDBStats);
addStr(rocksdb::DB::Properties::kSSTables);
addInt(rocksdb::DB::Properties::kNumRunningCompactions);
addInt(rocksdb::DB::Properties::kNumRunningFlushes);
addInt(rocksdb::DB::Properties::kIsFileDeletionsEnabled);
addInt(rocksdb::DB::Properties::kEstimatePendingCompactionBytes);
addIntAllCf(rocksdb::DB::Properties::kEstimatePendingCompactionBytes);
addInt(rocksdb::DB::Properties::kBaseLevel);
addInt(rocksdb::DB::Properties::kBlockCacheCapacity);
addInt(rocksdb::DB::Properties::kBlockCacheUsage);
addInt(rocksdb::DB::Properties::kBlockCachePinnedUsage);
addInt(rocksdb::DB::Properties::kTotalSstFilesSize);
addIntAllCf(rocksdb::DB::Properties::kTotalSstFilesSize);
addInt(rocksdb::DB::Properties::kActualDelayedWriteRate);
addInt(rocksdb::DB::Properties::kIsWriteStopped);
@ -2209,6 +2229,7 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
builder.add("cache.hit-rate-recent", VPackValue(rates.second >= 0.0 ? rates.second : 0.0));
// print column family statistics
// warning: output format limits numbers to 3 digits of precision or less.
builder.add("columnFamilies", VPackValue(VPackValueType::Object));
addCf("definitions", RocksDBColumnFamily::definitions());
addCf("documents", RocksDBColumnFamily::documents());
@ -2219,6 +2240,10 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
addCf("fulltext", RocksDBColumnFamily::fulltext());
builder.close();
if (_listener) {
builder.add("rocksdbengine.throttle.bps", VPackValue(_listener->GetThrottle()));
} // if
builder.close();
}

4
arangod/RocksDBEngine/RocksDBThrottle.h
View File

@ -85,6 +85,8 @@ class RocksDBThrottle : public rocksdb::EventListener {
void StopThread();
uint64_t GetThrottle() const {return _throttleBps;};
protected:
void Startup(rocksdb::DB* db);
@ -137,7 +139,7 @@ class RocksDBThrottle : public rocksdb::EventListener {
ThrottleData_t _throttleData[THROTTLE_INTERVALS];
size_t _replaceIdx;
uint64_t _throttleBps;
std::atomic<uint64_t> _throttleBps;
bool _firstThrottle;
std::unique_ptr<WriteControllerToken> _delayToken;