Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
arangodb/arangod/RocksDBEngine/RocksDBIndex.cpp

333 lines
11 KiB
C++
Raw Blame History

////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2017 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "RocksDBIndex.h"
#include "Basics/VelocyPackHelper.h"
#include "Cache/CacheManagerFeature.h"
#include "Cache/Common.h"
#include "Cache/Manager.h"
#include "Cache/TransactionalCache.h"
#include "RocksDBEngine/RocksDBColumnFamily.h"
#include "RocksDBEngine/RocksDBCommon.h"
#include "RocksDBEngine/RocksDBComparator.h"
#include "RocksDBEngine/RocksDBMethods.h"
#include "RocksDBEngine/RocksDBTransactionState.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "VocBase/LogicalCollection.h"
#include "VocBase/ticks.h"
#include <rocksdb/comparator.h>
using namespace arangodb;
using namespace arangodb::rocksutils;
// This is the number of distinct elements the index estimator can reliably
// store
// This correlates directly with the memory of the estimator:
// memory == ESTIMATOR_SIZE * 6 bytes
uint64_t const arangodb::RocksDBIndex::ESTIMATOR_SIZE = 4096;
RocksDBIndex::RocksDBIndex(
TRI_idx_iid_t id, LogicalCollection* collection,
std::vector<std::vector<arangodb::basics::AttributeName>> const& attributes,
bool unique, bool sparse, rocksdb::ColumnFamilyHandle* cf,
uint64_t objectId, bool useCache)
: Index(id, collection, attributes, unique, sparse),
_objectId((objectId != 0) ? objectId : TRI_NewTickServer()),
_cf(cf),
_cache(nullptr),
_cachePresent(false),
_useCache(useCache) {
TRI_ASSERT(cf != nullptr && cf != RocksDBColumnFamily::definitions());
if (_useCache) {
createCache();
}
}
RocksDBIndex::RocksDBIndex(TRI_idx_iid_t id, LogicalCollection* collection,
VPackSlice const& info,
rocksdb::ColumnFamilyHandle* cf, bool useCache)
: Index(id, collection, info),
_objectId(basics::VelocyPackHelper::stringUInt64(info.get("objectId"))),
_cf(cf),
_cache(nullptr),
_cachePresent(false),
_useCache(useCache) {
TRI_ASSERT(cf != nullptr && cf != RocksDBColumnFamily::definitions());
if (_objectId == 0) {
_objectId = TRI_NewTickServer();
}
if (_useCache) {
createCache();
}
}
RocksDBIndex::~RocksDBIndex() {
if (useCache()) {
try {
TRI_ASSERT(_cache != nullptr);
TRI_ASSERT(CacheManagerFeature::MANAGER != nullptr);
CacheManagerFeature::MANAGER->destroyCache(_cache);
} catch (...) {
}
}
}
rocksdb::Comparator const* RocksDBIndex::comparator() const {
return _cf->GetComparator();
}
void RocksDBIndex::toVelocyPackFigures(VPackBuilder& builder) const {
TRI_ASSERT(builder.isOpenObject());
Index::toVelocyPackFigures(builder);
builder.add("cacheInUse", VPackValue(useCache()));
if (useCache()) {
builder.add("cacheSize", VPackValue(_cache->size()));
auto hitRates = _cache->hitRates();
double rate = hitRates.first;
rate = std::isnan(rate) ? 0.0 : rate;
builder.add("cacheLifeTimeHitRate", VPackValue(rate));
rate = hitRates.second;
rate = std::isnan(rate) ? 0.0 : rate;
builder.add("cacheWindowedHitRate", VPackValue(rate));
} else {
builder.add("cacheSize", VPackValue(0));
}
}
void RocksDBIndex::load() {
if (_useCache) {
createCache();
TRI_ASSERT(_cachePresent);
}
}
void RocksDBIndex::unload() {
if (useCache()) {
// LOG_TOPIC(ERR, Logger::FIXME) << "unload cache";
disableCache();
TRI_ASSERT(!_cachePresent);
}
}
/// @brief return a VelocyPack representation of the index
void RocksDBIndex::toVelocyPack(VPackBuilder& builder, bool withFigures,
bool forPersistence) const {
Index::toVelocyPack(builder, withFigures, forPersistence);
if (forPersistence) {
// If we store it, it cannot be 0
TRI_ASSERT(_objectId != 0);
builder.add("objectId", VPackValue(std::to_string(_objectId)));
}
}
void RocksDBIndex::createCache() {
if (!_useCache || _cachePresent) {
// we leave this if we do not need the cache
// or if cache already created
return;
}
TRI_ASSERT(_useCache);
TRI_ASSERT(_cache.get() == nullptr);
TRI_ASSERT(CacheManagerFeature::MANAGER != nullptr);
_cache = CacheManagerFeature::MANAGER->createCache(
cache::CacheType::Transactional);
_cachePresent = (_cache.get() != nullptr);
TRI_ASSERT(_useCache);
}
void RocksDBIndex::disableCache() {
if (!_cachePresent) {
return;
}
TRI_ASSERT(CacheManagerFeature::MANAGER != nullptr);
// must have a cache...
TRI_ASSERT(_useCache);
TRI_ASSERT(_cachePresent);
TRI_ASSERT(_cache.get() != nullptr);
CacheManagerFeature::MANAGER->destroyCache(_cache);
_cache.reset();
_cachePresent = false;
TRI_ASSERT(_useCache);
}
void RocksDBIndex::serializeEstimate(std::string&) const {
// All indexes that do not have an estimator do not serialize anything.
}
bool RocksDBIndex::deserializeEstimate(RocksDBCounterManager*) {
// All indexes that do not have an estimator do not deserialize anything.
// So the estimate is always recreatable.
// We do not advance anything here.
return true;
}
void RocksDBIndex::recalculateEstimates() {
// Nothing to do.
return;
}
int RocksDBIndex::drop() {
// edge index needs to be dropped with prefix_same_as_start = false
// otherwise full index scan will not work
bool prefix_same_as_start = this->type() != Index::TRI_IDX_TYPE_EDGE_INDEX;
arangodb::Result r = rocksutils::removeLargeRange(
rocksutils::globalRocksDB(), this->getBounds(), prefix_same_as_start);
// Try to drop the cache as well.
if (_cachePresent) {
try {
TRI_ASSERT(CacheManagerFeature::MANAGER != nullptr);
CacheManagerFeature::MANAGER->destroyCache(_cache);
// Reset flag
_cache.reset();
_cachePresent = false;
} catch (...) {
}
}
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
//check if documents have been deleted
size_t numDocs = rocksutils::countKeyRange(rocksutils::globalRocksDB(),
this->getBounds(), prefix_same_as_start);
if (numDocs > 0) {
std::string errorMsg("deletion check in index drop failed - not all documents in the index have been deleted. remaining: ");
errorMsg.append(std::to_string(numDocs));
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, errorMsg);
}
#endif
return r.errorNumber();
}
void RocksDBIndex::truncate(transaction::Methods* trx) {
auto* mthds = RocksDBTransactionState::toMethods(trx);
RocksDBKeyBounds indexBounds = getBounds(type(), _objectId, _unique);
rocksdb::ReadOptions options = mthds->readOptions();
rocksdb::Slice end = indexBounds.end();
rocksdb::Comparator const* cmp = this->comparator();
options.iterate_upper_bound = &end;
if (type() == RocksDBIndex::TRI_IDX_TYPE_EDGE_INDEX) {
options.prefix_same_as_start = false;
options.total_order_seek = true;
}
options.verify_checksums = false;
options.fill_cache = false;
std::unique_ptr<rocksdb::Iterator> iter = mthds->NewIterator(options, _cf);
iter->Seek(indexBounds.start());
while (iter->Valid() && cmp->Compare(iter->key(), end) < 0) {
TRI_ASSERT(_objectId == RocksDBKey::objectId(iter->key()));
Result r = mthds->Delete(_cf, RocksDBKey(iter->key()));
if (!r.ok()) {
THROW_ARANGO_EXCEPTION(r);
}
r = postprocessRemove(trx, iter->key(), iter->value());
if (!r.ok()) {
THROW_ARANGO_EXCEPTION(r);
}
iter->Next();
}
}
/// @brief return the memory usage of the index
size_t RocksDBIndex::memory() const {
rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
RocksDBKeyBounds bounds = getBounds();
TRI_ASSERT(_cf == bounds.columnFamily());
rocksdb::Range r(bounds.start(), bounds.end());
uint64_t out;
db->GetApproximateSizes(_cf, &r, 1, &out,
static_cast<uint8_t>(
rocksdb::DB::SizeApproximationFlags::INCLUDE_MEMTABLES |
rocksdb::DB::SizeApproximationFlags::INCLUDE_FILES));
return static_cast<size_t>(out);
}
/// compact the index, should reduce read amplification
void RocksDBIndex::cleanup() {
rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
rocksdb::CompactRangeOptions opts;
RocksDBKeyBounds bounds = this->getBounds();
TRI_ASSERT(_cf == bounds.columnFamily());
rocksdb::Slice b = bounds.start(), e = bounds.end();
db->CompactRange(opts, _cf, &b, &e);
}
Result RocksDBIndex::postprocessRemove(transaction::Methods* trx,
rocksdb::Slice const& key,
rocksdb::Slice const& value) {
return {TRI_ERROR_NO_ERROR};
}
// blacklist given key from transactional cache
void RocksDBIndex::blackListKey(char const* data, std::size_t len) {
if (useCache()) {
TRI_ASSERT(_cache != nullptr);
bool blacklisted = false;
while (!blacklisted) {
auto status = _cache->blacklist(data, static_cast<uint32_t>(len));
if (status.ok()) {
blacklisted = true;
} else if (status.errorNumber() == TRI_ERROR_SHUTTING_DOWN) {
disableCache();
break;
}
}
}
}
RocksDBKeyBounds RocksDBIndex::getBounds(Index::IndexType type,
uint64_t objectId, bool unique) {
switch (type) {
case RocksDBIndex::TRI_IDX_TYPE_PRIMARY_INDEX:
return RocksDBKeyBounds::PrimaryIndex(objectId);
case RocksDBIndex::TRI_IDX_TYPE_EDGE_INDEX:
return RocksDBKeyBounds::EdgeIndex(objectId);
case RocksDBIndex::TRI_IDX_TYPE_HASH_INDEX:
case RocksDBIndex::TRI_IDX_TYPE_SKIPLIST_INDEX:
case RocksDBIndex::TRI_IDX_TYPE_PERSISTENT_INDEX:
if (unique) {
return RocksDBKeyBounds::UniqueVPackIndex(objectId);
}
return RocksDBKeyBounds::VPackIndex(objectId);
case RocksDBIndex::TRI_IDX_TYPE_FULLTEXT_INDEX:
return RocksDBKeyBounds::FulltextIndex(objectId);
case RocksDBIndex::TRI_IDX_TYPE_GEO1_INDEX:
case RocksDBIndex::TRI_IDX_TYPE_GEO2_INDEX:
return RocksDBKeyBounds::GeoIndex(objectId);
case RocksDBIndex::TRI_IDX_TYPE_UNKNOWN:
default:
THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
}
}
Reference in New Issue View Git Blame Copy Permalink