control RocksDB block cache filling for the primary and edge indexes (#2532)

arangod/RocksDBEngine/RocksDBEdgeIndex.cpp

@@ -61,6 +61,10 @@
 using namespace arangodb;
 using namespace arangodb::basics;
 
+namespace {
+constexpr bool EdgeIndexFillBlockCache = false;
+}
+
 RocksDBEdgeIndexIterator::RocksDBEdgeIndexIterator(
     LogicalCollection* collection, transaction::Methods* trx,
     ManagedDocumentResult* mmdr, arangodb::RocksDBEdgeIndex const* index,
@@ -69,15 +73,17 @@ RocksDBEdgeIndexIterator::RocksDBEdgeIndexIterator(
       _keys(keys.get()),
       _keysIterator(_keys->slice()),
       _index(index),
-      _iterator(
-          rocksutils::toRocksMethods(trx)->NewIterator(index->columnFamily())),
       _bounds(RocksDBKeyBounds::EdgeIndex(0)),
       _cache(cache),
       _builderIterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue()) {
   keys.release();  // now we have ownership for _keys
   TRI_ASSERT(_keys != nullptr);
   TRI_ASSERT(_keys->slice().isArray());
-  resetInplaceMemory();
+
+  auto* rocksMethods = rocksutils::toRocksMethods(trx);
+  rocksdb::ReadOptions options = rocksMethods->readOptions(); // intentional copy of the options
+  options.fill_cache = EdgeIndexFillBlockCache;
+  _iterator = std::move(rocksMethods->NewIterator(options, index->columnFamily()));
 }
 
 RocksDBEdgeIndexIterator::~RocksDBEdgeIndexIterator() {
@@ -296,7 +302,7 @@ void RocksDBEdgeIndexIterator::lookupInRocksDB(StringRef fromTo) {
   resetInplaceMemory();
   rocksdb::Comparator const* cmp = _index->comparator();
 
-  cache::Cache *cc = _cache.get();
+  cache::Cache* cc = _cache.get();
   _builder.openArray(true);
   auto end = _bounds.end();
   while (_iterator->Valid() && (cmp->Compare(_iterator->key(), end) < 0)) {
@@ -620,6 +626,10 @@ void RocksDBEdgeIndex::warmup(arangodb::transaction::Methods* trx) {
   ManagedDocumentResult mmdr;
   bool needsInsert = false;
 
+  // intentional copy of the read options
+  RocksDBMethods* mthds = rocksutils::toRocksMethods(trx);
+  auto options = mthds->readOptions();
+  options.fill_cache = EdgeIndexFillBlockCache;
   cache::Cache* cc = _cache.get();
   rocksutils::iterateBounds(
       bounds,
@@ -691,7 +701,8 @@ void RocksDBEdgeIndex::warmup(arangodb::transaction::Methods* trx) {
           }
         }
       },
-      RocksDBColumnFamily::edge());
+      RocksDBColumnFamily::edge(),
+      options);
 
   if (!previous.empty() && needsInsert) {
     // We still have something to store

arangod/RocksDBEngine/RocksDBFulltextIndex.cpp

@@ -61,7 +61,7 @@ DocumentIdentifierToken RocksDBFulltextIndex::toDocumentIdentifierToken(
 RocksDBFulltextIndex::RocksDBFulltextIndex(
     TRI_idx_iid_t iid, arangodb::LogicalCollection* collection,
     VPackSlice const& info)
-    : RocksDBIndex(iid, collection, info, RocksDBColumnFamily::geo()),
+    : RocksDBIndex(iid, collection, info, RocksDBColumnFamily::geo(), false),
       _minWordLength(TRI_FULLTEXT_MIN_WORD_LENGTH_DEFAULT) {
   TRI_ASSERT(iid != 0);
 

arangod/RocksDBEngine/RocksDBGeoIndex.cpp

@@ -235,7 +235,7 @@ void RocksDBGeoIndexIterator::reset() { replaceCursor(nullptr); }
 RocksDBGeoIndex::RocksDBGeoIndex(TRI_idx_iid_t iid,
                                  arangodb::LogicalCollection* collection,
                                  VPackSlice const& info)
-    : RocksDBIndex(iid, collection, info, RocksDBColumnFamily::geo()),
+    : RocksDBIndex(iid, collection, info, RocksDBColumnFamily::geo(), false),
       _variant(INDEX_GEO_INDIVIDUAL_LAT_LON),
       _geoJson(false),
       _geoIndex(nullptr) {

arangod/RocksDBEngine/RocksDBIndex.h

@@ -53,12 +53,12 @@ class RocksDBIndex : public Index {
                    attributes,
                bool unique, bool sparse,
                rocksdb::ColumnFamilyHandle* cf,
-               uint64_t objectId = 0,
-               bool useCache = false);
+               uint64_t objectId,
+               bool useCache);
 
   RocksDBIndex(TRI_idx_iid_t, LogicalCollection*,
                arangodb::velocypack::Slice const&, rocksdb::ColumnFamilyHandle* cf,
-               bool useCache = false);
+               bool useCache);
 
  public:
   ~RocksDBIndex();

arangod/RocksDBEngine/RocksDBIterators.cpp

@@ -29,6 +29,11 @@
 #include "RocksDBEngine/RocksDBTransactionState.h"
 
 using namespace arangodb;
+  
+namespace {
+constexpr bool AllIteratorFillBlockCache = true;
+constexpr bool AnyIteratorFillBlockCache = false;
+}
 
 // ================ All Iterator ==================
 
@@ -49,7 +54,7 @@ RocksDBAllIndexIterator::RocksDBAllIndexIterator(
   rocksdb::ReadOptions options = mthds->readOptions();
   TRI_ASSERT(options.snapshot != nullptr);
   TRI_ASSERT(options.prefix_same_as_start);
-  options.fill_cache = true;
+  options.fill_cache = AllIteratorFillBlockCache;
   options.verify_checksums = false;  // TODO evaluate
   // options.readahead_size = 4 * 1024 * 1024;
   _iterator = mthds->NewIterator(options, cf);
@@ -161,12 +166,12 @@ RocksDBAnyIndexIterator::RocksDBAnyIndexIterator(
           static_cast<RocksDBCollection*>(col->getPhysical())->objectId())),
       _total(0),
       _returned(0) {
-  // intentional copy of the read options
   RocksDBMethods* mthds = rocksutils::toRocksMethods(trx);
+  // intentional copy of the read options
   auto options = mthds->readOptions();
   TRI_ASSERT(options.snapshot != nullptr);
   TRI_ASSERT(options.prefix_same_as_start);
-  options.fill_cache = false;
+  options.fill_cache = AnyIteratorFillBlockCache;
   options.verify_checksums = false;  // TODO evaluate
   _iterator = mthds->NewIterator(options, RocksDBColumnFamily::documents());
 

arangod/RocksDBEngine/RocksDBPrimaryIndex.cpp

@@ -61,6 +61,10 @@
 
 using namespace arangodb;
 
+namespace {
+constexpr bool PrimaryIndexFillBlockCache = false;
+}
+
 // ================ Primary Index Iterator ================
 
 /// @brief hard-coded vector of the index attributes
@@ -185,7 +189,8 @@ RocksDBToken RocksDBPrimaryIndex::lookupKey(transaction::Methods* trx,
 
   // acquire rocksdb transaction
   RocksDBMethods* mthds = rocksutils::toRocksMethods(trx);
-  auto& options = mthds->readOptions();
+  auto options = mthds->readOptions(); // intentional copy
+  options.fill_cache = PrimaryIndexFillBlockCache;
   TRI_ASSERT(options.snapshot != nullptr);
 
   arangodb::Result r = mthds->Get(_cf, key, value.buffer());

arangod/RocksDBEngine/RocksDBVPackIndex.cpp

@@ -171,7 +171,7 @@ uint64_t RocksDBVPackIndex::HashForKey(const rocksdb::Slice& key) {
 RocksDBVPackIndex::RocksDBVPackIndex(TRI_idx_iid_t iid,
                                      arangodb::LogicalCollection* collection,
                                      arangodb::velocypack::Slice const& info)
-    : RocksDBIndex(iid, collection, info, RocksDBColumnFamily::index()),
+    : RocksDBIndex(iid, collection, info, RocksDBColumnFamily::index(), false),
       _useExpansion(false),
       _allowPartialIndex(true),
       _estimator(nullptr) {