Feature/faster any (#4306)

CHANGELOG

@@ -1,6 +1,8 @@
 devel
 -----
 
+* fix internal issue #1439: improve performance of any-iterator for RocksDB
+
 * fixed issue #4308: Crash when getter for error.name throws an error (on Windows)
 
 * UI: fixed an issue where a collection name gets wrongly cached within the

arangod/RocksDBEngine/RocksDBIterators.cpp

@@ -145,7 +145,7 @@ bool RocksDBAllIndexIterator::nextDocument(
 
   return true;
 }
-  
+
 void RocksDBAllIndexIterator::skip(uint64_t count, uint64_t& skipped) {
   TRI_ASSERT(_trx->state()->isRunning());
 
@@ -172,7 +172,6 @@ void RocksDBAllIndexIterator::reset() {
 }
 
 // ================ Any Iterator ================
-
 RocksDBAnyIndexIterator::RocksDBAnyIndexIterator(
     LogicalCollection* col, transaction::Methods* trx,
     RocksDBPrimaryIndex const* index)
@@ -193,26 +192,53 @@ RocksDBAnyIndexIterator::RocksDBAnyIndexIterator(
   TRI_ASSERT(_iterator);
 
   _total = col->numberDocuments(trx);
-  uint64_t off = RandomGenerator::interval(_total - 1);
+  _forward = RandomGenerator::interval(uint16_t(1)) ? true : false;
+
+  //initial seek
   if (_total > 0) {
-    if (off <= _total / 2) {
+    uint64_t steps = RandomGenerator::interval(_total - 1) % 500;
-      _iterator->Seek(_bounds.start());
+    auto initialKey = RocksDBKey();
-      while (_iterator->Valid() && off-- > 0) {
+    initialKey.constructDocument(
-        _iterator->Next();
+      static_cast<RocksDBCollection*>(col->getPhysical())->objectId(),
+      RandomGenerator::interval(UINT64_MAX)
+    );
+    _iterator->Seek(initialKey.string());
+
+    if (checkIter()) {
+      if (_forward) {
+        while (steps-- > 0) {
+          _iterator->Next();
+          if(!checkIter()) { break; }
+        }
+      } else {
+        while (steps-- > 0) {
+          _iterator->Prev();
+          if(!checkIter()) { break; }
+        }
       }
-    } else {
-      off = _total - (off + 1);
-      _iterator->SeekForPrev(_bounds.end());
-      while (_iterator->Valid() && off-- > 0) {
-        _iterator->Prev();
-      }
-    }
-    if (!_iterator->Valid() || outOfRange()) {
-      _iterator->Seek(_bounds.start());
     }
   }
 }
 
+bool RocksDBAnyIndexIterator::checkIter(){
+  if ( /* not  valid */            !_iterator->Valid() ||
+       /* out of range forward */  ( _forward && _cmp->Compare(_iterator->key(), _bounds.end())   > 0) ||
+       /* out of range backward */ (!_forward && _cmp->Compare(_iterator->key(), _bounds.start()) < 0)  ) {
+
+    if (_forward) {
+      _iterator->Seek(_bounds.start());
+    } else {
+      _iterator->SeekForPrev(_bounds.end());
+    }
+
+    if(!_iterator->Valid()){
+      return false;
+    }
+  }
+  return true;
+}
+
+
 bool RocksDBAnyIndexIterator::next(LocalDocumentIdCallback const& cb, size_t limit) {
   TRI_ASSERT(_trx->state()->isRunning());
 
@@ -284,7 +310,7 @@ RocksDBSortedAllIterator::RocksDBSortedAllIterator(
       _trx(trx),
       _bounds(RocksDBKeyBounds::PrimaryIndex(index->objectId())),
       _cmp(index->comparator()) {
- 
+
   RocksDBMethods* mthds = RocksDBTransactionState::toMethods(trx);
   // intentional copy of the read options
   auto options = mthds->readOptions();

arangod/RocksDBEngine/RocksDBIterators.h

@@ -88,11 +88,13 @@ class RocksDBAnyIndexIterator final : public IndexIterator {
   static uint64_t newOffset(LogicalCollection* collection,
                             transaction::Methods* trx);
 
+  bool checkIter();
   rocksdb::Comparator const* _cmp;
   std::unique_ptr<rocksdb::Iterator> _iterator;
   RocksDBKeyBounds const _bounds;
   uint64_t _total;
   uint64_t _returned;
+  bool _forward;
 };
 
 /// @brief iterates over the primary index and does lookups