Merge branch 'engine-api' of https://github.com/arangodb/arangodb into engine-api

arangod/Cache/Table.cpp

@@ -153,7 +153,7 @@ std::shared_ptr<Table> Table::setAuxiliary(std::shared_ptr<Table> table) {
   return result;
 }
 
-void* Table::primaryBucket(uint32_t index) {
+void* Table::primaryBucket(uint64_t index) {
   if (!isEnabled()) {
     return nullptr;
   }

arangod/Cache/Table.h

@@ -132,7 +132,7 @@ class Table : public std::enable_shared_from_this<Table> {
   /// @brief Returns a pointer to the specified bucket in the primary table,
   /// regardless of migration status.
   //////////////////////////////////////////////////////////////////////////////
-  void* primaryBucket(uint32_t index);
+  void* primaryBucket(uint64_t index);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief Returns a subtable in the auxiliary index which corresponds to the

arangod/Pregel/Algos/DMID/DMID.cpp

@@ -523,7 +523,8 @@ struct DMIDComputation
             float const ttt = pair.second / getEdges().size();
             if (ttt > *threshold) {
               /** its profitable to become a member, set value */
-              vertexState->membershipDegree[pair.first] = 1.0f / std::pow(*iterationCounter / 3, 2);
+              float deg = 1.0f / std::pow(*iterationCounter / 3.0f, 2.0f);
+              vertexState->membershipDegree[pair.first] = deg;
               aggregate<bool>(NEW_MEMBER_AGG, true);
             }
           }

arangod/RocksDBEngine/RocksDBPrimaryIndex.cpp

@@ -176,21 +176,6 @@ bool RocksDBAllIndexIterator::outOfRange() const {
 
 // ================ Any Iterator ================
 
-uint64_t RocksDBAnyIndexIterator::OFFSET = 0;
-
-uint64_t RocksDBAnyIndexIterator::newOffset(LogicalCollection* collection,
-                                            transaction::Methods* trx) {
-  auto count = collection->numberDocuments(trx);
-  /*auto adjustment = RandomGenerator::interval(count);
-  OFFSET = (OFFSET + adjustment) % count;
-  return OFFSET;*/
-  if (count == 0) {
-    return 0;
-  }
-
-  return RandomGenerator::interval(count);
-}
-
 RocksDBAnyIndexIterator::RocksDBAnyIndexIterator(
     LogicalCollection* collection, transaction::Methods* trx,
     ManagedDocumentResult* mmdr, RocksDBPrimaryIndex const* index)
@@ -203,18 +188,24 @@ RocksDBAnyIndexIterator::RocksDBAnyIndexIterator(
   auto options = state->readOptions();
 
   _iterator.reset(rtrx->GetIterator(options));
-  _iterator->Seek(_bounds.start());
-
-  // not thread safe by design
-  uint64_t off = newOffset(collection, trx);
-  if (off > 0) {
-    while (_iterator->Valid() && --off > 0) {
-      _iterator->Next();
+  _total = collection->numberDocuments(trx);
+  if (_total > 0) {
+    uint64_t off = RandomGenerator::interval(_total);
+    if (off < _total/2) {
+      _iterator->Seek(_bounds.start());
+      while (_iterator->Valid() && --off > 0) {
+        _iterator->Next();
+      }
+    } else {
+      off /= 2;
+      _iterator->SeekForPrev(_bounds.end());
+      while (_iterator->Valid() && --off > 0) {
+        _iterator->Prev();
+      }
+    }
+    if (!_iterator->Valid()) {
+      _iterator->Seek(_bounds.start());
     }
-  }
-  if (!_iterator->Valid()) {
-    // OFFSET = 0;
-    _iterator->Seek(_bounds.start());
   }
 }
 
@@ -231,12 +222,16 @@ bool RocksDBAnyIndexIterator::next(TokenCallback const& cb, size_t limit) {
     cb(token);
 
     --limit;
+    _returned++;
     _iterator->Next();
     if (!_iterator->Valid() || outOfRange()) {
+      if (_returned < _total) {
+        _iterator->Seek(_bounds.start());
+        continue;
+      }
       return false;
     }
   }
-
   return true;
 }
 

arangod/RocksDBEngine/RocksDBPrimaryIndex.h

@@ -118,7 +118,7 @@ class RocksDBAnyIndexIterator final : public IndexIterator {
   RocksDBComparator const* _cmp;
   std::unique_ptr<rocksdb::Iterator> _iterator;
   RocksDBKeyBounds _bounds;
-  static uint64_t OFFSET;
+  uint64_t _total, _returned;
 };
 
 class RocksDBPrimaryIndex final : public RocksDBIndex {

arangod/RocksDBEngine/RocksDBVPackIndex.cpp

@@ -31,6 +31,7 @@
 #include "Basics/StaticStrings.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Indexes/IndexLookupContext.h"
+#include "Indexes/SimpleAttributeEqualityMatcher.h"
 #include "RocksDBEngine/RocksDBCollection.h"
 #include "RocksDBEngine/RocksDBCommon.h"
 #include "RocksDBEngine/RocksDBComparator.h"
@@ -778,6 +779,12 @@ bool RocksDBVPackIndex::accessFitsIndex(
       TRI_IF_FAILURE("PersistentIndex::accessFitsIndex") {
         THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
       }
+      TRI_IF_FAILURE("SkiplistIndex::accessFitsIndex") {
+        THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+      }
+      TRI_IF_FAILURE("HashIndex::accessFitsIndex") {
+        THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+      }
 
       return true;
     }
@@ -831,6 +838,13 @@ bool RocksDBVPackIndex::supportsFilterCondition(
     arangodb::aql::AstNode const* node,
     arangodb::aql::Variable const* reference, size_t itemsInIndex,
     size_t& estimatedItems, double& estimatedCost) const {
+  // HashIndex has different semantics
+  if (this->type() == Index::TRI_IDX_TYPE_HASH_INDEX) {
+    SimpleAttributeEqualityMatcher matcher(_fields);
+    return matcher.matchAll(this, node, reference, itemsInIndex, estimatedItems,
+                            estimatedCost);
+  }
+  
   std::unordered_map<size_t, std::vector<arangodb::aql::AstNode const*>> found;
   std::unordered_set<std::string> nonNullAttributes;
   size_t values = 0;
@@ -988,6 +1002,13 @@ IndexIterator* RocksDBVPackIndex::iteratorForCondition(
     TRI_IF_FAILURE("PersistentIndex::noSortIterator") {
       THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
     }
+    TRI_IF_FAILURE("SkiplistIndex::noSortIterator") {
+      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+    }
+    TRI_IF_FAILURE("HashIndex::noSortIterator") {
+      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+    }
+    
   } else {
     // Create the search Values for the lookup
     VPackArrayBuilder guard(&searchValues);
@@ -1051,6 +1072,12 @@ IndexIterator* RocksDBVPackIndex::iteratorForCondition(
         TRI_IF_FAILURE("PersistentIndex::permutationEQ") {
           THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
         }
+        TRI_IF_FAILURE("SkiplistIndex::permutationEQ") {
+          THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+        }
+        TRI_IF_FAILURE("HashIndex::permutationEQ") {
+          THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+        }
       } else if (comp->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN) {
         if (isAttributeExpanded(usedFields)) {
           searchValues.openObject();
@@ -1058,6 +1085,12 @@ IndexIterator* RocksDBVPackIndex::iteratorForCondition(
           TRI_IF_FAILURE("PersistentIndex::permutationArrayIN") {
             THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
           }
+          TRI_IF_FAILURE("SkiplistIndex::permutationArrayIN") {
+            THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+          }
+          TRI_IF_FAILURE("HashIndex::permutationArrayIN") {
+            THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+          }
         } else {
           needNormalize = true;
           searchValues.openObject();
@@ -1130,6 +1163,12 @@ IndexIterator* RocksDBVPackIndex::iteratorForCondition(
   TRI_IF_FAILURE("PersistentIndex::noIterator") {
     THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
   }
+  TRI_IF_FAILURE("SkiplistIndex::noIterator") {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+  }
+  TRI_IF_FAILURE("HashIndex::noIterator") {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+  }
 
   if (needNormalize) {
     VPackBuilder expandedSearchValues;
@@ -1169,6 +1208,12 @@ IndexIterator* RocksDBVPackIndex::iteratorForCondition(
 arangodb::aql::AstNode* RocksDBVPackIndex::specializeCondition(
     arangodb::aql::AstNode* node,
     arangodb::aql::Variable const* reference) const {
+  // HashIndex uses slightly different semantics
+  if (this->type() == Index::TRI_IDX_TYPE_HASH_INDEX) {
+    SimpleAttributeEqualityMatcher matcher(_fields);
+    return matcher.specializeAll(this, node, reference);
+  }
+  
   std::unordered_map<size_t, std::vector<arangodb::aql::AstNode const*>> found;
   std::unordered_set<std::string> nonNullAttributes;
   size_t values = 0;