Merge branch 'generic-col-types' of https://github.com/arangodb/arangodb into generic-col-types

arangod/Agency/Agent.cpp

@@ -279,10 +279,11 @@ void Agent::sendAppendEntriesRPC() {
       std::vector<log_t> unconfirmed = _state.get(last_confirmed);
       index_t highest = unconfirmed.back().index;
 
-      if (highest == _lastHighest[followerId] &&
-          (long)(500.0e6 * _config.minPing()) >
-              (std::chrono::system_clock::now() - _lastSent[followerId])
-                  .count()) {
+      std::chrono::duration<double> m =
+        std::chrono::system_clock::now() - _lastSent[followerId];
+      
+      if (highest == _lastHighest[followerId]
+          && 0.5 * _config.minPing() > m.count()) {
         continue;
       }
 

arangod/Agency/Agent.h

@@ -170,7 +170,7 @@ class Agent : public arangodb::Thread {
   /// @brief Activate this agent in single agent mode.
   bool activateAgency();
 
-  /// @brief Activate new agent in pool to replace failed agent
+  /// @brief Activate new agent in pool to replace failed
   bool activateStandbyAgent();
 
   /// @brief Assignment of persisted state

arangod/Agency/Node.cpp

@@ -661,11 +661,14 @@ bool Node::applies(VPackSlice const& slice) {
   return true;
 }
 
-void Node::toBuilder(Builder& builder) const {
+void Node::toBuilder(Builder& builder, bool showHidden) const {
   try {
     if (type() == NODE) {
       VPackObjectBuilder guard(&builder);
       for (auto const& child : _children) {
+        if (child.first[0] == '.' && !showHidden) {
+          continue;
+        }
         builder.add(VPackValue(child.first));
         child.second->toBuilder(builder);
       }

arangod/Agency/Node.h

@@ -164,7 +164,7 @@ class Node {
   bool handle(arangodb::velocypack::Slice const&);
 
   /// @brief Create Builder representing this store
-  void toBuilder(Builder&) const;
+  void toBuilder(Builder&, bool showHidden = false) const;
 
   /// @brief Access children
   Children& children();

arangod/Agency/Store.cpp

@@ -466,7 +466,7 @@ query_t Store::clearExpired() const {
 /// Dump internal data to builder
 void Store::dumpToBuilder(Builder& builder) const {
   MUTEX_LOCKER(storeLocker, _storeLock);
-  toBuilder(builder);
+  toBuilder(builder, true);
   {
     VPackObjectBuilder guard(&builder);
     for (auto const& i : _timeTable) {
@@ -591,7 +591,8 @@ Store& Store::operator=(VPackSlice const& slice) {
 }
 
 /// Put key value store in velocypack
-void Store::toBuilder(Builder& b) const { _node.toBuilder(b); }
+void Store::toBuilder(Builder& b, bool showHidden) const {
+  _node.toBuilder(b, showHidden); }
 
 /// Get kv-store at path vector
 Node Store::operator()(std::vector<std::string> const& pv) { return _node(pv); }

arangod/Agency/Store.h

@@ -100,7 +100,7 @@ class Store : public arangodb::Thread {
   bool applies(arangodb::velocypack::Slice const&);
 
   /// @brief Create Builder representing this store
-  void toBuilder(Builder&) const;
+  void toBuilder(Builder&, bool showHidden = false) const;
 
   /// @brief Copy out a node
   Node const get(std::string const& path) const;

arangod/Aql/TraversalNode.cpp

@@ -899,16 +899,19 @@ void TraversalNode::prepareOptions() {
   _options->_baseLookupInfos.reserve(numEdgeColls);
   // Compute Edge Indexes. First default indexes:
   for (size_t i = 0; i < numEdgeColls; ++i) {
+    std::string usedField;
     auto dir = _directions[i];
     // TODO we can optimize here. indexCondition and Expression could be
     // made non-overlapping.
     traverser::TraverserOptions::LookupInfo info;
     switch (dir) {
       case TRI_EDGE_IN:
+        usedField = StaticStrings::ToString;
         info.indexCondition =
             globalEdgeConditionBuilder.getInboundCondition()->clone(ast);
         break;
       case TRI_EDGE_OUT:
+        usedField = StaticStrings::FromString;
         info.indexCondition =
             globalEdgeConditionBuilder.getOutboundCondition()->clone(ast);
         break;
@@ -922,6 +925,29 @@ void TraversalNode::prepareOptions() {
         info.idxHandles[0]);
     TRI_ASSERT(res);  // Right now we have an enforced edge index which will
                       // always fit.
+
+    // We now have to check if we need _from / _to inside the index lookup and which position
+    // it is used in. Such that the traverser can update the respective string value
+    // in-place
+    // TODO This place can be optimized.
+    if (info.idxHandles[0].isEdgeIndex()) {
+      // Special case for edge index....
+      // It serves two attributes, but can only be asked for one of them...
+      info.conditionNeedUpdate = true;
+      info.conditionMemberToUpdate = 0;
+    } else {
+      std::vector<std::vector<std::string>> fieldNames =
+          info.idxHandles[0].fieldNames();
+      for (size_t i = 0; i < fieldNames.size(); ++i) {
+        auto f = fieldNames[i];
+        if (f.size() == 1 && f[0] == usedField) {
+          // we only work for _from and _to not _from.foo which would be null anyways...
+          info.conditionNeedUpdate = true;
+          info.conditionMemberToUpdate = i;
+          break;
+        }
+      }
+    }
     _options->_baseLookupInfos.emplace_back(std::move(info));
   }
 
@@ -938,15 +964,18 @@ void TraversalNode::prepareOptions() {
     auto& builder = it.second;
 
     for (size_t i = 0; i < numEdgeColls; ++i) {
+      std::string usedField;
       auto dir = _directions[i];
       // TODO we can optimize here. indexCondition and Expression could be
       // made non-overlapping.
       traverser::TraverserOptions::LookupInfo info;
       switch (dir) {
         case TRI_EDGE_IN:
+          usedField = StaticStrings::ToString;
           info.indexCondition = builder->getInboundCondition()->clone(ast);
           break;
         case TRI_EDGE_OUT:
+          usedField = StaticStrings::FromString;
           info.indexCondition = builder->getOutboundCondition()->clone(ast);
           break;
         case TRI_EDGE_ANY:
@@ -960,6 +989,29 @@ void TraversalNode::prepareOptions() {
           info.idxHandles[0]);
       TRI_ASSERT(res);  // Right now we have an enforced edge index which will
                         // always fit.
+      // We now have to check if we need _from / _to inside the index lookup and which position
+      // it is used in. Such that the traverser can update the respective string value
+      // in-place
+      // TODO This place can be optimized.
+      if (info.idxHandles[0].isEdgeIndex()) {
+        // Special case for edge index....
+        // It serves two attributes, but can only be asked for one of them...
+        info.conditionNeedUpdate = true;
+        info.conditionMemberToUpdate = 0;
+      } else {
+        std::vector<std::vector<std::string>> fieldNames =
+            info.idxHandles[0].fieldNames();
+        for (size_t i = 0; i < fieldNames.size(); ++i) {
+          auto f = fieldNames[i];
+          if (f.size() == 1 && f[0] == usedField) {
+            // we only work for _from and _to not _from.foo which would be null anyways...
+            info.conditionNeedUpdate = true;
+            info.conditionMemberToUpdate = i;
+            break;
+          }
+        }
+      }
+
       infos.emplace_back(std::move(info));
     }
   }

arangod/Utils/Transaction.cpp

@@ -67,6 +67,22 @@
 
 using namespace arangodb;
   
+//////////////////////////////////////////////////////////////////////////////
+/// @brief Get the field names of the used index
+//////////////////////////////////////////////////////////////////////////////
+
+std::vector<std::vector<std::string>> Transaction::IndexHandle::fieldNames() const {
+  return _index->fieldNames();
+}
+
+//////////////////////////////////////////////////////////////////////////////
+/// @brief Only required by traversal should be removed ASAP
+//////////////////////////////////////////////////////////////////////////////
+
+bool Transaction::IndexHandle::isEdgeIndex() const {
+  return _index->type() == Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX;
+}
+
 //////////////////////////////////////////////////////////////////////////////
 /// @brief IndexHandle getter method
 //////////////////////////////////////////////////////////////////////////////

arangod/Utils/Transaction.h

@@ -97,6 +97,10 @@ class Transaction {
     }
     explicit IndexHandle(std::shared_ptr<arangodb::Index> idx) : _index(idx) {
     }
+    std::vector<std::vector<std::string>> fieldNames() const;
+
+    bool isEdgeIndex() const;
+
    private:
     std::shared_ptr<arangodb::Index> getIndex() const;
   };

arangod/VocBase/TraverserOptions.cpp

@@ -36,7 +36,10 @@
 using VPackHelper = arangodb::basics::VelocyPackHelper;
 
 arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo()
-    : expression(nullptr), indexCondition(nullptr) {
+    : expression(nullptr),
+      indexCondition(nullptr),
+      conditionNeedUpdate(false),
+      conditionMemberToUpdate(0) {
   // NOTE: We need exactly one in this case for the optimizer to update
   idxHandles.resize(1);
 };
@@ -52,6 +55,12 @@ arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(
   TRI_ASSERT(shards.isArray());
   idxHandles.reserve(shards.length());
 
+  conditionNeedUpdate = arangodb::basics::VelocyPackHelper::getBooleanValue(
+      info, "condNeedUpdate", false);
+  conditionMemberToUpdate =
+      arangodb::basics::VelocyPackHelper::getNumericValue<size_t>(
+          info, "condMemberToUpdate", 0);
+  
   VPackSlice read = info.get("handle");
   if (!read.isObject()) {
     THROW_ARANGO_EXCEPTION_MESSAGE(
@@ -84,6 +93,7 @@ arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(
         "Each lookup requires expression to be an object");
   }
 
+
   expression = new aql::Expression(query->ast(), read);
 
   read = info.get("condition");
@@ -95,10 +105,13 @@ arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(
   indexCondition = new aql::AstNode(query->ast(), read); 
 }
 
-arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(LookupInfo const& other) 
+arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo(
+    LookupInfo const& other)
     : idxHandles(other.idxHandles),
       expression(nullptr),
-      indexCondition(other.indexCondition) {
+      indexCondition(other.indexCondition),
+      conditionNeedUpdate(other.conditionNeedUpdate),
+      conditionMemberToUpdate(other.conditionMemberToUpdate) {
   expression = other.expression->clone();
 }
 
@@ -118,6 +131,8 @@ void arangodb::traverser::TraverserOptions::LookupInfo::buildEngineInfo(
   result.close();
   result.add(VPackValue("condition"));
   indexCondition->toVelocyPack(result, true);
+  result.add("condNeedUpdate", VPackValue(conditionNeedUpdate));
+  result.add("condMemberToUpdate", VPackValue(conditionMemberToUpdate));
   result.close();
 }
 
@@ -560,14 +575,17 @@ arangodb::traverser::TraverserOptions::nextCursorLocal(
   for (auto& info : list) {
     auto& node = info.indexCondition;
     TRI_ASSERT(node->numMembers() > 0);
-    auto dirCmp = node->getMemberUnchecked(node->numMembers() - 1);
-    TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ);
-    TRI_ASSERT(dirCmp->numMembers() == 2);
+    if (info.conditionNeedUpdate) {
+      // We have to inject _from/_to iff the condition needs it
+      auto dirCmp = node->getMemberUnchecked(info.conditionMemberToUpdate);
+      TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ);
+      TRI_ASSERT(dirCmp->numMembers() == 2);
 
-    auto idNode = dirCmp->getMemberUnchecked(1);
-    TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE);
-    TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING));
-    idNode->setStringValue(vid, vidLength);
+      auto idNode = dirCmp->getMemberUnchecked(1);
+      TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE);
+      TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING));
+      idNode->setStringValue(vid, vidLength);
+    }
     std::vector<OperationCursor*> csrs;
     csrs.reserve(info.idxHandles.size());
     for (auto const& it : info.idxHandles) {

arangod/VocBase/TraverserOptions.h

@@ -75,6 +75,10 @@ struct TraverserOptions {
     std::vector<arangodb::Transaction::IndexHandle> idxHandles;
     aql::Expression* expression;
     aql::AstNode* indexCondition;
+    // Flag if we have to update _from / _to in the index search condition
+    bool conditionNeedUpdate;
+    // Position of _from / _to in the index search condition
+    size_t conditionMemberToUpdate;
 
     LookupInfo();
     ~LookupInfo();

js/server/tests/aql/aql-graph-traverser.js

@@ -1488,8 +1488,14 @@ function optimizeInSuite () {
 
       result = db._query(edgeQuery, bindVars);
       extra = result.getExtra();
-      // We have only 10 valid elements in the array.
-      assertEqual(extra.stats.filtered, 90);
+      if (isCluster) {
+        // The cluster uses a different index no filtering on _key
+        assertEqual(extra.stats.filtered, 0);
+      } else {
+        // We have only 10 valid elements in the array.
+        assertEqual(extra.stats.filtered, 90);
+      }
+
       assertEqual(result.count(), 1000);
 
       // if the rule is disabled we expect to do way more filtering
@@ -1544,14 +1550,26 @@ function optimizeInSuite () {
 
       result = db._query(edgeQuery, bindVars);
       extra = result.getExtra();
-      // We have only 10 valid elements in the array.
-      assertEqual(extra.stats.filtered, 90);
+
+      if (isCluster) {
+        // The cluster uses a different index no filtering on _key
+        assertEqual(extra.stats.filtered, 0);
+      } else {
+        // We have only 10 valid elements in the array.
+        assertEqual(extra.stats.filtered, 90);
+      }
       assertEqual(result.count(), 1000);
 
       result = db._query(mixedQuery1, bindVars);
       extra = result.getExtra();
-      // We have only 10 valid elements in the array.
-      assertEqual(extra.stats.filtered, 90);
+
+      if (isCluster) {
+        // The cluster uses a different index no filtering on _key
+        assertEqual(extra.stats.filtered, 0);
+      } else {
+        // We have only 10 valid elements in the array.
+        assertEqual(extra.stats.filtered, 90);
+      }
       assertEqual(result.count(), 1000);
 
       result = db._query(mixedQuery2, bindVars);

lib/Rest/VppRequest.h

@@ -101,7 +101,6 @@ class VppRequest final : public GeneralRequest {
   std::unordered_map<std::string, std::string> _values;
   std::unordered_map<std::string, std::vector<std::string>> _arrayValues;
   uint64_t _messageId;
-  const std::unordered_map<std::string, std::string> _cookies;  // TODO remove
 
   void parseHeaderInformation();
 };