From 2091dae8758e915a71b7f1b9e3af954b63513abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Fri, 24 Mar 2017 15:14:12 +0100 Subject: [PATCH 01/27] Pregel: Fix concurrent creation of aggregator --- arangod/Pregel/AggregatorHandler.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arangod/Pregel/AggregatorHandler.cpp b/arangod/Pregel/AggregatorHandler.cpp index 143caca951..ec49b71c56 100644 --- a/arangod/Pregel/AggregatorHandler.cpp +++ b/arangod/Pregel/AggregatorHandler.cpp @@ -46,15 +46,17 @@ IAggregator* AggregatorHandler::getAggregator(AggregatorID const& name) { } } // aggregator doesn't exists, create it - { + std::unique_ptr agg(_algorithm->aggregator(name)); + if (agg) { WRITE_LOCKER(guard, _lock); - std::unique_ptr agg(_algorithm->aggregator(name)); - if (agg) { - _values[name] = agg.get(); + if (_values.find(name) == _values.end()) { + _values.emplace(name, agg.get()); return agg.release(); } + return _values[name]; + } else { + return nullptr; } - return nullptr; } void AggregatorHandler::aggregate(AggregatorID const& name, From f7eb96bc8dbda9d4499d47781772379c46561f9c Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 24 Mar 2017 15:31:39 +0100 Subject: [PATCH 02/27] Squashed commit. Moved over SmartSearch Shortest Path feature --- CHANGELOG | 4 + arangod/Aql/ExecutionEngine.cpp | 106 ++- arangod/Aql/ExecutionPlan.cpp | 25 +- arangod/Aql/GraphNode.cpp | 537 ++++++++++++++++ arangod/Aql/GraphNode.h | 216 +++++++ arangod/Aql/OptimizerRules.cpp | 11 +- arangod/Aql/ShortestPathBlock.cpp | 137 ++-- arangod/Aql/ShortestPathBlock.h | 15 +- arangod/Aql/ShortestPathNode.cpp | 450 ++++--------- arangod/Aql/ShortestPathNode.h | 89 +-- arangod/Aql/ShortestPathOptions.cpp | 54 -- arangod/Aql/ShortestPathOptions.h | 54 -- arangod/Aql/TraversalNode.cpp | 601 ++---------------- arangod/Aql/TraversalNode.h | 129 +--- arangod/CMakeLists.txt | 3 +- arangod/Cluster/TraverserEngine.cpp | 39 +- arangod/Cluster/TraverserEngine.h | 17 +- .../InternalRestTraverserHandler.cpp | 47 +- .../InternalRestTraverserHandler.h | 7 + arangod/V8Server/V8Traverser.cpp | 63 -- arangod/V8Server/V8Traverser.h | 89 --- arangod/VocBase/Traverser.cpp | 30 +- arangod/VocBase/Traverser.h | 8 + arangod/VocBase/TraverserOptions.cpp | 515 ++++++++++----- arangod/VocBase/TraverserOptions.h | 191 +++++- 25 files changed, 1765 insertions(+), 1672 deletions(-) create mode 100644 arangod/Aql/GraphNode.cpp create mode 100644 arangod/Aql/GraphNode.h delete mode 100644 arangod/Aql/ShortestPathOptions.cpp delete mode 100644 arangod/Aql/ShortestPathOptions.h delete mode 100644 arangod/V8Server/V8Traverser.cpp delete mode 100644 arangod/V8Server/V8Traverser.h diff --git a/CHANGELOG b/CHANGELOG index c0f6c34930..f06e5e4f68 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,10 @@ devel * increase default collection lock timeout from 30 to 900 seconds +* made shortest_path computation aware of SmartGraph features (Enterprise + Only) which will give a good performance boost on computing single-pair + shortest-paths in a sharded SmartGraph. + * added function `db._engine()` for retrieval of storage engine information at server runtime diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index d8d9d2eeb7..c4e1ead4ff 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -54,6 +54,27 @@ using namespace arangodb; using namespace arangodb::aql; +// Used in Mapping ServerID => Responsible Shards +struct TraversalInfoMapping { + // All forward computed edge shards + std::vector> forwardEdgeShards; + // All backward computed edge shards (only ShortestPath) + std::vector> backwardEdgeShards; + // All shards for vertices + std::unordered_map> vertexShards; + + TraversalInfoMapping() = delete; + TraversalInfoMapping(size_t length) { + // We always know the exact size of the shards. + // Furthermore the lengths of the vectors have to + // be correct, as accessed is based on indexes. + forwardEdgeShards.resize(length); + backwardEdgeShards.resize(length); + } + + ~TraversalInfoMapping() {} +}; + /// @brief helper function to create a block static ExecutionBlock* CreateBlock( ExecutionEngine* engine, ExecutionNode const* en, @@ -207,9 +228,10 @@ struct Instanciator final : public WalkerWorker { virtual void after(ExecutionNode* en) override final { ExecutionBlock* block = nullptr; { - if (en->getType() == ExecutionNode::TRAVERSAL) { + if (en->getType() == ExecutionNode::TRAVERSAL || + en->getType() == ExecutionNode::SHORTEST_PATH) { // We have to prepare the options before we build the block - static_cast(en)->prepareOptions(); + static_cast(en)->prepareOptions(); } std::unique_ptr eb(CreateBlock(engine, en, cache, std::unordered_set())); @@ -824,9 +846,10 @@ struct CoordinatorInstanciator : public WalkerWorker { /// @brief Build traverser engines on DBServers. Coordinator still uses /// traversal block. - void buildTraverserEnginesForNode(TraversalNode* en) { + void buildTraverserEnginesForNode(GraphNode* en) { // We have to initialize all options. After this point the node // is not cloneable any more. + // Required prior to creation of engines. en->prepareOptions(); VPackBuilder optsBuilder; auto opts = en->options(); @@ -841,10 +864,7 @@ struct CoordinatorInstanciator : public WalkerWorker { // For edgeCollections the Ordering is important for the index access. // Also the same edgeCollection can be included twice (iff direction is ANY) auto clusterInfo = arangodb::ClusterInfo::instance(); - std::unordered_map< - ServerID, - std::pair>, - std::unordered_map>>> + std::unordered_map mappingServerToCollections; auto servers = clusterInfo->getCurrentDBServers(); size_t length = edges.size(); @@ -855,18 +875,32 @@ struct CoordinatorInstanciator : public WalkerWorker { for (auto s : servers) { // We insert at lease an empty vector for every edge collection // Used in the traverser. - auto& info = mappingServerToCollections[s]; - // We need to exactly maintain the ordering. - // A server my be responsible for a shard in edge collection 1 but not 0 or 2. - info.first.resize(length); + mappingServerToCollections.emplace(s, length); } + for (size_t i = 0; i < length; ++i) { auto shardIds = edges[i]->shardIds(_includedShards); for (auto const& shard : *shardIds) { auto serverList = clusterInfo->getResponsibleServer(shard); TRI_ASSERT(!serverList->empty()); - auto& pair = mappingServerToCollections[(*serverList)[0]]; - pair.first[i].emplace_back(shard); + auto map = mappingServerToCollections.find((*serverList)[0]); + TRI_ASSERT(map != mappingServerToCollections.end()); + map->second.forwardEdgeShards[i].emplace_back(shard); + } + } + + std::vector> const& reverseEdges = en->reverseEdgeColls(); + if (!reverseEdges.empty()) { + TRI_ASSERT(reverseEdges.size() == length); + for (size_t i = 0; i < length; ++i) { + auto shardIds = reverseEdges[i]->shardIds(); + for (auto const& shard : *shardIds) { + auto serverList = clusterInfo->getResponsibleServer(shard); + TRI_ASSERT(!serverList->empty()); + auto map = mappingServerToCollections.find((*serverList)[0]); + TRI_ASSERT(map != mappingServerToCollections.end()); + map->second.backwardEdgeShards[i].emplace_back(shard); + } } } @@ -882,8 +916,8 @@ struct CoordinatorInstanciator : public WalkerWorker { auto cs = query->collections()->collections(); for (auto const& collection : (*cs)) { for (auto& entry : mappingServerToCollections) { - entry.second.second.emplace(collection.second->getName(), - std::vector()); + entry.second.vertexShards.emplace(collection.second->getName(), + std::vector()); } if (knownEdges.find(collection.second->getName()) == knownEdges.end()) { // This collection is not one of the edge collections used in this @@ -892,8 +926,9 @@ struct CoordinatorInstanciator : public WalkerWorker { for (auto const& shard : *shardIds) { auto serverList = clusterInfo->getResponsibleServer(shard); TRI_ASSERT(!serverList->empty()); - auto& pair = mappingServerToCollections[(*serverList)[0]]; - pair.second[collection.second->getName()].emplace_back(shard); + auto map = mappingServerToCollections.find((*serverList)[0]); + TRI_ASSERT(map != mappingServerToCollections.end()); + map->second.vertexShards[collection.second->getName()].emplace_back(shard); } } } @@ -901,15 +936,15 @@ struct CoordinatorInstanciator : public WalkerWorker { // This Traversal is started with a GRAPH. It knows all relevant collections. for (auto const& it : vertices) { for (auto& entry : mappingServerToCollections) { - entry.second.second.emplace(it->getName(), - std::vector()); + entry.second.vertexShards.emplace(it->getName(), + std::vector()); } auto shardIds = it->shardIds(_includedShards); for (auto const& shard : *shardIds) { auto serverList = clusterInfo->getResponsibleServer(shard); TRI_ASSERT(!serverList->empty()); - auto& pair = mappingServerToCollections[(*serverList)[0]]; - pair.second[it->getName()].emplace_back(shard); + auto map = mappingServerToCollections.find((*serverList)[0]); + map->second.vertexShards[it->getName()].emplace_back(shard); } } } @@ -934,7 +969,11 @@ struct CoordinatorInstanciator : public WalkerWorker { // "vertices" : { // "v1": [], // may be empty // "v2": [] // may be empty - // } + // }, + // "reverseEdges" : [ + // [ ], + // [ ] + // ] // } // } @@ -975,7 +1014,7 @@ struct CoordinatorInstanciator : public WalkerWorker { engineInfo.openObject(); engineInfo.add(VPackValue("vertices")); engineInfo.openObject(); - for (auto const& col : list.second.second) { + for (auto const& col : list.second.vertexShards) { engineInfo.add(VPackValue(col.first)); engineInfo.openArray(); for (auto const& v : col.second) { @@ -988,7 +1027,7 @@ struct CoordinatorInstanciator : public WalkerWorker { engineInfo.add(VPackValue("edges")); engineInfo.openArray(); - for (auto const& edgeShards : list.second.first) { + for (auto const& edgeShards : list.second.forwardEdgeShards) { engineInfo.openArray(); for (auto const& e : edgeShards) { shardSet.emplace(e); @@ -998,6 +1037,19 @@ struct CoordinatorInstanciator : public WalkerWorker { } engineInfo.close(); // edges + engineInfo.add(VPackValue("reverseEdges")); + + engineInfo.openArray(); + for (auto const& edgeShards : list.second.backwardEdgeShards) { + engineInfo.openArray(); + for (auto const& e : edgeShards) { + shardSet.emplace(e); + engineInfo.add(VPackValue(e)); + } + engineInfo.close(); + } + engineInfo.close(); // reverseEdges + engineInfo.close(); // shards en->enhanceEngineInfo(engineInfo); @@ -1128,8 +1180,10 @@ struct CoordinatorInstanciator : public WalkerWorker { engines.emplace_back(currentLocation, currentEngineId, part, en->id()); } - if (nodeType == ExecutionNode::TRAVERSAL) { - buildTraverserEnginesForNode(static_cast(en)); + if (nodeType == ExecutionNode::TRAVERSAL || + nodeType == ExecutionNode::SHORTEST_PATH) { + // Now build traverser engines. + buildTraverserEnginesForNode(static_cast(en)); } return false; diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index 652368760d..9dd2e0255a 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -35,7 +35,6 @@ #include "Aql/OptimizerRulesFeature.h" #include "Aql/Query.h" #include "Aql/ShortestPathNode.h" -#include "Aql/ShortestPathOptions.h" #include "Aql/SortNode.h" #include "Aql/TraversalNode.h" #include "Aql/Variable.h" @@ -145,8 +144,9 @@ static std::unique_ptr CreateTraversalOptions( return options; } -static ShortestPathOptions CreateShortestPathOptions(AstNode const* node) { - ShortestPathOptions options; +static std::unique_ptr +CreateShortestPathOptions(transaction::Methods* trx, AstNode const* node) { + auto options = std::make_unique(trx); if (node != nullptr && node->type == NODE_TYPE_OBJECT) { size_t n = node->numMembers(); @@ -161,10 +161,10 @@ static ShortestPathOptions CreateShortestPathOptions(AstNode const* node) { TRI_ASSERT(value->isConstant()); if (name == "weightAttribute" && value->isStringValue()) { - options.weightAttribute = - std::string(value->getStringValue(), value->getStringLength()); + options->setWeightAttribute( + std::string(value->getStringValue(), value->getStringLength())); } else if (name == "defaultWeight" && value->isNumericValue()) { - options.defaultWeight = value->getDoubleValue(); + options->setDefaultWeight(value->getDoubleValue()); } } } @@ -723,8 +723,9 @@ ExecutionNode* ExecutionPlan::fromNodeTraversal(ExecutionNode* previous, node->getMember(3)); // First create the node - auto travNode = new TraversalNode(this, nextId(), _ast->query()->vocbase(), - direction, start, graph, options); + auto travNode = + new TraversalNode(this, nextId(), _ast->query()->vocbase(), + direction->getMember(0), start, graph, options.release()); auto variable = node->getMember(4); TRI_ASSERT(variable->type == NODE_TYPE_VARIABLE); @@ -795,13 +796,13 @@ ExecutionNode* ExecutionPlan::fromNodeShortestPath(ExecutionNode* previous, AstNode const* target = parseTraversalVertexNode(previous, node->getMember(2)); AstNode const* graph = node->getMember(3); - ShortestPathOptions options = CreateShortestPathOptions(node->getMember(4)); - + std::unique_ptr options = + CreateShortestPathOptions(getAst()->query()->trx(), node->getMember(4)); // First create the node auto spNode = new ShortestPathNode(this, nextId(), _ast->query()->vocbase(), - direction->getIntValue(), start, target, - graph, options); + direction, start, target, + graph, options.release()); auto variable = node->getMember(5); TRI_ASSERT(variable->type == NODE_TYPE_VARIABLE); diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp new file mode 100644 index 0000000000..c67d80cd2e --- /dev/null +++ b/arangod/Aql/GraphNode.cpp @@ -0,0 +1,537 @@ +/// @brief Implementation of Shortest Path Execution Node +/// +/// @file arangod/Aql/ShortestPathNode.cpp +/// +/// DISCLAIMER +/// +/// Copyright 2010-2014 triagens GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "GraphNode.h" +#include "Aql/Ast.h" +#include "Aql/ExecutionPlan.h" +#include "Aql/Query.h" +#include "Cluster/ClusterInfo.h" +#include "Cluster/TraverserEngineRegistry.h" +#include "Utils/CollectionNameResolver.h" + +using namespace arangodb::basics; +using namespace arangodb::aql; + +static TRI_edge_direction_e parseDirection (AstNode const* node) { + TRI_ASSERT(node->isIntValue()); + auto dirNum = node->getIntValue(); + + switch (dirNum) { + case 0: + return TRI_EDGE_ANY; + case 1: + return TRI_EDGE_IN; + case 2: + return TRI_EDGE_OUT; + default: + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_QUERY_PARSE, + "direction can only be INBOUND, OUTBOUND or ANY"); + } +} + +GraphNode::GraphNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + traverser::BaseTraverserOptions* options) + : ExecutionNode(plan, id), + _vocbase(vocbase), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), + _graphObj(nullptr), + _tmpObjVariable(_plan->getAst()->variables()->createTemporaryVariable()), + _tmpObjVarNode(_plan->getAst()->createNodeReference(_tmpObjVariable)), + _tmpIdNode(_plan->getAst()->createNodeValueString("", 0)), + _fromCondition(nullptr), + _toCondition(nullptr), + _optionsBuild(false), + _isSmart(false), + _options(options) { + TRI_ASSERT(_vocbase != nullptr); + TRI_ASSERT(options != nullptr); + TRI_ASSERT(_options.get() != nullptr); +} + +GraphNode::GraphNode(ExecutionPlan* plan, + arangodb::velocypack::Slice const& base) + : ExecutionNode(plan, base), + _vocbase(plan->getAst()->query()->vocbase()), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), + _graphObj(nullptr), + _tmpObjVariable(nullptr), + _tmpObjVarNode(nullptr), + _tmpIdNode(nullptr), + _fromCondition(nullptr), + _toCondition(nullptr), + _optionsBuild(false), + _isSmart(false), + _options(nullptr) { + // NOTE: options have to be created by subclass. They differ + // Directions + VPackSlice dirList = base.get("directions"); + for (auto const& it : VPackArrayIterator(dirList)) { + uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it); + TRI_edge_direction_e d; + switch (dir) { + case 0: + TRI_ASSERT(false); + break; + case 1: + d = TRI_EDGE_IN; + break; + case 2: + d = TRI_EDGE_OUT; + break; + default: + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "Invalid direction value"); + break; + } + _directions.emplace_back(d); + } + + // TODO: Can we remove this? + std::string graphName; + if (base.hasKey("graph") && (base.get("graph").isString())) { + graphName = base.get("graph").copyString(); + if (base.hasKey("graphDefinition")) { + _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); + + if (_graphObj == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); + } + } else { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "missing graphDefinition."); + } + } else { + _graphInfo.add(base.get("graph")); + if (!_graphInfo.slice().isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "graph has to be an array."); + } + } + + VPackSlice list = base.get("edgeCollections"); + if (!list.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "traverser needs an array of edge collections."); + } + + for (auto const& it : VPackArrayIterator(list)) { + std::string e = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + _edgeColls.emplace_back( + std::make_unique(e, _vocbase, AccessMode::Type::READ)); + } + + list = base.get("vertexCollections"); + + if (!list.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "traverser needs an array of vertex collections."); + } + + for (auto const& it : VPackArrayIterator(list)) { + std::string v = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + _vertexColls.emplace_back( + std::make_unique(v, _vocbase, AccessMode::Type::READ)); + } + + + // Out variables + if (base.hasKey("vertexOutVariable")) { + _vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable"); + } + if (base.hasKey("edgeOutVariable")) { + _edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable"); + } + + // Temporary Filter Objects + TRI_ASSERT(base.hasKey("tmpObjVariable")); + _tmpObjVariable = varFromVPack(plan->getAst(), base, "tmpObjVariable"); + + TRI_ASSERT(base.hasKey("tmpObjVarNode")); + _tmpObjVarNode = new AstNode(plan->getAst(), base.get("tmpObjVarNode")); + + TRI_ASSERT(base.hasKey("tmpIdNode")); + _tmpIdNode = new AstNode(plan->getAst(), base.get("tmpIdNode")); + + // Filter Condition Parts + TRI_ASSERT(base.hasKey("fromCondition")); + _fromCondition = new AstNode(plan->getAst(), base.get("fromCondition")); + + TRI_ASSERT(base.hasKey("toCondition")); + _toCondition = new AstNode(plan->getAst(), base.get("toCondition")); + _isSmart = VelocyPackHelper::getBooleanValue(base, "isSmart", false); +} + +GraphNode::GraphNode( + ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + std::vector> const& edgeColls, + std::vector> const& vertexColls, + std::vector const& directions, + traverser::BaseTraverserOptions* options) + : ExecutionNode(plan, id), + _vocbase(vocbase), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), + _directions(directions), + _graphObj(nullptr), + _tmpObjVariable(nullptr), + _tmpObjVarNode(nullptr), + _tmpIdNode(nullptr), + _fromCondition(nullptr), + _toCondition(nullptr), + _optionsBuild(false), + _isSmart(false), + _options(options) { + _graphInfo.openArray(); + for (auto const& it : edgeColls) { + // Collections cannot be copied. So we need to create new ones to prevent leaks + _edgeColls.emplace_back(std::make_unique( + it->getName(), _vocbase, AccessMode::Type::READ)); + _graphInfo.add(VPackValue(it->getName())); + } + for (auto& it : vertexColls) { + // Collections cannot be copied. So we need to create new ones to prevent leaks + _vertexColls.emplace_back(std::make_unique( + it->getName(), _vocbase, AccessMode::Type::READ)); + } + + _graphInfo.close(); +} + +/// @brief the cost of a traversal node +double GraphNode::estimateCost(size_t& nrItems) const { + size_t incoming = 0; + double depCost = _dependencies.at(0)->getCost(incoming); + return depCost + _options->estimateCost(nrItems); +} + + + +#ifndef USE_ENTERPRISE +void GraphNode::enhanceEngineInfo(VPackBuilder& builder) const { + if (_graphObj != nullptr) { + _graphObj->enhanceEngineInfo(builder); + } +} +#endif + +void GraphNode::addEngine(traverser::TraverserEngineID const& engine, + arangodb::ServerID const& server) { + TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); + _engines.emplace(server, engine); +} + +/// @brief check if all directions are equal +bool GraphNode::allDirectionsEqual() const { + if (_directions.empty()) { + // no directions! + return false; + } + size_t const n = _directions.size(); + TRI_edge_direction_e const expected = _directions[0]; + + for (size_t i = 1; i < n; ++i) { + if (_directions[i] != expected) { + return false; + } + } + return true; +} + + + +AstNode* GraphNode::getTemporaryRefNode() const { + return _tmpObjVarNode; +} + +Variable const* GraphNode::getTemporaryVariable() const { + return _tmpObjVariable; +} + +void GraphNode::baseToVelocyPackHelper(VPackBuilder& nodes, + bool verbose) const { + ExecutionNode::toVelocyPackHelperGeneric(nodes, + verbose); // call base class method + + nodes.add("database", VPackValue(_vocbase->name())); + + nodes.add("graph", _graphInfo.slice()); + nodes.add(VPackValue("directions")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& d : _directions) { + nodes.add(VPackValue(d)); + } + } + + nodes.add(VPackValue("edgeCollections")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& e : _edgeColls) { + nodes.add(VPackValue(e->getName())); + } + } + + nodes.add(VPackValue("vertexCollections")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& v : _vertexColls) { + nodes.add(VPackValue(v->getName())); + } + } + + if (_graphObj != nullptr) { + nodes.add(VPackValue("graphDefinition")); + _graphObj->toVelocyPack(nodes, verbose); + } + + // Out variables + if (usesVertexOutVariable()) { + nodes.add(VPackValue("vertexOutVariable")); + vertexOutVariable()->toVelocyPack(nodes); + } + if (usesEdgeOutVariable()) { + nodes.add(VPackValue("edgeOutVariable")); + edgeOutVariable()->toVelocyPack(nodes); + } + + // Traversal Filter Conditions + + TRI_ASSERT(_tmpObjVariable != nullptr); + nodes.add(VPackValue("tmpObjVariable")); + _tmpObjVariable->toVelocyPack(nodes); + + TRI_ASSERT(_tmpObjVarNode != nullptr); + nodes.add(VPackValue("tmpObjVarNode")); + _tmpObjVarNode->toVelocyPack(nodes, verbose); + + TRI_ASSERT(_tmpIdNode != nullptr); + nodes.add(VPackValue("tmpIdNode")); + _tmpIdNode->toVelocyPack(nodes, verbose); + + TRI_ASSERT(_fromCondition != nullptr); + nodes.add(VPackValue("fromCondition")); + _fromCondition->toVelocyPack(nodes, verbose); + + TRI_ASSERT(_toCondition != nullptr); + nodes.add(VPackValue("toCondition")); + _toCondition->toVelocyPack(nodes, verbose); +} + +#ifndef USE_ENTERPRISE +void GraphNode::addEdgeColl(std::string const& n, TRI_edge_direction_e dir) { + if (dir == TRI_EDGE_ANY) { + _directions.emplace_back(TRI_EDGE_OUT); + _edgeColls.emplace_back( + std::make_unique(n, _vocbase, AccessMode::Type::READ)); + + _directions.emplace_back(TRI_EDGE_IN); + _edgeColls.emplace_back( + std::make_unique(n, _vocbase, AccessMode::Type::READ)); + } else { + _directions.emplace_back(dir); + _edgeColls.emplace_back( + std::make_unique(n, _vocbase, AccessMode::Type::READ)); + } +} +#endif + +void GraphNode::parseGraphAstNodes(AstNode const* direction, AstNode const* graph) { + TRI_ASSERT(graph != nullptr); + TRI_ASSERT(direction != nullptr); + + TRI_edge_direction_e baseDirection = parseDirection(direction); + + std::unordered_map seenCollections; + + + if (graph->type == NODE_TYPE_COLLECTION_LIST) { + size_t edgeCollectionCount = graph->numMembers(); + + _graphInfo.openArray(); + _edgeColls.reserve(edgeCollectionCount); + _directions.reserve(edgeCollectionCount); + + // First determine whether all edge collections are smart and sharded + // like a common collection: + auto ci = ClusterInfo::instance(); + if (ServerState::instance()->isRunningInCluster()) { + _isSmart = true; + std::string distributeShardsLike; + for (size_t i = 0; i < edgeCollectionCount; ++i) { + auto col = graph->getMember(i); + if (col->type == NODE_TYPE_DIRECTION) { + col = col->getMember(1); // The first member always is the collection + } + std::string n = col->getString(); + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart() || c->distributeShardsLike().empty()) { + _isSmart = false; + break; + } + if (distributeShardsLike.empty()) { + distributeShardsLike = c->distributeShardsLike(); + } else if (distributeShardsLike != c->distributeShardsLike()) { + _isSmart = false; + break; + } + } + } + + auto resolver = std::make_unique(_vocbase); + // List of edge collection names + for (size_t i = 0; i < edgeCollectionCount; ++i) { + auto col = graph->getMember(i); + TRI_edge_direction_e dir = TRI_EDGE_ANY; + + if (col->type == NODE_TYPE_DIRECTION) { + // We have a collection with special direction. + dir = parseDirection(col->getMember(0)); + col = col->getMember(1); + } else { + dir = baseDirection; + } + + std::string eColName = col->getString(); + + // now do some uniqueness checks for the specified collections + auto it = seenCollections.find(eColName); + if (it != seenCollections.end()) { + if ((*it).second != dir) { + std::string msg("conflicting directions specified for collection '" + + std::string(eColName)); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, + msg); + } + // do not re-add the same collection! + continue; + } + seenCollections.emplace(eColName, dir); + + if (resolver->getCollectionTypeCluster(eColName) != TRI_COL_TYPE_EDGE) { + std::string msg("collection type invalid for collection '" + + std::string(eColName) + + ": expecting collection type 'edge'"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, + msg); + } + + _graphInfo.add(VPackValue(eColName)); + if (ServerState::instance()->isRunningInCluster()) { + auto c = ci->getCollection(_vocbase->name(), eColName); + if (!c->isSmart()) { + addEdgeColl(eColName, dir); + } else { + std::vector names; + if (_isSmart) { + names = c->realNames(); + } else { + names = c->realNamesForRead(); + } + for (auto const& name : names) { + addEdgeColl(name, dir); + } + } + } else { + addEdgeColl(eColName, dir); + } + } + _graphInfo.close(); + } else { + if (_edgeColls.empty()) { + if (graph->isStringValue()) { + std::string graphName = graph->getString(); + _graphInfo.add(VPackValue(graphName)); + _graphObj = plan()->getAst()->query()->lookupGraphByName(graphName); + + if (_graphObj == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); + } + + auto eColls = _graphObj->edgeCollections(); + size_t length = eColls.size(); + if (length == 0) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); + } + + // First determine whether all edge collections are smart and sharded + // like a common collection: + auto ci = ClusterInfo::instance(); + if (ServerState::instance()->isRunningInCluster()) { + _isSmart = true; + std::string distributeShardsLike; + for (auto const& n : eColls) { + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart() || c->distributeShardsLike().empty()) { + _isSmart = false; + break; + } + if (distributeShardsLike.empty()) { + distributeShardsLike = c->distributeShardsLike(); + } else if (distributeShardsLike != c->distributeShardsLike()) { + _isSmart = false; + break; + } + } + } + + for (const auto& n : eColls) { + if (ServerState::instance()->isRunningInCluster()) { + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart()) { + addEdgeColl(n, baseDirection); + } else { + std::vector names; + if (_isSmart) { + names = c->realNames(); + } else { + names = c->realNamesForRead(); + } + for (auto const& name : names) { + addEdgeColl(name, baseDirection); + } + } + } else { + addEdgeColl(n, baseDirection); + } + } + + auto vColls = _graphObj->vertexCollections(); + length = vColls.size(); + if (length == 0) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); + } + _vertexColls.reserve(length); + for (auto const& v : vColls) { + _vertexColls.emplace_back(std::make_unique( + v, _vocbase, AccessMode::Type::READ)); + } + } + } + } +} diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h new file mode 100644 index 0000000000..28437acec3 --- /dev/null +++ b/arangod/Aql/GraphNode.h @@ -0,0 +1,216 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_AQL_GRAPH_NODE_H +#define ARANGOD_AQL_GRAPH_NODE_H 1 + +#include "Aql/Collection.h" +#include "Aql/ExecutionNode.h" +#include "Aql/Graphs.h" +#include "Cluster/TraverserEngineRegistry.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/TraverserOptions.h" + +/// NOTE: This Node is purely virtual and is used to unify graph parsing for +/// Traversal and ShortestPath node. It shall never be part of any plan +/// nor will their be a Block to implement it. + +namespace arangodb { + +namespace aql { + +class GraphNode : public ExecutionNode { + protected: + /// @brief Constructor for a new node parsed from AQL + GraphNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + traverser::BaseTraverserOptions* options); + + /// @brief Deserializer for node from VPack + GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); + + /// @brief Internal constructor to clone the node. + GraphNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + std::vector> const& edgeColls, + std::vector> const& vertexColls, + std::vector const& directions, + traverser::BaseTraverserOptions* options); + + public: + virtual ~GraphNode() {} + + /// @brief flag if smart search can be used (Enterprise only) + bool isSmart() const { + return _isSmart; + } + + /// @brief return the database + TRI_vocbase_t* vocbase() const { return _vocbase; } + + /// @brief return the vertex out variable + Variable const* vertexOutVariable() const { return _vertexOutVariable; } + + /// @brief checks if the vertex out variable is used + bool usesVertexOutVariable() const { return _vertexOutVariable != nullptr; } + + /// @brief set the vertex out variable + void setVertexOutput(Variable const* outVar) { _vertexOutVariable = outVar; } + + + /// @brief return the edge out variable + Variable const* edgeOutVariable() const { return _edgeOutVariable; } + + /// @brief checks if the edge out variable is used + bool usesEdgeOutVariable() const { return _edgeOutVariable != nullptr; } + + /// @brief set the edge out variable + void setEdgeOutput(Variable const* outVar) { _edgeOutVariable = outVar; } + + + std::vector> const& edgeColls() const { + return _edgeColls; + } + + std::vector> const& reverseEdgeColls() const { + return _reverseEdgeColls; + } + + std::vector> const& vertexColls() const { + return _vertexColls; + } + + bool allDirectionsEqual() const; + + AstNode* getTemporaryRefNode() const; + + Variable const* getTemporaryVariable() const; + + void enhanceEngineInfo(arangodb::velocypack::Builder&) const; + + virtual traverser::BaseTraverserOptions* options() const = 0; + + virtual void getConditionVariables(std::vector&) const {}; + + /// @brief estimate the cost of this graph node + double estimateCost(size_t& nrItems) const final; + + /// @brief Compute the traversal options containing the expressions + /// MUST! be called after optimization and before creation + /// of blocks. + virtual void prepareOptions() = 0; + + /// @brief Add a traverser engine Running on a DBServer to this node. + /// The block will communicate with them (CLUSTER ONLY) + void addEngine(traverser::TraverserEngineID const&, ServerID const&); + + /// @brief Returns a reference to the engines. (CLUSTER ONLY) + std::unordered_map const* engines() + const { + TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); + return &_engines; + } + + protected: + + //////////////////////////////////////////////////////////////////////////////// + /// @brief Export to VelocyPack + //////////////////////////////////////////////////////////////////////////////// + + void baseToVelocyPackHelper(arangodb::velocypack::Builder&, bool) const; + + //////////////////////////////////////////////////////////////////////////////// + /// @brief Helper function to parse all collection names / directions + //////////////////////////////////////////////////////////////////////////////// + + virtual void addEdgeColl(std::string const& name, TRI_edge_direction_e dir); + + //////////////////////////////////////////////////////////////////////////////// + /// @brief Helper function to parse the graph and direction nodes + //////////////////////////////////////////////////////////////////////////////// + + void parseGraphAstNodes(AstNode const* direction, AstNode const* graph); + + //////////////////////////////////////////////////////////////////////////////// + /// SECTION Shared subclass variables + //////////////////////////////////////////////////////////////////////////////// + protected: + /// @brief the database + TRI_vocbase_t* _vocbase; + + /// @brief vertex output variable + Variable const* _vertexOutVariable; + + /// @brief edge output variable + Variable const* _edgeOutVariable; + + /// @brief input graphInfo only used for serialisation & info + arangodb::velocypack::Builder _graphInfo; + + /// @brief The directions edges are followed + std::vector _directions; + + /// @brief the edge collections + std::vector> _edgeColls; + + /// @brief the reverse edge collections (ShortestPathOnly) + std::vector> _reverseEdgeColls; + + /// @brief the vertex collection names + std::vector> _vertexColls; + + /// @brief our graph... + Graph const* _graphObj; + + /// @brief Temporary pseudo variable for the currently traversed object. + Variable const* _tmpObjVariable; + + /// @brief Reference to the pseudo variable + AstNode* _tmpObjVarNode; + + /// @brief Pseudo string value node to hold the last visted vertex id. + AstNode* _tmpIdNode; + + /// @brief The hard coded condition on _from + /// NOTE: Created by sub classes, as it differs for class + AstNode* _fromCondition; + + /// @brief The hard coded condition on _to + /// NOTE: Created by sub classes, as it differs for class + AstNode* _toCondition; + + /// @brief Flag if options are already prepared. After + /// this flag was set the node cannot be cloned + /// any more. + bool _optionsBuild; + + /// @brief The list of traverser engines grouped by server. + std::unordered_map _engines; + + /// @brief flag, if traversal is smart (enterprise edition only!) + bool _isSmart; + + std::unique_ptr _options; + +}; +} +} +#endif diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 61bc514c7e..7bb33edb59 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -3764,8 +3764,10 @@ void arangodb::aql::prepareTraversalsRule(Optimizer* opt, SmallVector::allocator_type::arena_type a; SmallVector tNodes{a}; plan->findNodesOfType(tNodes, EN::TRAVERSAL, true); + SmallVector sNodes{a}; + plan->findNodesOfType(sNodes, EN::SHORTEST_PATH, true); - if (tNodes.empty()) { + if (tNodes.empty() && sNodes.empty()) { // no traversals present opt->addPlan(std::move(plan), rule, false); return; @@ -3776,6 +3778,13 @@ void arangodb::aql::prepareTraversalsRule(Optimizer* opt, for (auto const& n : tNodes) { TraversalNode* traversal = static_cast(n); traversal->prepareOptions(); + + } + // second make a pass over all shortest path nodes and remove unused + // variables from them + for (auto const& n : sNodes) { + ShortestPathNode* node = static_cast(n); + node->prepareOptions(); } opt->addPlan(std::move(plan), rule, true); diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index e768762516..24bb83f423 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -23,32 +23,35 @@ #include "ShortestPathBlock.h" #include "Aql/AqlItemBlock.h" +#include "Aql/Collection.h" #include "Aql/ExecutionEngine.h" #include "Aql/ExecutionPlan.h" #include "Aql/Query.h" +#include "Cluster/ClusterComm.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Cluster/SmartGraphPathFinder.h" +#endif #include "Utils/OperationCursor.h" #include "Transaction/Methods.h" #include "VocBase/EdgeCollectionInfo.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ManagedDocumentResult.h" +#include "VocBase/ticks.h" #include #include /// @brief typedef the template instantiation of the PathFinder typedef arangodb::basics::DynamicDistanceFinder< - arangodb::velocypack::Slice, arangodb::velocypack::Slice, double, - arangodb::traverser::ShortestPath> ArangoDBPathFinder; + VPackSlice, VPackSlice, double, arangodb::traverser::ShortestPath> + ArangoDBPathFinder; -typedef arangodb::basics::ConstDistanceFinder +typedef arangodb::basics::ConstDistanceFinder< + VPackSlice, VPackSlice, arangodb::basics::VelocyPackHelper::VPackStringHash, + arangodb::basics::VelocyPackHelper::VPackStringEqual, + arangodb::traverser::ShortestPath> ArangoDBConstDistancePathFinder; - - using namespace arangodb::aql; /// @brief Local class to expand edges. @@ -317,7 +320,7 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _vertexReg(ExecutionNode::MaxRegisterId), _edgeVar(nullptr), _edgeReg(ExecutionNode::MaxRegisterId), - _opts(_trx), + _opts(ep->options()), _posInPath(0), _pathLength(0), _path(nullptr), @@ -326,9 +329,7 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _targetReg(ExecutionNode::MaxRegisterId), _useTargetRegister(false), _usedConstant(false) { - - ep->fillOptions(_opts); - _mmdr.reset(new ManagedDocumentResult); + _mmdr.reset(new ManagedDocumentResult()); size_t count = ep->_edgeColls.size(); TRI_ASSERT(ep->_directions.size()); @@ -336,8 +337,8 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, for (size_t j = 0; j < count; ++j) { auto info = std::make_unique( - _trx, ep->_edgeColls[j], ep->_directions[j], _opts.weightAttribute, - _opts.defaultWeight); + _trx, ep->_edgeColls[j]->getName(), ep->_directions[j], + _opts->weightAttribute(), _opts->defaultWeight()); _collectionInfos.emplace_back(info.get()); info.release(); } @@ -370,34 +371,34 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _path = std::make_unique(); if (arangodb::ServerState::instance()->isCoordinator()) { - if (_opts.useWeight) { - _finder.reset(new arangodb::basics::DynamicDistanceFinder< - arangodb::velocypack::Slice, arangodb::velocypack::Slice, - double, arangodb::traverser::ShortestPath>( - EdgeWeightExpanderCluster(this, false), - EdgeWeightExpanderCluster(this, true), _opts.bidirectional)); + _engines = ep->engines(); + +#ifdef USE_ENTERPRISE + // TODO This feature is NOT imlpemented for weighted graphs + if (ep->isSmart() && !_opts->usesWeight()) { + _finder.reset(new arangodb::traverser::SmartGraphConstDistanceFinder( + _opts, _engines, engine->getQuery()->trx()->resolver())); } else { - _finder.reset(new arangodb::basics::ConstDistanceFinder< - arangodb::velocypack::Slice, arangodb::velocypack::Slice, - arangodb::basics::VelocyPackHelper::VPackStringHash, - arangodb::basics::VelocyPackHelper::VPackStringEqual, - arangodb::traverser::ShortestPath>( - ConstDistanceExpanderCluster(this, false), - ConstDistanceExpanderCluster(this, true))); +#endif + if (_opts->usesWeight()) { + _finder.reset(new ArangoDBPathFinder( + EdgeWeightExpanderCluster(this, false), + EdgeWeightExpanderCluster(this, true), true)); + } else { + _finder.reset(new ArangoDBConstDistancePathFinder( + ConstDistanceExpanderCluster(this, false), + ConstDistanceExpanderCluster(this, true))); + } +#ifdef USE_ENTERPRISE } +#endif } else { - if (_opts.useWeight) { - _finder.reset(new arangodb::basics::DynamicDistanceFinder< - arangodb::velocypack::Slice, arangodb::velocypack::Slice, - double, arangodb::traverser::ShortestPath>( + if (_opts->usesWeight()) { + _finder.reset(new ArangoDBPathFinder( EdgeWeightExpanderLocal(this, false), - EdgeWeightExpanderLocal(this, true), _opts.bidirectional)); + EdgeWeightExpanderLocal(this, true), true)); } else { - _finder.reset(new arangodb::basics::ConstDistanceFinder< - arangodb::velocypack::Slice, arangodb::velocypack::Slice, - arangodb::basics::VelocyPackHelper::VPackStringHash, - arangodb::basics::VelocyPackHelper::VPackStringEqual, - arangodb::traverser::ShortestPath>( + _finder.reset(new ArangoDBConstDistancePathFinder( ConstDistanceExpanderLocal(this, false), ConstDistanceExpanderLocal(this, true))); } @@ -443,6 +444,39 @@ int ShortestPathBlock::initializeCursor(AqlItemBlock* items, size_t pos) { return ExecutionBlock::initializeCursor(items, pos); } +/// @brief shutdown: Inform all traverser Engines to destroy themselves +int ShortestPathBlock::shutdown(int errorCode) { + DEBUG_BEGIN_BLOCK(); + // We have to clean up the engines in Coordinator Case. + if (arangodb::ServerState::instance()->isCoordinator()) { + auto cc = arangodb::ClusterComm::instance(); + std::string const url( + "/_db/" + arangodb::basics::StringUtils::urlEncode(_trx->vocbase()->name()) + + "/_internal/traverser/"); + for (auto const& it : *_engines) { + arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer(); + std::unordered_map headers; + auto res = cc->syncRequest( + "", coordTransactionID, "server:" + it.first, RequestType::DELETE_REQ, + url + arangodb::basics::StringUtils::itoa(it.second), "", headers, + 30.0); + if (res->status != CL_COMM_SENT) { + // Note If there was an error on server side we do not have CL_COMM_SENT + std::string message("Could not destruct all traversal engines"); + if (res->errorMessage.length() > 0) { + message += std::string(" : ") + res->errorMessage; + } + LOG_TOPIC(ERR, arangodb::Logger::FIXME) << message; + } + } + } + + return ExecutionBlock::shutdown(errorCode); + + // cppcheck-suppress style + DEBUG_END_BLOCK(); +} + bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { if (_usedConstant) { // Both source and target are constant. @@ -454,6 +488,12 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { // Both are constant, after this computation we are done _usedConstant = true; } + + _startBuilder.clear(); + _targetBuilder.clear(); + + VPackSlice start; + VPackSlice end; if (!_useStartRegister) { auto pos = _startVertexId.find('/'); if (pos == std::string::npos) { @@ -463,13 +503,15 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { "_id are allowed"); return false; } else { - _opts.setStart(_startVertexId); + _startBuilder.add(VPackValue(_startVertexId)); + start = _startBuilder.slice(); } } else { AqlValue const& in = items->getValueReference(_pos, _startReg); if (in.isObject()) { try { - _opts.setStart(_trx->extractIdString(in.slice())); + _startBuilder.add(VPackValue(_trx->extractIdString(in.slice()))); + start = _startBuilder.slice(); } catch (...) { // _id or _key not present... ignore this error and fall through @@ -477,8 +519,7 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { return false; } } else if (in.isString()) { - _startVertexId = in.slice().copyString(); - _opts.setStart(_startVertexId); + start = in.slice(); } else { _engine->getQuery()->registerWarning( TRI_ERROR_BAD_PARAMETER, "Invalid input for Shortest Path: Only " @@ -498,14 +539,15 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { "_id are allowed"); return false; } else { - _opts.setEnd(_targetVertexId); + _targetBuilder.add(VPackValue(_targetVertexId)); + end = _targetBuilder.slice(); } } else { AqlValue const& in = items->getValueReference(_pos, _targetReg); if (in.isObject()) { try { - std::string idString = _trx->extractIdString(in.slice()); - _opts.setEnd(idString); + _targetBuilder.add(VPackValue(_trx->extractIdString(in.slice()))); + end = _targetBuilder.slice(); } catch (...) { // _id or _key not present... ignore this error and fall through @@ -513,8 +555,7 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { return false; } } else if (in.isString()) { - _targetVertexId = in.slice().copyString(); - _opts.setEnd(_targetVertexId); + end = in.slice(); } else { _engine->getQuery()->registerWarning( TRI_ERROR_BAD_PARAMETER, "Invalid input for Shortest Path: Only " @@ -524,8 +565,6 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { } } - VPackSlice start = _opts.getStart(); - VPackSlice end = _opts.getEnd(); TRI_ASSERT(_finder != nullptr); // We do not need this data anymore. Result has been processed. // Save some memory. diff --git a/arangod/Aql/ShortestPathBlock.h b/arangod/Aql/ShortestPathBlock.h index 49fca8e6b2..0a4be14ea9 100644 --- a/arangod/Aql/ShortestPathBlock.h +++ b/arangod/Aql/ShortestPathBlock.h @@ -26,13 +26,14 @@ #include "Aql/ExecutionBlock.h" #include "Aql/ShortestPathNode.h" -#include "V8Server/V8Traverser.h" +#include "Basics/ShortestPathFinder.h" namespace arangodb { class ManagedDocumentResult; namespace traverser { class EdgeCollectionInfo; +class ShortestPath; } namespace aql { @@ -56,6 +57,9 @@ class ShortestPathBlock : public ExecutionBlock { /// @brief initializeCursor int initializeCursor(AqlItemBlock* items, size_t pos) override; + /// @brief shutdown send destroy to all engines. + int shutdown(int errorCode) override final; + /// @brief getSome AqlItemBlock* getSome(size_t atLeast, size_t atMost) override final; @@ -94,7 +98,7 @@ class ShortestPathBlock : public ExecutionBlock { std::unique_ptr _mmdr; /// @brief options to compute the shortest path - traverser::ShortestPathOptions _opts; + traverser::ShortestPathOptions* _opts; /// @brief list of edge collection infos used to compute the path std::vector _collectionInfos; @@ -144,6 +148,13 @@ class ShortestPathBlock : public ExecutionBlock { /// @brief Cache for edges send over the network std::vector>> _coordinatorCache; + /// @brief Builder to make sure that source velocypack does not get out of scope + arangodb::velocypack::Builder _startBuilder; + + /// @brief Builder to make sure that target velocypack does not get out of scope + arangodb::velocypack::Builder _targetBuilder; + + std::unordered_map const* _engines; }; } // namespace arangodb::aql diff --git a/arangod/Aql/ShortestPathNode.cpp b/arangod/Aql/ShortestPathNode.cpp index 8087b181de..822c7eaf42 100644 --- a/arangod/Aql/ShortestPathNode.cpp +++ b/arangod/Aql/ShortestPathNode.cpp @@ -32,8 +32,6 @@ #include "Cluster/ClusterComm.h" #include "Indexes/Index.h" #include "Utils/CollectionNameResolver.h" -#include "VocBase/LogicalCollection.h" -#include "V8Server/V8Traverser.h" #include #include @@ -64,243 +62,99 @@ static void parseNodeInput(AstNode const* node, std::string& id, } } -static TRI_edge_direction_e parseDirection (uint64_t dirNum) { - switch (dirNum) { - case 0: - return TRI_EDGE_ANY; - case 1: - return TRI_EDGE_IN; - case 2: - return TRI_EDGE_OUT; - default: - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_PARSE, - "direction can only be INBOUND, OUTBOUND or ANY"); - } -} - -ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, size_t id, - TRI_vocbase_t* vocbase, uint64_t direction, - AstNode const* start, AstNode const* target, - AstNode const* graph, - ShortestPathOptions const& options) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), +ShortestPathNode::ShortestPathNode( + ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, AstNode const* direction, + AstNode const* start, AstNode const* target, AstNode const* graph, + traverser::ShortestPathOptions* options) + : GraphNode(plan, id, vocbase, options), _inStartVariable(nullptr), - _inTargetVariable(nullptr), - _graphObj(nullptr), - _options(options) { - - TRI_ASSERT(_vocbase != nullptr); + _inTargetVariable(nullptr) { TRI_ASSERT(start != nullptr); TRI_ASSERT(target != nullptr); - TRI_ASSERT(graph != nullptr); - TRI_edge_direction_e baseDirection = parseDirection(direction); + // We have to call this here because we need a specific implementation + // for shortest_path_node. + parseGraphAstNodes(direction, graph); - std::unordered_map seenCollections; - auto addEdgeColl = [&](std::string const& n, TRI_edge_direction_e dir) -> void { - if (dir == TRI_EDGE_ANY) { - _directions.emplace_back(TRI_EDGE_OUT); - _edgeColls.emplace_back(n); + auto ast = _plan->getAst(); + // Let us build the conditions on _from and _to. Just in case we need them. + { + auto const* access = ast->createNodeAttributeAccess( + _tmpObjVarNode, StaticStrings::FromString.c_str(), + StaticStrings::FromString.length()); + auto const* condition = ast->createNodeBinaryOperator( + NODE_TYPE_OPERATOR_BINARY_EQ, access, _tmpIdNode); + _fromCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); + _fromCondition->addMember(condition); - _directions.emplace_back(TRI_EDGE_IN); - _edgeColls.emplace_back(std::move(n)); - } else { - _directions.emplace_back(dir); - _edgeColls.emplace_back(std::move(n)); - } - }; - - auto ci = ClusterInfo::instance(); - - if (graph->type == NODE_TYPE_COLLECTION_LIST) { - size_t edgeCollectionCount = graph->numMembers(); - auto resolver = std::make_unique(vocbase); - _graphInfo.openArray(); - _edgeColls.reserve(edgeCollectionCount); - _directions.reserve(edgeCollectionCount); - - // List of edge collection names - for (size_t i = 0; i < edgeCollectionCount; ++i) { - TRI_edge_direction_e dir = TRI_EDGE_ANY; - auto col = graph->getMember(i); - - if (col->type == NODE_TYPE_DIRECTION) { - TRI_ASSERT(col->numMembers() == 2); - auto dirNode = col->getMember(0); - // We have a collection with special direction. - TRI_ASSERT(dirNode->isIntValue()); - dir = parseDirection(dirNode->getIntValue()); - col = col->getMember(1); - } else { - dir = baseDirection; - } - - std::string eColName = col->getString(); - - // now do some uniqueness checks for the specified collections - auto it = seenCollections.find(eColName); - if (it != seenCollections.end()) { - if ((*it).second != dir) { - std::string msg("conflicting directions specified for collection '" + - std::string(eColName)); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, - msg); - } - // do not re-add the same collection! - continue; - } - seenCollections.emplace(eColName, dir); - - auto eColType = resolver->getCollectionTypeCluster(eColName); - if (eColType != TRI_COL_TYPE_EDGE) { - std::string msg("collection type invalid for collection '" + - std::string(eColName) + - ": expecting collection type 'edge'"); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, - msg); - } - - _graphInfo.add(VPackValue(eColName)); - if (ServerState::instance()->isRunningInCluster()) { - auto c = ci->getCollection(_vocbase->name(), eColName); - if (!c->isSmart()) { - addEdgeColl(eColName, dir); - } else { - std::vector names; - names = c->realNamesForRead(); - for (auto const& name : names) { - addEdgeColl(name, dir); - } - } - } else { - addEdgeColl(eColName, dir); - } - - if (dir == TRI_EDGE_ANY) { - // collection with direction ANY must be added again - _graphInfo.add(VPackValue(eColName)); - } - - } - _graphInfo.close(); - } else { - if (_edgeColls.empty()) { - if (graph->isStringValue()) { - std::string graphName = graph->getString(); - _graphInfo.add(VPackValue(graphName)); - _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); - - if (_graphObj == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); - } - - auto eColls = _graphObj->edgeCollections(); - size_t length = eColls.size(); - if (length == 0) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); - } - _edgeColls.reserve(length); - _directions.reserve(length); - - for (const auto& n : eColls) { - if (ServerState::instance()->isRunningInCluster()) { - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart()) { - addEdgeColl(n, baseDirection); - } else { - std::vector names; - names = c->realNamesForRead(); - for (auto const& name : names) { - addEdgeColl(name, baseDirection); - } - } - } else { - addEdgeColl(n, baseDirection); - } - } - } - } } + TRI_ASSERT(_fromCondition != nullptr); + { + auto const* access = ast->createNodeAttributeAccess( + _tmpObjVarNode, StaticStrings::ToString.c_str(), + StaticStrings::ToString.length()); + auto const* condition = ast->createNodeBinaryOperator( + NODE_TYPE_OPERATOR_BINARY_EQ, access, _tmpIdNode); + _toCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); + _toCondition->addMember(condition); + } + TRI_ASSERT(_toCondition != nullptr); parseNodeInput(start, _startVertexId, _inStartVariable); parseNodeInput(target, _targetVertexId, _inTargetVariable); } -ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, size_t id, - TRI_vocbase_t* vocbase, - std::vector const& edgeColls, - std::vector const& directions, - Variable const* inStartVariable, - std::string const& startVertexId, - Variable const* inTargetVariable, - std::string const& targetVertexId, - ShortestPathOptions const& options) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), +ShortestPathNode::ShortestPathNode( + ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + std::vector> const& edgeColls, + std::vector> const& reverseEdgeColls, + std::vector> const& vertexColls, + std::vector const& directions, + Variable const* inStartVariable, std::string const& startVertexId, + Variable const* inTargetVariable, std::string const& targetVertexId, + traverser::ShortestPathOptions* options) + : GraphNode(plan, id, vocbase, edgeColls, vertexColls, directions, options), _inStartVariable(inStartVariable), _startVertexId(startVertexId), _inTargetVariable(inTargetVariable), - _targetVertexId(targetVertexId), - _directions(directions), - _graphObj(nullptr), - _options(options) { - - _graphInfo.openArray(); - for (auto const& it : edgeColls) { - _edgeColls.emplace_back(it); - _graphInfo.add(VPackValue(it)); + _targetVertexId(targetVertexId) { + for (auto const& it : reverseEdgeColls) { + // Collections cannot be copied. So we need to create new ones to prevent leaks + _reverseEdgeColls.emplace_back(std::make_unique( + it->getName(), _vocbase, AccessMode::Type::READ)); } - _graphInfo.close(); } -void ShortestPathNode::fillOptions(arangodb::traverser::ShortestPathOptions& opts) const { - if (!_options.weightAttribute.empty()) { - opts.useWeight = true; - opts.weightAttribute = _options.weightAttribute; - opts.defaultWeight = _options.defaultWeight; - } else { - opts.useWeight = false; - } +ShortestPathNode::~ShortestPathNode() { +} + +arangodb::traverser::ShortestPathOptions* ShortestPathNode::options() + const { + return static_cast(_options.get()); } ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base) - : ExecutionNode(plan, base), - _vocbase(plan->getAst()->query()->vocbase()), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), + : GraphNode(plan, base), _inStartVariable(nullptr), - _inTargetVariable(nullptr), - _graphObj(nullptr) { - // Directions - VPackSlice dirList = base.get("directions"); - for (auto const& it : VPackArrayIterator(dirList)) { - uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it); - TRI_edge_direction_e d; - switch (dir) { - case 0: - d = TRI_EDGE_ANY; - break; - case 1: - d = TRI_EDGE_IN; - break; - case 2: - d = TRI_EDGE_OUT; - break; - default: - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "Invalid direction value"); - break; - } - _directions.emplace_back(d); + _inTargetVariable(nullptr) { + + // Reverse Collections + VPackSlice list = base.get("reverseEdgeCollections"); + if (!list.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_QUERY_BAD_JSON_PLAN, + "shortest path needs an array of reverse edge collections."); } + for (auto const& it : VPackArrayIterator(list)) { + std::string e = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + _reverseEdgeColls.emplace_back( + std::make_unique(e, _vocbase, AccessMode::Type::READ)); + } + + _options = std::make_unique( + plan->getAst()->query()->trx()); // Start Vertex if (base.hasKey("startInVariable")) { _inStartVariable = varFromVPack(plan->getAst(), base, "startInVariable"); @@ -334,80 +188,15 @@ ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, } } - std::string graphName; - if (base.hasKey("graph") && (base.get("graph").isString())) { - graphName = base.get("graph").copyString(); - if (base.hasKey("graphDefinition")) { - _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); - - if (_graphObj == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); - } - - auto const& eColls = _graphObj->edgeCollections(); - for (auto const& it : eColls) { - _edgeColls.push_back(it); - - // if there are twice as many directions as collections, this means we - // have a shortest path with direction ANY. we must add each collection - // twice then - if (_directions.size() == 2 * eColls.size()) { - // add collection again - _edgeColls.push_back(it); - } - } - } else { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "missing graphDefinition."); - } - } else { - _graphInfo.add(base.get("graph")); - if (!_graphInfo.slice().isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "graph has to be an array."); - } - // List of edge collection names - for (auto const& it : VPackArrayIterator(_graphInfo.slice())) { - if (!it.isString()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "graph has to be an array of strings."); - } - _edgeColls.emplace_back(it.copyString()); - } - if (_edgeColls.empty()) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_BAD_JSON_PLAN, - "graph has to be a non empty array of strings."); - } - } - - // Out variables - if (base.hasKey("vertexOutVariable")) { - _vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable"); - } - if (base.hasKey("edgeOutVariable")) { - _edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable"); - } - // Flags if (base.hasKey("shortestPathFlags")) { - _options = ShortestPathOptions(base); + // _options = ShortestPathOptions(base); } } void ShortestPathNode::toVelocyPackHelper(VPackBuilder& nodes, bool verbose) const { - ExecutionNode::toVelocyPackHelperGeneric(nodes, - verbose); // call base class method - nodes.add("database", VPackValue(_vocbase->name())); - nodes.add("graph", _graphInfo.slice()); - nodes.add(VPackValue("directions")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& d : _directions) { - nodes.add(VPackValue(d)); - } - } + baseToVelocyPackHelper(nodes, verbose); // In variables if (usesStartInVariable()) { @@ -424,23 +213,18 @@ void ShortestPathNode::toVelocyPackHelper(VPackBuilder& nodes, nodes.add("targetVertexId", VPackValue(_targetVertexId)); } - if (_graphObj != nullptr) { - nodes.add(VPackValue("graphDefinition")); - _graphObj->toVelocyPack(nodes, verbose); + // Reverse collections + nodes.add(VPackValue("reverseEdgeCollections")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& e : _edgeColls) { + nodes.add(VPackValue(e->getName())); + } } - // Out variables - if (usesVertexOutVariable()) { - nodes.add(VPackValue("vertexOutVariable")); - vertexOutVariable()->toVelocyPack(nodes); - } - if (usesEdgeOutVariable()) { - nodes.add(VPackValue("edgeOutVariable")); - edgeOutVariable()->toVelocyPack(nodes); - } - nodes.add(VPackValue("shortestPathFlags")); - _options.toVelocyPack(nodes); + // nodes.add(VPackValue("shortestPathFlags")); + // _options.toVelocyPack(nodes); // And close it: nodes.close(); @@ -449,9 +233,14 @@ void ShortestPathNode::toVelocyPackHelper(VPackBuilder& nodes, ExecutionNode* ShortestPathNode::clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const { - auto c = new ShortestPathNode(plan, _id, _vocbase, _edgeColls, _directions, + auto tmp = + std::make_unique(*options()); + auto c = new ShortestPathNode(plan, _id, _vocbase, _edgeColls, + _reverseEdgeColls, _vertexColls, _directions, _inStartVariable, _startVertexId, - _inTargetVariable, _targetVertexId, _options); + _inTargetVariable, _targetVertexId, tmp.get()); + tmp.release(); + if (usesVertexOutVariable()) { auto vertexOutVariable = _vertexOutVariable; if (withProperties) { @@ -472,31 +261,56 @@ ExecutionNode* ShortestPathNode::clone(ExecutionPlan* plan, c->setEdgeOutput(edgeOutVariable); } + // Temporary Filter Objects + c->_tmpObjVariable = _tmpObjVariable; + c->_tmpObjVarNode = _tmpObjVarNode; + c->_tmpIdNode = _tmpIdNode; + + // Filter Condition Parts + c->_fromCondition = _fromCondition->clone(_plan->getAst()); + c->_toCondition = _toCondition->clone(_plan->getAst()); + cloneHelper(c, plan, withDependencies, withProperties); return static_cast(c); } -double ShortestPathNode::estimateCost(size_t& nrItems) const { - // Standard estimation for Shortest path is O(|E| + |V|*LOG(|V|)) - // At this point we know |E| but do not know |V|. - size_t incoming = 0; - double depCost = _dependencies.at(0)->getCost(incoming); - auto collections = _plan->getAst()->query()->collections(); - size_t edgesCount = 0; - - TRI_ASSERT(collections != nullptr); - - for (auto const& it : _edgeColls) { - auto collection = collections->get(it); - - if (collection == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "unexpected pointer for collection"); - } - edgesCount += collection->count(); +void ShortestPathNode::prepareOptions() { + if (_optionsBuild) { + return; } - // Hard-Coded number of vertices edges / 10 - nrItems = edgesCount + static_cast(std::log2(edgesCount / 10) * (edgesCount / 10)); - return depCost + nrItems; + + size_t numEdgeColls = _edgeColls.size(); + TRI_ASSERT(!_optionsBuild); + _options->setVariable(_tmpObjVariable); + + auto opts = dynamic_cast(_options.get()); + + Ast* ast = _plan->getAst(); + + TRI_ASSERT(numEdgeColls == _directions.size()); + TRI_ASSERT(numEdgeColls == _reverseEdgeColls.size()); + // FIXME: _options->_baseLookupInfos.reserve(numEdgeColls); + // Compute Edge Indexes. First default indexes: + for (size_t i = 0; i < numEdgeColls; ++i) { + switch (_directions[i]) { + case TRI_EDGE_IN: + opts->addLookupInfo(ast, _edgeColls[i]->getName(), + StaticStrings::ToString, _toCondition); + opts->addReverseLookupInfo(ast, _reverseEdgeColls[i]->getName(), + StaticStrings::FromString, _fromCondition); + break; + case TRI_EDGE_OUT: + opts->addLookupInfo(ast, _edgeColls[i]->getName(), + StaticStrings::FromString, _fromCondition); + opts->addReverseLookupInfo(ast, _reverseEdgeColls[i]->getName(), + StaticStrings::ToString, _toCondition); + break; + case TRI_EDGE_ANY: + TRI_ASSERT(false); + break; + } + } + + _optionsBuild = true; } diff --git a/arangod/Aql/ShortestPathNode.h b/arangod/Aql/ShortestPathNode.h index 0ffb462047..92105c1905 100644 --- a/arangod/Aql/ShortestPathNode.h +++ b/arangod/Aql/ShortestPathNode.h @@ -24,21 +24,16 @@ #ifndef ARANGOD_AQL_SHORTEST_PATH_NODE_H #define ARANGOD_AQL_SHORTEST_PATH_NODE_H 1 -#include "Aql/ExecutionNode.h" -#include "Aql/Graphs.h" -#include "Aql/ShortestPathOptions.h" +#include "Aql/GraphNode.h" +#include "VocBase/TraverserOptions.h" #include namespace arangodb { - -namespace traverser { -struct ShortestPathOptions; -} namespace aql { /// @brief class ShortestPathNode -class ShortestPathNode : public ExecutionNode { +class ShortestPathNode : public GraphNode { friend class ExecutionBlock; friend class RedundantCalculationsReplacer; friend class ShortestPathBlock; @@ -46,23 +41,24 @@ class ShortestPathNode : public ExecutionNode { /// @brief constructor with a vocbase and a collection name public: ShortestPathNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - uint64_t direction, AstNode const* start, AstNode const* target, - AstNode const* graph, ShortestPathOptions const& options); + AstNode const* direction, AstNode const* start, AstNode const* target, + AstNode const* graph, traverser::ShortestPathOptions* options); ShortestPathNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); - ~ShortestPathNode() {} + ~ShortestPathNode(); /// @brief Internal constructor to clone the node. private: - ShortestPathNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - std::vector const& edgeColls, - std::vector const& directions, - Variable const* inStartVariable, - std::string const& startVertexId, - Variable const* inTargetVariable, - std::string const& targetVertexId, - ShortestPathOptions const& options); + ShortestPathNode( + ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + std::vector> const& edgeColls, + std::vector> const& reverseEdgeColls, + std::vector> const& vertexColls, + std::vector const& directions, + Variable const* inStartVariable, std::string const& startVertexId, + Variable const* inTargetVariable, std::string const& targetVertexId, + traverser::ShortestPathOptions* options); public: /// @brief return the type of the node @@ -76,9 +72,6 @@ class ShortestPathNode : public ExecutionNode { ExecutionNode* clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const override final; - /// @brief the cost of a traversal node - double estimateCost(size_t&) const override final; - /// @brief Test if this node uses an in variable or constant for start bool usesStartInVariable() const { return _inStartVariable != nullptr; @@ -99,24 +92,6 @@ class ShortestPathNode : public ExecutionNode { std::string const getTargetVertex() const { return _targetVertexId; } - /// @brief return the vertex out variable - Variable const* vertexOutVariable() const { return _vertexOutVariable; } - - /// @brief checks if the vertex out variable is used - bool usesVertexOutVariable() const { return _vertexOutVariable != nullptr; } - - /// @brief set the vertex out variable - void setVertexOutput(Variable const* outVar) { _vertexOutVariable = outVar; } - - /// @brief return the edge out variable - Variable const* edgeOutVariable() const { return _edgeOutVariable; } - - /// @brief checks if the edge out variable is used - bool usesEdgeOutVariable() const { return _edgeOutVariable != nullptr; } - - /// @brief set the edge out variable - void setEdgeOutput(Variable const* outVar) { _edgeOutVariable = outVar; } - /// @brief getVariablesSetHere std::vector getVariablesSetHere() const override final { std::vector vars; @@ -152,19 +127,19 @@ class ShortestPathNode : public ExecutionNode { } } - void fillOptions(arangodb::traverser::ShortestPathOptions&) const; + traverser::ShortestPathOptions* options() const override; + /// @brief Compute the traversal options containing the expressions + /// MUST! be called after optimization and before creation + /// of blocks. + void prepareOptions() override; + + /// @brief Helper function to parse all collection names / directions +#ifdef USE_ENTERPRISE + void addEdgeColl(std::string const& name, TRI_edge_direction_e dir) override; +#endif private: - /// @brief the database - TRI_vocbase_t* _vocbase; - - /// @brief vertex output variable - Variable const* _vertexOutVariable; - - /// @brief vertex output variable - Variable const* _edgeOutVariable; - /// @brief input variable only used if _vertexId is unused Variable const* _inStartVariable; @@ -177,20 +152,6 @@ class ShortestPathNode : public ExecutionNode { /// @brief input vertexId only used if _inVariable is unused std::string _targetVertexId; - /// @brief input graphInfo only used for serialisation & info - arangodb::velocypack::Builder _graphInfo; - - /// @brief The directions edges are followed - std::vector _directions; - - /// @brief the edge collection names - std::vector _edgeColls; - - /// @brief our graph... - Graph const* _graphObj; - - /// @brief Options for traversals - ShortestPathOptions _options; }; } // namespace arangodb::aql diff --git a/arangod/Aql/ShortestPathOptions.cpp b/arangod/Aql/ShortestPathOptions.cpp deleted file mode 100644 index 4e2374d0d2..0000000000 --- a/arangod/Aql/ShortestPathOptions.cpp +++ /dev/null @@ -1,54 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2016-2016 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -//////////////////////////////////////////////////////////////////////////////// - -#include "Aql/ShortestPathOptions.h" - -#include - -using namespace arangodb::aql; - -ShortestPathOptions::ShortestPathOptions(VPackSlice const& slice) - : weightAttribute(), defaultWeight(1) { - VPackSlice obj = slice.get("shortestPathFlags"); - - if (obj.isObject()) { - if (obj.hasKey("weightAttribute")) { - VPackSlice v = obj.get("weightAttribute"); - if (v.isString()) { - weightAttribute = v.copyString(); - } - } - - if (obj.hasKey("defaultWeight")) { - VPackSlice v = obj.get("defaultWeight"); - if (v.isNumber()) { - defaultWeight = v.getNumericValue(); - } - } - } -} - -void ShortestPathOptions::toVelocyPack(VPackBuilder& builder) const { - VPackObjectBuilder guard(&builder); - builder.add("weightAttribute", VPackValue(weightAttribute)); - builder.add("defaultWeight", VPackValue(defaultWeight)); -} diff --git a/arangod/Aql/ShortestPathOptions.h b/arangod/Aql/ShortestPathOptions.h deleted file mode 100644 index 040675182b..0000000000 --- a/arangod/Aql/ShortestPathOptions.h +++ /dev/null @@ -1,54 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2016-2016 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGOD_AQL_SHORTEST_PATH_OPTIONS_H -#define ARANGOD_AQL_SHORTEST_PATH_OPTIONS_H 1 - -#include "Basics/Common.h" - -#include -#include - -namespace arangodb { -namespace aql { - -/// @brief TraversalOptions -struct ShortestPathOptions { - - /// @brief constructor - explicit ShortestPathOptions(arangodb::velocypack::Slice const&); - - /// @brief constructor, using default values - ShortestPathOptions() - : weightAttribute(), - defaultWeight(1) {} - - void toVelocyPack(arangodb::velocypack::Builder&) const; - - std::string weightAttribute; - double defaultWeight; -}; - -} // namespace arangodb::aql -} // namespace arangodb -#endif - diff --git a/arangod/Aql/TraversalNode.cpp b/arangod/Aql/TraversalNode.cpp index 9c0d702069..ca17ab963c 100644 --- a/arangod/Aql/TraversalNode.cpp +++ b/arangod/Aql/TraversalNode.cpp @@ -82,51 +82,21 @@ void TraversalNode::TraversalEdgeConditionBuilder::toVelocyPack( _modCondition->toVelocyPack(builder, verbose); } -static TRI_edge_direction_e parseDirection (AstNode const* node) { - TRI_ASSERT(node->isIntValue()); - auto dirNum = node->getIntValue(); - - switch (dirNum) { - case 0: - return TRI_EDGE_ANY; - case 1: - return TRI_EDGE_IN; - case 2: - return TRI_EDGE_OUT; - default: - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_PARSE, - "direction can only be INBOUND, OUTBOUND or ANY"); - } -} - TraversalNode::TraversalNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, AstNode const* direction, AstNode const* start, AstNode const* graph, - std::unique_ptr& options) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), + TraverserOptions* options) + : GraphNode(plan, id, vocbase, options), _pathOutVariable(nullptr), _inVariable(nullptr), - _graphObj(nullptr), - _condition(nullptr), - _tmpObjVariable(_plan->getAst()->variables()->createTemporaryVariable()), - _tmpObjVarNode(_plan->getAst()->createNodeReference(_tmpObjVariable)), - _tmpIdNode(_plan->getAst()->createNodeValueString("", 0)), - _fromCondition(nullptr), - _toCondition(nullptr), - _optionsBuild(false), - _isSmart(false) { - TRI_ASSERT(_vocbase != nullptr); - TRI_ASSERT(direction != nullptr); + _condition(nullptr) { TRI_ASSERT(start != nullptr); - TRI_ASSERT(graph != nullptr); - _options.reset(options.release()); + // We have to call this here because we need a specific implementation + // for shortest_path_node. + parseGraphAstNodes(direction, graph); - auto ast = _plan->getAst(); // Let us build the conditions on _from and _to. Just in case we need them. + auto ast = _plan->getAst(); { auto const* access = ast->createNodeAttributeAccess( _tmpObjVarNode, StaticStrings::FromString.c_str(), @@ -147,215 +117,6 @@ TraversalNode::TraversalNode(ExecutionPlan* plan, size_t id, TRI_ASSERT(_toCondition != nullptr); TRI_ASSERT(_toCondition->type == NODE_TYPE_OPERATOR_BINARY_EQ); - auto resolver = std::make_unique(vocbase); - - // Parse Steps and direction - TRI_ASSERT(direction->type == NODE_TYPE_DIRECTION); - TRI_ASSERT(direction->numMembers() == 2); - // Member 0 is the direction. Already the correct Integer. - // Is not inserted by user but by enum. - TRI_edge_direction_e baseDirection = parseDirection(direction->getMember(0)); - - std::unordered_map seenCollections; - - auto addEdgeColl = [&](std::string const& n, TRI_edge_direction_e dir) -> void { - if (_isSmart) { - if (n.compare(0, 6, "_from_") == 0) { - if (dir != TRI_EDGE_IN) { - _directions.emplace_back(TRI_EDGE_OUT); - _edgeColls.emplace_back(std::make_unique( - n, _vocbase, AccessMode::Type::READ)); - } - return; - } else if (n.compare(0, 4, "_to_") == 0) { - if (dir != TRI_EDGE_OUT) { - _directions.emplace_back(TRI_EDGE_IN); - _edgeColls.emplace_back(std::make_unique( - n, _vocbase, AccessMode::Type::READ)); - } - return; - } - } - - if (dir == TRI_EDGE_ANY) { - _directions.emplace_back(TRI_EDGE_OUT); - _edgeColls.emplace_back(std::make_unique( - n, _vocbase, AccessMode::Type::READ)); - - _directions.emplace_back(TRI_EDGE_IN); - _edgeColls.emplace_back(std::make_unique( - n, _vocbase, AccessMode::Type::READ)); - } else { - _directions.emplace_back(dir); - _edgeColls.emplace_back(std::make_unique( - n, _vocbase, AccessMode::Type::READ)); - } - }; - - if (graph->type == NODE_TYPE_COLLECTION_LIST) { - size_t edgeCollectionCount = graph->numMembers(); - - _graphInfo.openArray(); - _edgeColls.reserve(edgeCollectionCount); - _directions.reserve(edgeCollectionCount); - - // First determine whether all edge collections are smart and sharded - // like a common collection: - auto ci = ClusterInfo::instance(); - if (ServerState::instance()->isRunningInCluster()) { - _isSmart = true; - std::string distributeShardsLike; - for (size_t i = 0; i < edgeCollectionCount; ++i) { - auto col = graph->getMember(i); - if (col->type == NODE_TYPE_DIRECTION) { - col = col->getMember(1); // The first member always is the collection - } - std::string n = col->getString(); - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart() || c->distributeShardsLike().empty()) { - _isSmart = false; - break; - } - if (distributeShardsLike.empty()) { - distributeShardsLike = c->distributeShardsLike(); - } else if (distributeShardsLike != c->distributeShardsLike()) { - _isSmart = false; - break; - } - } - } - - // List of edge collection names - for (size_t i = 0; i < edgeCollectionCount; ++i) { - auto col = graph->getMember(i); - TRI_edge_direction_e dir = TRI_EDGE_ANY; - - if (col->type == NODE_TYPE_DIRECTION) { - // We have a collection with special direction. - dir = parseDirection(col->getMember(0)); - col = col->getMember(1); - } else { - dir = baseDirection; - } - - std::string eColName = col->getString(); - - // now do some uniqueness checks for the specified collections - auto it = seenCollections.find(eColName); - if (it != seenCollections.end()) { - if ((*it).second != dir) { - std::string msg("conflicting directions specified for collection '" + - std::string(eColName)); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, - msg); - } - // do not re-add the same collection! - continue; - } - seenCollections.emplace(eColName, dir); - - if (resolver->getCollectionTypeCluster(eColName) != TRI_COL_TYPE_EDGE) { - std::string msg("collection type invalid for collection '" + - std::string(eColName) + - ": expecting collection type 'edge'"); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, - msg); - } - - _graphInfo.add(VPackValue(eColName)); - if (ServerState::instance()->isRunningInCluster()) { - auto c = ci->getCollection(_vocbase->name(), eColName); - if (!c->isSmart()) { - addEdgeColl(eColName, dir); - } else { - std::vector names; - if (_isSmart) { - names = c->realNames(); - } else { - names = c->realNamesForRead(); - } - for (auto const& name : names) { - addEdgeColl(name, dir); - } - } - } else { - addEdgeColl(eColName, dir); - } - } - _graphInfo.close(); - } else { - if (_edgeColls.empty()) { - if (graph->isStringValue()) { - std::string graphName = graph->getString(); - _graphInfo.add(VPackValue(graphName)); - _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); - - if (_graphObj == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); - } - - auto eColls = _graphObj->edgeCollections(); - size_t length = eColls.size(); - if (length == 0) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); - } - - // First determine whether all edge collections are smart and sharded - // like a common collection: - auto ci = ClusterInfo::instance(); - if (ServerState::instance()->isRunningInCluster()) { - _isSmart = true; - std::string distributeShardsLike; - for (auto const& n : eColls) { - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart() || c->distributeShardsLike().empty()) { - _isSmart = false; - break; - } - if (distributeShardsLike.empty()) { - distributeShardsLike = c->distributeShardsLike(); - } else if (distributeShardsLike != c->distributeShardsLike()) { - _isSmart = false; - break; - } - } - } - - for (const auto& n : eColls) { - if (ServerState::instance()->isRunningInCluster()) { - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart()) { - addEdgeColl(n, baseDirection); - } else { - std::vector names; - if (_isSmart) { - names = c->realNames(); - } else { - names = c->realNamesForRead(); - } - for (auto const& name : names) { - addEdgeColl(name, baseDirection); - } - } - } else { - addEdgeColl(n, baseDirection); - } - } - - auto vColls = _graphObj->vertexCollections(); - length = vColls.size(); - if (length == 0) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); - } - _vertexColls.reserve(length); - for (auto const& v : vColls) { - _vertexColls.emplace_back(std::make_unique( - v, _vocbase, AccessMode::Type::READ)); - } - } - } - } - // Parse start node switch (start->type) { case NODE_TYPE_REFERENCE: @@ -391,85 +152,22 @@ TraversalNode::TraversalNode( std::vector> const& vertexColls, Variable const* inVariable, std::string const& vertexId, std::vector const& directions, - std::unique_ptr& options) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), + TraverserOptions* options) + : GraphNode(plan, id, vocbase, edgeColls, vertexColls, directions, options), _pathOutVariable(nullptr), _inVariable(inVariable), _vertexId(vertexId), - _directions(directions), - _graphObj(nullptr), - _condition(nullptr), - _tmpObjVariable(nullptr), - _tmpObjVarNode(nullptr), - _tmpIdNode(nullptr), - _fromCondition(nullptr), - _toCondition(nullptr), - _optionsBuild(false), - _isSmart(false) { - _options.reset(options.release()); - _graphInfo.openArray(); - - for (auto& it : edgeColls) { - // Collections cannot be copied. So we need to create new ones to prevent leaks - _edgeColls.emplace_back(std::make_unique( - it->getName(), _vocbase, AccessMode::Type::READ)); - _graphInfo.add(VPackValue(it->getName())); - } - - for (auto& it : vertexColls) { - // Collections cannot be copied. So we need to create new ones to prevent leaks - _vertexColls.emplace_back(std::make_unique( - it->getName(), _vocbase, AccessMode::Type::READ)); - } - - _graphInfo.close(); + _condition(nullptr) { } TraversalNode::TraversalNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base) - : ExecutionNode(plan, base), - _vocbase(plan->getAst()->query()->vocbase()), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), + : GraphNode(plan, base), _pathOutVariable(nullptr), _inVariable(nullptr), - _graphObj(nullptr), - _condition(nullptr), - _options(std::make_unique( - _plan->getAst()->query()->trx(), base)), - _tmpObjVariable(nullptr), - _tmpObjVarNode(nullptr), - _tmpIdNode(nullptr), - _fromCondition(nullptr), - _toCondition(nullptr), - _optionsBuild(false), - _isSmart(false) { - - VPackSlice dirList = base.get("directions"); - for (auto const& it : VPackArrayIterator(dirList)) { - uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it); - TRI_edge_direction_e d; - switch (dir) { - case 0: - TRI_ASSERT(false); - break; - case 1: - d = TRI_EDGE_IN; - break; - case 2: - d = TRI_EDGE_OUT; - break; - default: - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "Invalid direction value"); - break; - } - _directions.emplace_back(d); - } - + _condition(nullptr) { + _options = std::make_unique( + _plan->getAst()->query()->trx(), base); // In Vertex if (base.hasKey("inVariable")) { _inVariable = varFromVPack(plan->getAst(), base, "inVariable"); @@ -503,81 +201,11 @@ TraversalNode::TraversalNode(ExecutionPlan* plan, } } - // TODO: Can we remove this? - std::string graphName; - if (base.hasKey("graph") && (base.get("graph").isString())) { - graphName = base.get("graph").copyString(); - if (base.hasKey("graphDefinition")) { - _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); - - if (_graphObj == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); - } - } else { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "missing graphDefinition."); - } - } else { - _graphInfo.add(base.get("graph")); - if (!_graphInfo.slice().isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "graph has to be an array."); - } - } - - list = base.get("edgeCollections"); - if (!list.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "traverser needs an array of edge collections."); - } - - for (auto const& it : VPackArrayIterator(list)) { - std::string e = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); - _edgeColls.emplace_back( - std::make_unique(e, _vocbase, AccessMode::Type::READ)); - } - - list = base.get("vertexCollections"); - - if (!list.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "traverser needs an array of vertex collections."); - } - - for (auto const& it : VPackArrayIterator(list)) { - std::string v = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); - _vertexColls.emplace_back( - std::make_unique(v, _vocbase, AccessMode::Type::READ)); - } - // Out variables - if (base.hasKey("vertexOutVariable")) { - _vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable"); - } - if (base.hasKey("edgeOutVariable")) { - _edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable"); - } if (base.hasKey("pathOutVariable")) { _pathOutVariable = varFromVPack(plan->getAst(), base, "pathOutVariable"); } - // Temporary Filter Objects - TRI_ASSERT(base.hasKey("tmpObjVariable")); - _tmpObjVariable = varFromVPack(plan->getAst(), base, "tmpObjVariable"); - - TRI_ASSERT(base.hasKey("tmpObjVarNode")); - _tmpObjVarNode = new AstNode(plan->getAst(), base.get("tmpObjVarNode")); - - TRI_ASSERT(base.hasKey("tmpIdNode")); - _tmpIdNode = new AstNode(plan->getAst(), base.get("tmpIdNode")); - - // Filter Condition Parts - TRI_ASSERT(base.hasKey("fromCondition")); - _fromCondition = new AstNode(plan->getAst(), base.get("fromCondition")); - - TRI_ASSERT(base.hasKey("toCondition")); - _toCondition = new AstNode(plan->getAst(), base.get("toCondition")); - list = base.get("globalEdgeConditions"); if (list.isArray()) { for (auto const& cond : VPackArrayIterator(list)) { @@ -637,60 +265,16 @@ int TraversalNode::checkIsOutVariable(size_t variableId) const { /// @brief check whether an access is inside the specified range bool TraversalNode::isInRange(uint64_t depth, bool isEdge) const { + uint64_t max = static_cast(_options.get())->maxDepth; if (isEdge) { - return (depth < _options->maxDepth); + return (depth < max); } - return (depth <= _options->maxDepth); -} - -/// @brief check if all directions are equal -bool TraversalNode::allDirectionsEqual() const { - if (_directions.empty()) { - // no directions! - return false; - } - size_t const n = _directions.size(); - TRI_edge_direction_e const expected = _directions[0]; - - for (size_t i = 1; i < n; ++i) { - if (_directions[i] != expected) { - return false; - } - } - return true; + return (depth <= max); } void TraversalNode::toVelocyPackHelper(arangodb::velocypack::Builder& nodes, bool verbose) const { - ExecutionNode::toVelocyPackHelperGeneric(nodes, - verbose); // call base class method - - nodes.add("database", VPackValue(_vocbase->name())); - - nodes.add("graph", _graphInfo.slice()); - nodes.add(VPackValue("directions")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& d : _directions) { - nodes.add(VPackValue(d)); - } - } - - nodes.add(VPackValue("edgeCollections")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& e : _edgeColls) { - nodes.add(VPackValue(e->getName())); - } - } - - nodes.add(VPackValue("vertexCollections")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& v : _vertexColls) { - nodes.add(VPackValue(v->getName())); - } - } + baseToVelocyPackHelper(nodes, verbose); // In variable if (usesInVariable()) { @@ -714,21 +298,8 @@ void TraversalNode::toVelocyPackHelper(arangodb::velocypack::Builder& nodes, nodes.close(); } - if (_graphObj != nullptr) { - nodes.add(VPackValue("graphDefinition")); - _graphObj->toVelocyPack(nodes, verbose); - } - - // Out variables - if (usesVertexOutVariable()) { - nodes.add(VPackValue("vertexOutVariable")); - vertexOutVariable()->toVelocyPack(nodes); - } - if (usesEdgeOutVariable()) { - nodes.add(VPackValue("edgeOutVariable")); - edgeOutVariable()->toVelocyPack(nodes); - } - if (usesPathOutVariable()) { + // Out variables + if (usesPathOutVariable()) { nodes.add(VPackValue("pathOutVariable")); pathOutVariable()->toVelocyPack(nodes); } @@ -736,29 +307,7 @@ void TraversalNode::toVelocyPackHelper(arangodb::velocypack::Builder& nodes, nodes.add(VPackValue("traversalFlags")); _options->toVelocyPack(nodes); - // Traversal Filter Conditions - - TRI_ASSERT(_tmpObjVariable != nullptr); - nodes.add(VPackValue("tmpObjVariable")); - _tmpObjVariable->toVelocyPack(nodes); - - TRI_ASSERT(_tmpObjVarNode != nullptr); - nodes.add(VPackValue("tmpObjVarNode")); - _tmpObjVarNode->toVelocyPack(nodes, verbose); - - TRI_ASSERT(_tmpIdNode != nullptr); - nodes.add(VPackValue("tmpIdNode")); - _tmpIdNode->toVelocyPack(nodes, verbose); - - TRI_ASSERT(_fromCondition != nullptr); - nodes.add(VPackValue("fromCondition")); - _fromCondition->toVelocyPack(nodes, verbose); - - TRI_ASSERT(_toCondition != nullptr); - nodes.add(VPackValue("toCondition")); - _toCondition->toVelocyPack(nodes, verbose); - - if (!_globalEdgeConditions.empty()) { + if (!_globalEdgeConditions.empty()) { nodes.add(VPackValue("globalEdgeConditions")); nodes.openArray(); for (auto const& it : _globalEdgeConditions) { @@ -808,9 +357,10 @@ ExecutionNode* TraversalNode::clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const { TRI_ASSERT(!_optionsBuild); auto tmp = - std::make_unique(*_options.get()); + std::make_unique(*options()); auto c = new TraversalNode(plan, _id, _vocbase, _edgeColls, _vertexColls, - _inVariable, _vertexId, _directions, tmp); + _inVariable, _vertexId, _directions, tmp.get()); + tmp.release(); if (usesVertexOutVariable()) { auto vertexOutVariable = _vertexOutVariable; @@ -887,17 +437,12 @@ ExecutionNode* TraversalNode::clone(ExecutionPlan* plan, bool withDependencies, return static_cast(c); } -/// @brief the cost of a traversal node -double TraversalNode::estimateCost(size_t& nrItems) const { - return _options->estimateCost(nrItems); -} - void TraversalNode::prepareOptions() { if (_optionsBuild) { return; } TRI_ASSERT(!_optionsBuild); - _options->_tmpVar = _tmpObjVariable; + _options->setVariable(_tmpObjVariable); size_t numEdgeColls = _edgeColls.size(); bool res = false; @@ -910,71 +455,29 @@ void TraversalNode::prepareOptions() { Ast* ast = _plan->getAst(); auto trx = ast->query()->trx(); - _options->_baseLookupInfos.reserve(numEdgeColls); + // FIXME: _options->_baseLookupInfos.reserve(numEdgeColls); // Compute Edge Indexes. First default indexes: for (size_t i = 0; i < numEdgeColls; ++i) { - std::string usedField; - auto dir = _directions[i]; - // TODO we can optimize here. indexCondition and Expression could be - // made non-overlapping. - traverser::TraverserOptions::LookupInfo info; - switch (dir) { + switch (_directions[i]) { case TRI_EDGE_IN: - usedField = StaticStrings::ToString; - info.indexCondition = - globalEdgeConditionBuilder.getInboundCondition()->clone(ast); + _options->addLookupInfo( + ast, _edgeColls[i]->getName(), StaticStrings::ToString, + globalEdgeConditionBuilder.getInboundCondition()->clone(ast)); break; case TRI_EDGE_OUT: - usedField = StaticStrings::FromString; - info.indexCondition = - globalEdgeConditionBuilder.getOutboundCondition()->clone(ast); + _options->addLookupInfo( + ast, _edgeColls[i]->getName(), StaticStrings::FromString, + globalEdgeConditionBuilder.getOutboundCondition()->clone(ast)); break; case TRI_EDGE_ANY: TRI_ASSERT(false); break; } - info.expression = new Expression(ast, info.indexCondition->clone(ast)); - res = trx->getBestIndexHandleForFilterCondition( - _edgeColls[i]->getName(), info.indexCondition, _tmpObjVariable, 1000, - info.idxHandles[0]); - TRI_ASSERT(res); // Right now we have an enforced edge index which will - // always fit. - if (!res) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "expected edge index not found"); - } - - // We now have to check if we need _from / _to inside the index lookup and which position - // it is used in. Such that the traverser can update the respective string value - // in-place - std::pair> pathCmp; - for (size_t i = 0; i < info.indexCondition->numMembers(); ++i) { - // We search through the nary-and and look for EQ - _from/_to - auto eq = info.indexCondition->getMemberUnchecked(i); - if (eq->type != NODE_TYPE_OPERATOR_BINARY_EQ) { - // No equality. Skip - continue; - } - TRI_ASSERT(eq->numMembers() == 2); - // It is sufficient to only check member one. - // We build the condition this way. - auto mem = eq->getMemberUnchecked(0); - if (mem->isAttributeAccessForVariable(pathCmp)) { - if (pathCmp.first != _tmpObjVariable) { - continue; - } - if (pathCmp.second.size() == 1 && pathCmp.second[0].name == usedField) { - info.conditionNeedUpdate = true; - info.conditionMemberToUpdate = i; - break; - } - continue; - } - } - _options->_baseLookupInfos.emplace_back(std::move(info)); } + auto opts = static_cast(_options.get()); for (auto& it : _edgeConditions) { - auto ins = _options->_depthLookupInfo.emplace( + auto ins = opts->_depthLookupInfo.emplace( it.first, std::vector()); // We probably have to adopt minDepth. We cannot fulfill a condition of larger depth anyway TRI_ASSERT(ins.second); @@ -1053,27 +556,21 @@ void TraversalNode::prepareOptions() { for (auto const& jt : _globalVertexConditions) { it.second->addMember(jt); } - _options->_vertexExpressions.emplace(it.first, new Expression(ast, it.second)); - TRI_ASSERT(!_options->_vertexExpressions[it.first]->isV8()); + opts->_vertexExpressions.emplace(it.first, new Expression(ast, it.second)); + TRI_ASSERT(!opts->_vertexExpressions[it.first]->isV8()); } if (!_globalVertexConditions.empty()) { auto cond = _plan->getAst()->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); for (auto const& it : _globalVertexConditions) { cond->addMember(it); } - _options->_baseVertexExpression = new Expression(ast, cond); - TRI_ASSERT(!_options->_baseVertexExpression->isV8()); + opts->_baseVertexExpression = new Expression(ast, cond); + TRI_ASSERT(!opts->_baseVertexExpression->isV8()); } _optionsBuild = true; } -void TraversalNode::addEngine(TraverserEngineID const& engine, - arangodb::ServerID const& server) { - TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); - _engines.emplace(server, engine); -} - /// @brief remember the condition to execute for early traversal abortion. void TraversalNode::setCondition(arangodb::aql::Condition* condition) { std::unordered_set varsUsedByCondition; @@ -1128,15 +625,7 @@ void TraversalNode::registerGlobalCondition(bool isConditionOnEdge, } arangodb::traverser::TraverserOptions* TraversalNode::options() const { - return _options.get(); -} - -AstNode* TraversalNode::getTemporaryRefNode() const { - return _tmpObjVarNode; -} - -Variable const* TraversalNode::getTemporaryVariable() const { - return _tmpObjVariable; + return static_cast(_options.get()); } void TraversalNode::getConditionVariables( @@ -1148,16 +637,6 @@ void TraversalNode::getConditionVariables( } } -#ifndef USE_ENTERPRISE -void TraversalNode::enhanceEngineInfo(VPackBuilder& builder) const { - if (_graphObj != nullptr) { - _graphObj->enhanceEngineInfo(builder); - } else { - // TODO enhance the Info based on EdgeCollections. - } -} -#endif - #ifdef TRI_ENABLE_MAINTAINER_MODE void TraversalNode::checkConditionsDefined() const { TRI_ASSERT(_tmpObjVariable != nullptr); diff --git a/arangod/Aql/TraversalNode.h b/arangod/Aql/TraversalNode.h index e7a21bd1b8..0fa3769c1b 100644 --- a/arangod/Aql/TraversalNode.h +++ b/arangod/Aql/TraversalNode.h @@ -24,23 +24,19 @@ #ifndef ARANGOD_AQL_TRAVERSAL_NODE_H #define ARANGOD_AQL_TRAVERSAL_NODE_H 1 -#include "Aql/ExecutionNode.h" -#include "Aql/Collection.h" +#include "Aql/GraphNode.h" #include "Aql/Condition.h" -#include "Aql/Graphs.h" #include "Cluster/ServerState.h" -#include "Cluster/TraverserEngineRegistry.h" -#include "VocBase/LogicalCollection.h" #include "VocBase/TraverserOptions.h" namespace arangodb { namespace aql { /// @brief class TraversalNode -class TraversalNode : public ExecutionNode { +class TraversalNode : public GraphNode { + class TraversalEdgeConditionBuilder final : public EdgeConditionBuilder { private: - /// @brief reference to the outer traversal node TraversalNode const* _tn; protected: @@ -73,7 +69,7 @@ class TraversalNode : public ExecutionNode { TraversalNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, AstNode const* direction, AstNode const* start, AstNode const* graph, - std::unique_ptr& options); + traverser::TraverserOptions* options); TraversalNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); @@ -86,17 +82,12 @@ class TraversalNode : public ExecutionNode { std::vector> const& vertexColls, Variable const* inVariable, std::string const& vertexId, std::vector const& directions, - std::unique_ptr& options); + traverser::TraverserOptions* options); public: /// @brief return the type of the node NodeType getType() const override final { return TRAVERSAL; } - /// @brief flag, if smart traversal (enterprise edition only!) is done - bool isSmart() const { - return _isSmart; - } - /// @brief export to VelocyPack void toVelocyPackHelper(arangodb::velocypack::Builder&, bool) const override final; @@ -105,9 +96,6 @@ class TraversalNode : public ExecutionNode { ExecutionNode* clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const override final; - /// @brief the cost of a traversal node - double estimateCost(size_t&) const override final; - /// @brief Test if this node uses an in variable or constant bool usesInVariable() const { return _inVariable != nullptr; } @@ -161,27 +149,6 @@ class TraversalNode : public ExecutionNode { return vars; } - /// @brief return the database - TRI_vocbase_t* vocbase() const { return _vocbase; } - - /// @brief return the vertex out variable - Variable const* vertexOutVariable() const { return _vertexOutVariable; } - - /// @brief checks if the vertex out variable is used - bool usesVertexOutVariable() const { return _vertexOutVariable != nullptr; } - - /// @brief set the vertex out variable - void setVertexOutput(Variable const* outVar) { _vertexOutVariable = outVar; } - - /// @brief return the edge out variable - Variable const* edgeOutVariable() const { return _edgeOutVariable; } - - /// @brief checks if the edge out variable is used - bool usesEdgeOutVariable() const { return _edgeOutVariable != nullptr; } - - /// @brief set the edge out variable - void setEdgeOutput(Variable const* outVar) { _edgeOutVariable = outVar; } - /// @brief checks if the path out variable is used bool usesPathOutVariable() const { return _pathOutVariable != nullptr; } @@ -196,14 +163,6 @@ class TraversalNode : public ExecutionNode { std::string const getStartVertex() const { return _vertexId; } - std::vector> const& edgeColls() const { - return _edgeColls; - } - - std::vector> const& vertexColls() const { - return _vertexColls; - } - /// @brief remember the condition to execute for early traversal abortion. void setCondition(Condition* condition); @@ -226,34 +185,14 @@ class TraversalNode : public ExecutionNode { /// The condition will contain the local variable for it's accesses. void registerGlobalCondition(bool, AstNode const*); - bool allDirectionsEqual() const; + traverser::TraverserOptions* options() const override; - traverser::TraverserOptions* options() const; - - AstNode* getTemporaryRefNode() const; - - Variable const* getTemporaryVariable() const; - - void getConditionVariables(std::vector&) const; - - void enhanceEngineInfo(arangodb::velocypack::Builder&) const; + void getConditionVariables(std::vector&) const override; /// @brief Compute the traversal options containing the expressions /// MUST! be called after optimization and before creation /// of blocks. - void prepareOptions(); - - /// @brief Add a traverser engine Running on a DBServer to this node. - /// The block will communicate with them (CLUSTER ONLY) - void addEngine(traverser::TraverserEngineID const&, ServerID const&); - - - /// @brief Returns a reference to the engines. (CLUSTER ONLY) - std::unordered_map const* engines() - const { - TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); - return &_engines; - } + void prepareOptions() override; private: @@ -264,15 +203,6 @@ class TraversalNode : public ExecutionNode { private: - /// @brief the database - TRI_vocbase_t* _vocbase; - - /// @brief vertex output variable - Variable const* _vertexOutVariable; - - /// @brief vertex output variable - Variable const* _edgeOutVariable; - /// @brief vertex output variable Variable const* _pathOutVariable; @@ -282,45 +212,12 @@ class TraversalNode : public ExecutionNode { /// @brief input vertexId only used if _inVariable is unused std::string _vertexId; - /// @brief input graphInfo only used for serialization & info - arangodb::velocypack::Builder _graphInfo; - - /// @brief The directions edges are followed - std::vector _directions; - - /// @brief the edge collection names - std::vector> _edgeColls; - - /// @brief the vertex collection names - std::vector> _vertexColls; - - /// @brief our graph - Graph const* _graphObj; - /// @brief early abort traversal conditions: Condition* _condition; /// @brief variables that are inside of the condition std::unordered_set _conditionVariables; - /// @brief Options for traversals - std::unique_ptr _options; - - /// @brief Temporary pseudo variable for the currently traversed object. - Variable const* _tmpObjVariable; - - /// @brief Reference to the pseudo variable - AstNode* _tmpObjVarNode; - - /// @brief Pseudo string value node to hold the last visted vertex id. - AstNode* _tmpIdNode; - - /// @brief The hard coded condition on _from - AstNode* _fromCondition; - - /// @brief The hard coded condition on _to - AstNode* _toCondition; - /// @brief The global edge condition. Does not contain /// _from and _to checks std::vector _globalEdgeConditions; @@ -335,16 +232,6 @@ class TraversalNode : public ExecutionNode { /// @brief List of all depth specific conditions for vertices std::unordered_map _vertexConditions; - /// @brief Flag if options are already prepared. After - /// this flag was set the node cannot be cloned - /// any more. - bool _optionsBuild; - - /// @brief The list of traverser engines grouped by server. - std::unordered_map _engines; - - /// @brief flag, if traversal is smart (enterprise edition only!) - bool _isSmart; }; } // namespace arangodb::aql diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index b6079257a9..f3b9d0b943 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -134,6 +134,7 @@ SET(ARANGOD_SOURCES Aql/Function.cpp Aql/Functions.cpp Aql/Graphs.cpp + Aql/GraphNode.cpp Aql/IndexBlock.cpp Aql/IndexNode.cpp Aql/ModificationBlocks.cpp @@ -159,7 +160,6 @@ SET(ARANGOD_SOURCES Aql/ShortStringStorage.cpp Aql/ShortestPathBlock.cpp Aql/ShortestPathNode.cpp - Aql/ShortestPathOptions.cpp Aql/SortBlock.cpp Aql/SortCondition.cpp Aql/SortNode.cpp @@ -311,7 +311,6 @@ SET(ARANGOD_SOURCES V8Server/FoxxQueuesFeature.cpp V8Server/V8Context.cpp V8Server/V8DealerFeature.cpp - V8Server/V8Traverser.cpp V8Server/v8-actions.cpp V8Server/v8-collection-util.cpp V8Server/v8-collection.cpp diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index f86df70a24..474d063933 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -40,7 +40,9 @@ using namespace arangodb::traverser; static const std::string OPTIONS = "options"; static const std::string SHARDS = "shards"; +static const std::string TYPE = "type"; static const std::string EDGES = "edges"; +static const std::string BACKWARDEDGES = "reverseEdges"; static const std::string VARIABLES = "variables"; static const std::string VERTICES = "vertices"; @@ -144,13 +146,14 @@ void BaseTraverserEngine::getEdges(VPackSlice vertex, size_t depth, VPackBuilder builder.openObject(); builder.add(VPackValue("edges")); builder.openArray(); + auto opts = static_cast(_opts.get()); if (vertex.isArray()) { for (VPackSlice v : VPackArrayIterator(vertex)) { TRI_ASSERT(v.isString()); result.clear(); - auto edgeCursor = _opts->nextCursor(&mmdr, v, depth); + auto edgeCursor = opts->nextCursor(&mmdr, v, depth); while (edgeCursor->next(result, cursorId)) { - if (!_opts->evaluateEdgeExpression(result.back(), v, depth, cursorId)) { + if (!opts->evaluateEdgeExpression(result.back(), v, depth, cursorId)) { filtered++; result.pop_back(); } @@ -161,10 +164,10 @@ void BaseTraverserEngine::getEdges(VPackSlice vertex, size_t depth, VPackBuilder // Result now contains all valid edges, probably multiples. } } else if (vertex.isString()) { - std::unique_ptr edgeCursor(_opts->nextCursor(&mmdr, vertex, depth)); + std::unique_ptr edgeCursor(opts->nextCursor(&mmdr, vertex, depth)); while (edgeCursor->next(result, cursorId)) { - if (!_opts->evaluateEdgeExpression(result.back(), vertex, depth, cursorId)) { + if (!opts->evaluateEdgeExpression(result.back(), vertex, depth, cursorId)) { filtered++; result.pop_back(); } @@ -321,18 +324,24 @@ TraverserEngine::TraverserEngine(TRI_vocbase_t* vocbase, } VPackSlice shardsSlice = info.get(SHARDS); VPackSlice edgesSlice = shardsSlice.get(EDGES); - - _opts.reset(new TraverserOptions(_query, optsSlice, edgesSlice)); + VPackSlice backwardSlice = shardsSlice.get(BACKWARDEDGES); + VPackSlice typeSlice = optsSlice.get(TYPE); + if (!typeSlice.isString()) { + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_BAD_PARAMETER, + "The body requires an " + TYPE + " attribute."); + } + StringRef type(typeSlice); + if (type == "shortest") { + _opts.reset(new ShortestPathOptions(_query, optsSlice, edgesSlice, backwardSlice)); + } else if (type == "traversal") { + _opts.reset(new TraverserOptions(_query, optsSlice, edgesSlice)); + } else { + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_BAD_PARAMETER, + "The " + TYPE + " has to be shortest or traversal."); + } } - TraverserEngine::~TraverserEngine() { } - -void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} - -void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index 18a856a2d7..4e779dcb90 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -27,6 +27,7 @@ #include "Basics/Common.h" #include "Aql/Collections.h" +#include "VocBase/TraverserOptions.h" struct TRI_vocbase_t; @@ -49,7 +50,7 @@ class Builder; class Slice; } namespace traverser { -struct TraverserOptions; +struct BaseTraverserOptions; class BaseTraverserEngine { friend class TraverserEngineRegistry; @@ -77,18 +78,12 @@ class BaseTraverserEngine { void getVertexData(arangodb::velocypack::Slice, size_t, arangodb::velocypack::Builder&); - virtual void smartSearch(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; - - virtual void smartSearchBFS(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; - bool lockCollection(std::string const&); std::shared_ptr context() const; protected: - std::unique_ptr _opts; + std::unique_ptr _opts; arangodb::aql::Query* _query; transaction::Methods* _trx; arangodb::aql::Collections _collections; @@ -109,12 +104,6 @@ class TraverserEngine : public BaseTraverserEngine { TraverserEngine(TRI_vocbase_t*, arangodb::velocypack::Slice); public: ~TraverserEngine(); - - void smartSearch(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) override; - - void smartSearchBFS(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) override; }; } // namespace traverser diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index 5eda69d3f5..2bafce5c64 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -129,12 +129,29 @@ void InternalRestTraverserHandler::queryEngine() { return; } - traverser::BaseTraverserEngine* engine = _registry->get(engineId); - if (engine == nullptr) { - generateError( - ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "invalid TraverserEngineId"); - return; + double start = TRI_microtime(); + traverser::BaseTraverserEngine* engine = nullptr; + while (true) { + try { + engine = _registry->get(engineId); + if (engine == nullptr) { + generateError( + ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + "invalid TraverserEngineId"); + return; + } + break; + } catch (arangodb::basics::Exception const& ex) { /* check type */ + if (ex.code() == TRI_ERROR_DEADLOCK) { + // Define timeout properly. + if (TRI_microtime() - start > 600.0) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_TIMEOUT); + } + usleep(100000); + } else { + throw ex; + } + } } auto& registry = _registry; // For the guard @@ -222,10 +239,18 @@ void InternalRestTraverserHandler::queryEngine() { } engine->getVertexData(keysSlice, depthSlice.getNumericValue(), result); } - } else if (option == "smartSearch") { - engine->smartSearch(body, result); - } else if (option == "smartSearchBFS") { - engine->smartSearchBFS(body, result); + } else { + enterpriseSmartSearch(option, engine, body, result); + } + generateResult(ResponseCode::OK, result.slice(), engine->context()); +} + +#ifndef USE_ENTERPRISE +void InternalRestTraverserHandler::enterpriseSmartSearch( + std::string const& option, traverser::BaseTraverserEngine* engine, + VPackSlice body, VPackBuilder& result) { + if (option == "smartSearch" || option == "smartSearchBFS" || "smartShortestPath") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } else { // PATH Info wrong other error generateError( @@ -233,8 +258,8 @@ void InternalRestTraverserHandler::queryEngine() { ""); return; } - generateResult(ResponseCode::OK, result.slice(), engine->context()); } +#endif void InternalRestTraverserHandler::destroyEngine() { std::vector const& suffixes = _request->decodedSuffixes(); diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.h b/arangod/InternalRestHandler/InternalRestTraverserHandler.h index 42c2117882..ac7587e8fb 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.h +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.h @@ -28,6 +28,7 @@ namespace arangodb { namespace traverser { +class BaseTraverserEngine; class TraverserEngineRegistry; } @@ -51,6 +52,12 @@ class InternalRestTraverserHandler : public RestVocbaseBaseHandler { // @brief Destroy an existing Traverser Engine. void destroyEngine(); + // @brief Do a smart search (EE only) + void enterpriseSmartSearch(std::string const& option, + traverser::BaseTraverserEngine* engine, + arangodb::velocypack::Slice body, + arangodb::velocypack::Builder& result); + private: traverser::TraverserEngineRegistry* _registry; }; diff --git a/arangod/V8Server/V8Traverser.cpp b/arangod/V8Server/V8Traverser.cpp deleted file mode 100644 index f6f0b5d727..0000000000 --- a/arangod/V8Server/V8Traverser.cpp +++ /dev/null @@ -1,63 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -//////////////////////////////////////////////////////////////////////////////// - -#include "V8Traverser.h" -#include "VocBase/LogicalCollection.h" -#include "VocBase/SingleServerTraverser.h" - -#include -#include - -using namespace arangodb; -using namespace arangodb::basics; -using namespace arangodb::traverser; - -ShortestPathOptions::ShortestPathOptions(transaction::Methods* trx) - : BasicOptions(trx), - direction("outbound"), - useWeight(false), - weightAttribute(""), - defaultWeight(1), - bidirectional(true), - multiThreaded(true) { -} - -void ShortestPathOptions::setStart(std::string const& id) { - start = id; - startBuilder.clear(); - startBuilder.add(VPackValue(id)); -} - -void ShortestPathOptions::setEnd(std::string const& id) { - end = id; - endBuilder.clear(); - endBuilder.add(VPackValue(id)); -} - -VPackSlice ShortestPathOptions::getStart() const { - return startBuilder.slice(); -} - -VPackSlice ShortestPathOptions::getEnd() const { - return endBuilder.slice(); -} diff --git a/arangod/V8Server/V8Traverser.h b/arangod/V8Server/V8Traverser.h deleted file mode 100644 index 402082dd52..0000000000 --- a/arangod/V8Server/V8Traverser.h +++ /dev/null @@ -1,89 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGOD_V8_SERVER_V8_TRAVERSER_H -#define ARANGOD_V8_SERVER_V8_TRAVERSER_H 1 - -#include "Basics/VelocyPackHelper.h" -#include "VocBase/Traverser.h" - -namespace arangodb { - -namespace velocypack { -class Slice; -} -} - -namespace arangodb { -namespace traverser { - -// A collection of shared options used in several functions. -// Should not be used directly, use specialization instead. -struct BasicOptions { - - transaction::Methods* _trx; - - protected: - - explicit BasicOptions(transaction::Methods* trx) - : _trx(trx) {} - - virtual ~BasicOptions() { - } - - public: - std::string start; - - - public: - - transaction::Methods* trx() { return _trx; } - -}; - -struct ShortestPathOptions : BasicOptions { - public: - std::string direction; - bool useWeight; - std::string weightAttribute; - double defaultWeight; - bool bidirectional; - bool multiThreaded; - std::string end; - arangodb::velocypack::Builder startBuilder; - arangodb::velocypack::Builder endBuilder; - - explicit ShortestPathOptions(transaction::Methods* trx); - - void setStart(std::string const&); - void setEnd(std::string const&); - - arangodb::velocypack::Slice getStart() const; - arangodb::velocypack::Slice getEnd() const; - -}; - -} -} - -#endif diff --git a/arangod/VocBase/Traverser.cpp b/arangod/VocBase/Traverser.cpp index 044784a0db..6c69671392 100644 --- a/arangod/VocBase/Traverser.cpp +++ b/arangod/VocBase/Traverser.cpp @@ -58,20 +58,26 @@ void arangodb::traverser::ShortestPath::vertexToVelocyPack(transaction::Methods* size_t position, VPackBuilder& builder) { TRI_ASSERT(position < length()); VPackSlice v = _vertices[position]; - TRI_ASSERT(v.isString()); - std::string collection = v.copyString(); - size_t p = collection.find("/"); - TRI_ASSERT(p != std::string::npos); + TRI_ASSERT(v.isString() || v.isObject() || v.isNull()); + if (v.isString()) { + // In this case we have to fetch the vertex + std::string collection = v.copyString(); + size_t p = collection.find("/"); + TRI_ASSERT(p != std::string::npos); - transaction::BuilderLeaser searchBuilder(trx); - searchBuilder->add(VPackValue(collection.substr(p + 1))); - collection = collection.substr(0, p); + transaction::BuilderLeaser searchBuilder(trx); + searchBuilder->add(VPackValue(collection.substr(p + 1))); + collection = collection.substr(0, p); - int res = - trx->documentFastPath(collection, mmdr, searchBuilder->slice(), builder, true); - if (res != TRI_ERROR_NO_ERROR) { - builder.clear(); // Just in case... - builder.add(basics::VelocyPackHelper::NullValue()); + int res = + trx->documentFastPath(collection, mmdr, searchBuilder->slice(), builder, true); + if (res != TRI_ERROR_NO_ERROR) { + builder.clear(); // Just in case... + builder.add(basics::VelocyPackHelper::NullValue()); + } + } else { + // We already have the vertex data. + builder.add(v); } } diff --git a/arangod/VocBase/Traverser.h b/arangod/VocBase/Traverser.h index 40baf7f1fc..63e92d0ccc 100644 --- a/arangod/VocBase/Traverser.h +++ b/arangod/VocBase/Traverser.h @@ -54,6 +54,10 @@ class Query; } namespace traverser { +#ifdef USE_ENTERPRISE +class SmartGraphConstDistanceFinder; +#endif + struct TraverserOptions; class ShortestPath { @@ -65,6 +69,10 @@ class ShortestPath { arangodb::basics::VelocyPackHelper::VPackStringHash, arangodb::basics::VelocyPackHelper::VPackStringEqual, ShortestPath>; +#ifdef USE_ENTERPRISE + friend class arangodb::traverser::SmartGraphConstDistanceFinder; +#endif + public: ////////////////////////////////////////////////////////////////////////////// /// @brief Constructor. This is an abstract only class. diff --git a/arangod/VocBase/TraverserOptions.cpp b/arangod/VocBase/TraverserOptions.cpp index a44fcaf35c..340c6da33e 100644 --- a/arangod/VocBase/TraverserOptions.cpp +++ b/arangod/VocBase/TraverserOptions.cpp @@ -24,19 +24,24 @@ #include "TraverserOptions.h" #include "Aql/Ast.h" +#include "Aql/AstNode.h" #include "Aql/Expression.h" #include "Aql/Query.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ClusterEdgeCursor.h" #include "Indexes/Index.h" +#include "Transaction/Methods.h" #include "VocBase/SingleServerTraverser.h" #include #include using VPackHelper = arangodb::basics::VelocyPackHelper; +using TraverserOptions = arangodb::traverser::TraverserOptions; +using ShortestPathOptions = arangodb::traverser::ShortestPathOptions; +using BaseTraverserOptions = arangodb::traverser::BaseTraverserOptions; -arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo() +BaseTraverserOptions::LookupInfo::LookupInfo() : expression(nullptr), indexCondition(nullptr), conditionNeedUpdate(false), @@ -45,13 +50,13 @@ arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo() idxHandles.resize(1); }; -arangodb::traverser::TraverserOptions::LookupInfo::~LookupInfo() { +BaseTraverserOptions::LookupInfo::~LookupInfo() { if (expression != nullptr) { delete expression; } } -arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( +BaseTraverserOptions::LookupInfo::LookupInfo( arangodb::aql::Query* query, VPackSlice const& info, VPackSlice const& shards) { TRI_ASSERT(shards.isArray()); idxHandles.reserve(shards.length()); @@ -105,7 +110,7 @@ arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( indexCondition = new aql::AstNode(query->ast(), read); } -arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( +BaseTraverserOptions::LookupInfo::LookupInfo( LookupInfo const& other) : idxHandles(other.idxHandles), expression(nullptr), @@ -115,7 +120,7 @@ arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( expression = other.expression->clone(nullptr); } -void arangodb::traverser::TraverserOptions::LookupInfo::buildEngineInfo( +void BaseTraverserOptions::LookupInfo::buildEngineInfo( VPackBuilder& result) const { result.openObject(); result.add(VPackValue("handle")); @@ -136,7 +141,7 @@ void arangodb::traverser::TraverserOptions::LookupInfo::buildEngineInfo( result.close(); } -double arangodb::traverser::TraverserOptions::LookupInfo::estimateCost(size_t& nrItems) const { +double TraverserOptions::LookupInfo::estimateCost(size_t& nrItems) const { // If we do not have an index yet we cannot do anything. // Should NOT be the case TRI_ASSERT(!idxHandles.empty()); @@ -151,14 +156,10 @@ double arangodb::traverser::TraverserOptions::LookupInfo::estimateCost(size_t& n return 1000.0; } -arangodb::traverser::TraverserOptions::TraverserOptions( +TraverserOptions::TraverserOptions( transaction::Methods* trx, VPackSlice const& slice) - : _trx(trx), + : BaseTraverserOptions(trx), _baseVertexExpression(nullptr), - _tmpVar(nullptr), - _ctx(new aql::FixedVarExpressionContext()), - _traverser(nullptr), - _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -174,7 +175,7 @@ arangodb::traverser::TraverserOptions::TraverserOptions( std::string tmp = VPackHelper::getStringValue(obj, "uniqueVertices", ""); if (tmp == "path") { uniqueVertices = - arangodb::traverser::TraverserOptions::UniquenessLevel::PATH; + TraverserOptions::UniquenessLevel::PATH; } else if (tmp == "global") { if (!useBreadthFirst) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, @@ -183,35 +184,230 @@ arangodb::traverser::TraverserOptions::TraverserOptions( "unpredictable results."); } uniqueVertices = - arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL; + TraverserOptions::UniquenessLevel::GLOBAL; } else { uniqueVertices = - arangodb::traverser::TraverserOptions::UniquenessLevel::NONE; + TraverserOptions::UniquenessLevel::NONE; } tmp = VPackHelper::getStringValue(obj, "uniqueEdges", ""); if (tmp == "none") { uniqueEdges = - arangodb::traverser::TraverserOptions::UniquenessLevel::NONE; + TraverserOptions::UniquenessLevel::NONE; } else if (tmp == "global") { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "uniqueEdges: 'global' is not supported, " "due to unpredictable results. Use 'path' " "or 'none' instead"); + uniqueEdges = + TraverserOptions::UniquenessLevel::GLOBAL; } else { uniqueEdges = - arangodb::traverser::TraverserOptions::UniquenessLevel::PATH; + TraverserOptions::UniquenessLevel::PATH; } } -arangodb::traverser::TraverserOptions::TraverserOptions( - arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) - : _trx(query->trx()), - _baseVertexExpression(nullptr), +BaseTraverserOptions::BaseTraverserOptions(BaseTraverserOptions const& other) + : _ctx(new aql::FixedVarExpressionContext()), + _trx(other._trx), _tmpVar(nullptr), - _ctx(new aql::FixedVarExpressionContext()), - _traverser(nullptr), - _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), + _isCoordinator(arangodb::ServerState::instance()->isCoordinator()) { + TRI_ASSERT(other._baseLookupInfos.empty()); + TRI_ASSERT(other._tmpVar == nullptr); +} + + +BaseTraverserOptions::BaseTraverserOptions( + arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) + : _ctx(new aql::FixedVarExpressionContext()), + _trx(query->trx()), + _tmpVar(nullptr), + _isCoordinator(arangodb::ServerState::instance()->isCoordinator()) { + VPackSlice read = info.get("tmpVar"); + if (!read.isObject()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "The options require a tmpVar"); + } + _tmpVar = query->ast()->variables()->createVariable(read); + + read = info.get("baseLookupInfos"); + if (!read.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "The options require a baseLookupInfos"); + } + + size_t length = read.length(); + TRI_ASSERT(read.length() == collections.length()); + _baseLookupInfos.reserve(length); + for (size_t j = 0; j < length; ++j) { + _baseLookupInfos.emplace_back(query, read.at(j), collections.at(j)); + } +} + +BaseTraverserOptions::~BaseTraverserOptions() { + delete _ctx; +} + + +void BaseTraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { + builder.openObject(); + injectVelocyPackIndexes(builder); + builder.close(); +} + +void BaseTraverserOptions::buildEngineInfo(VPackBuilder& result) const { + result.openObject(); + injectEngineInfo(result); + result.close(); +} + +void BaseTraverserOptions::setVariable(aql::Variable const* variable) { + _tmpVar = variable; +} + +void BaseTraverserOptions::addLookupInfo(aql::Ast* ast, + std::string const& collectionName, + std::string const& attributeName, + aql::AstNode* condition) { + injectLookupInfoInList(_baseLookupInfos, ast, collectionName, attributeName, condition); +} + +void BaseTraverserOptions::injectLookupInfoInList( + std::vector& list, aql::Ast* ast, + std::string const& collectionName, std::string const& attributeName, + aql::AstNode* condition) { + traverser::TraverserOptions::LookupInfo info; + info.indexCondition = condition; + info.expression = new aql::Expression(ast, condition->clone(ast)); + bool res = _trx->getBestIndexHandleForFilterCondition( + collectionName, info.indexCondition, _tmpVar, 1000, info.idxHandles[0]); + TRI_ASSERT(res); // Right now we have an enforced edge index which will + // always fit. + if (!res) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "expected edge index not found"); + } + + // We now have to check if we need _from / _to inside the index lookup and + // which position + // it is used in. Such that the traverser can update the respective string + // value in-place + + std::pair> pathCmp; + for (size_t i = 0; i < info.indexCondition->numMembers(); ++i) { + // We search through the nary-and and look for EQ - _from/_to + auto eq = info.indexCondition->getMemberUnchecked(i); + if (eq->type != arangodb::aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ) { + // No equality. Skip + continue; + } + TRI_ASSERT(eq->numMembers() == 2); + // It is sufficient to only check member one. + // We build the condition this way. + auto mem = eq->getMemberUnchecked(0); + if (mem->isAttributeAccessForVariable(pathCmp)) { + if (pathCmp.first != _tmpVar) { + continue; + } + if (pathCmp.second.size() == 1 && pathCmp.second[0].name == attributeName) { + info.conditionNeedUpdate = true; + info.conditionMemberToUpdate = i; + break; + } + continue; + } + } + list.emplace_back(std::move(info)); +} + +void BaseTraverserOptions::clearVariableValues() { + _ctx->clearVariableValues(); +} + +void BaseTraverserOptions::setVariableValue( + aql::Variable const* var, aql::AqlValue const value) { + _ctx->setVariableValue(var, value); +} + +void BaseTraverserOptions::serializeVariables( + VPackBuilder& builder) const { + TRI_ASSERT(builder.isOpenArray()); + _ctx->serializeAllVariables(_trx, builder); +} + +arangodb::transaction::Methods* BaseTraverserOptions::trx() const { + return _trx; +} + +void BaseTraverserOptions::injectVelocyPackIndexes(VPackBuilder& builder) const { + TRI_ASSERT(builder.isOpenObject()); + + // base indexes + builder.add("base", VPackValue(VPackValueType::Array)); + for (auto const& it : _baseLookupInfos) { + for (auto const& it2 : it.idxHandles) { + builder.openObject(); + it2.getIndex()->toVelocyPack(builder, false); + builder.close(); + } + } + builder.close(); +} + +void BaseTraverserOptions::injectEngineInfo(VPackBuilder& result) const { + TRI_ASSERT(result.isOpenObject()); + result.add(VPackValue("baseLookupInfos")); + result.openArray(); + for (auto const& it : _baseLookupInfos) { + it.buildEngineInfo(result); + } + result.close(); + + result.add(VPackValue("tmpVar")); + _tmpVar->toVelocyPack(result); +} + +arangodb::aql::Expression* BaseTraverserOptions::getEdgeExpression( + size_t cursorId) const { + TRI_ASSERT(!_baseLookupInfos.empty()); + TRI_ASSERT(_baseLookupInfos.size() > cursorId); + return _baseLookupInfos[cursorId].expression; +} + +bool BaseTraverserOptions::evaluateExpression( + arangodb::aql::Expression* expression, VPackSlice value) const { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(!expression->isV8()); + expression->setVariable(_tmpVar, value); + bool mustDestroy = false; + aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); + TRI_ASSERT(res.isBoolean()); + bool result = res.toBoolean(); + expression->clearVariable(_tmpVar); + if (mustDestroy) { + res.destroy(); + } + return result; +} + +double BaseTraverserOptions::costForLookupInfoList( + std::vector const& list, + size_t& createItems) const { + double cost = 0; + createItems = 0; + for (auto const& li : list) { + cost += li.estimateCost(createItems); + } + return cost; +} + +TraverserOptions::TraverserOptions( + arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) + : BaseTraverserOptions(query, info, collections), + _baseVertexExpression(nullptr), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -239,13 +435,6 @@ arangodb::traverser::TraverserOptions::TraverserOptions( } useBreadthFirst = read.getBool(); - read = info.get("tmpVar"); - if (!read.isObject()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "The options require a tmpVar"); - } - _tmpVar = query->ast()->variables()->createVariable(read); - read = info.get("uniqueVertices"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, @@ -288,19 +477,6 @@ arangodb::traverser::TraverserOptions::TraverserOptions( "The options require a uniqueEdges"); } - read = info.get("baseLookupInfos"); - if (!read.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "The options require a baseLookupInfos"); - } - - size_t length = read.length(); - TRI_ASSERT(read.length() == collections.length()); - _baseLookupInfos.reserve(length); - for (size_t j = 0; j < length; ++j) { - _baseLookupInfos.emplace_back(query, read.at(j), collections.at(j)); - } - read = info.get("depthLookupInfo"); if (!read.isNone()) { if (!read.isObject()) { @@ -308,6 +484,7 @@ arangodb::traverser::TraverserOptions::TraverserOptions( TRI_ERROR_BAD_PARAMETER, "The options require depthLookupInfo to be an object"); } + size_t length = collections.length(); _depthLookupInfo.reserve(read.length()); for (auto const& depth : VPackObjectIterator(read)) { uint64_t d = basics::StringUtils::uint64(depth.key.copyString()); @@ -355,95 +532,78 @@ arangodb::traverser::TraverserOptions::TraverserOptions( } // Check for illegal option combination: TRI_ASSERT(uniqueEdges != - arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL); + TraverserOptions::UniquenessLevel::GLOBAL); TRI_ASSERT( uniqueVertices != - arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL || + TraverserOptions::UniquenessLevel::GLOBAL || useBreadthFirst); } -arangodb::traverser::TraverserOptions::TraverserOptions( +TraverserOptions::TraverserOptions( TraverserOptions const& other) - : _trx(other._trx), + : BaseTraverserOptions(other), _baseVertexExpression(nullptr), - _tmpVar(nullptr), - _ctx(new aql::FixedVarExpressionContext()), - _traverser(nullptr), - _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(other.minDepth), maxDepth(other.maxDepth), useBreadthFirst(other.useBreadthFirst), uniqueVertices(other.uniqueVertices), uniqueEdges(other.uniqueEdges) { - TRI_ASSERT(other._baseLookupInfos.empty()); TRI_ASSERT(other._depthLookupInfo.empty()); TRI_ASSERT(other._vertexExpressions.empty()); - TRI_ASSERT(other._tmpVar == nullptr); TRI_ASSERT(other._baseVertexExpression == nullptr); // Check for illegal option combination: TRI_ASSERT(uniqueEdges != - arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL); + TraverserOptions::UniquenessLevel::GLOBAL); TRI_ASSERT( uniqueVertices != - arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL || + TraverserOptions::UniquenessLevel::GLOBAL || useBreadthFirst); } -arangodb::traverser::TraverserOptions::~TraverserOptions() { +TraverserOptions::~TraverserOptions() { for (auto& pair : _vertexExpressions) { delete pair.second; } delete _baseVertexExpression; - delete _ctx; } -void arangodb::traverser::TraverserOptions::toVelocyPack(VPackBuilder& builder) const { +void TraverserOptions::toVelocyPack(VPackBuilder& builder) const { VPackObjectBuilder guard(&builder); - builder.add("minDepth", VPackValue(minDepth)); builder.add("maxDepth", VPackValue(maxDepth)); builder.add("bfs", VPackValue(useBreadthFirst)); switch (uniqueVertices) { - case arangodb::traverser::TraverserOptions::UniquenessLevel::NONE: + case TraverserOptions::UniquenessLevel::NONE: builder.add("uniqueVertices", VPackValue("none")); break; - case arangodb::traverser::TraverserOptions::UniquenessLevel::PATH: + case TraverserOptions::UniquenessLevel::PATH: builder.add("uniqueVertices", VPackValue("path")); break; - case arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL: + case TraverserOptions::UniquenessLevel::GLOBAL: builder.add("uniqueVertices", VPackValue("global")); break; } switch (uniqueEdges) { - case arangodb::traverser::TraverserOptions::UniquenessLevel::NONE: + case TraverserOptions::UniquenessLevel::NONE: builder.add("uniqueEdges", VPackValue("none")); break; - case arangodb::traverser::TraverserOptions::UniquenessLevel::PATH: + case TraverserOptions::UniquenessLevel::PATH: builder.add("uniqueEdges", VPackValue("path")); break; - case arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL: + case TraverserOptions::UniquenessLevel::GLOBAL: builder.add("uniqueEdges", VPackValue("global")); break; } + builder.add("type", VPackValue("traversal")); } -void arangodb::traverser::TraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { +void TraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { VPackObjectBuilder guard(&builder); - - // base indexes - builder.add("base", VPackValue(VPackValueType::Array)); - for (auto const& it : _baseLookupInfos) { - for (auto const& it2 : it.idxHandles) { - builder.openObject(); - it2.getIndex()->toVelocyPack(builder, false); - builder.close(); - } - } - builder.close(); - + BaseTraverserOptions::injectVelocyPackIndexes(builder); + // depth lookup indexes builder.add("levels", VPackValue(VPackValueType::Object)); for (auto const& it : _depthLookupInfo) { @@ -461,8 +621,9 @@ void arangodb::traverser::TraverserOptions::toVelocyPackIndexes(VPackBuilder& bu builder.close(); } -void arangodb::traverser::TraverserOptions::buildEngineInfo(VPackBuilder& result) const { +void TraverserOptions::buildEngineInfo(VPackBuilder& result) const { result.openObject(); + BaseTraverserOptions::injectEngineInfo(result); result.add("minDepth", VPackValue(minDepth)); result.add("maxDepth", VPackValue(maxDepth)); result.add("bfs", VPackValue(useBreadthFirst)); @@ -493,13 +654,6 @@ void arangodb::traverser::TraverserOptions::buildEngineInfo(VPackBuilder& result break; } - result.add(VPackValue("baseLookupInfos")); - result.openArray(); - for (auto const& it: _baseLookupInfos) { - it.buildEngineInfo(result); - } - result.close(); - if (!_depthLookupInfo.empty()) { result.add(VPackValue("depthLookupInfo")); result.openObject(); @@ -534,10 +688,7 @@ void arangodb::traverser::TraverserOptions::buildEngineInfo(VPackBuilder& result _baseVertexExpression->toVelocyPack(result, true); result.close(); } - - result.add(VPackValue("tmpVar")); - _tmpVar->toVelocyPack(result); - + result.add("type", VPackValue("traversal")); result.close(); } @@ -549,7 +700,7 @@ bool arangodb::traverser::TraverserOptions::vertexHasFilter( return _vertexExpressions.find(depth) != _vertexExpressions.end(); } -bool arangodb::traverser::TraverserOptions::evaluateEdgeExpression( +bool TraverserOptions::evaluateEdgeExpression( arangodb::velocypack::Slice edge, arangodb::velocypack::Slice vertex, uint64_t depth, size_t cursorId) const { if (_isCoordinator) { @@ -565,44 +716,33 @@ bool arangodb::traverser::TraverserOptions::evaluateEdgeExpression( TRI_ASSERT(specific->second.size() > cursorId); expression = specific->second[cursorId].expression; } else { - TRI_ASSERT(!_baseLookupInfos.empty()); - TRI_ASSERT(_baseLookupInfos.size() > cursorId); - expression = _baseLookupInfos[cursorId].expression; + expression = getEdgeExpression(cursorId); } - if (expression != nullptr) { - TRI_ASSERT(!expression->isV8()); - expression->setVariable(_tmpVar, edge); - - VPackValueLength vidLength; - char const* vid = vertex.getString(vidLength); - - // inject _from/_to value - auto node = expression->nodeForModification(); - - TRI_ASSERT(node->numMembers() > 0); - auto dirCmp = node->getMemberUnchecked(node->numMembers() - 1); - TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); - TRI_ASSERT(dirCmp->numMembers() == 2); - - auto idNode = dirCmp->getMemberUnchecked(1); - TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); - TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); - idNode->stealComputedValue(); - idNode->setStringValue(vid, vidLength); - - bool mustDestroy = false; - aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); - expression->clearVariable(_tmpVar); - bool result = res.toBoolean(); - if (mustDestroy) { - res.destroy(); - } - return result; + if (expression == nullptr) { + return true; } - return true; + + VPackValueLength vidLength; + char const* vid = vertex.getString(vidLength); + + // inject _from/_to value + auto node = expression->nodeForModification(); + + TRI_ASSERT(node->numMembers() > 0); + auto dirCmp = node->getMemberUnchecked(node->numMembers() - 1); + TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); + TRI_ASSERT(dirCmp->numMembers() == 2); + + auto idNode = dirCmp->getMemberUnchecked(1); + TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); + TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); + idNode->stealComputedValue(); + idNode->setStringValue(vid, vidLength); + + return evaluateExpression(expression, edge); } -bool arangodb::traverser::TraverserOptions::evaluateVertexExpression( +bool TraverserOptions::evaluateVertexExpression( arangodb::velocypack::Slice vertex, uint64_t depth) const { arangodb::aql::Expression* expression = nullptr; @@ -614,25 +754,11 @@ bool arangodb::traverser::TraverserOptions::evaluateVertexExpression( expression = _baseVertexExpression; } - if (expression == nullptr) { - return true; - } - - TRI_ASSERT(!expression->isV8()); - expression->setVariable(_tmpVar, vertex); - bool mustDestroy = false; - aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); - TRI_ASSERT(res.isBoolean()); - bool result = res.toBoolean(); - expression->clearVariable(_tmpVar); - if (mustDestroy) { - res.destroy(); - } - return result; + return evaluateExpression(expression, vertex); } arangodb::traverser::EdgeCursor* -arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, +TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, VPackSlice vertex, uint64_t depth) const { if (_isCoordinator) { @@ -650,7 +776,7 @@ arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, } arangodb::traverser::EdgeCursor* -arangodb::traverser::TraverserOptions::nextCursorLocal(ManagedDocumentResult* mmdr, +TraverserOptions::nextCursorLocal(ManagedDocumentResult* mmdr, VPackSlice vertex, uint64_t depth, std::vector& list) const { TRI_ASSERT(mmdr != nullptr); auto allCursor = std::make_unique(mmdr, _trx, list.size()); @@ -683,45 +809,19 @@ arangodb::traverser::TraverserOptions::nextCursorLocal(ManagedDocumentResult* mm } arangodb::traverser::EdgeCursor* -arangodb::traverser::TraverserOptions::nextCursorCoordinator( +TraverserOptions::nextCursorCoordinator( VPackSlice vertex, uint64_t depth) const { TRI_ASSERT(_traverser != nullptr); auto cursor = std::make_unique(vertex, depth, _traverser); return cursor.release(); } -void arangodb::traverser::TraverserOptions::clearVariableValues() { - _ctx->clearVariableValues(); -} - -void arangodb::traverser::TraverserOptions::setVariableValue( - aql::Variable const* var, aql::AqlValue const value) { - _ctx->setVariableValue(var, value); -} - -void arangodb::traverser::TraverserOptions::linkTraverser( +void TraverserOptions::linkTraverser( arangodb::traverser::ClusterTraverser* trav) { _traverser = trav; } -void arangodb::traverser::TraverserOptions::serializeVariables( - VPackBuilder& builder) const { - TRI_ASSERT(builder.isOpenArray()); - _ctx->serializeAllVariables(_trx, builder); -} - -double arangodb::traverser::TraverserOptions::costForLookupInfoList( - std::vector const& list, - size_t& createItems) const { - double cost = 0; - createItems = 0; - for (auto const& li : list) { - cost += li.estimateCost(createItems); - } - return cost; -} - -double arangodb::traverser::TraverserOptions::estimateCost(size_t& nrItems) const { +double TraverserOptions::estimateCost(size_t& nrItems) const { size_t count = 1; double cost = 0; size_t baseCreateItems = 0; @@ -743,3 +843,74 @@ double arangodb::traverser::TraverserOptions::estimateCost(size_t& nrItems) cons nrItems = count; return cost; } + +ShortestPathOptions::ShortestPathOptions(arangodb::aql::Query* query, + VPackSlice info, + VPackSlice collections, + VPackSlice reverseCollections) + : BaseTraverserOptions(query, info, collections), + _defaultWeight(1), + _weightAttribute("") { + VPackSlice read = info.get("reverseLookupInfos"); + + if (!read.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "The options require a reverseLookupInfos"); + } + size_t length = read.length(); + TRI_ASSERT(read.length() == reverseCollections.length()); + _reverseLookupInfos.reserve(length); + for (size_t j = 0; j < length; ++j) { + _reverseLookupInfos.emplace_back(query, read.at(j), reverseCollections.at(j)); + } + + read = info.get("weightAttribute"); + if (read.isString()) { + _weightAttribute = read.copyString(); + + read = info.get("defaultWeight"); + if (read.isNumber()) { + _defaultWeight = read.getNumericValue(); + } + } +} + +void ShortestPathOptions::toVelocyPack(VPackBuilder& builder) const { + VPackObjectBuilder guard(&builder); + builder.add("type", VPackValue("shortest")); + // FIXME +} + +void ShortestPathOptions::buildEngineInfo(VPackBuilder& result) const { + VPackObjectBuilder guard(&result); + BaseTraverserOptions::injectEngineInfo(result); + result.add(VPackValue("reverseLookupInfos")); + result.openArray(); + for (auto const& it : _reverseLookupInfos) { + it.buildEngineInfo(result); + } + result.close(); + if (usesWeight()) { + result.add("weightAttribute", VPackValue(weightAttribute())); + result.add("defaultWeight", VPackValue(defaultWeight())); + } + result.add("type", VPackValue("shortest")); +} + +void ShortestPathOptions::addReverseLookupInfo( + aql::Ast* ast, std::string const& collectionName, + std::string const& attributeName, aql::AstNode* condition) { + injectLookupInfoInList(_reverseLookupInfos, ast, collectionName, + attributeName, condition); +} + +double ShortestPathOptions::estimateCost(size_t& nrItems) const { + // We estimate the cost with the 7 degrees of separation: + // The shortest path between two vertices is most likely + // less than seven hops long. + size_t edgesCount = 0; + double baseCost = costForLookupInfoList(_baseLookupInfos, edgesCount); + baseCost = std::pow(baseCost, 7); + nrItems = static_cast(std::pow(edgesCount, 7)); + return nrItems; +} diff --git a/arangod/VocBase/TraverserOptions.h b/arangod/VocBase/TraverserOptions.h index 3a0d685c9e..4bc2090c9c 100644 --- a/arangod/VocBase/TraverserOptions.h +++ b/arangod/VocBase/TraverserOptions.h @@ -62,12 +62,7 @@ class EdgeCursor { size_t&) = 0; }; - -struct TraverserOptions { - friend class arangodb::aql::TraversalNode; - - public: - enum UniquenessLevel { NONE, PATH, GLOBAL }; +struct BaseTraverserOptions { protected: @@ -98,18 +93,95 @@ struct TraverserOptions { }; - public: - transaction::Methods* _trx; - protected: - std::vector _baseLookupInfos; - std::unordered_map> _depthLookupInfo; - std::unordered_map _vertexExpressions; - aql::Expression* _baseVertexExpression; - aql::Variable const* _tmpVar; + private: aql::FixedVarExpressionContext* _ctx; - arangodb::traverser::ClusterTraverser* _traverser; + + protected: + transaction::Methods* _trx; + std::vector _baseLookupInfos; + aql::Variable const* _tmpVar; bool const _isCoordinator; + public: + explicit BaseTraverserOptions(transaction::Methods* trx) + : _ctx(new aql::FixedVarExpressionContext()), + _trx(trx), + _tmpVar(nullptr), + _isCoordinator(arangodb::ServerState::instance()->isCoordinator()) {} + + /// @brief This copy constructor is only working during planning phase. + /// After planning this node should not be copied anywhere. + explicit BaseTraverserOptions(BaseTraverserOptions const&); + + BaseTraverserOptions(arangodb::aql::Query*, arangodb::velocypack::Slice, + arangodb::velocypack::Slice); + + virtual ~BaseTraverserOptions(); + + // Creates a complete Object containing all EngineInfo + // in the given builder. + virtual void buildEngineInfo(arangodb::velocypack::Builder&) const; + + void setVariable(aql::Variable const*); + + void addLookupInfo(aql::Ast* ast, std::string const& collectionName, + std::string const& attributeName, aql::AstNode* condition); + + void clearVariableValues(); + + void setVariableValue(aql::Variable const*, aql::AqlValue const); + + void serializeVariables(arangodb::velocypack::Builder&) const; + + transaction::Methods* trx() const; + + /// @brief Build a velocypack for cloning in the plan. + virtual void toVelocyPack(arangodb::velocypack::Builder&) const = 0; + + // Creates a complete Object containing all index information + // in the given builder. + virtual void toVelocyPackIndexes(arangodb::velocypack::Builder&) const; + + /// @brief Estimate the total cost for this operation + virtual double estimateCost(size_t& nrItems) const = 0; + + protected: + + double costForLookupInfoList(std::vector const& list, + size_t& createItems) const; + + // Requires an open Object in the given builder an + // will inject index information into it. + // Does not close the builder. + void injectVelocyPackIndexes(arangodb::velocypack::Builder&) const; + + // Requires an open Object in the given builder an + // will inject EngineInfo into it. + // Does not close the builder. + void injectEngineInfo(arangodb::velocypack::Builder&) const; + + aql::Expression* getEdgeExpression(size_t cursorId) const; + + bool evaluateExpression(aql::Expression*, arangodb::velocypack::Slice varValue) const; + + void injectLookupInfoInList(std::vector&, aql::Ast* ast, + std::string const& collectionName, + std::string const& attributeName, + aql::AstNode* condition); +}; + +struct TraverserOptions : public BaseTraverserOptions { + friend class arangodb::aql::TraversalNode; + + public: + enum UniquenessLevel { NONE, PATH, GLOBAL }; + + protected: + std::unordered_map> _depthLookupInfo; + std::unordered_map _vertexExpressions; + aql::Expression* _baseVertexExpression; + arangodb::traverser::ClusterTraverser* _traverser; + public: uint64_t minDepth; @@ -122,12 +194,9 @@ struct TraverserOptions { UniquenessLevel uniqueEdges; explicit TraverserOptions(transaction::Methods* trx) - : _trx(trx), + : BaseTraverserOptions(trx), _baseVertexExpression(nullptr), - _tmpVar(nullptr), - _ctx(new aql::FixedVarExpressionContext()), _traverser(nullptr), - _isCoordinator(trx->state()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -146,14 +215,14 @@ struct TraverserOptions { virtual ~TraverserOptions(); /// @brief Build a velocypack for cloning in the plan. - void toVelocyPack(arangodb::velocypack::Builder&) const; + void toVelocyPack(arangodb::velocypack::Builder&) const override; /// @brief Build a velocypack for indexes - void toVelocyPackIndexes(arangodb::velocypack::Builder&) const; + void toVelocyPackIndexes(arangodb::velocypack::Builder&) const override; /// @brief Build a velocypack containing all relevant information /// for DBServer traverser engines. - void buildEngineInfo(arangodb::velocypack::Builder&) const; + void buildEngineInfo(arangodb::velocypack::Builder&) const override; bool vertexHasFilter(uint64_t) const; @@ -165,21 +234,12 @@ struct TraverserOptions { EdgeCursor* nextCursor(ManagedDocumentResult*, arangodb::velocypack::Slice, uint64_t) const; - void clearVariableValues(); - - void setVariableValue(aql::Variable const*, aql::AqlValue const); - void linkTraverser(arangodb::traverser::ClusterTraverser*); - void serializeVariables(arangodb::velocypack::Builder&) const; - - double estimateCost(size_t& nrItems) const; + double estimateCost(size_t& nrItems) const override; private: - double costForLookupInfoList(std::vector const& list, - size_t& createItems) const; - EdgeCursor* nextCursorLocal(ManagedDocumentResult*, arangodb::velocypack::Slice, uint64_t, std::vector&) const; @@ -187,6 +247,73 @@ struct TraverserOptions { EdgeCursor* nextCursorCoordinator(arangodb::velocypack::Slice, uint64_t) const; }; +struct ShortestPathOptions : public BaseTraverserOptions { + + protected: + + double _defaultWeight; + std::string _weightAttribute; + std::vector _reverseLookupInfos; + + public: + + enum DIRECTION {FORWARD, BACKWARD}; + + explicit ShortestPathOptions(transaction::Methods* trx) + : BaseTraverserOptions(trx), + _defaultWeight(1), + _weightAttribute("") {} + + ShortestPathOptions(arangodb::aql::Query*, arangodb::velocypack::Slice info, + arangodb::velocypack::Slice collections, + arangodb::velocypack::Slice reverseCollections); + + void setWeightAttribute(std::string const& attr) { + _weightAttribute = attr; + } + + void setDefaultWeight(double weight) { + _defaultWeight = weight; + } + + bool usesWeight() const { + return !_weightAttribute.empty(); + } + + std::string const weightAttribute() const { + return _weightAttribute; + } + + double defaultWeight() const { + return _defaultWeight; + } + + /// @brief Build a velocypack for cloning in the plan. + void toVelocyPack(arangodb::velocypack::Builder&) const override; + + /// @brief Build a velocypack containing all relevant information + /// for DBServer traverser engines. + void buildEngineInfo(arangodb::velocypack::Builder&) const override; + + double estimateCost(size_t& nrItems) const override; + + /// @brief Add a lookup info for reverse direction + void addReverseLookupInfo(aql::Ast* ast, std::string const& collectionName, + std::string const& attributeName, + aql::AstNode* condition); + + template + EdgeCursor* nextCursor(ManagedDocumentResult*, arangodb::velocypack::Slice) const; + + private: + + template + EdgeCursor* nextCursorLocal(ManagedDocumentResult*, + arangodb::velocypack::Slice, + std::vector&) const; + +}; + } } #endif From 81889bfc54915067b32b48c9c87ed6edfd632202 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 24 Mar 2017 16:17:36 +0100 Subject: [PATCH 03/27] Work on testingjs to make it fit for storage engine and others: - add storage engine switch: - filter for files with `-rocksdb` or `-mmfiles` depending on the used storage engine. - fix missing color reset on error messages - make --test a substring match instead of a full match. - print an error message if --test did not match any testcase and thus no tests were run - disable crash check for the recovery testcases that are actually intended to crash so we can recover from the failed state - scan for recovery testcases instead of keeping a hand written list - make test-scan function public --- js/client/modules/@arangodb/testing.js | 226 +++++++++++-------------- 1 file changed, 95 insertions(+), 131 deletions(-) diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 06a2276257..2efdf8cc78 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -101,6 +101,8 @@ const optionsDocumentation = [ ' - `loopSleepWhen`: sleep every nth iteration', ' - `loopSleepSec`: sleep seconds between iterations', '', + ' - `storageEngine`: set to `rocksdb` or `mmfiles` - defaults to `mmfiles`', + '', ' - `server`: server_url (e.g. tcp://127.0.0.1:8529) for external server', ' - `cluster`: if set to true the tests are run with the coordinator', ' of a small local cluster', @@ -190,6 +192,7 @@ const optionsDefaults = { 'skipShebang': false, 'skipSsl': false, 'skipTimeCritical': false, + 'storageEngine': 'mmfiles', 'test': undefined, 'testBuckets': undefined, 'username': 'root', @@ -261,6 +264,25 @@ let LOGS_DIR; let UNITTESTS_DIR; let GDB_OUTPUT = ""; +function doOnePathInner(path) { + return _.filter(fs.list(makePathUnix(path)), + function (p) { + return p.substr(-3) === '.js'; + }) + .map(function (x) { + return fs.join(makePathUnix(path), x); + }).sort(); +} + +function scanTestPath(path) { + var community = doOnePathInner(path); + if (global.ARANGODB_CLIENT_VERSION(true)['enterprise-version']) { + return community.concat(doOnePathInner('enterprise/' + path)); + } else { + return community; + } +} + function makeResults (testname, instanceInfo) { const startTime = time(); @@ -322,13 +344,17 @@ function makeArgsArangod (options, appDir, role) { config = "arangod-" + role + ".conf"; } - return { + let args = { 'configuration': fs.join(CONFIG_DIR, config), 'define': 'TOP_DIR=' + TOP_DIR, 'wal.flush-timeout': options.walFlushTimeout, 'javascript.app-path': appDir, 'http.trusted-origin': options.httpTrustedOrigin || 'all' }; + if (options.storageEngine !== 'mmfiles') { + args['server.storage-engine'] = 'rocksdb'; + } + return args; } // ////////////////////////////////////////////////////////////////////////////// @@ -545,7 +571,7 @@ function analyzeCrash (binary, arangod, options, checkStr) { var cp = corePattern.asciiSlice(0, corePattern.length); if (matchApport.exec(cp) != null) { - print(RED + "apport handles corefiles on your system. Uninstall it if you want us to get corefiles for analysis."); + print(RED + "apport handles corefiles on your system. Uninstall it if you want us to get corefiles for analysis." + RESET); return; } @@ -556,7 +582,7 @@ function analyzeCrash (binary, arangod, options, checkStr) { options.coreDirectory = cp.replace("%e", "*").replace("%t", "*").replace("%p", arangod.pid); } else { - print(RED + "Don't know howto locate corefiles in your system. '" + cpf + "' contains: '" + cp + "'"); + print(RED + "Don't know howto locate corefiles in your system. '" + cpf + "' contains: '" + cp + "'" + RESET); return; } } @@ -573,7 +599,7 @@ function analyzeCrash (binary, arangod, options, checkStr) { storeArangodPath + ' for later analysis.\n' + 'Server shut down with :\n' + yaml.safeDump(arangod) + - 'marking build as crashy.'); + 'marking build as crashy.' + RESET); let corePath = (options.coreDirectory === '') ? 'core' @@ -826,6 +852,7 @@ function performTests (options, testList, testname, runFn) { let results = {}; let continueTesting = true; + let count = 0; for (let i = 0; i < testList.length; i++) { let te = testList[i]; @@ -834,6 +861,7 @@ function performTests (options, testList, testname, runFn) { if (filterTestcaseByOptions(te, options, filtered)) { let first = true; let loopCount = 0; + count += 1; while (first || options.loopEternal) { if (!continueTesting) { @@ -895,6 +923,15 @@ function performTests (options, testList, testname, runFn) { } } + if (count === 0) { + results["ALLTESTS"] = { + status: false, + skipped: true + }; + results.status = false; + print(RED + "No testcase matched the filter." + RESET); + } + print('Shutting down...'); shutdownInstance(instanceInfo, options); print('done.'); @@ -1029,7 +1066,7 @@ function executeArangod (cmd, args, options) { // / @brief executes a command and wait for result // ////////////////////////////////////////////////////////////////////////////// -function executeAndWait (cmd, args, options, valgrindTest, rootDir) { +function executeAndWait (cmd, args, options, valgrindTest, rootDir, disableCoreCheck = false) { if (valgrindTest && options.valgrind) { let valgrindOpts = {}; @@ -1072,7 +1109,8 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir) { let errorMessage = ' - '; - if (res.hasOwnProperty('signal') && + if (!disableCoreCheck && + res.hasOwnProperty('signal') && ((res.signal === 11) || (res.signal === 6) || // Windows sometimes has random numbers in signal... @@ -1793,11 +1831,13 @@ function rubyTests (options, ssl) { } }; + let count = 0; for (let i = 0; i < files.length; i++) { const te = files[i]; if (te.substr(0, 4) === 'api-' && te.substr(-3) === '.rb') { if (filterTestcaseByOptions(te, options, filtered)) { + count += 1; if (!continueTesting) { print('Skipping ' + te + ' server is gone.'); @@ -1870,6 +1910,15 @@ function rubyTests (options, ssl) { print('Shutting down...'); + if (count === 0) { + result["ALLTESTS"] = { + status: false, + skipped: true + }; + result.status = false; + print(RED + "No testcase matched the filter." + RESET); + } + fs.remove(tmpname); shutdownInstance(instanceInfo, options); print('done.'); @@ -1886,56 +1935,37 @@ let testsCases = { }; function findTests () { - function doOnePathInner(path) { - return _.filter(fs.list(makePathUnix(path)), - function (p) { - return p.substr(-3) === '.js'; - }) - .map(function (x) { - return fs.join(makePathUnix(path), x); - }).sort(); - } - - function doOnePath(path) { - var community = doOnePathInner(path); - if (global.ARANGODB_CLIENT_VERSION(true)['enterprise-version']) { - return community.concat(doOnePathInner('enterprise/' + path)); - } else { - return community; - } - } - if (testsCases.setup) { return; } - testsCases.common = doOnePath('js/common/tests/shell'); + testsCases.common = scanTestPath('js/common/tests/shell'); - testsCases.server_only = doOnePath('js/server/tests/shell'); + testsCases.server_only = scanTestPath('js/server/tests/shell'); - testsCases.client_only = doOnePath('js/client/tests/shell'); + testsCases.client_only = scanTestPath('js/client/tests/shell'); - testsCases.server_aql = doOnePath('js/server/tests/aql'); + testsCases.server_aql = scanTestPath('js/server/tests/aql'); testsCases.server_aql = _.filter(testsCases.server_aql, function(p) { return p.indexOf('ranges-combined') === -1; }); - testsCases.server_aql_extended = doOnePath('js/server/tests/aql'); + testsCases.server_aql_extended = scanTestPath('js/server/tests/aql'); testsCases.server_aql_extended = _.filter(testsCases.server_aql_extended, function(p) { return p.indexOf('ranges-combined') !== -1; }); - testsCases.server_aql_performance = doOnePath('js/server/perftests'); + testsCases.server_aql_performance = scanTestPath('js/server/perftests'); - testsCases.server_http = doOnePath('js/common/tests/http'); + testsCases.server_http = scanTestPath('js/common/tests/http'); - testsCases.replication = doOnePath('js/common/tests/replication'); + testsCases.replication = scanTestPath('js/common/tests/replication'); - testsCases.agency = doOnePath('js/client/tests/agency'); + testsCases.agency = scanTestPath('js/client/tests/agency'); - testsCases.resilience = doOnePath('js/server/tests/resilience'); + testsCases.resilience = scanTestPath('js/server/tests/resilience'); - testsCases.client_resilience = doOnePath('js/client/tests/resilience'); - testsCases.cluster_sync = doOnePath('js/server/tests/cluster-sync'); + testsCases.client_resilience = scanTestPath('js/client/tests/resilience'); + testsCases.cluster_sync = scanTestPath('js/server/tests/cluster-sync'); testsCases.server = testsCases.common.concat(testsCases.server_only); testsCases.client = testsCases.common.concat(testsCases.client_only); @@ -1950,7 +1980,7 @@ function findTests () { function filterTestcaseByOptions (testname, options, whichFilter) { if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) { whichFilter.filter = 'testcase'; - return testname === options.test; + return testname.search(options.test) >= 0; } if (options.replication) { @@ -2016,6 +2046,14 @@ function filterTestcaseByOptions (testname, options, whichFilter) { return false; } + if ((testname.indexOf('-mmfiles') !== -1) && options.storageEngine === "rocksdb") { + whichFilter.filter = 'skip when running as rocksdb'; + return false; + } + if ((testname.indexOf('-rocksdb') !== -1) && options.storageEngine === "mmfiles") { + whichFilter.filter = 'skip when running as mmfiles'; + return false; + } return true; } @@ -3466,8 +3504,7 @@ function runArangodRecovery (instanceInfo, options, script, setup) { } argv = argv.concat([ - '--javascript.script', - fs.join('.', 'js', 'server', 'tests', 'recovery', script + '.js') + '--javascript.script', script ]); let binary = ARANGOD_BIN; @@ -3476,94 +3513,9 @@ function runArangodRecovery (instanceInfo, options, script, setup) { argv.unshift(ARANGOD_BIN); } - instanceInfo.pid = executeAndWait(binary, argv, options, "recovery", instanceInfo.rootDir); + instanceInfo.pid = executeAndWait(binary, argv, options, "recovery", instanceInfo.rootDir, setup); } -const recoveryTests = [ - 'insert-update-replace', - 'die-during-collector', - 'disk-full-logfile', - 'disk-full-logfile-data', - 'disk-full-datafile', - 'collection-drop-recreate', - 'collection-duplicate-name', - 'create-with-temp', - 'create-with-temp-old', - 'create-collection-fail', - 'create-collection-tmpfile', - 'create-database-existing', - 'create-database-fail', - 'empty-datafiles', - 'flush-drop-database-and-fail', - 'drop-database-flush-and-fail', - 'drop-database-only-tmp', - 'create-databases', - 'recreate-databases', - 'drop-databases', - 'create-and-drop-databases', - 'drop-database-and-fail', - 'flush-drop-database-and-fail', - 'collection-rename-recreate', - 'collection-rename-recreate-flush', - 'collection-unload', - 'resume-recovery-multi-flush', - 'resume-recovery-simple', - 'resume-recovery-all', - 'resume-recovery-other', - 'resume-recovery', - 'foxx-directories', - 'collection-duplicate', - 'collection-rename', - 'collection-properties', - 'empty-logfiles', - 'many-logs', - 'multiple-logs', - 'collection-recreate', - 'drop-index', - 'drop-index-shutdown', - 'drop-indexes', - 'create-indexes', - 'create-collections', - 'recreate-collection', - 'drop-single-collection', - 'drop-collections', - 'collections-reuse', - 'collections-different-attributes', - 'indexes-after-flush', - 'indexes-hash', - 'indexes-rocksdb', - 'indexes-rocksdb-nosync', - 'indexes-rocksdb-restore', - 'indexes-sparse-hash', - 'indexes-skiplist', - 'indexes-sparse-skiplist', - 'indexes-geo', - 'edges', - 'indexes', - 'many-inserts', - 'many-updates', - 'wait-for-sync', - 'attributes', - 'no-journal', - 'write-throttling', - 'collector-oom', - 'transaction-no-abort', - 'transaction-no-commit', - 'transaction-just-committed', - 'multi-database-durability', - 'disk-full-no-collection-journal', - 'no-shutdown-info-with-flush', - 'no-shutdown-info-no-flush', - 'no-shutdown-info-multiple-logs', - 'insert-update-remove', - 'insert-update-remove-distance', - 'big-transaction-durability', - 'transaction-durability', - 'transaction-durability-multiple', - 'corrupt-wal-marker-multiple', - 'corrupt-wal-marker-single' -]; - testFuncs.recovery = function (options) { let results = {}; @@ -3577,11 +3529,16 @@ testFuncs.recovery = function (options) { let status = true; + let recoveryTests = scanTestPath('js/server/tests/recovery'); + let count = 0; + for (let i = 0; i < recoveryTests.length; ++i) { let test = recoveryTests[i]; + let filtered = {}; - if (options.test === undefined || options.test === test) { + if (filterTestcaseByOptions (test, options, filtered )) { let instanceInfo = {}; + count += 1; runArangodRecovery(instanceInfo, options, test, true); @@ -3597,13 +3554,20 @@ testFuncs.recovery = function (options) { status = false; } } else { - results[test] = { - status: true, - skipped: true - }; + if (options.extremeVerbosity) { + print('Skipped ' + test + ' because of ' + filtered.filter); + } } } + if (count === 0) { + results["ALLTESTS"] = { + status: false, + skipped: true + }; + status = false; + print(RED + "No testcase matched the filter." + RESET); + } results.status = status; return { From f67ddb7bab4207b9ad5dc992a3cbd1bbe170b8f0 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 24 Mar 2017 16:22:10 +0100 Subject: [PATCH 04/27] split testcase to work around macos instabilities --- .../tests/shell/shell-foxx-response-2-spec.js | 100 ++++++++++++++++++ .../tests/shell/shell-foxx-response-spec.js | 87 --------------- 2 files changed, 100 insertions(+), 87 deletions(-) create mode 100644 js/server/tests/shell/shell-foxx-response-2-spec.js diff --git a/js/server/tests/shell/shell-foxx-response-2-spec.js b/js/server/tests/shell/shell-foxx-response-2-spec.js new file mode 100644 index 0000000000..c7d1efaae5 --- /dev/null +++ b/js/server/tests/shell/shell-foxx-response-2-spec.js @@ -0,0 +1,100 @@ +/* global describe, it */ +'use strict'; +const expect = require('chai').expect; +const sinon = require('sinon'); +const statuses = require('statuses'); +const path = require('path'); +const fs = require('fs'); +const internal = require('internal'); +const crypto = require('@arangodb/crypto'); +const SyntheticResponse = require('@arangodb/foxx/router/response'); + +describe('SyntheticResponse', function () { + describe('cookie', function () { + it('adds a cookie', function () { + require("console").log('adds a cookie'); + const rawRes = {}; + const res = new SyntheticResponse(rawRes, {}); + res.cookie('hello', 'banana'); + expect(rawRes.cookies).to.eql([ + {name: 'hello', value: 'banana'} + ]); + }); + it('optionally adds a TTL', function () { + require("console").log('optionally adds a TTL'); + const rawRes = {}; + const res = new SyntheticResponse(rawRes, {}); + res.cookie('hello', 'banana', {ttl: 22}); + expect(rawRes.cookies).to.eql([ + {name: 'hello', value: 'banana', lifeTime: 22} + ]); + }); + it('optionally adds some metadata', function () { + require("console").log('optionally adds some metadata'); + const rawRes = {}; + require("console").log("1"); + const res = new SyntheticResponse(rawRes, {}); + require("console").log("2"); + res.cookie('hello', 'banana', { + path: '/path', + domain: 'cats.example', + secure: true, + httpOnly: true + }); + require("console").log("3"); + expect(rawRes.cookies).to.eql([ + { + name: 'hello', + value: 'banana', + path: '/path', + domain: 'cats.example', + secure: true, + httpOnly: true + } + ]); + require("console").log("4"); + }); + it('supports signed cookies when a secret is provided', function () { + require("console").log('supports signed cookies when a secret is provided'); + const rawRes = {}; + const res = new SyntheticResponse(rawRes, {}); + res.cookie('hello', 'banana', {secret: 'potato'}); + expect(rawRes.cookies).to.eql([ + {name: 'hello', value: 'banana'}, + {name: 'hello.sig', value: crypto.hmac('potato', 'banana')} + ]); + }); + it('supports signed cookies with different algorithms', function () { + require("console").log('supports signed cookies with different algorithms'); + const rawRes = {}; + const res = new SyntheticResponse(rawRes, {}); + res.cookie('hello', 'banana', { + secret: 'potato', + algorithm: 'sha512' + }); + expect(rawRes.cookies).to.eql([ + {name: 'hello', value: 'banana'}, + {name: 'hello.sig', value: crypto.hmac('potato', 'banana', 'sha512')} + ]); + }); + it('treats options string as a secret', function () { + require("console").log('treats options string as a secret'); + const rawRes = {}; + const res = new SyntheticResponse(rawRes, {}); + res.cookie('hello', 'banana', 'potato'); + expect(rawRes.cookies).to.eql([ + {name: 'hello', value: 'banana'}, + {name: 'hello.sig', value: crypto.hmac('potato', 'banana')} + ]); + }); + it('treats options number as a TTL value', function () { + require("console").log('treats options number as a TTL value'); + const rawRes = {}; + const res = new SyntheticResponse(rawRes, {}); + res.cookie('hello', 'banana', 22); + expect(rawRes.cookies).to.eql([ + {name: 'hello', value: 'banana', lifeTime: 22} + ]); + }); + }); +}); diff --git a/js/server/tests/shell/shell-foxx-response-spec.js b/js/server/tests/shell/shell-foxx-response-spec.js index 7d8f344ada..f117fb21e8 100644 --- a/js/server/tests/shell/shell-foxx-response-spec.js +++ b/js/server/tests/shell/shell-foxx-response-spec.js @@ -694,91 +694,4 @@ describe('SyntheticResponse', function () { expect(res.headers).to.have.a.property('vary', '*'); }); }); - describe('cookie', function () { - it('adds a cookie', function () { - require("console").log('adds a cookie'); - const rawRes = {}; - const res = new SyntheticResponse(rawRes, {}); - res.cookie('hello', 'banana'); - expect(rawRes.cookies).to.eql([ - {name: 'hello', value: 'banana'} - ]); - }); - it('optionally adds a TTL', function () { - require("console").log('optionally adds a TTL'); - const rawRes = {}; - const res = new SyntheticResponse(rawRes, {}); - res.cookie('hello', 'banana', {ttl: 22}); - expect(rawRes.cookies).to.eql([ - {name: 'hello', value: 'banana', lifeTime: 22} - ]); - }); - it('optionally adds some metadata', function () { - require("console").log('optionally adds some metadata'); - const rawRes = {}; - require("console").log("1"); - const res = new SyntheticResponse(rawRes, {}); - require("console").log("2"); - res.cookie('hello', 'banana', { - path: '/path', - domain: 'cats.example', - secure: true, - httpOnly: true - }); - require("console").log("3"); - expect(rawRes.cookies).to.eql([ - { - name: 'hello', - value: 'banana', - path: '/path', - domain: 'cats.example', - secure: true, - httpOnly: true - } - ]); - require("console").log("4"); - }); - it('supports signed cookies when a secret is provided', function () { - require("console").log('supports signed cookies when a secret is provided'); - const rawRes = {}; - const res = new SyntheticResponse(rawRes, {}); - res.cookie('hello', 'banana', {secret: 'potato'}); - expect(rawRes.cookies).to.eql([ - {name: 'hello', value: 'banana'}, - {name: 'hello.sig', value: crypto.hmac('potato', 'banana')} - ]); - }); - it('supports signed cookies with different algorithms', function () { - require("console").log('supports signed cookies with different algorithms'); - const rawRes = {}; - const res = new SyntheticResponse(rawRes, {}); - res.cookie('hello', 'banana', { - secret: 'potato', - algorithm: 'sha512' - }); - expect(rawRes.cookies).to.eql([ - {name: 'hello', value: 'banana'}, - {name: 'hello.sig', value: crypto.hmac('potato', 'banana', 'sha512')} - ]); - }); - it('treats options string as a secret', function () { - require("console").log('treats options string as a secret'); - const rawRes = {}; - const res = new SyntheticResponse(rawRes, {}); - res.cookie('hello', 'banana', 'potato'); - expect(rawRes.cookies).to.eql([ - {name: 'hello', value: 'banana'}, - {name: 'hello.sig', value: crypto.hmac('potato', 'banana')} - ]); - }); - it('treats options number as a TTL value', function () { - require("console").log('treats options number as a TTL value'); - const rawRes = {}; - const res = new SyntheticResponse(rawRes, {}); - res.cookie('hello', 'banana', 22); - expect(rawRes.cookies).to.eql([ - {name: 'hello', value: 'banana', lifeTime: 22} - ]); - }); - }); }); From 95f4265d2524a73d699c7588f5e3dadd8d9f55eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Fri, 24 Mar 2017 17:15:36 +0100 Subject: [PATCH 05/27] Windows fix --- arangod/Aql/GraphNode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index c67d80cd2e..1e3276ce4a 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -32,6 +32,7 @@ #include "Cluster/TraverserEngineRegistry.h" #include "Utils/CollectionNameResolver.h" +using namespace arangodb; using namespace arangodb::basics; using namespace arangodb::aql; From 7007db2f448ed3f2e937e14d3df866aa7d79580a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Fri, 24 Mar 2017 17:23:31 +0100 Subject: [PATCH 06/27] Windows: Fix ShortestPathNode --- arangod/Aql/ShortestPathNode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/arangod/Aql/ShortestPathNode.cpp b/arangod/Aql/ShortestPathNode.cpp index 822c7eaf42..76c023f825 100644 --- a/arangod/Aql/ShortestPathNode.cpp +++ b/arangod/Aql/ShortestPathNode.cpp @@ -36,6 +36,7 @@ #include #include +using namespace arangodb; using namespace arangodb::basics; using namespace arangodb::aql; From cb1efe12ddb992ace672870224474767b2464648 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 24 Mar 2017 17:47:02 +0100 Subject: [PATCH 07/27] colorize warnings/errors for better readability --- Documentation/Scripts/generateMdFiles.py | 72 ++++++++++++++---------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/Documentation/Scripts/generateMdFiles.py b/Documentation/Scripts/generateMdFiles.py index 68ca7547c5..cb81ccf926 100644 --- a/Documentation/Scripts/generateMdFiles.py +++ b/Documentation/Scripts/generateMdFiles.py @@ -4,6 +4,19 @@ import os import json #import MarkdownPP + +RESET = '\033[0m' +def make_std_color(No): + # defined for 1 through 7 + return '\033[3' + No+ 'm' +def make_color(No): + # defined for 1 through 255 + return '\033[38;5;'+ No + 'm' + +WRN_COLOR = make_std_color('3') +ERR_COLOR = make_std_color('1') +STD_COLOR = make_color('8') + ################################################################################ ### @brief length of the swagger definition namespace ################################################################################ @@ -31,8 +44,7 @@ def getReference(name, source, verb): try: ref = name['$ref'][defLen:] except Exception as x: - print >>sys.stderr, "No reference in: " - print >>sys.stderr, name + print >>sys.stderr, ERR_COLOR + "No reference in: " + name + RESET raise if not ref in swagger['definitions']: fn = '' @@ -40,7 +52,7 @@ def getReference(name, source, verb): fn = swagger['paths'][route][verb]['x-filename'] else: fn = swagger['definitions'][source]['x-filename'] - print >> sys.stderr, json.dumps(swagger['definitions'], indent=4, separators=(', ',': '), sort_keys=True) + print >> sys.stderr, STD_COLOR + json.dumps(swagger['definitions'], indent=4, separators=(', ',': '), sort_keys=True) + RESET raise Exception("invalid reference: " + ref + " in " + fn) return ref @@ -85,8 +97,8 @@ def unwrapPostJson(reference, layer): try: subStructRef = getReference(thisParam['items'], reference, None) except: - print >>sys.stderr, "while analyzing: " + param - print >>sys.stderr, thisParam + print >>sys.stderr, ERR_COLOR + "while analyzing: " + param + RESET + print >>sys.stderr, WRN_COLOR + thisParam + RESET rc += "\n" + unwrapPostJson(subStructRef, layer + 1) else: rc += ' ' * layer + " - **" + param + "**: " + TrimThisParam(thisParam['description'], layer) + '\n' @@ -122,8 +134,8 @@ def getRestReplyBodyParam(param): try: rc += unwrapPostJson(getReference(thisVerb['responses'][param]['schema'], route, verb), 0) except Exception: - print >>sys.stderr,"failed to search " + param + " in: " - print >>sys.stderr,json.dumps(thisVerb, indent=4, separators=(', ',': '), sort_keys=True) + print >>sys.stderr, ERR_COLOR + "failed to search " + param + " in: " + RESET + print >>sys.stderr, WRN_COLOR + json.dumps(thisVerb, indent=4, separators=(', ',': '), sort_keys=True) + RESET raise return rc + "\n" @@ -273,14 +285,14 @@ def replaceCode(lines, blockName): (verb,route) = headerMatch.group(1).split(',')[0].split(' ') verb = verb.lower() except: - print >> sys.stderr, "failed to parse header from: " + headerMatch.group(1) + " while analysing " + blockName + print >> sys.stderr, ERR_COLOR + "failed to parse header from: " + headerMatch.group(1) + " while analysing " + blockName + RESET raise try: thisVerb = swagger['paths'][route][verb] except: - print >> sys.stderr, "failed to locate route in the swagger json: [" + verb + " " + route + "]" + " while analysing " + blockName - print >> sys.stderr, lines + print >> sys.stderr, ERR_COLOR + "failed to locate route in the swagger json: [" + verb + " " + route + "]" + " while analysing " + blockName + RESET + print >> sys.stderr, WRN_COLOR + lines + RESET raise for (oneRX, repl) in RX: @@ -395,7 +407,7 @@ def walk_on_files(inDirPath, outDirPath): mdpp.close() md.close() findStartCode(md, outFileFull) - print "Processed %d files, skipped %d" % (count, skipped) + print STD_COLOR + "Processed %d files, skipped %d" % (count, skipped) + RESET def findStartCode(fd,full_path): inFD = open(full_path, "r") @@ -422,7 +434,7 @@ def findStartCode(fd,full_path): try: textFile = replaceCodeFullFile(textFile) except: - print >>sys.stderr, "while parsing :\n" + textFile + print >>sys.stderr, ERR_COLOR + "while parsing : " + full_path + RESET raise #print "9" * 80 #print textFile @@ -438,10 +450,10 @@ def replaceText(text, pathOfFile, searchText): #print '7'*80 global dokuBlocks if not searchText in dokuBlocks[0]: - print >> sys.stderr, "Failed to locate the docublock '%s' for replacing it into the file '%s'\n have:" % (searchText, pathOfFile) - print >> sys.stderr, dokuBlocks[0].keys() - print >> sys.stderr, '*' * 80 - print >> sys.stderr, text + print >> sys.stderr, "%sFailed to locate the docublock '%s' for replacing it into the file '%s'\n have:%s" % (ERR_COLOR, searchText, pathOfFile, RESET) + print >> sys.stderr, WRN_COLOR + dokuBlocks[0].keys() + RESET + print >> sys.stderr, ERR_COLOR + '*' * 80 + RESET + print >> sys.stderr, WRN_COLOR + text + RESET exit(1) #print '7'*80 #print dokuBlocks[0][searchText] @@ -453,22 +465,22 @@ def replaceTextInline(text, pathOfFile, searchText): ''' reads the mdpp and generates the md ''' global dokuBlocks if not searchText in dokuBlocks[1]: - print >> sys.stderr, "Failed to locate the inline docublock '%s' for replacing it into the file '%s'\n have:" % (searchText, pathOfFile) - print >> sys.stderr, dokuBlocks[1].keys() - print >> sys.stderr, '*' * 80 - print >> sys.stderr, text + print >> sys.stderr, "%sFailed to locate the inline docublock '%s' for replacing it into the file '%s'\n have: %s" % (ERR_COLOR, searchText, pathOfFile, RESET) + print >> sys.stderr, "%s%s%s" %(WRN_COLOR, dokuBlocks[1].keys(), RESET) + print >> sys.stderr, ERR_COLOR + '*' * 80 + RESET + print >> sys.stderr, WRN_COLOR + text + RESET exit(1) rePattern = r'(?s)\s*@startDocuBlockInline\s+'+ searchText +'\s.*?@endDocuBlock\s' + searchText # (?s) is equivalent to flags=re.DOTALL but works in Python 2.6 match = re.search(rePattern, text) if (match == None): - print >> sys.stderr, "failed to match with '%s' for %s in file %s in: \n%s" % (rePattern, searchText, pathOfFile, text) + print >> sys.stderr, "%sfailed to match with '%s' for %s in file %s in: \n%s" % (ERR_COLOR, rePattern, searchText, pathOfFile, text, RESET) exit(1) subtext = match.group(0) if (len(re.findall('@startDocuBlock', subtext)) > 1): - print >> sys.stderr, "failed to snap with '%s' on end docublock for %s in %s our match is:\n%s" % (rePattern, searchText, pathOfFile, subtext) + print >> sys.stderr, "%sfailed to snap with '%s' on end docublock for %s in %s our match is:\n%s" % (ERR_COLOR, rePattern, searchText, pathOfFile, subtext, RESET) exit(1) return re.sub(rePattern, dokuBlocks[1][searchText], text) @@ -495,7 +507,7 @@ def readStartLine(line): try: thisBlockName = SEARCH_START.search(line).group(1).strip() except: - print >> sys.stderr, "failed to read startDocuBlock: [" + line + "]" + print >> sys.stderr, ERR_COLOR + "failed to read startDocuBlock: [" + line + "]" + RESET exit(1) dokuBlocks[thisBlockType][thisBlockName] = "" return STATE_SEARCH_END @@ -525,10 +537,10 @@ def loadDokuBlocks(): if blockFilter != None: remainBlocks= {} - print "filtering blocks" + print STD_COLOR + "filtering blocks" + RESET for oneBlock in dokuBlocks[0]: if blockFilter.match(oneBlock) != None: - print "found block %s" % oneBlock + print "%sfound block %s%s" % (STD_COLOR, oneBlock, RESET) #print dokuBlocks[0][oneBlock] remainBlocks[oneBlock] = dokuBlocks[0][oneBlock] dokuBlocks[0] = remainBlocks @@ -541,14 +553,14 @@ def loadDokuBlocks(): #print dokuBlocks[0][oneBlock] #print "6"*80 except: - print >>sys.stderr, "while parsing :\n" + oneBlock + print >>sys.stderr, ERR_COLOR + "while parsing :\n" + oneBlock + RESET raise for oneBlock in dokuBlocks[1]: try: dokuBlocks[1][oneBlock] = replaceCode(dokuBlocks[1][oneBlock], oneBlock) except: - print >>sys.stderr, "while parsing :\n" + oneBlock + print >>sys.stderr, WRN_COLOR + "while parsing :\n" + oneBlock + RESET raise @@ -560,15 +572,15 @@ if __name__ == '__main__': outDir = sys.argv[2] swaggerJson = sys.argv[3] if len(sys.argv) > 4 and sys.argv[4].strip() != '': - print "filtering " + sys.argv[4] + print STD_COLOR + "filtering " + sys.argv[4] + RESET fileFilter = re.compile(sys.argv[4]) if len(sys.argv) > 5 and sys.argv[5].strip() != '': - print "filtering Docublocks: " + sys.argv[5] + print STD_COLOR + "filtering Docublocks: " + sys.argv[5] + RESET blockFilter = re.compile(sys.argv[5]) f=open(swaggerJson, 'rU') swagger= json.load(f) f.close() loadDokuBlocks() - print "loaded %d / %d docu blocks" % (len(dokuBlocks[0]), len(dokuBlocks[1])) + print "%sloaded %d / %d docu blocks%s" % (STD_COLOR, len(dokuBlocks[0]), len(dokuBlocks[1]), RESET) #print dokuBlocks[0].keys() walk_on_files(inDir, outDir) From aadcad3a000dd69dd29d1c6bc417d0f94f3712c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gra=CC=88tzer?= Date: Fri, 24 Mar 2017 17:52:50 +0100 Subject: [PATCH 08/27] Fix various conversion warnings --- arangod/Pregel/Algos/EffectiveCloseness/HLLCounter.cpp | 2 +- arangod/Pregel/Algos/HITS.cpp | 8 ++++---- arangod/Pregel/Algos/PageRank.cpp | 10 +++++----- arangod/Pregel/CommonFormats.h | 2 +- arangod/Pregel/Worker.cpp | 6 +++--- arangod/VocBase/TraverserOptions.cpp | 1 + 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/arangod/Pregel/Algos/EffectiveCloseness/HLLCounter.cpp b/arangod/Pregel/Algos/EffectiveCloseness/HLLCounter.cpp index 7a51d64b72..cd94874f26 100644 --- a/arangod/Pregel/Algos/EffectiveCloseness/HLLCounter.cpp +++ b/arangod/Pregel/Algos/EffectiveCloseness/HLLCounter.cpp @@ -88,7 +88,7 @@ uint32_t HLLCounter::getCount() { } else if (estimate > (1.0 / 30.0) * pow_2_32) { estimate = neg_pow_2_32 * log(1.0 - (estimate / pow_2_32)); } - return estimate; + return (uint32_t) estimate; } void HLLCounter::merge(HLLCounter const& other) { diff --git a/arangod/Pregel/Algos/HITS.cpp b/arangod/Pregel/Algos/HITS.cpp index 446a697ed5..4104476dd8 100644 --- a/arangod/Pregel/Algos/HITS.cpp +++ b/arangod/Pregel/Algos/HITS.cpp @@ -56,13 +56,13 @@ struct HITSComputation void compute( MessageIterator> const& messages) override { - double auth = 0.0f; - double hub = 0.0f; + double auth = 0.0; + double hub = 0.0; // we don't know our incoming neighbours in step 0, therfore we need step 0 // as 'initialization' before actually starting to converge if (globalSuperstep() <= 1) { - auth = 1.0f; - hub = 1.0f; + auth = 1.0; + hub = 1.0; } else { HITSWorkerContext const* ctx = static_cast(context()); for (SenderMessage const* message : messages) { diff --git a/arangod/Pregel/Algos/PageRank.cpp b/arangod/Pregel/Algos/PageRank.cpp index 176859fda4..1035719359 100644 --- a/arangod/Pregel/Algos/PageRank.cpp +++ b/arangod/Pregel/Algos/PageRank.cpp @@ -32,7 +32,7 @@ using namespace arangodb; using namespace arangodb::pregel; using namespace arangodb::pregel::algos; -static float EPS = 0.00001; +static float EPS = 0.00001f; static std::string const kConvergence = "convergence"; struct PRWorkerContext : public WorkerContext { @@ -41,9 +41,9 @@ struct PRWorkerContext : public WorkerContext { float commonProb = 0; void preGlobalSuperstep(uint64_t gss) override { if (gss == 0) { - commonProb = 1.0 / vertexCount(); + commonProb = 1.0f / vertexCount(); } else { - commonProb = 0.15 / vertexCount(); + commonProb = 0.15f / vertexCount(); } } }; @@ -64,11 +64,11 @@ struct PRComputation : public VertexComputation { if (globalSuperstep() == 0) { *ptr = ctx->commonProb; } else { - float sum = 0.0; + float sum = 0.0f; for (const float* msg : messages) { sum += *msg; } - *ptr = 0.85 * sum + ctx->commonProb; + *ptr = 0.85f * sum + ctx->commonProb; } float diff = fabs(copy - *ptr); aggregate(kConvergence, diff); diff --git a/arangod/Pregel/CommonFormats.h b/arangod/Pregel/CommonFormats.h index d116f859f5..b6cdc097a1 100644 --- a/arangod/Pregel/CommonFormats.h +++ b/arangod/Pregel/CommonFormats.h @@ -115,7 +115,7 @@ struct SenderMessageFormat : public MessageFormat> { SenderMessageFormat() {} void unwrapValue(VPackSlice s, SenderMessage& senderVal) const override { VPackArrayIterator array(s); - senderVal.senderId.shard = (*array).getUInt(); + senderVal.senderId.shard = (PregelShard) ((*array).getUInt()); senderVal.senderId.key = (*(++array)).copyString(); senderVal.value = (*(++array)).getNumber(); } diff --git a/arangod/Pregel/Worker.cpp b/arangod/Pregel/Worker.cpp index 13e60943a6..119e079db7 100644 --- a/arangod/Pregel/Worker.cpp +++ b/arangod/Pregel/Worker.cpp @@ -522,9 +522,9 @@ void Worker::_finishedProcessing() { // async adaptive message buffering _messageBatchSize = _algorithm->messageBatchSize(_config, _messageStats); } else { - uint32_t tn = _config.parallelism(); - uint32_t s = _messageStats.sendCount / tn / 2UL; - _messageBatchSize = s > 1000 ? s : 1000; + uint64_t tn = _config.parallelism(); + uint64_t s = _messageStats.sendCount / tn / 2UL; + _messageBatchSize = s > 1000 ? (uint32_t)s : 1000; } _messageStats.resetTracking(); LOG_TOPIC(DEBUG, Logger::PREGEL) << "Batch size: " << _messageBatchSize; diff --git a/arangod/VocBase/TraverserOptions.cpp b/arangod/VocBase/TraverserOptions.cpp index 340c6da33e..b616d924fc 100644 --- a/arangod/VocBase/TraverserOptions.cpp +++ b/arangod/VocBase/TraverserOptions.cpp @@ -40,6 +40,7 @@ using VPackHelper = arangodb::basics::VelocyPackHelper; using TraverserOptions = arangodb::traverser::TraverserOptions; using ShortestPathOptions = arangodb::traverser::ShortestPathOptions; using BaseTraverserOptions = arangodb::traverser::BaseTraverserOptions; +using namespace arangodb::transaction; BaseTraverserOptions::LookupInfo::LookupInfo() : expression(nullptr), From 3925f946c548656600ccafdade183267502bd3af Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 24 Mar 2017 18:23:20 +0100 Subject: [PATCH 09/27] Revert "Squashed commit. Moved over SmartSearch Shortest Path feature" This reverts commit f7eb96bc8dbda9d4499d47781772379c46561f9c. --- CHANGELOG | 4 - arangod/Aql/ExecutionEngine.cpp | 106 +-- arangod/Aql/ExecutionPlan.cpp | 25 +- arangod/Aql/GraphNode.cpp | 538 ---------------- arangod/Aql/GraphNode.h | 216 ------- arangod/Aql/OptimizerRules.cpp | 11 +- arangod/Aql/ShortestPathBlock.cpp | 137 ++-- arangod/Aql/ShortestPathBlock.h | 15 +- arangod/Aql/ShortestPathNode.cpp | 442 +++++++++---- arangod/Aql/ShortestPathNode.h | 89 ++- arangod/Aql/ShortestPathOptions.cpp | 54 ++ arangod/Aql/ShortestPathOptions.h | 54 ++ arangod/Aql/TraversalNode.cpp | 601 ++++++++++++++++-- arangod/Aql/TraversalNode.h | 129 +++- arangod/CMakeLists.txt | 3 +- arangod/Cluster/TraverserEngine.cpp | 39 +- arangod/Cluster/TraverserEngine.h | 17 +- .../InternalRestTraverserHandler.cpp | 47 +- .../InternalRestTraverserHandler.h | 7 - arangod/V8Server/V8Traverser.cpp | 63 ++ arangod/V8Server/V8Traverser.h | 89 +++ arangod/VocBase/Traverser.cpp | 30 +- arangod/VocBase/Traverser.h | 8 - arangod/VocBase/TraverserOptions.cpp | 514 +++++---------- arangod/VocBase/TraverserOptions.h | 187 +----- 25 files changed, 1666 insertions(+), 1759 deletions(-) delete mode 100644 arangod/Aql/GraphNode.cpp delete mode 100644 arangod/Aql/GraphNode.h create mode 100644 arangod/Aql/ShortestPathOptions.cpp create mode 100644 arangod/Aql/ShortestPathOptions.h create mode 100644 arangod/V8Server/V8Traverser.cpp create mode 100644 arangod/V8Server/V8Traverser.h diff --git a/CHANGELOG b/CHANGELOG index 0be784780c..9b8dfb4621 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,10 +3,6 @@ devel * increase default collection lock timeout from 30 to 900 seconds -* made shortest_path computation aware of SmartGraph features (Enterprise - Only) which will give a good performance boost on computing single-pair - shortest-paths in a sharded SmartGraph. - * added function `db._engine()` for retrieval of storage engine information at server runtime diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index c4e1ead4ff..d8d9d2eeb7 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -54,27 +54,6 @@ using namespace arangodb; using namespace arangodb::aql; -// Used in Mapping ServerID => Responsible Shards -struct TraversalInfoMapping { - // All forward computed edge shards - std::vector> forwardEdgeShards; - // All backward computed edge shards (only ShortestPath) - std::vector> backwardEdgeShards; - // All shards for vertices - std::unordered_map> vertexShards; - - TraversalInfoMapping() = delete; - TraversalInfoMapping(size_t length) { - // We always know the exact size of the shards. - // Furthermore the lengths of the vectors have to - // be correct, as accessed is based on indexes. - forwardEdgeShards.resize(length); - backwardEdgeShards.resize(length); - } - - ~TraversalInfoMapping() {} -}; - /// @brief helper function to create a block static ExecutionBlock* CreateBlock( ExecutionEngine* engine, ExecutionNode const* en, @@ -228,10 +207,9 @@ struct Instanciator final : public WalkerWorker { virtual void after(ExecutionNode* en) override final { ExecutionBlock* block = nullptr; { - if (en->getType() == ExecutionNode::TRAVERSAL || - en->getType() == ExecutionNode::SHORTEST_PATH) { + if (en->getType() == ExecutionNode::TRAVERSAL) { // We have to prepare the options before we build the block - static_cast(en)->prepareOptions(); + static_cast(en)->prepareOptions(); } std::unique_ptr eb(CreateBlock(engine, en, cache, std::unordered_set())); @@ -846,10 +824,9 @@ struct CoordinatorInstanciator : public WalkerWorker { /// @brief Build traverser engines on DBServers. Coordinator still uses /// traversal block. - void buildTraverserEnginesForNode(GraphNode* en) { + void buildTraverserEnginesForNode(TraversalNode* en) { // We have to initialize all options. After this point the node // is not cloneable any more. - // Required prior to creation of engines. en->prepareOptions(); VPackBuilder optsBuilder; auto opts = en->options(); @@ -864,7 +841,10 @@ struct CoordinatorInstanciator : public WalkerWorker { // For edgeCollections the Ordering is important for the index access. // Also the same edgeCollection can be included twice (iff direction is ANY) auto clusterInfo = arangodb::ClusterInfo::instance(); - std::unordered_map + std::unordered_map< + ServerID, + std::pair>, + std::unordered_map>>> mappingServerToCollections; auto servers = clusterInfo->getCurrentDBServers(); size_t length = edges.size(); @@ -875,32 +855,18 @@ struct CoordinatorInstanciator : public WalkerWorker { for (auto s : servers) { // We insert at lease an empty vector for every edge collection // Used in the traverser. - mappingServerToCollections.emplace(s, length); + auto& info = mappingServerToCollections[s]; + // We need to exactly maintain the ordering. + // A server my be responsible for a shard in edge collection 1 but not 0 or 2. + info.first.resize(length); } - for (size_t i = 0; i < length; ++i) { auto shardIds = edges[i]->shardIds(_includedShards); for (auto const& shard : *shardIds) { auto serverList = clusterInfo->getResponsibleServer(shard); TRI_ASSERT(!serverList->empty()); - auto map = mappingServerToCollections.find((*serverList)[0]); - TRI_ASSERT(map != mappingServerToCollections.end()); - map->second.forwardEdgeShards[i].emplace_back(shard); - } - } - - std::vector> const& reverseEdges = en->reverseEdgeColls(); - if (!reverseEdges.empty()) { - TRI_ASSERT(reverseEdges.size() == length); - for (size_t i = 0; i < length; ++i) { - auto shardIds = reverseEdges[i]->shardIds(); - for (auto const& shard : *shardIds) { - auto serverList = clusterInfo->getResponsibleServer(shard); - TRI_ASSERT(!serverList->empty()); - auto map = mappingServerToCollections.find((*serverList)[0]); - TRI_ASSERT(map != mappingServerToCollections.end()); - map->second.backwardEdgeShards[i].emplace_back(shard); - } + auto& pair = mappingServerToCollections[(*serverList)[0]]; + pair.first[i].emplace_back(shard); } } @@ -916,8 +882,8 @@ struct CoordinatorInstanciator : public WalkerWorker { auto cs = query->collections()->collections(); for (auto const& collection : (*cs)) { for (auto& entry : mappingServerToCollections) { - entry.second.vertexShards.emplace(collection.second->getName(), - std::vector()); + entry.second.second.emplace(collection.second->getName(), + std::vector()); } if (knownEdges.find(collection.second->getName()) == knownEdges.end()) { // This collection is not one of the edge collections used in this @@ -926,9 +892,8 @@ struct CoordinatorInstanciator : public WalkerWorker { for (auto const& shard : *shardIds) { auto serverList = clusterInfo->getResponsibleServer(shard); TRI_ASSERT(!serverList->empty()); - auto map = mappingServerToCollections.find((*serverList)[0]); - TRI_ASSERT(map != mappingServerToCollections.end()); - map->second.vertexShards[collection.second->getName()].emplace_back(shard); + auto& pair = mappingServerToCollections[(*serverList)[0]]; + pair.second[collection.second->getName()].emplace_back(shard); } } } @@ -936,15 +901,15 @@ struct CoordinatorInstanciator : public WalkerWorker { // This Traversal is started with a GRAPH. It knows all relevant collections. for (auto const& it : vertices) { for (auto& entry : mappingServerToCollections) { - entry.second.vertexShards.emplace(it->getName(), - std::vector()); + entry.second.second.emplace(it->getName(), + std::vector()); } auto shardIds = it->shardIds(_includedShards); for (auto const& shard : *shardIds) { auto serverList = clusterInfo->getResponsibleServer(shard); TRI_ASSERT(!serverList->empty()); - auto map = mappingServerToCollections.find((*serverList)[0]); - map->second.vertexShards[it->getName()].emplace_back(shard); + auto& pair = mappingServerToCollections[(*serverList)[0]]; + pair.second[it->getName()].emplace_back(shard); } } } @@ -969,11 +934,7 @@ struct CoordinatorInstanciator : public WalkerWorker { // "vertices" : { // "v1": [], // may be empty // "v2": [] // may be empty - // }, - // "reverseEdges" : [ - // [ ], - // [ ] - // ] + // } // } // } @@ -1014,7 +975,7 @@ struct CoordinatorInstanciator : public WalkerWorker { engineInfo.openObject(); engineInfo.add(VPackValue("vertices")); engineInfo.openObject(); - for (auto const& col : list.second.vertexShards) { + for (auto const& col : list.second.second) { engineInfo.add(VPackValue(col.first)); engineInfo.openArray(); for (auto const& v : col.second) { @@ -1027,7 +988,7 @@ struct CoordinatorInstanciator : public WalkerWorker { engineInfo.add(VPackValue("edges")); engineInfo.openArray(); - for (auto const& edgeShards : list.second.forwardEdgeShards) { + for (auto const& edgeShards : list.second.first) { engineInfo.openArray(); for (auto const& e : edgeShards) { shardSet.emplace(e); @@ -1037,19 +998,6 @@ struct CoordinatorInstanciator : public WalkerWorker { } engineInfo.close(); // edges - engineInfo.add(VPackValue("reverseEdges")); - - engineInfo.openArray(); - for (auto const& edgeShards : list.second.backwardEdgeShards) { - engineInfo.openArray(); - for (auto const& e : edgeShards) { - shardSet.emplace(e); - engineInfo.add(VPackValue(e)); - } - engineInfo.close(); - } - engineInfo.close(); // reverseEdges - engineInfo.close(); // shards en->enhanceEngineInfo(engineInfo); @@ -1180,10 +1128,8 @@ struct CoordinatorInstanciator : public WalkerWorker { engines.emplace_back(currentLocation, currentEngineId, part, en->id()); } - if (nodeType == ExecutionNode::TRAVERSAL || - nodeType == ExecutionNode::SHORTEST_PATH) { - // Now build traverser engines. - buildTraverserEnginesForNode(static_cast(en)); + if (nodeType == ExecutionNode::TRAVERSAL) { + buildTraverserEnginesForNode(static_cast(en)); } return false; diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index 9dd2e0255a..652368760d 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -35,6 +35,7 @@ #include "Aql/OptimizerRulesFeature.h" #include "Aql/Query.h" #include "Aql/ShortestPathNode.h" +#include "Aql/ShortestPathOptions.h" #include "Aql/SortNode.h" #include "Aql/TraversalNode.h" #include "Aql/Variable.h" @@ -144,9 +145,8 @@ static std::unique_ptr CreateTraversalOptions( return options; } -static std::unique_ptr -CreateShortestPathOptions(transaction::Methods* trx, AstNode const* node) { - auto options = std::make_unique(trx); +static ShortestPathOptions CreateShortestPathOptions(AstNode const* node) { + ShortestPathOptions options; if (node != nullptr && node->type == NODE_TYPE_OBJECT) { size_t n = node->numMembers(); @@ -161,10 +161,10 @@ CreateShortestPathOptions(transaction::Methods* trx, AstNode const* node) { TRI_ASSERT(value->isConstant()); if (name == "weightAttribute" && value->isStringValue()) { - options->setWeightAttribute( - std::string(value->getStringValue(), value->getStringLength())); + options.weightAttribute = + std::string(value->getStringValue(), value->getStringLength()); } else if (name == "defaultWeight" && value->isNumericValue()) { - options->setDefaultWeight(value->getDoubleValue()); + options.defaultWeight = value->getDoubleValue(); } } } @@ -723,9 +723,8 @@ ExecutionNode* ExecutionPlan::fromNodeTraversal(ExecutionNode* previous, node->getMember(3)); // First create the node - auto travNode = - new TraversalNode(this, nextId(), _ast->query()->vocbase(), - direction->getMember(0), start, graph, options.release()); + auto travNode = new TraversalNode(this, nextId(), _ast->query()->vocbase(), + direction, start, graph, options); auto variable = node->getMember(4); TRI_ASSERT(variable->type == NODE_TYPE_VARIABLE); @@ -796,13 +795,13 @@ ExecutionNode* ExecutionPlan::fromNodeShortestPath(ExecutionNode* previous, AstNode const* target = parseTraversalVertexNode(previous, node->getMember(2)); AstNode const* graph = node->getMember(3); - std::unique_ptr options = - CreateShortestPathOptions(getAst()->query()->trx(), node->getMember(4)); + ShortestPathOptions options = CreateShortestPathOptions(node->getMember(4)); + // First create the node auto spNode = new ShortestPathNode(this, nextId(), _ast->query()->vocbase(), - direction, start, target, - graph, options.release()); + direction->getIntValue(), start, target, + graph, options); auto variable = node->getMember(5); TRI_ASSERT(variable->type == NODE_TYPE_VARIABLE); diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp deleted file mode 100644 index 1e3276ce4a..0000000000 --- a/arangod/Aql/GraphNode.cpp +++ /dev/null @@ -1,538 +0,0 @@ -/// @brief Implementation of Shortest Path Execution Node -/// -/// @file arangod/Aql/ShortestPathNode.cpp -/// -/// DISCLAIMER -/// -/// Copyright 2010-2014 triagens GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany -//////////////////////////////////////////////////////////////////////////////// - -#include "GraphNode.h" -#include "Aql/Ast.h" -#include "Aql/ExecutionPlan.h" -#include "Aql/Query.h" -#include "Cluster/ClusterInfo.h" -#include "Cluster/TraverserEngineRegistry.h" -#include "Utils/CollectionNameResolver.h" - -using namespace arangodb; -using namespace arangodb::basics; -using namespace arangodb::aql; - -static TRI_edge_direction_e parseDirection (AstNode const* node) { - TRI_ASSERT(node->isIntValue()); - auto dirNum = node->getIntValue(); - - switch (dirNum) { - case 0: - return TRI_EDGE_ANY; - case 1: - return TRI_EDGE_IN; - case 2: - return TRI_EDGE_OUT; - default: - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_PARSE, - "direction can only be INBOUND, OUTBOUND or ANY"); - } -} - -GraphNode::GraphNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - traverser::BaseTraverserOptions* options) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), - _graphObj(nullptr), - _tmpObjVariable(_plan->getAst()->variables()->createTemporaryVariable()), - _tmpObjVarNode(_plan->getAst()->createNodeReference(_tmpObjVariable)), - _tmpIdNode(_plan->getAst()->createNodeValueString("", 0)), - _fromCondition(nullptr), - _toCondition(nullptr), - _optionsBuild(false), - _isSmart(false), - _options(options) { - TRI_ASSERT(_vocbase != nullptr); - TRI_ASSERT(options != nullptr); - TRI_ASSERT(_options.get() != nullptr); -} - -GraphNode::GraphNode(ExecutionPlan* plan, - arangodb::velocypack::Slice const& base) - : ExecutionNode(plan, base), - _vocbase(plan->getAst()->query()->vocbase()), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), - _graphObj(nullptr), - _tmpObjVariable(nullptr), - _tmpObjVarNode(nullptr), - _tmpIdNode(nullptr), - _fromCondition(nullptr), - _toCondition(nullptr), - _optionsBuild(false), - _isSmart(false), - _options(nullptr) { - // NOTE: options have to be created by subclass. They differ - // Directions - VPackSlice dirList = base.get("directions"); - for (auto const& it : VPackArrayIterator(dirList)) { - uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it); - TRI_edge_direction_e d; - switch (dir) { - case 0: - TRI_ASSERT(false); - break; - case 1: - d = TRI_EDGE_IN; - break; - case 2: - d = TRI_EDGE_OUT; - break; - default: - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "Invalid direction value"); - break; - } - _directions.emplace_back(d); - } - - // TODO: Can we remove this? - std::string graphName; - if (base.hasKey("graph") && (base.get("graph").isString())) { - graphName = base.get("graph").copyString(); - if (base.hasKey("graphDefinition")) { - _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); - - if (_graphObj == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); - } - } else { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "missing graphDefinition."); - } - } else { - _graphInfo.add(base.get("graph")); - if (!_graphInfo.slice().isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "graph has to be an array."); - } - } - - VPackSlice list = base.get("edgeCollections"); - if (!list.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "traverser needs an array of edge collections."); - } - - for (auto const& it : VPackArrayIterator(list)) { - std::string e = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); - _edgeColls.emplace_back( - std::make_unique(e, _vocbase, AccessMode::Type::READ)); - } - - list = base.get("vertexCollections"); - - if (!list.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, - "traverser needs an array of vertex collections."); - } - - for (auto const& it : VPackArrayIterator(list)) { - std::string v = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); - _vertexColls.emplace_back( - std::make_unique(v, _vocbase, AccessMode::Type::READ)); - } - - - // Out variables - if (base.hasKey("vertexOutVariable")) { - _vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable"); - } - if (base.hasKey("edgeOutVariable")) { - _edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable"); - } - - // Temporary Filter Objects - TRI_ASSERT(base.hasKey("tmpObjVariable")); - _tmpObjVariable = varFromVPack(plan->getAst(), base, "tmpObjVariable"); - - TRI_ASSERT(base.hasKey("tmpObjVarNode")); - _tmpObjVarNode = new AstNode(plan->getAst(), base.get("tmpObjVarNode")); - - TRI_ASSERT(base.hasKey("tmpIdNode")); - _tmpIdNode = new AstNode(plan->getAst(), base.get("tmpIdNode")); - - // Filter Condition Parts - TRI_ASSERT(base.hasKey("fromCondition")); - _fromCondition = new AstNode(plan->getAst(), base.get("fromCondition")); - - TRI_ASSERT(base.hasKey("toCondition")); - _toCondition = new AstNode(plan->getAst(), base.get("toCondition")); - _isSmart = VelocyPackHelper::getBooleanValue(base, "isSmart", false); -} - -GraphNode::GraphNode( - ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - std::vector> const& edgeColls, - std::vector> const& vertexColls, - std::vector const& directions, - traverser::BaseTraverserOptions* options) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _vertexOutVariable(nullptr), - _edgeOutVariable(nullptr), - _directions(directions), - _graphObj(nullptr), - _tmpObjVariable(nullptr), - _tmpObjVarNode(nullptr), - _tmpIdNode(nullptr), - _fromCondition(nullptr), - _toCondition(nullptr), - _optionsBuild(false), - _isSmart(false), - _options(options) { - _graphInfo.openArray(); - for (auto const& it : edgeColls) { - // Collections cannot be copied. So we need to create new ones to prevent leaks - _edgeColls.emplace_back(std::make_unique( - it->getName(), _vocbase, AccessMode::Type::READ)); - _graphInfo.add(VPackValue(it->getName())); - } - for (auto& it : vertexColls) { - // Collections cannot be copied. So we need to create new ones to prevent leaks - _vertexColls.emplace_back(std::make_unique( - it->getName(), _vocbase, AccessMode::Type::READ)); - } - - _graphInfo.close(); -} - -/// @brief the cost of a traversal node -double GraphNode::estimateCost(size_t& nrItems) const { - size_t incoming = 0; - double depCost = _dependencies.at(0)->getCost(incoming); - return depCost + _options->estimateCost(nrItems); -} - - - -#ifndef USE_ENTERPRISE -void GraphNode::enhanceEngineInfo(VPackBuilder& builder) const { - if (_graphObj != nullptr) { - _graphObj->enhanceEngineInfo(builder); - } -} -#endif - -void GraphNode::addEngine(traverser::TraverserEngineID const& engine, - arangodb::ServerID const& server) { - TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); - _engines.emplace(server, engine); -} - -/// @brief check if all directions are equal -bool GraphNode::allDirectionsEqual() const { - if (_directions.empty()) { - // no directions! - return false; - } - size_t const n = _directions.size(); - TRI_edge_direction_e const expected = _directions[0]; - - for (size_t i = 1; i < n; ++i) { - if (_directions[i] != expected) { - return false; - } - } - return true; -} - - - -AstNode* GraphNode::getTemporaryRefNode() const { - return _tmpObjVarNode; -} - -Variable const* GraphNode::getTemporaryVariable() const { - return _tmpObjVariable; -} - -void GraphNode::baseToVelocyPackHelper(VPackBuilder& nodes, - bool verbose) const { - ExecutionNode::toVelocyPackHelperGeneric(nodes, - verbose); // call base class method - - nodes.add("database", VPackValue(_vocbase->name())); - - nodes.add("graph", _graphInfo.slice()); - nodes.add(VPackValue("directions")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& d : _directions) { - nodes.add(VPackValue(d)); - } - } - - nodes.add(VPackValue("edgeCollections")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& e : _edgeColls) { - nodes.add(VPackValue(e->getName())); - } - } - - nodes.add(VPackValue("vertexCollections")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& v : _vertexColls) { - nodes.add(VPackValue(v->getName())); - } - } - - if (_graphObj != nullptr) { - nodes.add(VPackValue("graphDefinition")); - _graphObj->toVelocyPack(nodes, verbose); - } - - // Out variables - if (usesVertexOutVariable()) { - nodes.add(VPackValue("vertexOutVariable")); - vertexOutVariable()->toVelocyPack(nodes); - } - if (usesEdgeOutVariable()) { - nodes.add(VPackValue("edgeOutVariable")); - edgeOutVariable()->toVelocyPack(nodes); - } - - // Traversal Filter Conditions - - TRI_ASSERT(_tmpObjVariable != nullptr); - nodes.add(VPackValue("tmpObjVariable")); - _tmpObjVariable->toVelocyPack(nodes); - - TRI_ASSERT(_tmpObjVarNode != nullptr); - nodes.add(VPackValue("tmpObjVarNode")); - _tmpObjVarNode->toVelocyPack(nodes, verbose); - - TRI_ASSERT(_tmpIdNode != nullptr); - nodes.add(VPackValue("tmpIdNode")); - _tmpIdNode->toVelocyPack(nodes, verbose); - - TRI_ASSERT(_fromCondition != nullptr); - nodes.add(VPackValue("fromCondition")); - _fromCondition->toVelocyPack(nodes, verbose); - - TRI_ASSERT(_toCondition != nullptr); - nodes.add(VPackValue("toCondition")); - _toCondition->toVelocyPack(nodes, verbose); -} - -#ifndef USE_ENTERPRISE -void GraphNode::addEdgeColl(std::string const& n, TRI_edge_direction_e dir) { - if (dir == TRI_EDGE_ANY) { - _directions.emplace_back(TRI_EDGE_OUT); - _edgeColls.emplace_back( - std::make_unique(n, _vocbase, AccessMode::Type::READ)); - - _directions.emplace_back(TRI_EDGE_IN); - _edgeColls.emplace_back( - std::make_unique(n, _vocbase, AccessMode::Type::READ)); - } else { - _directions.emplace_back(dir); - _edgeColls.emplace_back( - std::make_unique(n, _vocbase, AccessMode::Type::READ)); - } -} -#endif - -void GraphNode::parseGraphAstNodes(AstNode const* direction, AstNode const* graph) { - TRI_ASSERT(graph != nullptr); - TRI_ASSERT(direction != nullptr); - - TRI_edge_direction_e baseDirection = parseDirection(direction); - - std::unordered_map seenCollections; - - - if (graph->type == NODE_TYPE_COLLECTION_LIST) { - size_t edgeCollectionCount = graph->numMembers(); - - _graphInfo.openArray(); - _edgeColls.reserve(edgeCollectionCount); - _directions.reserve(edgeCollectionCount); - - // First determine whether all edge collections are smart and sharded - // like a common collection: - auto ci = ClusterInfo::instance(); - if (ServerState::instance()->isRunningInCluster()) { - _isSmart = true; - std::string distributeShardsLike; - for (size_t i = 0; i < edgeCollectionCount; ++i) { - auto col = graph->getMember(i); - if (col->type == NODE_TYPE_DIRECTION) { - col = col->getMember(1); // The first member always is the collection - } - std::string n = col->getString(); - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart() || c->distributeShardsLike().empty()) { - _isSmart = false; - break; - } - if (distributeShardsLike.empty()) { - distributeShardsLike = c->distributeShardsLike(); - } else if (distributeShardsLike != c->distributeShardsLike()) { - _isSmart = false; - break; - } - } - } - - auto resolver = std::make_unique(_vocbase); - // List of edge collection names - for (size_t i = 0; i < edgeCollectionCount; ++i) { - auto col = graph->getMember(i); - TRI_edge_direction_e dir = TRI_EDGE_ANY; - - if (col->type == NODE_TYPE_DIRECTION) { - // We have a collection with special direction. - dir = parseDirection(col->getMember(0)); - col = col->getMember(1); - } else { - dir = baseDirection; - } - - std::string eColName = col->getString(); - - // now do some uniqueness checks for the specified collections - auto it = seenCollections.find(eColName); - if (it != seenCollections.end()) { - if ((*it).second != dir) { - std::string msg("conflicting directions specified for collection '" + - std::string(eColName)); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, - msg); - } - // do not re-add the same collection! - continue; - } - seenCollections.emplace(eColName, dir); - - if (resolver->getCollectionTypeCluster(eColName) != TRI_COL_TYPE_EDGE) { - std::string msg("collection type invalid for collection '" + - std::string(eColName) + - ": expecting collection type 'edge'"); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, - msg); - } - - _graphInfo.add(VPackValue(eColName)); - if (ServerState::instance()->isRunningInCluster()) { - auto c = ci->getCollection(_vocbase->name(), eColName); - if (!c->isSmart()) { - addEdgeColl(eColName, dir); - } else { - std::vector names; - if (_isSmart) { - names = c->realNames(); - } else { - names = c->realNamesForRead(); - } - for (auto const& name : names) { - addEdgeColl(name, dir); - } - } - } else { - addEdgeColl(eColName, dir); - } - } - _graphInfo.close(); - } else { - if (_edgeColls.empty()) { - if (graph->isStringValue()) { - std::string graphName = graph->getString(); - _graphInfo.add(VPackValue(graphName)); - _graphObj = plan()->getAst()->query()->lookupGraphByName(graphName); - - if (_graphObj == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); - } - - auto eColls = _graphObj->edgeCollections(); - size_t length = eColls.size(); - if (length == 0) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); - } - - // First determine whether all edge collections are smart and sharded - // like a common collection: - auto ci = ClusterInfo::instance(); - if (ServerState::instance()->isRunningInCluster()) { - _isSmart = true; - std::string distributeShardsLike; - for (auto const& n : eColls) { - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart() || c->distributeShardsLike().empty()) { - _isSmart = false; - break; - } - if (distributeShardsLike.empty()) { - distributeShardsLike = c->distributeShardsLike(); - } else if (distributeShardsLike != c->distributeShardsLike()) { - _isSmart = false; - break; - } - } - } - - for (const auto& n : eColls) { - if (ServerState::instance()->isRunningInCluster()) { - auto c = ci->getCollection(_vocbase->name(), n); - if (!c->isSmart()) { - addEdgeColl(n, baseDirection); - } else { - std::vector names; - if (_isSmart) { - names = c->realNames(); - } else { - names = c->realNamesForRead(); - } - for (auto const& name : names) { - addEdgeColl(name, baseDirection); - } - } - } else { - addEdgeColl(n, baseDirection); - } - } - - auto vColls = _graphObj->vertexCollections(); - length = vColls.size(); - if (length == 0) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); - } - _vertexColls.reserve(length); - for (auto const& v : vColls) { - _vertexColls.emplace_back(std::make_unique( - v, _vocbase, AccessMode::Type::READ)); - } - } - } - } -} diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h deleted file mode 100644 index 28437acec3..0000000000 --- a/arangod/Aql/GraphNode.h +++ /dev/null @@ -1,216 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGOD_AQL_GRAPH_NODE_H -#define ARANGOD_AQL_GRAPH_NODE_H 1 - -#include "Aql/Collection.h" -#include "Aql/ExecutionNode.h" -#include "Aql/Graphs.h" -#include "Cluster/TraverserEngineRegistry.h" -#include "VocBase/LogicalCollection.h" -#include "VocBase/TraverserOptions.h" - -/// NOTE: This Node is purely virtual and is used to unify graph parsing for -/// Traversal and ShortestPath node. It shall never be part of any plan -/// nor will their be a Block to implement it. - -namespace arangodb { - -namespace aql { - -class GraphNode : public ExecutionNode { - protected: - /// @brief Constructor for a new node parsed from AQL - GraphNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - traverser::BaseTraverserOptions* options); - - /// @brief Deserializer for node from VPack - GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); - - /// @brief Internal constructor to clone the node. - GraphNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - std::vector> const& edgeColls, - std::vector> const& vertexColls, - std::vector const& directions, - traverser::BaseTraverserOptions* options); - - public: - virtual ~GraphNode() {} - - /// @brief flag if smart search can be used (Enterprise only) - bool isSmart() const { - return _isSmart; - } - - /// @brief return the database - TRI_vocbase_t* vocbase() const { return _vocbase; } - - /// @brief return the vertex out variable - Variable const* vertexOutVariable() const { return _vertexOutVariable; } - - /// @brief checks if the vertex out variable is used - bool usesVertexOutVariable() const { return _vertexOutVariable != nullptr; } - - /// @brief set the vertex out variable - void setVertexOutput(Variable const* outVar) { _vertexOutVariable = outVar; } - - - /// @brief return the edge out variable - Variable const* edgeOutVariable() const { return _edgeOutVariable; } - - /// @brief checks if the edge out variable is used - bool usesEdgeOutVariable() const { return _edgeOutVariable != nullptr; } - - /// @brief set the edge out variable - void setEdgeOutput(Variable const* outVar) { _edgeOutVariable = outVar; } - - - std::vector> const& edgeColls() const { - return _edgeColls; - } - - std::vector> const& reverseEdgeColls() const { - return _reverseEdgeColls; - } - - std::vector> const& vertexColls() const { - return _vertexColls; - } - - bool allDirectionsEqual() const; - - AstNode* getTemporaryRefNode() const; - - Variable const* getTemporaryVariable() const; - - void enhanceEngineInfo(arangodb::velocypack::Builder&) const; - - virtual traverser::BaseTraverserOptions* options() const = 0; - - virtual void getConditionVariables(std::vector&) const {}; - - /// @brief estimate the cost of this graph node - double estimateCost(size_t& nrItems) const final; - - /// @brief Compute the traversal options containing the expressions - /// MUST! be called after optimization and before creation - /// of blocks. - virtual void prepareOptions() = 0; - - /// @brief Add a traverser engine Running on a DBServer to this node. - /// The block will communicate with them (CLUSTER ONLY) - void addEngine(traverser::TraverserEngineID const&, ServerID const&); - - /// @brief Returns a reference to the engines. (CLUSTER ONLY) - std::unordered_map const* engines() - const { - TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); - return &_engines; - } - - protected: - - //////////////////////////////////////////////////////////////////////////////// - /// @brief Export to VelocyPack - //////////////////////////////////////////////////////////////////////////////// - - void baseToVelocyPackHelper(arangodb::velocypack::Builder&, bool) const; - - //////////////////////////////////////////////////////////////////////////////// - /// @brief Helper function to parse all collection names / directions - //////////////////////////////////////////////////////////////////////////////// - - virtual void addEdgeColl(std::string const& name, TRI_edge_direction_e dir); - - //////////////////////////////////////////////////////////////////////////////// - /// @brief Helper function to parse the graph and direction nodes - //////////////////////////////////////////////////////////////////////////////// - - void parseGraphAstNodes(AstNode const* direction, AstNode const* graph); - - //////////////////////////////////////////////////////////////////////////////// - /// SECTION Shared subclass variables - //////////////////////////////////////////////////////////////////////////////// - protected: - /// @brief the database - TRI_vocbase_t* _vocbase; - - /// @brief vertex output variable - Variable const* _vertexOutVariable; - - /// @brief edge output variable - Variable const* _edgeOutVariable; - - /// @brief input graphInfo only used for serialisation & info - arangodb::velocypack::Builder _graphInfo; - - /// @brief The directions edges are followed - std::vector _directions; - - /// @brief the edge collections - std::vector> _edgeColls; - - /// @brief the reverse edge collections (ShortestPathOnly) - std::vector> _reverseEdgeColls; - - /// @brief the vertex collection names - std::vector> _vertexColls; - - /// @brief our graph... - Graph const* _graphObj; - - /// @brief Temporary pseudo variable for the currently traversed object. - Variable const* _tmpObjVariable; - - /// @brief Reference to the pseudo variable - AstNode* _tmpObjVarNode; - - /// @brief Pseudo string value node to hold the last visted vertex id. - AstNode* _tmpIdNode; - - /// @brief The hard coded condition on _from - /// NOTE: Created by sub classes, as it differs for class - AstNode* _fromCondition; - - /// @brief The hard coded condition on _to - /// NOTE: Created by sub classes, as it differs for class - AstNode* _toCondition; - - /// @brief Flag if options are already prepared. After - /// this flag was set the node cannot be cloned - /// any more. - bool _optionsBuild; - - /// @brief The list of traverser engines grouped by server. - std::unordered_map _engines; - - /// @brief flag, if traversal is smart (enterprise edition only!) - bool _isSmart; - - std::unique_ptr _options; - -}; -} -} -#endif diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 7bb33edb59..61bc514c7e 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -3764,10 +3764,8 @@ void arangodb::aql::prepareTraversalsRule(Optimizer* opt, SmallVector::allocator_type::arena_type a; SmallVector tNodes{a}; plan->findNodesOfType(tNodes, EN::TRAVERSAL, true); - SmallVector sNodes{a}; - plan->findNodesOfType(sNodes, EN::SHORTEST_PATH, true); - if (tNodes.empty() && sNodes.empty()) { + if (tNodes.empty()) { // no traversals present opt->addPlan(std::move(plan), rule, false); return; @@ -3778,13 +3776,6 @@ void arangodb::aql::prepareTraversalsRule(Optimizer* opt, for (auto const& n : tNodes) { TraversalNode* traversal = static_cast(n); traversal->prepareOptions(); - - } - // second make a pass over all shortest path nodes and remove unused - // variables from them - for (auto const& n : sNodes) { - ShortestPathNode* node = static_cast(n); - node->prepareOptions(); } opt->addPlan(std::move(plan), rule, true); diff --git a/arangod/Aql/ShortestPathBlock.cpp b/arangod/Aql/ShortestPathBlock.cpp index 24bb83f423..e768762516 100644 --- a/arangod/Aql/ShortestPathBlock.cpp +++ b/arangod/Aql/ShortestPathBlock.cpp @@ -23,35 +23,32 @@ #include "ShortestPathBlock.h" #include "Aql/AqlItemBlock.h" -#include "Aql/Collection.h" #include "Aql/ExecutionEngine.h" #include "Aql/ExecutionPlan.h" #include "Aql/Query.h" -#include "Cluster/ClusterComm.h" -#ifdef USE_ENTERPRISE -#include "Enterprise/Cluster/SmartGraphPathFinder.h" -#endif #include "Utils/OperationCursor.h" #include "Transaction/Methods.h" #include "VocBase/EdgeCollectionInfo.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ManagedDocumentResult.h" -#include "VocBase/ticks.h" #include #include /// @brief typedef the template instantiation of the PathFinder typedef arangodb::basics::DynamicDistanceFinder< - VPackSlice, VPackSlice, double, arangodb::traverser::ShortestPath> - ArangoDBPathFinder; + arangodb::velocypack::Slice, arangodb::velocypack::Slice, double, + arangodb::traverser::ShortestPath> ArangoDBPathFinder; -typedef arangodb::basics::ConstDistanceFinder< - VPackSlice, VPackSlice, arangodb::basics::VelocyPackHelper::VPackStringHash, - arangodb::basics::VelocyPackHelper::VPackStringEqual, - arangodb::traverser::ShortestPath> +typedef arangodb::basics::ConstDistanceFinder ArangoDBConstDistancePathFinder; + + using namespace arangodb::aql; /// @brief Local class to expand edges. @@ -320,7 +317,7 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _vertexReg(ExecutionNode::MaxRegisterId), _edgeVar(nullptr), _edgeReg(ExecutionNode::MaxRegisterId), - _opts(ep->options()), + _opts(_trx), _posInPath(0), _pathLength(0), _path(nullptr), @@ -329,7 +326,9 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _targetReg(ExecutionNode::MaxRegisterId), _useTargetRegister(false), _usedConstant(false) { - _mmdr.reset(new ManagedDocumentResult()); + + ep->fillOptions(_opts); + _mmdr.reset(new ManagedDocumentResult); size_t count = ep->_edgeColls.size(); TRI_ASSERT(ep->_directions.size()); @@ -337,8 +336,8 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, for (size_t j = 0; j < count; ++j) { auto info = std::make_unique( - _trx, ep->_edgeColls[j]->getName(), ep->_directions[j], - _opts->weightAttribute(), _opts->defaultWeight()); + _trx, ep->_edgeColls[j], ep->_directions[j], _opts.weightAttribute, + _opts.defaultWeight); _collectionInfos.emplace_back(info.get()); info.release(); } @@ -371,34 +370,34 @@ ShortestPathBlock::ShortestPathBlock(ExecutionEngine* engine, _path = std::make_unique(); if (arangodb::ServerState::instance()->isCoordinator()) { - _engines = ep->engines(); - -#ifdef USE_ENTERPRISE - // TODO This feature is NOT imlpemented for weighted graphs - if (ep->isSmart() && !_opts->usesWeight()) { - _finder.reset(new arangodb::traverser::SmartGraphConstDistanceFinder( - _opts, _engines, engine->getQuery()->trx()->resolver())); + if (_opts.useWeight) { + _finder.reset(new arangodb::basics::DynamicDistanceFinder< + arangodb::velocypack::Slice, arangodb::velocypack::Slice, + double, arangodb::traverser::ShortestPath>( + EdgeWeightExpanderCluster(this, false), + EdgeWeightExpanderCluster(this, true), _opts.bidirectional)); } else { -#endif - if (_opts->usesWeight()) { - _finder.reset(new ArangoDBPathFinder( - EdgeWeightExpanderCluster(this, false), - EdgeWeightExpanderCluster(this, true), true)); - } else { - _finder.reset(new ArangoDBConstDistancePathFinder( - ConstDistanceExpanderCluster(this, false), - ConstDistanceExpanderCluster(this, true))); - } -#ifdef USE_ENTERPRISE + _finder.reset(new arangodb::basics::ConstDistanceFinder< + arangodb::velocypack::Slice, arangodb::velocypack::Slice, + arangodb::basics::VelocyPackHelper::VPackStringHash, + arangodb::basics::VelocyPackHelper::VPackStringEqual, + arangodb::traverser::ShortestPath>( + ConstDistanceExpanderCluster(this, false), + ConstDistanceExpanderCluster(this, true))); } -#endif } else { - if (_opts->usesWeight()) { - _finder.reset(new ArangoDBPathFinder( + if (_opts.useWeight) { + _finder.reset(new arangodb::basics::DynamicDistanceFinder< + arangodb::velocypack::Slice, arangodb::velocypack::Slice, + double, arangodb::traverser::ShortestPath>( EdgeWeightExpanderLocal(this, false), - EdgeWeightExpanderLocal(this, true), true)); + EdgeWeightExpanderLocal(this, true), _opts.bidirectional)); } else { - _finder.reset(new ArangoDBConstDistancePathFinder( + _finder.reset(new arangodb::basics::ConstDistanceFinder< + arangodb::velocypack::Slice, arangodb::velocypack::Slice, + arangodb::basics::VelocyPackHelper::VPackStringHash, + arangodb::basics::VelocyPackHelper::VPackStringEqual, + arangodb::traverser::ShortestPath>( ConstDistanceExpanderLocal(this, false), ConstDistanceExpanderLocal(this, true))); } @@ -444,39 +443,6 @@ int ShortestPathBlock::initializeCursor(AqlItemBlock* items, size_t pos) { return ExecutionBlock::initializeCursor(items, pos); } -/// @brief shutdown: Inform all traverser Engines to destroy themselves -int ShortestPathBlock::shutdown(int errorCode) { - DEBUG_BEGIN_BLOCK(); - // We have to clean up the engines in Coordinator Case. - if (arangodb::ServerState::instance()->isCoordinator()) { - auto cc = arangodb::ClusterComm::instance(); - std::string const url( - "/_db/" + arangodb::basics::StringUtils::urlEncode(_trx->vocbase()->name()) + - "/_internal/traverser/"); - for (auto const& it : *_engines) { - arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer(); - std::unordered_map headers; - auto res = cc->syncRequest( - "", coordTransactionID, "server:" + it.first, RequestType::DELETE_REQ, - url + arangodb::basics::StringUtils::itoa(it.second), "", headers, - 30.0); - if (res->status != CL_COMM_SENT) { - // Note If there was an error on server side we do not have CL_COMM_SENT - std::string message("Could not destruct all traversal engines"); - if (res->errorMessage.length() > 0) { - message += std::string(" : ") + res->errorMessage; - } - LOG_TOPIC(ERR, arangodb::Logger::FIXME) << message; - } - } - } - - return ExecutionBlock::shutdown(errorCode); - - // cppcheck-suppress style - DEBUG_END_BLOCK(); -} - bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { if (_usedConstant) { // Both source and target are constant. @@ -488,12 +454,6 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { // Both are constant, after this computation we are done _usedConstant = true; } - - _startBuilder.clear(); - _targetBuilder.clear(); - - VPackSlice start; - VPackSlice end; if (!_useStartRegister) { auto pos = _startVertexId.find('/'); if (pos == std::string::npos) { @@ -503,15 +463,13 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { "_id are allowed"); return false; } else { - _startBuilder.add(VPackValue(_startVertexId)); - start = _startBuilder.slice(); + _opts.setStart(_startVertexId); } } else { AqlValue const& in = items->getValueReference(_pos, _startReg); if (in.isObject()) { try { - _startBuilder.add(VPackValue(_trx->extractIdString(in.slice()))); - start = _startBuilder.slice(); + _opts.setStart(_trx->extractIdString(in.slice())); } catch (...) { // _id or _key not present... ignore this error and fall through @@ -519,7 +477,8 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { return false; } } else if (in.isString()) { - start = in.slice(); + _startVertexId = in.slice().copyString(); + _opts.setStart(_startVertexId); } else { _engine->getQuery()->registerWarning( TRI_ERROR_BAD_PARAMETER, "Invalid input for Shortest Path: Only " @@ -539,15 +498,14 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { "_id are allowed"); return false; } else { - _targetBuilder.add(VPackValue(_targetVertexId)); - end = _targetBuilder.slice(); + _opts.setEnd(_targetVertexId); } } else { AqlValue const& in = items->getValueReference(_pos, _targetReg); if (in.isObject()) { try { - _targetBuilder.add(VPackValue(_trx->extractIdString(in.slice()))); - end = _targetBuilder.slice(); + std::string idString = _trx->extractIdString(in.slice()); + _opts.setEnd(idString); } catch (...) { // _id or _key not present... ignore this error and fall through @@ -555,7 +513,8 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { return false; } } else if (in.isString()) { - end = in.slice(); + _targetVertexId = in.slice().copyString(); + _opts.setEnd(_targetVertexId); } else { _engine->getQuery()->registerWarning( TRI_ERROR_BAD_PARAMETER, "Invalid input for Shortest Path: Only " @@ -565,6 +524,8 @@ bool ShortestPathBlock::nextPath(AqlItemBlock const* items) { } } + VPackSlice start = _opts.getStart(); + VPackSlice end = _opts.getEnd(); TRI_ASSERT(_finder != nullptr); // We do not need this data anymore. Result has been processed. // Save some memory. diff --git a/arangod/Aql/ShortestPathBlock.h b/arangod/Aql/ShortestPathBlock.h index 0a4be14ea9..49fca8e6b2 100644 --- a/arangod/Aql/ShortestPathBlock.h +++ b/arangod/Aql/ShortestPathBlock.h @@ -26,14 +26,13 @@ #include "Aql/ExecutionBlock.h" #include "Aql/ShortestPathNode.h" -#include "Basics/ShortestPathFinder.h" +#include "V8Server/V8Traverser.h" namespace arangodb { class ManagedDocumentResult; namespace traverser { class EdgeCollectionInfo; -class ShortestPath; } namespace aql { @@ -57,9 +56,6 @@ class ShortestPathBlock : public ExecutionBlock { /// @brief initializeCursor int initializeCursor(AqlItemBlock* items, size_t pos) override; - /// @brief shutdown send destroy to all engines. - int shutdown(int errorCode) override final; - /// @brief getSome AqlItemBlock* getSome(size_t atLeast, size_t atMost) override final; @@ -98,7 +94,7 @@ class ShortestPathBlock : public ExecutionBlock { std::unique_ptr _mmdr; /// @brief options to compute the shortest path - traverser::ShortestPathOptions* _opts; + traverser::ShortestPathOptions _opts; /// @brief list of edge collection infos used to compute the path std::vector _collectionInfos; @@ -148,13 +144,6 @@ class ShortestPathBlock : public ExecutionBlock { /// @brief Cache for edges send over the network std::vector>> _coordinatorCache; - /// @brief Builder to make sure that source velocypack does not get out of scope - arangodb::velocypack::Builder _startBuilder; - - /// @brief Builder to make sure that target velocypack does not get out of scope - arangodb::velocypack::Builder _targetBuilder; - - std::unordered_map const* _engines; }; } // namespace arangodb::aql diff --git a/arangod/Aql/ShortestPathNode.cpp b/arangod/Aql/ShortestPathNode.cpp index 76c023f825..891b3b2440 100644 --- a/arangod/Aql/ShortestPathNode.cpp +++ b/arangod/Aql/ShortestPathNode.cpp @@ -32,6 +32,8 @@ #include "Cluster/ClusterComm.h" #include "Indexes/Index.h" #include "Utils/CollectionNameResolver.h" +#include "VocBase/LogicalCollection.h" +#include "V8Server/V8Traverser.h" #include #include @@ -63,99 +65,243 @@ static void parseNodeInput(AstNode const* node, std::string& id, } } -ShortestPathNode::ShortestPathNode( - ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, AstNode const* direction, - AstNode const* start, AstNode const* target, AstNode const* graph, - traverser::ShortestPathOptions* options) - : GraphNode(plan, id, vocbase, options), +static TRI_edge_direction_e parseDirection (uint64_t dirNum) { + switch (dirNum) { + case 0: + return TRI_EDGE_ANY; + case 1: + return TRI_EDGE_IN; + case 2: + return TRI_EDGE_OUT; + default: + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_QUERY_PARSE, + "direction can only be INBOUND, OUTBOUND or ANY"); + } +} + +ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, size_t id, + TRI_vocbase_t* vocbase, uint64_t direction, + AstNode const* start, AstNode const* target, + AstNode const* graph, + ShortestPathOptions const& options) + : ExecutionNode(plan, id), + _vocbase(vocbase), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), _inStartVariable(nullptr), - _inTargetVariable(nullptr) { + _inTargetVariable(nullptr), + _graphObj(nullptr), + _options(options) { + + TRI_ASSERT(_vocbase != nullptr); TRI_ASSERT(start != nullptr); TRI_ASSERT(target != nullptr); + TRI_ASSERT(graph != nullptr); - // We have to call this here because we need a specific implementation - // for shortest_path_node. - parseGraphAstNodes(direction, graph); + TRI_edge_direction_e baseDirection = parseDirection(direction); - auto ast = _plan->getAst(); - // Let us build the conditions on _from and _to. Just in case we need them. - { - auto const* access = ast->createNodeAttributeAccess( - _tmpObjVarNode, StaticStrings::FromString.c_str(), - StaticStrings::FromString.length()); - auto const* condition = ast->createNodeBinaryOperator( - NODE_TYPE_OPERATOR_BINARY_EQ, access, _tmpIdNode); - _fromCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); - _fromCondition->addMember(condition); + std::unordered_map seenCollections; + auto addEdgeColl = [&](std::string const& n, TRI_edge_direction_e dir) -> void { + if (dir == TRI_EDGE_ANY) { + _directions.emplace_back(TRI_EDGE_OUT); + _edgeColls.emplace_back(n); + _directions.emplace_back(TRI_EDGE_IN); + _edgeColls.emplace_back(std::move(n)); + } else { + _directions.emplace_back(dir); + _edgeColls.emplace_back(std::move(n)); + } + }; + + auto ci = ClusterInfo::instance(); + + if (graph->type == NODE_TYPE_COLLECTION_LIST) { + size_t edgeCollectionCount = graph->numMembers(); + auto resolver = std::make_unique(vocbase); + _graphInfo.openArray(); + _edgeColls.reserve(edgeCollectionCount); + _directions.reserve(edgeCollectionCount); + + // List of edge collection names + for (size_t i = 0; i < edgeCollectionCount; ++i) { + TRI_edge_direction_e dir = TRI_EDGE_ANY; + auto col = graph->getMember(i); + + if (col->type == NODE_TYPE_DIRECTION) { + TRI_ASSERT(col->numMembers() == 2); + auto dirNode = col->getMember(0); + // We have a collection with special direction. + TRI_ASSERT(dirNode->isIntValue()); + dir = parseDirection(dirNode->getIntValue()); + col = col->getMember(1); + } else { + dir = baseDirection; + } + + std::string eColName = col->getString(); + + // now do some uniqueness checks for the specified collections + auto it = seenCollections.find(eColName); + if (it != seenCollections.end()) { + if ((*it).second != dir) { + std::string msg("conflicting directions specified for collection '" + + std::string(eColName)); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, + msg); + } + // do not re-add the same collection! + continue; + } + seenCollections.emplace(eColName, dir); + + auto eColType = resolver->getCollectionTypeCluster(eColName); + if (eColType != TRI_COL_TYPE_EDGE) { + std::string msg("collection type invalid for collection '" + + std::string(eColName) + + ": expecting collection type 'edge'"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, + msg); + } + + _graphInfo.add(VPackValue(eColName)); + if (ServerState::instance()->isRunningInCluster()) { + auto c = ci->getCollection(_vocbase->name(), eColName); + if (!c->isSmart()) { + addEdgeColl(eColName, dir); + } else { + std::vector names; + names = c->realNamesForRead(); + for (auto const& name : names) { + addEdgeColl(name, dir); + } + } + } else { + addEdgeColl(eColName, dir); + } + + if (dir == TRI_EDGE_ANY) { + // collection with direction ANY must be added again + _graphInfo.add(VPackValue(eColName)); + } + + } + _graphInfo.close(); + } else { + if (_edgeColls.empty()) { + if (graph->isStringValue()) { + std::string graphName = graph->getString(); + _graphInfo.add(VPackValue(graphName)); + _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); + + if (_graphObj == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); + } + + auto eColls = _graphObj->edgeCollections(); + size_t length = eColls.size(); + if (length == 0) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); + } + _edgeColls.reserve(length); + _directions.reserve(length); + + for (const auto& n : eColls) { + if (ServerState::instance()->isRunningInCluster()) { + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart()) { + addEdgeColl(n, baseDirection); + } else { + std::vector names; + names = c->realNamesForRead(); + for (auto const& name : names) { + addEdgeColl(name, baseDirection); + } + } + } else { + addEdgeColl(n, baseDirection); + } + } + } + } } - TRI_ASSERT(_fromCondition != nullptr); - { - auto const* access = ast->createNodeAttributeAccess( - _tmpObjVarNode, StaticStrings::ToString.c_str(), - StaticStrings::ToString.length()); - auto const* condition = ast->createNodeBinaryOperator( - NODE_TYPE_OPERATOR_BINARY_EQ, access, _tmpIdNode); - _toCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); - _toCondition->addMember(condition); - } - TRI_ASSERT(_toCondition != nullptr); parseNodeInput(start, _startVertexId, _inStartVariable); parseNodeInput(target, _targetVertexId, _inTargetVariable); } -ShortestPathNode::ShortestPathNode( - ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - std::vector> const& edgeColls, - std::vector> const& reverseEdgeColls, - std::vector> const& vertexColls, - std::vector const& directions, - Variable const* inStartVariable, std::string const& startVertexId, - Variable const* inTargetVariable, std::string const& targetVertexId, - traverser::ShortestPathOptions* options) - : GraphNode(plan, id, vocbase, edgeColls, vertexColls, directions, options), +ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, size_t id, + TRI_vocbase_t* vocbase, + std::vector const& edgeColls, + std::vector const& directions, + Variable const* inStartVariable, + std::string const& startVertexId, + Variable const* inTargetVariable, + std::string const& targetVertexId, + ShortestPathOptions const& options) + : ExecutionNode(plan, id), + _vocbase(vocbase), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), _inStartVariable(inStartVariable), _startVertexId(startVertexId), _inTargetVariable(inTargetVariable), - _targetVertexId(targetVertexId) { - for (auto const& it : reverseEdgeColls) { - // Collections cannot be copied. So we need to create new ones to prevent leaks - _reverseEdgeColls.emplace_back(std::make_unique( - it->getName(), _vocbase, AccessMode::Type::READ)); + _targetVertexId(targetVertexId), + _directions(directions), + _graphObj(nullptr), + _options(options) { + + _graphInfo.openArray(); + for (auto const& it : edgeColls) { + _edgeColls.emplace_back(it); + _graphInfo.add(VPackValue(it)); } + _graphInfo.close(); } -ShortestPathNode::~ShortestPathNode() { -} - -arangodb::traverser::ShortestPathOptions* ShortestPathNode::options() - const { - return static_cast(_options.get()); +void ShortestPathNode::fillOptions(arangodb::traverser::ShortestPathOptions& opts) const { + if (!_options.weightAttribute.empty()) { + opts.useWeight = true; + opts.weightAttribute = _options.weightAttribute; + opts.defaultWeight = _options.defaultWeight; + } else { + opts.useWeight = false; + } } ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base) - : GraphNode(plan, base), + : ExecutionNode(plan, base), + _vocbase(plan->getAst()->query()->vocbase()), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), _inStartVariable(nullptr), - _inTargetVariable(nullptr) { - - // Reverse Collections - VPackSlice list = base.get("reverseEdgeCollections"); - if (!list.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_BAD_JSON_PLAN, - "shortest path needs an array of reverse edge collections."); + _inTargetVariable(nullptr), + _graphObj(nullptr) { + // Directions + VPackSlice dirList = base.get("directions"); + for (auto const& it : VPackArrayIterator(dirList)) { + uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it); + TRI_edge_direction_e d; + switch (dir) { + case 0: + d = TRI_EDGE_ANY; + break; + case 1: + d = TRI_EDGE_IN; + break; + case 2: + d = TRI_EDGE_OUT; + break; + default: + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "Invalid direction value"); + break; + } + _directions.emplace_back(d); } - for (auto const& it : VPackArrayIterator(list)) { - std::string e = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); - _reverseEdgeColls.emplace_back( - std::make_unique(e, _vocbase, AccessMode::Type::READ)); - } - - _options = std::make_unique( - plan->getAst()->query()->trx()); // Start Vertex if (base.hasKey("startInVariable")) { _inStartVariable = varFromVPack(plan->getAst(), base, "startInVariable"); @@ -189,15 +335,80 @@ ShortestPathNode::ShortestPathNode(ExecutionPlan* plan, } } + std::string graphName; + if (base.hasKey("graph") && (base.get("graph").isString())) { + graphName = base.get("graph").copyString(); + if (base.hasKey("graphDefinition")) { + _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); + + if (_graphObj == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); + } + + auto const& eColls = _graphObj->edgeCollections(); + for (auto const& it : eColls) { + _edgeColls.push_back(it); + + // if there are twice as many directions as collections, this means we + // have a shortest path with direction ANY. we must add each collection + // twice then + if (_directions.size() == 2 * eColls.size()) { + // add collection again + _edgeColls.push_back(it); + } + } + } else { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "missing graphDefinition."); + } + } else { + _graphInfo.add(base.get("graph")); + if (!_graphInfo.slice().isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "graph has to be an array."); + } + // List of edge collection names + for (auto const& it : VPackArrayIterator(_graphInfo.slice())) { + if (!it.isString()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "graph has to be an array of strings."); + } + _edgeColls.emplace_back(it.copyString()); + } + if (_edgeColls.empty()) { + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_QUERY_BAD_JSON_PLAN, + "graph has to be a non empty array of strings."); + } + } + + // Out variables + if (base.hasKey("vertexOutVariable")) { + _vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable"); + } + if (base.hasKey("edgeOutVariable")) { + _edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable"); + } + // Flags if (base.hasKey("shortestPathFlags")) { - // _options = ShortestPathOptions(base); + _options = ShortestPathOptions(base); } } void ShortestPathNode::toVelocyPackHelper(VPackBuilder& nodes, bool verbose) const { - baseToVelocyPackHelper(nodes, verbose); + ExecutionNode::toVelocyPackHelperGeneric(nodes, + verbose); // call base class method + nodes.add("database", VPackValue(_vocbase->name())); + nodes.add("graph", _graphInfo.slice()); + nodes.add(VPackValue("directions")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& d : _directions) { + nodes.add(VPackValue(d)); + } + } // In variables if (usesStartInVariable()) { @@ -214,18 +425,23 @@ void ShortestPathNode::toVelocyPackHelper(VPackBuilder& nodes, nodes.add("targetVertexId", VPackValue(_targetVertexId)); } - // Reverse collections - nodes.add(VPackValue("reverseEdgeCollections")); - { - VPackArrayBuilder guard(&nodes); - for (auto const& e : _edgeColls) { - nodes.add(VPackValue(e->getName())); - } + if (_graphObj != nullptr) { + nodes.add(VPackValue("graphDefinition")); + _graphObj->toVelocyPack(nodes, verbose); } + // Out variables + if (usesVertexOutVariable()) { + nodes.add(VPackValue("vertexOutVariable")); + vertexOutVariable()->toVelocyPack(nodes); + } + if (usesEdgeOutVariable()) { + nodes.add(VPackValue("edgeOutVariable")); + edgeOutVariable()->toVelocyPack(nodes); + } - // nodes.add(VPackValue("shortestPathFlags")); - // _options.toVelocyPack(nodes); + nodes.add(VPackValue("shortestPathFlags")); + _options.toVelocyPack(nodes); // And close it: nodes.close(); @@ -234,14 +450,9 @@ void ShortestPathNode::toVelocyPackHelper(VPackBuilder& nodes, ExecutionNode* ShortestPathNode::clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const { - auto tmp = - std::make_unique(*options()); - auto c = new ShortestPathNode(plan, _id, _vocbase, _edgeColls, - _reverseEdgeColls, _vertexColls, _directions, + auto c = new ShortestPathNode(plan, _id, _vocbase, _edgeColls, _directions, _inStartVariable, _startVertexId, - _inTargetVariable, _targetVertexId, tmp.get()); - tmp.release(); - + _inTargetVariable, _targetVertexId, _options); if (usesVertexOutVariable()) { auto vertexOutVariable = _vertexOutVariable; if (withProperties) { @@ -262,56 +473,31 @@ ExecutionNode* ShortestPathNode::clone(ExecutionPlan* plan, c->setEdgeOutput(edgeOutVariable); } - // Temporary Filter Objects - c->_tmpObjVariable = _tmpObjVariable; - c->_tmpObjVarNode = _tmpObjVarNode; - c->_tmpIdNode = _tmpIdNode; - - // Filter Condition Parts - c->_fromCondition = _fromCondition->clone(_plan->getAst()); - c->_toCondition = _toCondition->clone(_plan->getAst()); - cloneHelper(c, plan, withDependencies, withProperties); return static_cast(c); } -void ShortestPathNode::prepareOptions() { - if (_optionsBuild) { - return; - } +double ShortestPathNode::estimateCost(size_t& nrItems) const { + // Standard estimation for Shortest path is O(|E| + |V|*LOG(|V|)) + // At this point we know |E| but do not know |V|. + size_t incoming = 0; + double depCost = _dependencies.at(0)->getCost(incoming); + auto collections = _plan->getAst()->query()->collections(); + size_t edgesCount = 0; - size_t numEdgeColls = _edgeColls.size(); - TRI_ASSERT(!_optionsBuild); - _options->setVariable(_tmpObjVariable); + TRI_ASSERT(collections != nullptr); - auto opts = dynamic_cast(_options.get()); + for (auto const& it : _edgeColls) { + auto collection = collections->get(it); - Ast* ast = _plan->getAst(); - - TRI_ASSERT(numEdgeColls == _directions.size()); - TRI_ASSERT(numEdgeColls == _reverseEdgeColls.size()); - // FIXME: _options->_baseLookupInfos.reserve(numEdgeColls); - // Compute Edge Indexes. First default indexes: - for (size_t i = 0; i < numEdgeColls; ++i) { - switch (_directions[i]) { - case TRI_EDGE_IN: - opts->addLookupInfo(ast, _edgeColls[i]->getName(), - StaticStrings::ToString, _toCondition); - opts->addReverseLookupInfo(ast, _reverseEdgeColls[i]->getName(), - StaticStrings::FromString, _fromCondition); - break; - case TRI_EDGE_OUT: - opts->addLookupInfo(ast, _edgeColls[i]->getName(), - StaticStrings::FromString, _fromCondition); - opts->addReverseLookupInfo(ast, _reverseEdgeColls[i]->getName(), - StaticStrings::ToString, _toCondition); - break; - case TRI_EDGE_ANY: - TRI_ASSERT(false); - break; + if (collection == nullptr) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "unexpected pointer for collection"); } + edgesCount += collection->count(); } - - _optionsBuild = true; + // Hard-Coded number of vertices edges / 10 + nrItems = edgesCount + static_cast(std::log2(edgesCount / 10) * (edgesCount / 10)); + return depCost + nrItems; } diff --git a/arangod/Aql/ShortestPathNode.h b/arangod/Aql/ShortestPathNode.h index 92105c1905..0ffb462047 100644 --- a/arangod/Aql/ShortestPathNode.h +++ b/arangod/Aql/ShortestPathNode.h @@ -24,16 +24,21 @@ #ifndef ARANGOD_AQL_SHORTEST_PATH_NODE_H #define ARANGOD_AQL_SHORTEST_PATH_NODE_H 1 -#include "Aql/GraphNode.h" -#include "VocBase/TraverserOptions.h" +#include "Aql/ExecutionNode.h" +#include "Aql/Graphs.h" +#include "Aql/ShortestPathOptions.h" #include namespace arangodb { + +namespace traverser { +struct ShortestPathOptions; +} namespace aql { /// @brief class ShortestPathNode -class ShortestPathNode : public GraphNode { +class ShortestPathNode : public ExecutionNode { friend class ExecutionBlock; friend class RedundantCalculationsReplacer; friend class ShortestPathBlock; @@ -41,24 +46,23 @@ class ShortestPathNode : public GraphNode { /// @brief constructor with a vocbase and a collection name public: ShortestPathNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - AstNode const* direction, AstNode const* start, AstNode const* target, - AstNode const* graph, traverser::ShortestPathOptions* options); + uint64_t direction, AstNode const* start, AstNode const* target, + AstNode const* graph, ShortestPathOptions const& options); ShortestPathNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); - ~ShortestPathNode(); + ~ShortestPathNode() {} /// @brief Internal constructor to clone the node. private: - ShortestPathNode( - ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, - std::vector> const& edgeColls, - std::vector> const& reverseEdgeColls, - std::vector> const& vertexColls, - std::vector const& directions, - Variable const* inStartVariable, std::string const& startVertexId, - Variable const* inTargetVariable, std::string const& targetVertexId, - traverser::ShortestPathOptions* options); + ShortestPathNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, + std::vector const& edgeColls, + std::vector const& directions, + Variable const* inStartVariable, + std::string const& startVertexId, + Variable const* inTargetVariable, + std::string const& targetVertexId, + ShortestPathOptions const& options); public: /// @brief return the type of the node @@ -72,6 +76,9 @@ class ShortestPathNode : public GraphNode { ExecutionNode* clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const override final; + /// @brief the cost of a traversal node + double estimateCost(size_t&) const override final; + /// @brief Test if this node uses an in variable or constant for start bool usesStartInVariable() const { return _inStartVariable != nullptr; @@ -92,6 +99,24 @@ class ShortestPathNode : public GraphNode { std::string const getTargetVertex() const { return _targetVertexId; } + /// @brief return the vertex out variable + Variable const* vertexOutVariable() const { return _vertexOutVariable; } + + /// @brief checks if the vertex out variable is used + bool usesVertexOutVariable() const { return _vertexOutVariable != nullptr; } + + /// @brief set the vertex out variable + void setVertexOutput(Variable const* outVar) { _vertexOutVariable = outVar; } + + /// @brief return the edge out variable + Variable const* edgeOutVariable() const { return _edgeOutVariable; } + + /// @brief checks if the edge out variable is used + bool usesEdgeOutVariable() const { return _edgeOutVariable != nullptr; } + + /// @brief set the edge out variable + void setEdgeOutput(Variable const* outVar) { _edgeOutVariable = outVar; } + /// @brief getVariablesSetHere std::vector getVariablesSetHere() const override final { std::vector vars; @@ -127,19 +152,19 @@ class ShortestPathNode : public GraphNode { } } - traverser::ShortestPathOptions* options() const override; + void fillOptions(arangodb::traverser::ShortestPathOptions&) const; - /// @brief Compute the traversal options containing the expressions - /// MUST! be called after optimization and before creation - /// of blocks. - void prepareOptions() override; - - /// @brief Helper function to parse all collection names / directions -#ifdef USE_ENTERPRISE - void addEdgeColl(std::string const& name, TRI_edge_direction_e dir) override; -#endif private: + /// @brief the database + TRI_vocbase_t* _vocbase; + + /// @brief vertex output variable + Variable const* _vertexOutVariable; + + /// @brief vertex output variable + Variable const* _edgeOutVariable; + /// @brief input variable only used if _vertexId is unused Variable const* _inStartVariable; @@ -152,6 +177,20 @@ class ShortestPathNode : public GraphNode { /// @brief input vertexId only used if _inVariable is unused std::string _targetVertexId; + /// @brief input graphInfo only used for serialisation & info + arangodb::velocypack::Builder _graphInfo; + + /// @brief The directions edges are followed + std::vector _directions; + + /// @brief the edge collection names + std::vector _edgeColls; + + /// @brief our graph... + Graph const* _graphObj; + + /// @brief Options for traversals + ShortestPathOptions _options; }; } // namespace arangodb::aql diff --git a/arangod/Aql/ShortestPathOptions.cpp b/arangod/Aql/ShortestPathOptions.cpp new file mode 100644 index 0000000000..4e2374d0d2 --- /dev/null +++ b/arangod/Aql/ShortestPathOptions.cpp @@ -0,0 +1,54 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2016-2016 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "Aql/ShortestPathOptions.h" + +#include + +using namespace arangodb::aql; + +ShortestPathOptions::ShortestPathOptions(VPackSlice const& slice) + : weightAttribute(), defaultWeight(1) { + VPackSlice obj = slice.get("shortestPathFlags"); + + if (obj.isObject()) { + if (obj.hasKey("weightAttribute")) { + VPackSlice v = obj.get("weightAttribute"); + if (v.isString()) { + weightAttribute = v.copyString(); + } + } + + if (obj.hasKey("defaultWeight")) { + VPackSlice v = obj.get("defaultWeight"); + if (v.isNumber()) { + defaultWeight = v.getNumericValue(); + } + } + } +} + +void ShortestPathOptions::toVelocyPack(VPackBuilder& builder) const { + VPackObjectBuilder guard(&builder); + builder.add("weightAttribute", VPackValue(weightAttribute)); + builder.add("defaultWeight", VPackValue(defaultWeight)); +} diff --git a/arangod/Aql/ShortestPathOptions.h b/arangod/Aql/ShortestPathOptions.h new file mode 100644 index 0000000000..040675182b --- /dev/null +++ b/arangod/Aql/ShortestPathOptions.h @@ -0,0 +1,54 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2016-2016 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_AQL_SHORTEST_PATH_OPTIONS_H +#define ARANGOD_AQL_SHORTEST_PATH_OPTIONS_H 1 + +#include "Basics/Common.h" + +#include +#include + +namespace arangodb { +namespace aql { + +/// @brief TraversalOptions +struct ShortestPathOptions { + + /// @brief constructor + explicit ShortestPathOptions(arangodb::velocypack::Slice const&); + + /// @brief constructor, using default values + ShortestPathOptions() + : weightAttribute(), + defaultWeight(1) {} + + void toVelocyPack(arangodb::velocypack::Builder&) const; + + std::string weightAttribute; + double defaultWeight; +}; + +} // namespace arangodb::aql +} // namespace arangodb +#endif + diff --git a/arangod/Aql/TraversalNode.cpp b/arangod/Aql/TraversalNode.cpp index ca17ab963c..9c0d702069 100644 --- a/arangod/Aql/TraversalNode.cpp +++ b/arangod/Aql/TraversalNode.cpp @@ -82,21 +82,51 @@ void TraversalNode::TraversalEdgeConditionBuilder::toVelocyPack( _modCondition->toVelocyPack(builder, verbose); } +static TRI_edge_direction_e parseDirection (AstNode const* node) { + TRI_ASSERT(node->isIntValue()); + auto dirNum = node->getIntValue(); + + switch (dirNum) { + case 0: + return TRI_EDGE_ANY; + case 1: + return TRI_EDGE_IN; + case 2: + return TRI_EDGE_OUT; + default: + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_QUERY_PARSE, + "direction can only be INBOUND, OUTBOUND or ANY"); + } +} + TraversalNode::TraversalNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, AstNode const* direction, AstNode const* start, AstNode const* graph, - TraverserOptions* options) - : GraphNode(plan, id, vocbase, options), + std::unique_ptr& options) + : ExecutionNode(plan, id), + _vocbase(vocbase), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), _pathOutVariable(nullptr), _inVariable(nullptr), - _condition(nullptr) { + _graphObj(nullptr), + _condition(nullptr), + _tmpObjVariable(_plan->getAst()->variables()->createTemporaryVariable()), + _tmpObjVarNode(_plan->getAst()->createNodeReference(_tmpObjVariable)), + _tmpIdNode(_plan->getAst()->createNodeValueString("", 0)), + _fromCondition(nullptr), + _toCondition(nullptr), + _optionsBuild(false), + _isSmart(false) { + TRI_ASSERT(_vocbase != nullptr); + TRI_ASSERT(direction != nullptr); TRI_ASSERT(start != nullptr); - // We have to call this here because we need a specific implementation - // for shortest_path_node. - parseGraphAstNodes(direction, graph); + TRI_ASSERT(graph != nullptr); + _options.reset(options.release()); - // Let us build the conditions on _from and _to. Just in case we need them. auto ast = _plan->getAst(); + // Let us build the conditions on _from and _to. Just in case we need them. { auto const* access = ast->createNodeAttributeAccess( _tmpObjVarNode, StaticStrings::FromString.c_str(), @@ -117,6 +147,215 @@ TraversalNode::TraversalNode(ExecutionPlan* plan, size_t id, TRI_ASSERT(_toCondition != nullptr); TRI_ASSERT(_toCondition->type == NODE_TYPE_OPERATOR_BINARY_EQ); + auto resolver = std::make_unique(vocbase); + + // Parse Steps and direction + TRI_ASSERT(direction->type == NODE_TYPE_DIRECTION); + TRI_ASSERT(direction->numMembers() == 2); + // Member 0 is the direction. Already the correct Integer. + // Is not inserted by user but by enum. + TRI_edge_direction_e baseDirection = parseDirection(direction->getMember(0)); + + std::unordered_map seenCollections; + + auto addEdgeColl = [&](std::string const& n, TRI_edge_direction_e dir) -> void { + if (_isSmart) { + if (n.compare(0, 6, "_from_") == 0) { + if (dir != TRI_EDGE_IN) { + _directions.emplace_back(TRI_EDGE_OUT); + _edgeColls.emplace_back(std::make_unique( + n, _vocbase, AccessMode::Type::READ)); + } + return; + } else if (n.compare(0, 4, "_to_") == 0) { + if (dir != TRI_EDGE_OUT) { + _directions.emplace_back(TRI_EDGE_IN); + _edgeColls.emplace_back(std::make_unique( + n, _vocbase, AccessMode::Type::READ)); + } + return; + } + } + + if (dir == TRI_EDGE_ANY) { + _directions.emplace_back(TRI_EDGE_OUT); + _edgeColls.emplace_back(std::make_unique( + n, _vocbase, AccessMode::Type::READ)); + + _directions.emplace_back(TRI_EDGE_IN); + _edgeColls.emplace_back(std::make_unique( + n, _vocbase, AccessMode::Type::READ)); + } else { + _directions.emplace_back(dir); + _edgeColls.emplace_back(std::make_unique( + n, _vocbase, AccessMode::Type::READ)); + } + }; + + if (graph->type == NODE_TYPE_COLLECTION_LIST) { + size_t edgeCollectionCount = graph->numMembers(); + + _graphInfo.openArray(); + _edgeColls.reserve(edgeCollectionCount); + _directions.reserve(edgeCollectionCount); + + // First determine whether all edge collections are smart and sharded + // like a common collection: + auto ci = ClusterInfo::instance(); + if (ServerState::instance()->isRunningInCluster()) { + _isSmart = true; + std::string distributeShardsLike; + for (size_t i = 0; i < edgeCollectionCount; ++i) { + auto col = graph->getMember(i); + if (col->type == NODE_TYPE_DIRECTION) { + col = col->getMember(1); // The first member always is the collection + } + std::string n = col->getString(); + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart() || c->distributeShardsLike().empty()) { + _isSmart = false; + break; + } + if (distributeShardsLike.empty()) { + distributeShardsLike = c->distributeShardsLike(); + } else if (distributeShardsLike != c->distributeShardsLike()) { + _isSmart = false; + break; + } + } + } + + // List of edge collection names + for (size_t i = 0; i < edgeCollectionCount; ++i) { + auto col = graph->getMember(i); + TRI_edge_direction_e dir = TRI_EDGE_ANY; + + if (col->type == NODE_TYPE_DIRECTION) { + // We have a collection with special direction. + dir = parseDirection(col->getMember(0)); + col = col->getMember(1); + } else { + dir = baseDirection; + } + + std::string eColName = col->getString(); + + // now do some uniqueness checks for the specified collections + auto it = seenCollections.find(eColName); + if (it != seenCollections.end()) { + if ((*it).second != dir) { + std::string msg("conflicting directions specified for collection '" + + std::string(eColName)); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, + msg); + } + // do not re-add the same collection! + continue; + } + seenCollections.emplace(eColName, dir); + + if (resolver->getCollectionTypeCluster(eColName) != TRI_COL_TYPE_EDGE) { + std::string msg("collection type invalid for collection '" + + std::string(eColName) + + ": expecting collection type 'edge'"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID, + msg); + } + + _graphInfo.add(VPackValue(eColName)); + if (ServerState::instance()->isRunningInCluster()) { + auto c = ci->getCollection(_vocbase->name(), eColName); + if (!c->isSmart()) { + addEdgeColl(eColName, dir); + } else { + std::vector names; + if (_isSmart) { + names = c->realNames(); + } else { + names = c->realNamesForRead(); + } + for (auto const& name : names) { + addEdgeColl(name, dir); + } + } + } else { + addEdgeColl(eColName, dir); + } + } + _graphInfo.close(); + } else { + if (_edgeColls.empty()) { + if (graph->isStringValue()) { + std::string graphName = graph->getString(); + _graphInfo.add(VPackValue(graphName)); + _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); + + if (_graphObj == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); + } + + auto eColls = _graphObj->edgeCollections(); + size_t length = eColls.size(); + if (length == 0) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); + } + + // First determine whether all edge collections are smart and sharded + // like a common collection: + auto ci = ClusterInfo::instance(); + if (ServerState::instance()->isRunningInCluster()) { + _isSmart = true; + std::string distributeShardsLike; + for (auto const& n : eColls) { + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart() || c->distributeShardsLike().empty()) { + _isSmart = false; + break; + } + if (distributeShardsLike.empty()) { + distributeShardsLike = c->distributeShardsLike(); + } else if (distributeShardsLike != c->distributeShardsLike()) { + _isSmart = false; + break; + } + } + } + + for (const auto& n : eColls) { + if (ServerState::instance()->isRunningInCluster()) { + auto c = ci->getCollection(_vocbase->name(), n); + if (!c->isSmart()) { + addEdgeColl(n, baseDirection); + } else { + std::vector names; + if (_isSmart) { + names = c->realNames(); + } else { + names = c->realNamesForRead(); + } + for (auto const& name : names) { + addEdgeColl(name, baseDirection); + } + } + } else { + addEdgeColl(n, baseDirection); + } + } + + auto vColls = _graphObj->vertexCollections(); + length = vColls.size(); + if (length == 0) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_EMPTY); + } + _vertexColls.reserve(length); + for (auto const& v : vColls) { + _vertexColls.emplace_back(std::make_unique( + v, _vocbase, AccessMode::Type::READ)); + } + } + } + } + // Parse start node switch (start->type) { case NODE_TYPE_REFERENCE: @@ -152,22 +391,85 @@ TraversalNode::TraversalNode( std::vector> const& vertexColls, Variable const* inVariable, std::string const& vertexId, std::vector const& directions, - TraverserOptions* options) - : GraphNode(plan, id, vocbase, edgeColls, vertexColls, directions, options), + std::unique_ptr& options) + : ExecutionNode(plan, id), + _vocbase(vocbase), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), _pathOutVariable(nullptr), _inVariable(inVariable), _vertexId(vertexId), - _condition(nullptr) { + _directions(directions), + _graphObj(nullptr), + _condition(nullptr), + _tmpObjVariable(nullptr), + _tmpObjVarNode(nullptr), + _tmpIdNode(nullptr), + _fromCondition(nullptr), + _toCondition(nullptr), + _optionsBuild(false), + _isSmart(false) { + _options.reset(options.release()); + _graphInfo.openArray(); + + for (auto& it : edgeColls) { + // Collections cannot be copied. So we need to create new ones to prevent leaks + _edgeColls.emplace_back(std::make_unique( + it->getName(), _vocbase, AccessMode::Type::READ)); + _graphInfo.add(VPackValue(it->getName())); + } + + for (auto& it : vertexColls) { + // Collections cannot be copied. So we need to create new ones to prevent leaks + _vertexColls.emplace_back(std::make_unique( + it->getName(), _vocbase, AccessMode::Type::READ)); + } + + _graphInfo.close(); } TraversalNode::TraversalNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base) - : GraphNode(plan, base), + : ExecutionNode(plan, base), + _vocbase(plan->getAst()->query()->vocbase()), + _vertexOutVariable(nullptr), + _edgeOutVariable(nullptr), _pathOutVariable(nullptr), _inVariable(nullptr), - _condition(nullptr) { - _options = std::make_unique( - _plan->getAst()->query()->trx(), base); + _graphObj(nullptr), + _condition(nullptr), + _options(std::make_unique( + _plan->getAst()->query()->trx(), base)), + _tmpObjVariable(nullptr), + _tmpObjVarNode(nullptr), + _tmpIdNode(nullptr), + _fromCondition(nullptr), + _toCondition(nullptr), + _optionsBuild(false), + _isSmart(false) { + + VPackSlice dirList = base.get("directions"); + for (auto const& it : VPackArrayIterator(dirList)) { + uint64_t dir = arangodb::basics::VelocyPackHelper::stringUInt64(it); + TRI_edge_direction_e d; + switch (dir) { + case 0: + TRI_ASSERT(false); + break; + case 1: + d = TRI_EDGE_IN; + break; + case 2: + d = TRI_EDGE_OUT; + break; + default: + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "Invalid direction value"); + break; + } + _directions.emplace_back(d); + } + // In Vertex if (base.hasKey("inVariable")) { _inVariable = varFromVPack(plan->getAst(), base, "inVariable"); @@ -201,11 +503,81 @@ TraversalNode::TraversalNode(ExecutionPlan* plan, } } + // TODO: Can we remove this? + std::string graphName; + if (base.hasKey("graph") && (base.get("graph").isString())) { + graphName = base.get("graph").copyString(); + if (base.hasKey("graphDefinition")) { + _graphObj = plan->getAst()->query()->lookupGraphByName(graphName); + + if (_graphObj == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_NOT_FOUND); + } + } else { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "missing graphDefinition."); + } + } else { + _graphInfo.add(base.get("graph")); + if (!_graphInfo.slice().isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "graph has to be an array."); + } + } + + list = base.get("edgeCollections"); + if (!list.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "traverser needs an array of edge collections."); + } + + for (auto const& it : VPackArrayIterator(list)) { + std::string e = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + _edgeColls.emplace_back( + std::make_unique(e, _vocbase, AccessMode::Type::READ)); + } + + list = base.get("vertexCollections"); + + if (!list.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_BAD_JSON_PLAN, + "traverser needs an array of vertex collections."); + } + + for (auto const& it : VPackArrayIterator(list)) { + std::string v = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + _vertexColls.emplace_back( + std::make_unique(v, _vocbase, AccessMode::Type::READ)); + } + // Out variables + if (base.hasKey("vertexOutVariable")) { + _vertexOutVariable = varFromVPack(plan->getAst(), base, "vertexOutVariable"); + } + if (base.hasKey("edgeOutVariable")) { + _edgeOutVariable = varFromVPack(plan->getAst(), base, "edgeOutVariable"); + } if (base.hasKey("pathOutVariable")) { _pathOutVariable = varFromVPack(plan->getAst(), base, "pathOutVariable"); } + // Temporary Filter Objects + TRI_ASSERT(base.hasKey("tmpObjVariable")); + _tmpObjVariable = varFromVPack(plan->getAst(), base, "tmpObjVariable"); + + TRI_ASSERT(base.hasKey("tmpObjVarNode")); + _tmpObjVarNode = new AstNode(plan->getAst(), base.get("tmpObjVarNode")); + + TRI_ASSERT(base.hasKey("tmpIdNode")); + _tmpIdNode = new AstNode(plan->getAst(), base.get("tmpIdNode")); + + // Filter Condition Parts + TRI_ASSERT(base.hasKey("fromCondition")); + _fromCondition = new AstNode(plan->getAst(), base.get("fromCondition")); + + TRI_ASSERT(base.hasKey("toCondition")); + _toCondition = new AstNode(plan->getAst(), base.get("toCondition")); + list = base.get("globalEdgeConditions"); if (list.isArray()) { for (auto const& cond : VPackArrayIterator(list)) { @@ -265,16 +637,60 @@ int TraversalNode::checkIsOutVariable(size_t variableId) const { /// @brief check whether an access is inside the specified range bool TraversalNode::isInRange(uint64_t depth, bool isEdge) const { - uint64_t max = static_cast(_options.get())->maxDepth; if (isEdge) { - return (depth < max); + return (depth < _options->maxDepth); } - return (depth <= max); + return (depth <= _options->maxDepth); +} + +/// @brief check if all directions are equal +bool TraversalNode::allDirectionsEqual() const { + if (_directions.empty()) { + // no directions! + return false; + } + size_t const n = _directions.size(); + TRI_edge_direction_e const expected = _directions[0]; + + for (size_t i = 1; i < n; ++i) { + if (_directions[i] != expected) { + return false; + } + } + return true; } void TraversalNode::toVelocyPackHelper(arangodb::velocypack::Builder& nodes, bool verbose) const { - baseToVelocyPackHelper(nodes, verbose); + ExecutionNode::toVelocyPackHelperGeneric(nodes, + verbose); // call base class method + + nodes.add("database", VPackValue(_vocbase->name())); + + nodes.add("graph", _graphInfo.slice()); + nodes.add(VPackValue("directions")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& d : _directions) { + nodes.add(VPackValue(d)); + } + } + + nodes.add(VPackValue("edgeCollections")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& e : _edgeColls) { + nodes.add(VPackValue(e->getName())); + } + } + + nodes.add(VPackValue("vertexCollections")); + { + VPackArrayBuilder guard(&nodes); + for (auto const& v : _vertexColls) { + nodes.add(VPackValue(v->getName())); + } + } // In variable if (usesInVariable()) { @@ -298,8 +714,21 @@ void TraversalNode::toVelocyPackHelper(arangodb::velocypack::Builder& nodes, nodes.close(); } - // Out variables - if (usesPathOutVariable()) { + if (_graphObj != nullptr) { + nodes.add(VPackValue("graphDefinition")); + _graphObj->toVelocyPack(nodes, verbose); + } + + // Out variables + if (usesVertexOutVariable()) { + nodes.add(VPackValue("vertexOutVariable")); + vertexOutVariable()->toVelocyPack(nodes); + } + if (usesEdgeOutVariable()) { + nodes.add(VPackValue("edgeOutVariable")); + edgeOutVariable()->toVelocyPack(nodes); + } + if (usesPathOutVariable()) { nodes.add(VPackValue("pathOutVariable")); pathOutVariable()->toVelocyPack(nodes); } @@ -307,7 +736,29 @@ void TraversalNode::toVelocyPackHelper(arangodb::velocypack::Builder& nodes, nodes.add(VPackValue("traversalFlags")); _options->toVelocyPack(nodes); - if (!_globalEdgeConditions.empty()) { + // Traversal Filter Conditions + + TRI_ASSERT(_tmpObjVariable != nullptr); + nodes.add(VPackValue("tmpObjVariable")); + _tmpObjVariable->toVelocyPack(nodes); + + TRI_ASSERT(_tmpObjVarNode != nullptr); + nodes.add(VPackValue("tmpObjVarNode")); + _tmpObjVarNode->toVelocyPack(nodes, verbose); + + TRI_ASSERT(_tmpIdNode != nullptr); + nodes.add(VPackValue("tmpIdNode")); + _tmpIdNode->toVelocyPack(nodes, verbose); + + TRI_ASSERT(_fromCondition != nullptr); + nodes.add(VPackValue("fromCondition")); + _fromCondition->toVelocyPack(nodes, verbose); + + TRI_ASSERT(_toCondition != nullptr); + nodes.add(VPackValue("toCondition")); + _toCondition->toVelocyPack(nodes, verbose); + + if (!_globalEdgeConditions.empty()) { nodes.add(VPackValue("globalEdgeConditions")); nodes.openArray(); for (auto const& it : _globalEdgeConditions) { @@ -357,10 +808,9 @@ ExecutionNode* TraversalNode::clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const { TRI_ASSERT(!_optionsBuild); auto tmp = - std::make_unique(*options()); + std::make_unique(*_options.get()); auto c = new TraversalNode(plan, _id, _vocbase, _edgeColls, _vertexColls, - _inVariable, _vertexId, _directions, tmp.get()); - tmp.release(); + _inVariable, _vertexId, _directions, tmp); if (usesVertexOutVariable()) { auto vertexOutVariable = _vertexOutVariable; @@ -437,12 +887,17 @@ ExecutionNode* TraversalNode::clone(ExecutionPlan* plan, bool withDependencies, return static_cast(c); } +/// @brief the cost of a traversal node +double TraversalNode::estimateCost(size_t& nrItems) const { + return _options->estimateCost(nrItems); +} + void TraversalNode::prepareOptions() { if (_optionsBuild) { return; } TRI_ASSERT(!_optionsBuild); - _options->setVariable(_tmpObjVariable); + _options->_tmpVar = _tmpObjVariable; size_t numEdgeColls = _edgeColls.size(); bool res = false; @@ -455,29 +910,71 @@ void TraversalNode::prepareOptions() { Ast* ast = _plan->getAst(); auto trx = ast->query()->trx(); - // FIXME: _options->_baseLookupInfos.reserve(numEdgeColls); + _options->_baseLookupInfos.reserve(numEdgeColls); // Compute Edge Indexes. First default indexes: for (size_t i = 0; i < numEdgeColls; ++i) { - switch (_directions[i]) { + std::string usedField; + auto dir = _directions[i]; + // TODO we can optimize here. indexCondition and Expression could be + // made non-overlapping. + traverser::TraverserOptions::LookupInfo info; + switch (dir) { case TRI_EDGE_IN: - _options->addLookupInfo( - ast, _edgeColls[i]->getName(), StaticStrings::ToString, - globalEdgeConditionBuilder.getInboundCondition()->clone(ast)); + usedField = StaticStrings::ToString; + info.indexCondition = + globalEdgeConditionBuilder.getInboundCondition()->clone(ast); break; case TRI_EDGE_OUT: - _options->addLookupInfo( - ast, _edgeColls[i]->getName(), StaticStrings::FromString, - globalEdgeConditionBuilder.getOutboundCondition()->clone(ast)); + usedField = StaticStrings::FromString; + info.indexCondition = + globalEdgeConditionBuilder.getOutboundCondition()->clone(ast); break; case TRI_EDGE_ANY: TRI_ASSERT(false); break; } + info.expression = new Expression(ast, info.indexCondition->clone(ast)); + res = trx->getBestIndexHandleForFilterCondition( + _edgeColls[i]->getName(), info.indexCondition, _tmpObjVariable, 1000, + info.idxHandles[0]); + TRI_ASSERT(res); // Right now we have an enforced edge index which will + // always fit. + if (!res) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "expected edge index not found"); + } + + // We now have to check if we need _from / _to inside the index lookup and which position + // it is used in. Such that the traverser can update the respective string value + // in-place + std::pair> pathCmp; + for (size_t i = 0; i < info.indexCondition->numMembers(); ++i) { + // We search through the nary-and and look for EQ - _from/_to + auto eq = info.indexCondition->getMemberUnchecked(i); + if (eq->type != NODE_TYPE_OPERATOR_BINARY_EQ) { + // No equality. Skip + continue; + } + TRI_ASSERT(eq->numMembers() == 2); + // It is sufficient to only check member one. + // We build the condition this way. + auto mem = eq->getMemberUnchecked(0); + if (mem->isAttributeAccessForVariable(pathCmp)) { + if (pathCmp.first != _tmpObjVariable) { + continue; + } + if (pathCmp.second.size() == 1 && pathCmp.second[0].name == usedField) { + info.conditionNeedUpdate = true; + info.conditionMemberToUpdate = i; + break; + } + continue; + } + } + _options->_baseLookupInfos.emplace_back(std::move(info)); } - auto opts = static_cast(_options.get()); for (auto& it : _edgeConditions) { - auto ins = opts->_depthLookupInfo.emplace( + auto ins = _options->_depthLookupInfo.emplace( it.first, std::vector()); // We probably have to adopt minDepth. We cannot fulfill a condition of larger depth anyway TRI_ASSERT(ins.second); @@ -556,21 +1053,27 @@ void TraversalNode::prepareOptions() { for (auto const& jt : _globalVertexConditions) { it.second->addMember(jt); } - opts->_vertexExpressions.emplace(it.first, new Expression(ast, it.second)); - TRI_ASSERT(!opts->_vertexExpressions[it.first]->isV8()); + _options->_vertexExpressions.emplace(it.first, new Expression(ast, it.second)); + TRI_ASSERT(!_options->_vertexExpressions[it.first]->isV8()); } if (!_globalVertexConditions.empty()) { auto cond = _plan->getAst()->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); for (auto const& it : _globalVertexConditions) { cond->addMember(it); } - opts->_baseVertexExpression = new Expression(ast, cond); - TRI_ASSERT(!opts->_baseVertexExpression->isV8()); + _options->_baseVertexExpression = new Expression(ast, cond); + TRI_ASSERT(!_options->_baseVertexExpression->isV8()); } _optionsBuild = true; } +void TraversalNode::addEngine(TraverserEngineID const& engine, + arangodb::ServerID const& server) { + TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); + _engines.emplace(server, engine); +} + /// @brief remember the condition to execute for early traversal abortion. void TraversalNode::setCondition(arangodb::aql::Condition* condition) { std::unordered_set varsUsedByCondition; @@ -625,7 +1128,15 @@ void TraversalNode::registerGlobalCondition(bool isConditionOnEdge, } arangodb::traverser::TraverserOptions* TraversalNode::options() const { - return static_cast(_options.get()); + return _options.get(); +} + +AstNode* TraversalNode::getTemporaryRefNode() const { + return _tmpObjVarNode; +} + +Variable const* TraversalNode::getTemporaryVariable() const { + return _tmpObjVariable; } void TraversalNode::getConditionVariables( @@ -637,6 +1148,16 @@ void TraversalNode::getConditionVariables( } } +#ifndef USE_ENTERPRISE +void TraversalNode::enhanceEngineInfo(VPackBuilder& builder) const { + if (_graphObj != nullptr) { + _graphObj->enhanceEngineInfo(builder); + } else { + // TODO enhance the Info based on EdgeCollections. + } +} +#endif + #ifdef TRI_ENABLE_MAINTAINER_MODE void TraversalNode::checkConditionsDefined() const { TRI_ASSERT(_tmpObjVariable != nullptr); diff --git a/arangod/Aql/TraversalNode.h b/arangod/Aql/TraversalNode.h index 0fa3769c1b..e7a21bd1b8 100644 --- a/arangod/Aql/TraversalNode.h +++ b/arangod/Aql/TraversalNode.h @@ -24,19 +24,23 @@ #ifndef ARANGOD_AQL_TRAVERSAL_NODE_H #define ARANGOD_AQL_TRAVERSAL_NODE_H 1 -#include "Aql/GraphNode.h" +#include "Aql/ExecutionNode.h" +#include "Aql/Collection.h" #include "Aql/Condition.h" +#include "Aql/Graphs.h" #include "Cluster/ServerState.h" +#include "Cluster/TraverserEngineRegistry.h" +#include "VocBase/LogicalCollection.h" #include "VocBase/TraverserOptions.h" namespace arangodb { namespace aql { /// @brief class TraversalNode -class TraversalNode : public GraphNode { - +class TraversalNode : public ExecutionNode { class TraversalEdgeConditionBuilder final : public EdgeConditionBuilder { private: + /// @brief reference to the outer traversal node TraversalNode const* _tn; protected: @@ -69,7 +73,7 @@ class TraversalNode : public GraphNode { TraversalNode(ExecutionPlan* plan, size_t id, TRI_vocbase_t* vocbase, AstNode const* direction, AstNode const* start, AstNode const* graph, - traverser::TraverserOptions* options); + std::unique_ptr& options); TraversalNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); @@ -82,12 +86,17 @@ class TraversalNode : public GraphNode { std::vector> const& vertexColls, Variable const* inVariable, std::string const& vertexId, std::vector const& directions, - traverser::TraverserOptions* options); + std::unique_ptr& options); public: /// @brief return the type of the node NodeType getType() const override final { return TRAVERSAL; } + /// @brief flag, if smart traversal (enterprise edition only!) is done + bool isSmart() const { + return _isSmart; + } + /// @brief export to VelocyPack void toVelocyPackHelper(arangodb::velocypack::Builder&, bool) const override final; @@ -96,6 +105,9 @@ class TraversalNode : public GraphNode { ExecutionNode* clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const override final; + /// @brief the cost of a traversal node + double estimateCost(size_t&) const override final; + /// @brief Test if this node uses an in variable or constant bool usesInVariable() const { return _inVariable != nullptr; } @@ -149,6 +161,27 @@ class TraversalNode : public GraphNode { return vars; } + /// @brief return the database + TRI_vocbase_t* vocbase() const { return _vocbase; } + + /// @brief return the vertex out variable + Variable const* vertexOutVariable() const { return _vertexOutVariable; } + + /// @brief checks if the vertex out variable is used + bool usesVertexOutVariable() const { return _vertexOutVariable != nullptr; } + + /// @brief set the vertex out variable + void setVertexOutput(Variable const* outVar) { _vertexOutVariable = outVar; } + + /// @brief return the edge out variable + Variable const* edgeOutVariable() const { return _edgeOutVariable; } + + /// @brief checks if the edge out variable is used + bool usesEdgeOutVariable() const { return _edgeOutVariable != nullptr; } + + /// @brief set the edge out variable + void setEdgeOutput(Variable const* outVar) { _edgeOutVariable = outVar; } + /// @brief checks if the path out variable is used bool usesPathOutVariable() const { return _pathOutVariable != nullptr; } @@ -163,6 +196,14 @@ class TraversalNode : public GraphNode { std::string const getStartVertex() const { return _vertexId; } + std::vector> const& edgeColls() const { + return _edgeColls; + } + + std::vector> const& vertexColls() const { + return _vertexColls; + } + /// @brief remember the condition to execute for early traversal abortion. void setCondition(Condition* condition); @@ -185,14 +226,34 @@ class TraversalNode : public GraphNode { /// The condition will contain the local variable for it's accesses. void registerGlobalCondition(bool, AstNode const*); - traverser::TraverserOptions* options() const override; + bool allDirectionsEqual() const; - void getConditionVariables(std::vector&) const override; + traverser::TraverserOptions* options() const; + + AstNode* getTemporaryRefNode() const; + + Variable const* getTemporaryVariable() const; + + void getConditionVariables(std::vector&) const; + + void enhanceEngineInfo(arangodb::velocypack::Builder&) const; /// @brief Compute the traversal options containing the expressions /// MUST! be called after optimization and before creation /// of blocks. - void prepareOptions() override; + void prepareOptions(); + + /// @brief Add a traverser engine Running on a DBServer to this node. + /// The block will communicate with them (CLUSTER ONLY) + void addEngine(traverser::TraverserEngineID const&, ServerID const&); + + + /// @brief Returns a reference to the engines. (CLUSTER ONLY) + std::unordered_map const* engines() + const { + TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); + return &_engines; + } private: @@ -203,6 +264,15 @@ class TraversalNode : public GraphNode { private: + /// @brief the database + TRI_vocbase_t* _vocbase; + + /// @brief vertex output variable + Variable const* _vertexOutVariable; + + /// @brief vertex output variable + Variable const* _edgeOutVariable; + /// @brief vertex output variable Variable const* _pathOutVariable; @@ -212,12 +282,45 @@ class TraversalNode : public GraphNode { /// @brief input vertexId only used if _inVariable is unused std::string _vertexId; + /// @brief input graphInfo only used for serialization & info + arangodb::velocypack::Builder _graphInfo; + + /// @brief The directions edges are followed + std::vector _directions; + + /// @brief the edge collection names + std::vector> _edgeColls; + + /// @brief the vertex collection names + std::vector> _vertexColls; + + /// @brief our graph + Graph const* _graphObj; + /// @brief early abort traversal conditions: Condition* _condition; /// @brief variables that are inside of the condition std::unordered_set _conditionVariables; + /// @brief Options for traversals + std::unique_ptr _options; + + /// @brief Temporary pseudo variable for the currently traversed object. + Variable const* _tmpObjVariable; + + /// @brief Reference to the pseudo variable + AstNode* _tmpObjVarNode; + + /// @brief Pseudo string value node to hold the last visted vertex id. + AstNode* _tmpIdNode; + + /// @brief The hard coded condition on _from + AstNode* _fromCondition; + + /// @brief The hard coded condition on _to + AstNode* _toCondition; + /// @brief The global edge condition. Does not contain /// _from and _to checks std::vector _globalEdgeConditions; @@ -232,6 +335,16 @@ class TraversalNode : public GraphNode { /// @brief List of all depth specific conditions for vertices std::unordered_map _vertexConditions; + /// @brief Flag if options are already prepared. After + /// this flag was set the node cannot be cloned + /// any more. + bool _optionsBuild; + + /// @brief The list of traverser engines grouped by server. + std::unordered_map _engines; + + /// @brief flag, if traversal is smart (enterprise edition only!) + bool _isSmart; }; } // namespace arangodb::aql diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index f3b9d0b943..b6079257a9 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -134,7 +134,6 @@ SET(ARANGOD_SOURCES Aql/Function.cpp Aql/Functions.cpp Aql/Graphs.cpp - Aql/GraphNode.cpp Aql/IndexBlock.cpp Aql/IndexNode.cpp Aql/ModificationBlocks.cpp @@ -160,6 +159,7 @@ SET(ARANGOD_SOURCES Aql/ShortStringStorage.cpp Aql/ShortestPathBlock.cpp Aql/ShortestPathNode.cpp + Aql/ShortestPathOptions.cpp Aql/SortBlock.cpp Aql/SortCondition.cpp Aql/SortNode.cpp @@ -311,6 +311,7 @@ SET(ARANGOD_SOURCES V8Server/FoxxQueuesFeature.cpp V8Server/V8Context.cpp V8Server/V8DealerFeature.cpp + V8Server/V8Traverser.cpp V8Server/v8-actions.cpp V8Server/v8-collection-util.cpp V8Server/v8-collection.cpp diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 474d063933..f86df70a24 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -40,9 +40,7 @@ using namespace arangodb::traverser; static const std::string OPTIONS = "options"; static const std::string SHARDS = "shards"; -static const std::string TYPE = "type"; static const std::string EDGES = "edges"; -static const std::string BACKWARDEDGES = "reverseEdges"; static const std::string VARIABLES = "variables"; static const std::string VERTICES = "vertices"; @@ -146,14 +144,13 @@ void BaseTraverserEngine::getEdges(VPackSlice vertex, size_t depth, VPackBuilder builder.openObject(); builder.add(VPackValue("edges")); builder.openArray(); - auto opts = static_cast(_opts.get()); if (vertex.isArray()) { for (VPackSlice v : VPackArrayIterator(vertex)) { TRI_ASSERT(v.isString()); result.clear(); - auto edgeCursor = opts->nextCursor(&mmdr, v, depth); + auto edgeCursor = _opts->nextCursor(&mmdr, v, depth); while (edgeCursor->next(result, cursorId)) { - if (!opts->evaluateEdgeExpression(result.back(), v, depth, cursorId)) { + if (!_opts->evaluateEdgeExpression(result.back(), v, depth, cursorId)) { filtered++; result.pop_back(); } @@ -164,10 +161,10 @@ void BaseTraverserEngine::getEdges(VPackSlice vertex, size_t depth, VPackBuilder // Result now contains all valid edges, probably multiples. } } else if (vertex.isString()) { - std::unique_ptr edgeCursor(opts->nextCursor(&mmdr, vertex, depth)); + std::unique_ptr edgeCursor(_opts->nextCursor(&mmdr, vertex, depth)); while (edgeCursor->next(result, cursorId)) { - if (!opts->evaluateEdgeExpression(result.back(), vertex, depth, cursorId)) { + if (!_opts->evaluateEdgeExpression(result.back(), vertex, depth, cursorId)) { filtered++; result.pop_back(); } @@ -324,24 +321,18 @@ TraverserEngine::TraverserEngine(TRI_vocbase_t* vocbase, } VPackSlice shardsSlice = info.get(SHARDS); VPackSlice edgesSlice = shardsSlice.get(EDGES); - VPackSlice backwardSlice = shardsSlice.get(BACKWARDEDGES); - VPackSlice typeSlice = optsSlice.get(TYPE); - if (!typeSlice.isString()) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_BAD_PARAMETER, - "The body requires an " + TYPE + " attribute."); - } - StringRef type(typeSlice); - if (type == "shortest") { - _opts.reset(new ShortestPathOptions(_query, optsSlice, edgesSlice, backwardSlice)); - } else if (type == "traversal") { - _opts.reset(new TraverserOptions(_query, optsSlice, edgesSlice)); - } else { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_BAD_PARAMETER, - "The " + TYPE + " has to be shortest or traversal."); - } + + _opts.reset(new TraverserOptions(_query, optsSlice, edgesSlice)); } + TraverserEngine::~TraverserEngine() { } + +void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); +} + +void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); +} diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index 4e779dcb90..18a856a2d7 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -27,7 +27,6 @@ #include "Basics/Common.h" #include "Aql/Collections.h" -#include "VocBase/TraverserOptions.h" struct TRI_vocbase_t; @@ -50,7 +49,7 @@ class Builder; class Slice; } namespace traverser { -struct BaseTraverserOptions; +struct TraverserOptions; class BaseTraverserEngine { friend class TraverserEngineRegistry; @@ -78,12 +77,18 @@ class BaseTraverserEngine { void getVertexData(arangodb::velocypack::Slice, size_t, arangodb::velocypack::Builder&); + virtual void smartSearch(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) = 0; + + virtual void smartSearchBFS(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) = 0; + bool lockCollection(std::string const&); std::shared_ptr context() const; protected: - std::unique_ptr _opts; + std::unique_ptr _opts; arangodb::aql::Query* _query; transaction::Methods* _trx; arangodb::aql::Collections _collections; @@ -104,6 +109,12 @@ class TraverserEngine : public BaseTraverserEngine { TraverserEngine(TRI_vocbase_t*, arangodb::velocypack::Slice); public: ~TraverserEngine(); + + void smartSearch(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) override; + + void smartSearchBFS(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) override; }; } // namespace traverser diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index 2bafce5c64..5eda69d3f5 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -129,29 +129,12 @@ void InternalRestTraverserHandler::queryEngine() { return; } - double start = TRI_microtime(); - traverser::BaseTraverserEngine* engine = nullptr; - while (true) { - try { - engine = _registry->get(engineId); - if (engine == nullptr) { - generateError( - ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "invalid TraverserEngineId"); - return; - } - break; - } catch (arangodb::basics::Exception const& ex) { /* check type */ - if (ex.code() == TRI_ERROR_DEADLOCK) { - // Define timeout properly. - if (TRI_microtime() - start > 600.0) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_TIMEOUT); - } - usleep(100000); - } else { - throw ex; - } - } + traverser::BaseTraverserEngine* engine = _registry->get(engineId); + if (engine == nullptr) { + generateError( + ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + "invalid TraverserEngineId"); + return; } auto& registry = _registry; // For the guard @@ -239,18 +222,10 @@ void InternalRestTraverserHandler::queryEngine() { } engine->getVertexData(keysSlice, depthSlice.getNumericValue(), result); } - } else { - enterpriseSmartSearch(option, engine, body, result); - } - generateResult(ResponseCode::OK, result.slice(), engine->context()); -} - -#ifndef USE_ENTERPRISE -void InternalRestTraverserHandler::enterpriseSmartSearch( - std::string const& option, traverser::BaseTraverserEngine* engine, - VPackSlice body, VPackBuilder& result) { - if (option == "smartSearch" || option == "smartSearchBFS" || "smartShortestPath") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); + } else if (option == "smartSearch") { + engine->smartSearch(body, result); + } else if (option == "smartSearchBFS") { + engine->smartSearchBFS(body, result); } else { // PATH Info wrong other error generateError( @@ -258,8 +233,8 @@ void InternalRestTraverserHandler::enterpriseSmartSearch( ""); return; } + generateResult(ResponseCode::OK, result.slice(), engine->context()); } -#endif void InternalRestTraverserHandler::destroyEngine() { std::vector const& suffixes = _request->decodedSuffixes(); diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.h b/arangod/InternalRestHandler/InternalRestTraverserHandler.h index ac7587e8fb..42c2117882 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.h +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.h @@ -28,7 +28,6 @@ namespace arangodb { namespace traverser { -class BaseTraverserEngine; class TraverserEngineRegistry; } @@ -52,12 +51,6 @@ class InternalRestTraverserHandler : public RestVocbaseBaseHandler { // @brief Destroy an existing Traverser Engine. void destroyEngine(); - // @brief Do a smart search (EE only) - void enterpriseSmartSearch(std::string const& option, - traverser::BaseTraverserEngine* engine, - arangodb::velocypack::Slice body, - arangodb::velocypack::Builder& result); - private: traverser::TraverserEngineRegistry* _registry; }; diff --git a/arangod/V8Server/V8Traverser.cpp b/arangod/V8Server/V8Traverser.cpp new file mode 100644 index 0000000000..f6f0b5d727 --- /dev/null +++ b/arangod/V8Server/V8Traverser.cpp @@ -0,0 +1,63 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "V8Traverser.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/SingleServerTraverser.h" + +#include +#include + +using namespace arangodb; +using namespace arangodb::basics; +using namespace arangodb::traverser; + +ShortestPathOptions::ShortestPathOptions(transaction::Methods* trx) + : BasicOptions(trx), + direction("outbound"), + useWeight(false), + weightAttribute(""), + defaultWeight(1), + bidirectional(true), + multiThreaded(true) { +} + +void ShortestPathOptions::setStart(std::string const& id) { + start = id; + startBuilder.clear(); + startBuilder.add(VPackValue(id)); +} + +void ShortestPathOptions::setEnd(std::string const& id) { + end = id; + endBuilder.clear(); + endBuilder.add(VPackValue(id)); +} + +VPackSlice ShortestPathOptions::getStart() const { + return startBuilder.slice(); +} + +VPackSlice ShortestPathOptions::getEnd() const { + return endBuilder.slice(); +} diff --git a/arangod/V8Server/V8Traverser.h b/arangod/V8Server/V8Traverser.h new file mode 100644 index 0000000000..402082dd52 --- /dev/null +++ b/arangod/V8Server/V8Traverser.h @@ -0,0 +1,89 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_V8_SERVER_V8_TRAVERSER_H +#define ARANGOD_V8_SERVER_V8_TRAVERSER_H 1 + +#include "Basics/VelocyPackHelper.h" +#include "VocBase/Traverser.h" + +namespace arangodb { + +namespace velocypack { +class Slice; +} +} + +namespace arangodb { +namespace traverser { + +// A collection of shared options used in several functions. +// Should not be used directly, use specialization instead. +struct BasicOptions { + + transaction::Methods* _trx; + + protected: + + explicit BasicOptions(transaction::Methods* trx) + : _trx(trx) {} + + virtual ~BasicOptions() { + } + + public: + std::string start; + + + public: + + transaction::Methods* trx() { return _trx; } + +}; + +struct ShortestPathOptions : BasicOptions { + public: + std::string direction; + bool useWeight; + std::string weightAttribute; + double defaultWeight; + bool bidirectional; + bool multiThreaded; + std::string end; + arangodb::velocypack::Builder startBuilder; + arangodb::velocypack::Builder endBuilder; + + explicit ShortestPathOptions(transaction::Methods* trx); + + void setStart(std::string const&); + void setEnd(std::string const&); + + arangodb::velocypack::Slice getStart() const; + arangodb::velocypack::Slice getEnd() const; + +}; + +} +} + +#endif diff --git a/arangod/VocBase/Traverser.cpp b/arangod/VocBase/Traverser.cpp index 6c69671392..044784a0db 100644 --- a/arangod/VocBase/Traverser.cpp +++ b/arangod/VocBase/Traverser.cpp @@ -58,26 +58,20 @@ void arangodb::traverser::ShortestPath::vertexToVelocyPack(transaction::Methods* size_t position, VPackBuilder& builder) { TRI_ASSERT(position < length()); VPackSlice v = _vertices[position]; - TRI_ASSERT(v.isString() || v.isObject() || v.isNull()); - if (v.isString()) { - // In this case we have to fetch the vertex - std::string collection = v.copyString(); - size_t p = collection.find("/"); - TRI_ASSERT(p != std::string::npos); + TRI_ASSERT(v.isString()); + std::string collection = v.copyString(); + size_t p = collection.find("/"); + TRI_ASSERT(p != std::string::npos); - transaction::BuilderLeaser searchBuilder(trx); - searchBuilder->add(VPackValue(collection.substr(p + 1))); - collection = collection.substr(0, p); + transaction::BuilderLeaser searchBuilder(trx); + searchBuilder->add(VPackValue(collection.substr(p + 1))); + collection = collection.substr(0, p); - int res = - trx->documentFastPath(collection, mmdr, searchBuilder->slice(), builder, true); - if (res != TRI_ERROR_NO_ERROR) { - builder.clear(); // Just in case... - builder.add(basics::VelocyPackHelper::NullValue()); - } - } else { - // We already have the vertex data. - builder.add(v); + int res = + trx->documentFastPath(collection, mmdr, searchBuilder->slice(), builder, true); + if (res != TRI_ERROR_NO_ERROR) { + builder.clear(); // Just in case... + builder.add(basics::VelocyPackHelper::NullValue()); } } diff --git a/arangod/VocBase/Traverser.h b/arangod/VocBase/Traverser.h index 63e92d0ccc..40baf7f1fc 100644 --- a/arangod/VocBase/Traverser.h +++ b/arangod/VocBase/Traverser.h @@ -54,10 +54,6 @@ class Query; } namespace traverser { -#ifdef USE_ENTERPRISE -class SmartGraphConstDistanceFinder; -#endif - struct TraverserOptions; class ShortestPath { @@ -69,10 +65,6 @@ class ShortestPath { arangodb::basics::VelocyPackHelper::VPackStringHash, arangodb::basics::VelocyPackHelper::VPackStringEqual, ShortestPath>; -#ifdef USE_ENTERPRISE - friend class arangodb::traverser::SmartGraphConstDistanceFinder; -#endif - public: ////////////////////////////////////////////////////////////////////////////// /// @brief Constructor. This is an abstract only class. diff --git a/arangod/VocBase/TraverserOptions.cpp b/arangod/VocBase/TraverserOptions.cpp index b616d924fc..ba3f301d4b 100644 --- a/arangod/VocBase/TraverserOptions.cpp +++ b/arangod/VocBase/TraverserOptions.cpp @@ -24,25 +24,21 @@ #include "TraverserOptions.h" #include "Aql/Ast.h" -#include "Aql/AstNode.h" #include "Aql/Expression.h" #include "Aql/Query.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ClusterEdgeCursor.h" #include "Indexes/Index.h" -#include "Transaction/Methods.h" #include "VocBase/SingleServerTraverser.h" #include #include +using namespace arangodb::transaction; using VPackHelper = arangodb::basics::VelocyPackHelper; using TraverserOptions = arangodb::traverser::TraverserOptions; -using ShortestPathOptions = arangodb::traverser::ShortestPathOptions; -using BaseTraverserOptions = arangodb::traverser::BaseTraverserOptions; -using namespace arangodb::transaction; -BaseTraverserOptions::LookupInfo::LookupInfo() +arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo() : expression(nullptr), indexCondition(nullptr), conditionNeedUpdate(false), @@ -51,13 +47,13 @@ BaseTraverserOptions::LookupInfo::LookupInfo() idxHandles.resize(1); }; -BaseTraverserOptions::LookupInfo::~LookupInfo() { +arangodb::traverser::TraverserOptions::LookupInfo::~LookupInfo() { if (expression != nullptr) { delete expression; } } -BaseTraverserOptions::LookupInfo::LookupInfo( +arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( arangodb::aql::Query* query, VPackSlice const& info, VPackSlice const& shards) { TRI_ASSERT(shards.isArray()); idxHandles.reserve(shards.length()); @@ -111,7 +107,7 @@ BaseTraverserOptions::LookupInfo::LookupInfo( indexCondition = new aql::AstNode(query->ast(), read); } -BaseTraverserOptions::LookupInfo::LookupInfo( +arangodb::traverser::TraverserOptions::LookupInfo::LookupInfo( LookupInfo const& other) : idxHandles(other.idxHandles), expression(nullptr), @@ -121,7 +117,7 @@ BaseTraverserOptions::LookupInfo::LookupInfo( expression = other.expression->clone(nullptr); } -void BaseTraverserOptions::LookupInfo::buildEngineInfo( +void arangodb::traverser::TraverserOptions::LookupInfo::buildEngineInfo( VPackBuilder& result) const { result.openObject(); result.add(VPackValue("handle")); @@ -142,7 +138,7 @@ void BaseTraverserOptions::LookupInfo::buildEngineInfo( result.close(); } -double TraverserOptions::LookupInfo::estimateCost(size_t& nrItems) const { +double arangodb::traverser::TraverserOptions::LookupInfo::estimateCost(size_t& nrItems) const { // If we do not have an index yet we cannot do anything. // Should NOT be the case TRI_ASSERT(!idxHandles.empty()); @@ -157,10 +153,14 @@ double TraverserOptions::LookupInfo::estimateCost(size_t& nrItems) const { return 1000.0; } -TraverserOptions::TraverserOptions( +arangodb::traverser::TraverserOptions::TraverserOptions( transaction::Methods* trx, VPackSlice const& slice) - : BaseTraverserOptions(trx), + : _trx(trx), _baseVertexExpression(nullptr), + _tmpVar(nullptr), + _ctx(new aql::FixedVarExpressionContext()), + _traverser(nullptr), + _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -176,7 +176,7 @@ TraverserOptions::TraverserOptions( std::string tmp = VPackHelper::getStringValue(obj, "uniqueVertices", ""); if (tmp == "path") { uniqueVertices = - TraverserOptions::UniquenessLevel::PATH; + arangodb::traverser::TraverserOptions::UniquenessLevel::PATH; } else if (tmp == "global") { if (!useBreadthFirst) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, @@ -185,230 +185,35 @@ TraverserOptions::TraverserOptions( "unpredictable results."); } uniqueVertices = - TraverserOptions::UniquenessLevel::GLOBAL; + arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL; } else { uniqueVertices = - TraverserOptions::UniquenessLevel::NONE; + arangodb::traverser::TraverserOptions::UniquenessLevel::NONE; } tmp = VPackHelper::getStringValue(obj, "uniqueEdges", ""); if (tmp == "none") { uniqueEdges = - TraverserOptions::UniquenessLevel::NONE; + arangodb::traverser::TraverserOptions::UniquenessLevel::NONE; } else if (tmp == "global") { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "uniqueEdges: 'global' is not supported, " "due to unpredictable results. Use 'path' " "or 'none' instead"); - uniqueEdges = - TraverserOptions::UniquenessLevel::GLOBAL; } else { uniqueEdges = - TraverserOptions::UniquenessLevel::PATH; + arangodb::traverser::TraverserOptions::UniquenessLevel::PATH; } } -BaseTraverserOptions::BaseTraverserOptions(BaseTraverserOptions const& other) - : _ctx(new aql::FixedVarExpressionContext()), - _trx(other._trx), - _tmpVar(nullptr), - _isCoordinator(arangodb::ServerState::instance()->isCoordinator()) { - TRI_ASSERT(other._baseLookupInfos.empty()); - TRI_ASSERT(other._tmpVar == nullptr); -} - - -BaseTraverserOptions::BaseTraverserOptions( +arangodb::traverser::TraverserOptions::TraverserOptions( arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) - : _ctx(new aql::FixedVarExpressionContext()), - _trx(query->trx()), - _tmpVar(nullptr), - _isCoordinator(arangodb::ServerState::instance()->isCoordinator()) { - VPackSlice read = info.get("tmpVar"); - if (!read.isObject()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "The options require a tmpVar"); - } - _tmpVar = query->ast()->variables()->createVariable(read); - - read = info.get("baseLookupInfos"); - if (!read.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "The options require a baseLookupInfos"); - } - - size_t length = read.length(); - TRI_ASSERT(read.length() == collections.length()); - _baseLookupInfos.reserve(length); - for (size_t j = 0; j < length; ++j) { - _baseLookupInfos.emplace_back(query, read.at(j), collections.at(j)); - } -} - -BaseTraverserOptions::~BaseTraverserOptions() { - delete _ctx; -} - - -void BaseTraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { - builder.openObject(); - injectVelocyPackIndexes(builder); - builder.close(); -} - -void BaseTraverserOptions::buildEngineInfo(VPackBuilder& result) const { - result.openObject(); - injectEngineInfo(result); - result.close(); -} - -void BaseTraverserOptions::setVariable(aql::Variable const* variable) { - _tmpVar = variable; -} - -void BaseTraverserOptions::addLookupInfo(aql::Ast* ast, - std::string const& collectionName, - std::string const& attributeName, - aql::AstNode* condition) { - injectLookupInfoInList(_baseLookupInfos, ast, collectionName, attributeName, condition); -} - -void BaseTraverserOptions::injectLookupInfoInList( - std::vector& list, aql::Ast* ast, - std::string const& collectionName, std::string const& attributeName, - aql::AstNode* condition) { - traverser::TraverserOptions::LookupInfo info; - info.indexCondition = condition; - info.expression = new aql::Expression(ast, condition->clone(ast)); - bool res = _trx->getBestIndexHandleForFilterCondition( - collectionName, info.indexCondition, _tmpVar, 1000, info.idxHandles[0]); - TRI_ASSERT(res); // Right now we have an enforced edge index which will - // always fit. - if (!res) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "expected edge index not found"); - } - - // We now have to check if we need _from / _to inside the index lookup and - // which position - // it is used in. Such that the traverser can update the respective string - // value in-place - - std::pair> pathCmp; - for (size_t i = 0; i < info.indexCondition->numMembers(); ++i) { - // We search through the nary-and and look for EQ - _from/_to - auto eq = info.indexCondition->getMemberUnchecked(i); - if (eq->type != arangodb::aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ) { - // No equality. Skip - continue; - } - TRI_ASSERT(eq->numMembers() == 2); - // It is sufficient to only check member one. - // We build the condition this way. - auto mem = eq->getMemberUnchecked(0); - if (mem->isAttributeAccessForVariable(pathCmp)) { - if (pathCmp.first != _tmpVar) { - continue; - } - if (pathCmp.second.size() == 1 && pathCmp.second[0].name == attributeName) { - info.conditionNeedUpdate = true; - info.conditionMemberToUpdate = i; - break; - } - continue; - } - } - list.emplace_back(std::move(info)); -} - -void BaseTraverserOptions::clearVariableValues() { - _ctx->clearVariableValues(); -} - -void BaseTraverserOptions::setVariableValue( - aql::Variable const* var, aql::AqlValue const value) { - _ctx->setVariableValue(var, value); -} - -void BaseTraverserOptions::serializeVariables( - VPackBuilder& builder) const { - TRI_ASSERT(builder.isOpenArray()); - _ctx->serializeAllVariables(_trx, builder); -} - -arangodb::transaction::Methods* BaseTraverserOptions::trx() const { - return _trx; -} - -void BaseTraverserOptions::injectVelocyPackIndexes(VPackBuilder& builder) const { - TRI_ASSERT(builder.isOpenObject()); - - // base indexes - builder.add("base", VPackValue(VPackValueType::Array)); - for (auto const& it : _baseLookupInfos) { - for (auto const& it2 : it.idxHandles) { - builder.openObject(); - it2.getIndex()->toVelocyPack(builder, false); - builder.close(); - } - } - builder.close(); -} - -void BaseTraverserOptions::injectEngineInfo(VPackBuilder& result) const { - TRI_ASSERT(result.isOpenObject()); - result.add(VPackValue("baseLookupInfos")); - result.openArray(); - for (auto const& it : _baseLookupInfos) { - it.buildEngineInfo(result); - } - result.close(); - - result.add(VPackValue("tmpVar")); - _tmpVar->toVelocyPack(result); -} - -arangodb::aql::Expression* BaseTraverserOptions::getEdgeExpression( - size_t cursorId) const { - TRI_ASSERT(!_baseLookupInfos.empty()); - TRI_ASSERT(_baseLookupInfos.size() > cursorId); - return _baseLookupInfos[cursorId].expression; -} - -bool BaseTraverserOptions::evaluateExpression( - arangodb::aql::Expression* expression, VPackSlice value) const { - if (expression == nullptr) { - return true; - } - - TRI_ASSERT(!expression->isV8()); - expression->setVariable(_tmpVar, value); - bool mustDestroy = false; - aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); - TRI_ASSERT(res.isBoolean()); - bool result = res.toBoolean(); - expression->clearVariable(_tmpVar); - if (mustDestroy) { - res.destroy(); - } - return result; -} - -double BaseTraverserOptions::costForLookupInfoList( - std::vector const& list, - size_t& createItems) const { - double cost = 0; - createItems = 0; - for (auto const& li : list) { - cost += li.estimateCost(createItems); - } - return cost; -} - -TraverserOptions::TraverserOptions( - arangodb::aql::Query* query, VPackSlice info, VPackSlice collections) - : BaseTraverserOptions(query, info, collections), + : _trx(query->trx()), _baseVertexExpression(nullptr), + _tmpVar(nullptr), + _ctx(new aql::FixedVarExpressionContext()), + _traverser(nullptr), + _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -436,6 +241,13 @@ TraverserOptions::TraverserOptions( } useBreadthFirst = read.getBool(); + read = info.get("tmpVar"); + if (!read.isObject()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "The options require a tmpVar"); + } + _tmpVar = query->ast()->variables()->createVariable(read); + read = info.get("uniqueVertices"); if (!read.isInteger()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, @@ -478,6 +290,19 @@ TraverserOptions::TraverserOptions( "The options require a uniqueEdges"); } + read = info.get("baseLookupInfos"); + if (!read.isArray()) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, + "The options require a baseLookupInfos"); + } + + size_t length = read.length(); + TRI_ASSERT(read.length() == collections.length()); + _baseLookupInfos.reserve(length); + for (size_t j = 0; j < length; ++j) { + _baseLookupInfos.emplace_back(query, read.at(j), collections.at(j)); + } + read = info.get("depthLookupInfo"); if (!read.isNone()) { if (!read.isObject()) { @@ -485,7 +310,6 @@ TraverserOptions::TraverserOptions( TRI_ERROR_BAD_PARAMETER, "The options require depthLookupInfo to be an object"); } - size_t length = collections.length(); _depthLookupInfo.reserve(read.length()); for (auto const& depth : VPackObjectIterator(read)) { uint64_t d = basics::StringUtils::uint64(depth.key.copyString()); @@ -533,78 +357,95 @@ TraverserOptions::TraverserOptions( } // Check for illegal option combination: TRI_ASSERT(uniqueEdges != - TraverserOptions::UniquenessLevel::GLOBAL); + arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL); TRI_ASSERT( uniqueVertices != - TraverserOptions::UniquenessLevel::GLOBAL || + arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL || useBreadthFirst); } -TraverserOptions::TraverserOptions( +arangodb::traverser::TraverserOptions::TraverserOptions( TraverserOptions const& other) - : BaseTraverserOptions(other), + : _trx(other._trx), _baseVertexExpression(nullptr), + _tmpVar(nullptr), + _ctx(new aql::FixedVarExpressionContext()), + _traverser(nullptr), + _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), minDepth(other.minDepth), maxDepth(other.maxDepth), useBreadthFirst(other.useBreadthFirst), uniqueVertices(other.uniqueVertices), uniqueEdges(other.uniqueEdges) { + TRI_ASSERT(other._baseLookupInfos.empty()); TRI_ASSERT(other._depthLookupInfo.empty()); TRI_ASSERT(other._vertexExpressions.empty()); + TRI_ASSERT(other._tmpVar == nullptr); TRI_ASSERT(other._baseVertexExpression == nullptr); // Check for illegal option combination: TRI_ASSERT(uniqueEdges != - TraverserOptions::UniquenessLevel::GLOBAL); + arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL); TRI_ASSERT( uniqueVertices != - TraverserOptions::UniquenessLevel::GLOBAL || + arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL || useBreadthFirst); } -TraverserOptions::~TraverserOptions() { +arangodb::traverser::TraverserOptions::~TraverserOptions() { for (auto& pair : _vertexExpressions) { delete pair.second; } delete _baseVertexExpression; + delete _ctx; } -void TraverserOptions::toVelocyPack(VPackBuilder& builder) const { +void arangodb::traverser::TraverserOptions::toVelocyPack(VPackBuilder& builder) const { VPackObjectBuilder guard(&builder); + builder.add("minDepth", VPackValue(minDepth)); builder.add("maxDepth", VPackValue(maxDepth)); builder.add("bfs", VPackValue(useBreadthFirst)); switch (uniqueVertices) { - case TraverserOptions::UniquenessLevel::NONE: + case arangodb::traverser::TraverserOptions::UniquenessLevel::NONE: builder.add("uniqueVertices", VPackValue("none")); break; - case TraverserOptions::UniquenessLevel::PATH: + case arangodb::traverser::TraverserOptions::UniquenessLevel::PATH: builder.add("uniqueVertices", VPackValue("path")); break; - case TraverserOptions::UniquenessLevel::GLOBAL: + case arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL: builder.add("uniqueVertices", VPackValue("global")); break; } switch (uniqueEdges) { - case TraverserOptions::UniquenessLevel::NONE: + case arangodb::traverser::TraverserOptions::UniquenessLevel::NONE: builder.add("uniqueEdges", VPackValue("none")); break; - case TraverserOptions::UniquenessLevel::PATH: + case arangodb::traverser::TraverserOptions::UniquenessLevel::PATH: builder.add("uniqueEdges", VPackValue("path")); break; - case TraverserOptions::UniquenessLevel::GLOBAL: + case arangodb::traverser::TraverserOptions::UniquenessLevel::GLOBAL: builder.add("uniqueEdges", VPackValue("global")); break; } - builder.add("type", VPackValue("traversal")); } -void TraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { +void arangodb::traverser::TraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { VPackObjectBuilder guard(&builder); - BaseTraverserOptions::injectVelocyPackIndexes(builder); - + + // base indexes + builder.add("base", VPackValue(VPackValueType::Array)); + for (auto const& it : _baseLookupInfos) { + for (auto const& it2 : it.idxHandles) { + builder.openObject(); + it2.getIndex()->toVelocyPack(builder, false); + builder.close(); + } + } + builder.close(); + // depth lookup indexes builder.add("levels", VPackValue(VPackValueType::Object)); for (auto const& it : _depthLookupInfo) { @@ -622,9 +463,8 @@ void TraverserOptions::toVelocyPackIndexes(VPackBuilder& builder) const { builder.close(); } -void TraverserOptions::buildEngineInfo(VPackBuilder& result) const { +void arangodb::traverser::TraverserOptions::buildEngineInfo(VPackBuilder& result) const { result.openObject(); - BaseTraverserOptions::injectEngineInfo(result); result.add("minDepth", VPackValue(minDepth)); result.add("maxDepth", VPackValue(maxDepth)); result.add("bfs", VPackValue(useBreadthFirst)); @@ -655,6 +495,13 @@ void TraverserOptions::buildEngineInfo(VPackBuilder& result) const { break; } + result.add(VPackValue("baseLookupInfos")); + result.openArray(); + for (auto const& it: _baseLookupInfos) { + it.buildEngineInfo(result); + } + result.close(); + if (!_depthLookupInfo.empty()) { result.add(VPackValue("depthLookupInfo")); result.openObject(); @@ -689,7 +536,10 @@ void TraverserOptions::buildEngineInfo(VPackBuilder& result) const { _baseVertexExpression->toVelocyPack(result, true); result.close(); } - result.add("type", VPackValue("traversal")); + + result.add(VPackValue("tmpVar")); + _tmpVar->toVelocyPack(result); + result.close(); } @@ -701,7 +551,7 @@ bool arangodb::traverser::TraverserOptions::vertexHasFilter( return _vertexExpressions.find(depth) != _vertexExpressions.end(); } -bool TraverserOptions::evaluateEdgeExpression( +bool arangodb::traverser::TraverserOptions::evaluateEdgeExpression( arangodb::velocypack::Slice edge, arangodb::velocypack::Slice vertex, uint64_t depth, size_t cursorId) const { if (_isCoordinator) { @@ -717,33 +567,44 @@ bool TraverserOptions::evaluateEdgeExpression( TRI_ASSERT(specific->second.size() > cursorId); expression = specific->second[cursorId].expression; } else { - expression = getEdgeExpression(cursorId); + TRI_ASSERT(!_baseLookupInfos.empty()); + TRI_ASSERT(_baseLookupInfos.size() > cursorId); + expression = _baseLookupInfos[cursorId].expression; } - if (expression == nullptr) { - return true; + if (expression != nullptr) { + TRI_ASSERT(!expression->isV8()); + expression->setVariable(_tmpVar, edge); + + VPackValueLength vidLength; + char const* vid = vertex.getString(vidLength); + + // inject _from/_to value + auto node = expression->nodeForModification(); + + TRI_ASSERT(node->numMembers() > 0); + auto dirCmp = node->getMemberUnchecked(node->numMembers() - 1); + TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); + TRI_ASSERT(dirCmp->numMembers() == 2); + + auto idNode = dirCmp->getMemberUnchecked(1); + TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); + TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); + idNode->stealComputedValue(); + idNode->setStringValue(vid, vidLength); + + bool mustDestroy = false; + aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); + expression->clearVariable(_tmpVar); + bool result = res.toBoolean(); + if (mustDestroy) { + res.destroy(); + } + return result; } - - VPackValueLength vidLength; - char const* vid = vertex.getString(vidLength); - - // inject _from/_to value - auto node = expression->nodeForModification(); - - TRI_ASSERT(node->numMembers() > 0); - auto dirCmp = node->getMemberUnchecked(node->numMembers() - 1); - TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); - TRI_ASSERT(dirCmp->numMembers() == 2); - - auto idNode = dirCmp->getMemberUnchecked(1); - TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); - TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); - idNode->stealComputedValue(); - idNode->setStringValue(vid, vidLength); - - return evaluateExpression(expression, edge); + return true; } -bool TraverserOptions::evaluateVertexExpression( +bool arangodb::traverser::TraverserOptions::evaluateVertexExpression( arangodb::velocypack::Slice vertex, uint64_t depth) const { arangodb::aql::Expression* expression = nullptr; @@ -755,11 +616,25 @@ bool TraverserOptions::evaluateVertexExpression( expression = _baseVertexExpression; } - return evaluateExpression(expression, vertex); + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(!expression->isV8()); + expression->setVariable(_tmpVar, vertex); + bool mustDestroy = false; + aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); + TRI_ASSERT(res.isBoolean()); + bool result = res.toBoolean(); + expression->clearVariable(_tmpVar); + if (mustDestroy) { + res.destroy(); + } + return result; } arangodb::traverser::EdgeCursor* -TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, +arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, VPackSlice vertex, uint64_t depth) const { if (_isCoordinator) { @@ -777,7 +652,7 @@ TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, } arangodb::traverser::EdgeCursor* -TraverserOptions::nextCursorLocal(ManagedDocumentResult* mmdr, +arangodb::traverser::TraverserOptions::nextCursorLocal(ManagedDocumentResult* mmdr, VPackSlice vertex, uint64_t depth, std::vector& list) const { TRI_ASSERT(mmdr != nullptr); auto allCursor = std::make_unique(mmdr, _trx, list.size()); @@ -810,19 +685,45 @@ TraverserOptions::nextCursorLocal(ManagedDocumentResult* mmdr, } arangodb::traverser::EdgeCursor* -TraverserOptions::nextCursorCoordinator( +arangodb::traverser::TraverserOptions::nextCursorCoordinator( VPackSlice vertex, uint64_t depth) const { TRI_ASSERT(_traverser != nullptr); auto cursor = std::make_unique(vertex, depth, _traverser); return cursor.release(); } -void TraverserOptions::linkTraverser( +void arangodb::traverser::TraverserOptions::clearVariableValues() { + _ctx->clearVariableValues(); +} + +void arangodb::traverser::TraverserOptions::setVariableValue( + aql::Variable const* var, aql::AqlValue const value) { + _ctx->setVariableValue(var, value); +} + +void arangodb::traverser::TraverserOptions::linkTraverser( arangodb::traverser::ClusterTraverser* trav) { _traverser = trav; } -double TraverserOptions::estimateCost(size_t& nrItems) const { +void arangodb::traverser::TraverserOptions::serializeVariables( + VPackBuilder& builder) const { + TRI_ASSERT(builder.isOpenArray()); + _ctx->serializeAllVariables(_trx, builder); +} + +double arangodb::traverser::TraverserOptions::costForLookupInfoList( + std::vector const& list, + size_t& createItems) const { + double cost = 0; + createItems = 0; + for (auto const& li : list) { + cost += li.estimateCost(createItems); + } + return cost; +} + +double arangodb::traverser::TraverserOptions::estimateCost(size_t& nrItems) const { size_t count = 1; double cost = 0; size_t baseCreateItems = 0; @@ -844,74 +745,3 @@ double TraverserOptions::estimateCost(size_t& nrItems) const { nrItems = count; return cost; } - -ShortestPathOptions::ShortestPathOptions(arangodb::aql::Query* query, - VPackSlice info, - VPackSlice collections, - VPackSlice reverseCollections) - : BaseTraverserOptions(query, info, collections), - _defaultWeight(1), - _weightAttribute("") { - VPackSlice read = info.get("reverseLookupInfos"); - - if (!read.isArray()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, - "The options require a reverseLookupInfos"); - } - size_t length = read.length(); - TRI_ASSERT(read.length() == reverseCollections.length()); - _reverseLookupInfos.reserve(length); - for (size_t j = 0; j < length; ++j) { - _reverseLookupInfos.emplace_back(query, read.at(j), reverseCollections.at(j)); - } - - read = info.get("weightAttribute"); - if (read.isString()) { - _weightAttribute = read.copyString(); - - read = info.get("defaultWeight"); - if (read.isNumber()) { - _defaultWeight = read.getNumericValue(); - } - } -} - -void ShortestPathOptions::toVelocyPack(VPackBuilder& builder) const { - VPackObjectBuilder guard(&builder); - builder.add("type", VPackValue("shortest")); - // FIXME -} - -void ShortestPathOptions::buildEngineInfo(VPackBuilder& result) const { - VPackObjectBuilder guard(&result); - BaseTraverserOptions::injectEngineInfo(result); - result.add(VPackValue("reverseLookupInfos")); - result.openArray(); - for (auto const& it : _reverseLookupInfos) { - it.buildEngineInfo(result); - } - result.close(); - if (usesWeight()) { - result.add("weightAttribute", VPackValue(weightAttribute())); - result.add("defaultWeight", VPackValue(defaultWeight())); - } - result.add("type", VPackValue("shortest")); -} - -void ShortestPathOptions::addReverseLookupInfo( - aql::Ast* ast, std::string const& collectionName, - std::string const& attributeName, aql::AstNode* condition) { - injectLookupInfoInList(_reverseLookupInfos, ast, collectionName, - attributeName, condition); -} - -double ShortestPathOptions::estimateCost(size_t& nrItems) const { - // We estimate the cost with the 7 degrees of separation: - // The shortest path between two vertices is most likely - // less than seven hops long. - size_t edgesCount = 0; - double baseCost = costForLookupInfoList(_baseLookupInfos, edgesCount); - baseCost = std::pow(baseCost, 7); - nrItems = static_cast(std::pow(edgesCount, 7)); - return nrItems; -} diff --git a/arangod/VocBase/TraverserOptions.h b/arangod/VocBase/TraverserOptions.h index 4bc2090c9c..3a0d685c9e 100644 --- a/arangod/VocBase/TraverserOptions.h +++ b/arangod/VocBase/TraverserOptions.h @@ -62,7 +62,12 @@ class EdgeCursor { size_t&) = 0; }; -struct BaseTraverserOptions { + +struct TraverserOptions { + friend class arangodb::aql::TraversalNode; + + public: + enum UniquenessLevel { NONE, PATH, GLOBAL }; protected: @@ -93,94 +98,17 @@ struct BaseTraverserOptions { }; - private: - aql::FixedVarExpressionContext* _ctx; - - protected: + public: transaction::Methods* _trx; + protected: std::vector _baseLookupInfos; - aql::Variable const* _tmpVar; - bool const _isCoordinator; - - public: - explicit BaseTraverserOptions(transaction::Methods* trx) - : _ctx(new aql::FixedVarExpressionContext()), - _trx(trx), - _tmpVar(nullptr), - _isCoordinator(arangodb::ServerState::instance()->isCoordinator()) {} - - /// @brief This copy constructor is only working during planning phase. - /// After planning this node should not be copied anywhere. - explicit BaseTraverserOptions(BaseTraverserOptions const&); - - BaseTraverserOptions(arangodb::aql::Query*, arangodb::velocypack::Slice, - arangodb::velocypack::Slice); - - virtual ~BaseTraverserOptions(); - - // Creates a complete Object containing all EngineInfo - // in the given builder. - virtual void buildEngineInfo(arangodb::velocypack::Builder&) const; - - void setVariable(aql::Variable const*); - - void addLookupInfo(aql::Ast* ast, std::string const& collectionName, - std::string const& attributeName, aql::AstNode* condition); - - void clearVariableValues(); - - void setVariableValue(aql::Variable const*, aql::AqlValue const); - - void serializeVariables(arangodb::velocypack::Builder&) const; - - transaction::Methods* trx() const; - - /// @brief Build a velocypack for cloning in the plan. - virtual void toVelocyPack(arangodb::velocypack::Builder&) const = 0; - - // Creates a complete Object containing all index information - // in the given builder. - virtual void toVelocyPackIndexes(arangodb::velocypack::Builder&) const; - - /// @brief Estimate the total cost for this operation - virtual double estimateCost(size_t& nrItems) const = 0; - - protected: - - double costForLookupInfoList(std::vector const& list, - size_t& createItems) const; - - // Requires an open Object in the given builder an - // will inject index information into it. - // Does not close the builder. - void injectVelocyPackIndexes(arangodb::velocypack::Builder&) const; - - // Requires an open Object in the given builder an - // will inject EngineInfo into it. - // Does not close the builder. - void injectEngineInfo(arangodb::velocypack::Builder&) const; - - aql::Expression* getEdgeExpression(size_t cursorId) const; - - bool evaluateExpression(aql::Expression*, arangodb::velocypack::Slice varValue) const; - - void injectLookupInfoInList(std::vector&, aql::Ast* ast, - std::string const& collectionName, - std::string const& attributeName, - aql::AstNode* condition); -}; - -struct TraverserOptions : public BaseTraverserOptions { - friend class arangodb::aql::TraversalNode; - - public: - enum UniquenessLevel { NONE, PATH, GLOBAL }; - - protected: - std::unordered_map> _depthLookupInfo; - std::unordered_map _vertexExpressions; + std::unordered_map> _depthLookupInfo; + std::unordered_map _vertexExpressions; aql::Expression* _baseVertexExpression; + aql::Variable const* _tmpVar; + aql::FixedVarExpressionContext* _ctx; arangodb::traverser::ClusterTraverser* _traverser; + bool const _isCoordinator; public: uint64_t minDepth; @@ -194,9 +122,12 @@ struct TraverserOptions : public BaseTraverserOptions { UniquenessLevel uniqueEdges; explicit TraverserOptions(transaction::Methods* trx) - : BaseTraverserOptions(trx), + : _trx(trx), _baseVertexExpression(nullptr), + _tmpVar(nullptr), + _ctx(new aql::FixedVarExpressionContext()), _traverser(nullptr), + _isCoordinator(trx->state()->isCoordinator()), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -215,14 +146,14 @@ struct TraverserOptions : public BaseTraverserOptions { virtual ~TraverserOptions(); /// @brief Build a velocypack for cloning in the plan. - void toVelocyPack(arangodb::velocypack::Builder&) const override; + void toVelocyPack(arangodb::velocypack::Builder&) const; /// @brief Build a velocypack for indexes - void toVelocyPackIndexes(arangodb::velocypack::Builder&) const override; + void toVelocyPackIndexes(arangodb::velocypack::Builder&) const; /// @brief Build a velocypack containing all relevant information /// for DBServer traverser engines. - void buildEngineInfo(arangodb::velocypack::Builder&) const override; + void buildEngineInfo(arangodb::velocypack::Builder&) const; bool vertexHasFilter(uint64_t) const; @@ -234,12 +165,21 @@ struct TraverserOptions : public BaseTraverserOptions { EdgeCursor* nextCursor(ManagedDocumentResult*, arangodb::velocypack::Slice, uint64_t) const; + void clearVariableValues(); + + void setVariableValue(aql::Variable const*, aql::AqlValue const); + void linkTraverser(arangodb::traverser::ClusterTraverser*); - double estimateCost(size_t& nrItems) const override; + void serializeVariables(arangodb::velocypack::Builder&) const; + + double estimateCost(size_t& nrItems) const; private: + double costForLookupInfoList(std::vector const& list, + size_t& createItems) const; + EdgeCursor* nextCursorLocal(ManagedDocumentResult*, arangodb::velocypack::Slice, uint64_t, std::vector&) const; @@ -247,73 +187,6 @@ struct TraverserOptions : public BaseTraverserOptions { EdgeCursor* nextCursorCoordinator(arangodb::velocypack::Slice, uint64_t) const; }; -struct ShortestPathOptions : public BaseTraverserOptions { - - protected: - - double _defaultWeight; - std::string _weightAttribute; - std::vector _reverseLookupInfos; - - public: - - enum DIRECTION {FORWARD, BACKWARD}; - - explicit ShortestPathOptions(transaction::Methods* trx) - : BaseTraverserOptions(trx), - _defaultWeight(1), - _weightAttribute("") {} - - ShortestPathOptions(arangodb::aql::Query*, arangodb::velocypack::Slice info, - arangodb::velocypack::Slice collections, - arangodb::velocypack::Slice reverseCollections); - - void setWeightAttribute(std::string const& attr) { - _weightAttribute = attr; - } - - void setDefaultWeight(double weight) { - _defaultWeight = weight; - } - - bool usesWeight() const { - return !_weightAttribute.empty(); - } - - std::string const weightAttribute() const { - return _weightAttribute; - } - - double defaultWeight() const { - return _defaultWeight; - } - - /// @brief Build a velocypack for cloning in the plan. - void toVelocyPack(arangodb::velocypack::Builder&) const override; - - /// @brief Build a velocypack containing all relevant information - /// for DBServer traverser engines. - void buildEngineInfo(arangodb::velocypack::Builder&) const override; - - double estimateCost(size_t& nrItems) const override; - - /// @brief Add a lookup info for reverse direction - void addReverseLookupInfo(aql::Ast* ast, std::string const& collectionName, - std::string const& attributeName, - aql::AstNode* condition); - - template - EdgeCursor* nextCursor(ManagedDocumentResult*, arangodb::velocypack::Slice) const; - - private: - - template - EdgeCursor* nextCursorLocal(ManagedDocumentResult*, - arangodb::velocypack::Slice, - std::vector&) const; - -}; - } } #endif From c2f370f99b3f5683c646778d03575462a25193d1 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 24 Mar 2017 13:39:05 -0400 Subject: [PATCH 10/27] Fixed an alignment bug in cache. --- arangod/Cache/Table.cpp | 14 ++++++++------ arangod/Cache/Table.h | 4 +++- tests/Cache/Table.cpp | 6 ++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/arangod/Cache/Table.cpp b/arangod/Cache/Table.cpp index b3ae8102b4..59a2f01c97 100644 --- a/arangod/Cache/Table.cpp +++ b/arangod/Cache/Table.cpp @@ -79,19 +79,23 @@ Table::Table(uint32_t logSize) _size(static_cast(1) << _logSize), _shift(32 - _logSize), _mask((_size - 1) << _shift), - _buckets(new GenericBucket[_size]), + _buffer(new uint8_t[(_size * BUCKET_SIZE) + Table::padding]), + _buckets(reinterpret_cast( + reinterpret_cast((_buffer.get() + 63)) & + ~(static_cast(0x3fU)))), _auxiliary(nullptr), _bucketClearer(defaultClearer), _slotsTotal(_size), - _slotsUsed(0) { + _slotsUsed(static_cast(0)) { _state.lock(); _state.toggleFlag(State::Flag::disabled); - memset(_buckets.get(), 0, BUCKET_SIZE * _size); + memset(_buckets, 0, BUCKET_SIZE * _size); _state.unlock(); } uint64_t Table::allocationSize(uint32_t logSize) { - return sizeof(Table) + (BUCKET_SIZE * (static_cast(1) << logSize)); + return sizeof(Table) + (BUCKET_SIZE * (static_cast(1) << logSize)) + + Table::padding; } uint64_t Table::memoryUsage() const { return Table::allocationSize(_logSize); } @@ -108,7 +112,6 @@ std::pair> Table::fetchAndLockBucket( if (ok) { ok = !_state.isSet(State::Flag::disabled); if (ok) { - TRI_ASSERT(_buckets.get() != nullptr); bucket = &(_buckets[(hash & _mask) >> _shift]); source = shared_from_this(); ok = bucket->lock(maxTries); @@ -154,7 +157,6 @@ void* Table::primaryBucket(uint32_t index) { if (!isEnabled()) { return nullptr; } - TRI_ASSERT(_buckets.get() != nullptr); return &(_buckets[index]); } diff --git a/arangod/Cache/Table.h b/arangod/Cache/Table.h index 1682a93699..73557708ee 100644 --- a/arangod/Cache/Table.h +++ b/arangod/Cache/Table.h @@ -43,6 +43,7 @@ class Table : public std::enable_shared_from_this { static const uint32_t maxLogSize; static constexpr uint32_t standardLogSizeAdjustment = 6; static constexpr int64_t triesGuarantee = -1; + static constexpr uint64_t padding = 64; typedef std::function BucketClearer; @@ -187,7 +188,8 @@ class Table : public std::enable_shared_from_this
{ uint64_t _size; uint32_t _shift; uint32_t _mask; - std::unique_ptr _buckets; + std::unique_ptr _buffer; + GenericBucket* _buckets; std::shared_ptr
_auxiliary; diff --git a/tests/Cache/Table.cpp b/tests/Cache/Table.cpp index fea325b7ba..d07fa60532 100644 --- a/tests/Cache/Table.cpp +++ b/tests/Cache/Table.cpp @@ -40,7 +40,8 @@ using namespace arangodb::cache; TEST_CASE("cache::Table", "[cache]") { SECTION("test static allocation size method") { for (uint32_t i = Table::minLogSize; i <= Table::maxLogSize; i++) { - REQUIRE(Table::allocationSize(i) == (sizeof(Table) + (BUCKET_SIZE << i))); + REQUIRE(Table::allocationSize(i) == + (sizeof(Table) + (BUCKET_SIZE << i) + Table::padding)); } } @@ -48,7 +49,8 @@ TEST_CASE("cache::Table", "[cache]") { for (uint32_t i = Table::minLogSize; i <= 20; i++) { auto table = std::make_shared
(i); REQUIRE(table.get() != nullptr); - REQUIRE(table->memoryUsage() == (sizeof(Table) + (BUCKET_SIZE << i))); + REQUIRE(table->memoryUsage() == + (sizeof(Table) + (BUCKET_SIZE << i) + Table::padding)); REQUIRE(table->logSize() == i); REQUIRE(table->size() == (static_cast(1) << i)); } From f3ce18b848a0b9ec5b5fa27d8c642a439ca84ad7 Mon Sep 17 00:00:00 2001 From: hkernbach Date: Sat, 25 Mar 2017 18:03:08 +0100 Subject: [PATCH 11/27] ui - fixed edge save bug due to api conversion --- .../_admin/aardvark/APP/frontend/js/views/documentView.js | 6 ++++-- .../_admin/aardvark/APP/frontend/js/views/graphViewer.js | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/documentView.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/documentView.js index ef9f0d063d..1ed371ca8b 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/views/documentView.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/documentView.js @@ -256,8 +256,10 @@ } model = JSON.stringify(model); + console.log(model); + console.log(this.type); - if (this.type._from && this.type._to) { + if (this.type === 'edge' || this.type._from) { var callbackE = function (error, data) { if (error) { arangoHelper.arangoError('Error', data.responseJSON.errorMessage); @@ -267,7 +269,7 @@ } }.bind(this); - this.collection.saveEdge(this.colid, this.docid, this.type._from, this.type._to, model, callbackE); + this.collection.saveEdge(this.colid, this.docid, $('#document-from').html(), $('#document-to').html(), model, callbackE); } else { var callback = function (error, data) { if (error) { diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphViewer.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphViewer.js index 2f54d853cb..dbe2abd46d 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphViewer.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphViewer.js @@ -2018,11 +2018,11 @@ var callback = function (error, data, id) { if (!error) { var attributes = ''; - attributes += 'ID ' + data._id + ''; - if (Object.keys(data).length > 3) { + attributes += 'ID ' + data.documents[0]._id + ''; + if (Object.keys(data.documents[0]).length > 3) { attributes += 'ATTRIBUTES '; } - _.each(data, function (value, key) { + _.each(data.documents[0], function (value, key) { if (key !== '_key' && key !== '_id' && key !== '_rev' && key !== '_from' && key !== '_to') { attributes += '' + key + ''; } From d4c2bba930cfeb2850c5c420f6d27f1c199e8294 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Mon, 27 Mar 2017 09:26:17 +0200 Subject: [PATCH 12/27] create mock primary mock index for rocksdb --- arangod/CMakeLists.txt | 1 + arangod/RocksDBEngine/RocksDBCollection.cpp | 8 +- arangod/RocksDBEngine/RocksDBCollection.h | 4 +- arangod/RocksDBEngine/RocksDBIndexFactory.cpp | 6 +- .../RocksDBEngine/RocksDBPrimaryMockIndex.cpp | 205 ++++++++++++++++++ .../RocksDBEngine/RocksDBPrimaryMockIndex.h | 166 ++++++++++++++ 6 files changed, 381 insertions(+), 9 deletions(-) create mode 100644 arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp create mode 100644 arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 0936c7beba..894cc50796 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -438,6 +438,7 @@ set(ARANGOD_SOURCES RocksDBEngine/RocksDBEntry.cpp RocksDBEngine/RocksDBIndexFactory.cpp RocksDBEngine/RocksDBPrimaryIndex.cpp + RocksDBEngine/RocksDBPrimaryMockIndex.cpp RocksDBEngine/RocksDBTransactionCollection.cpp RocksDBEngine/RocksDBTransactionState.cpp RocksDBEngine/RocksDBTypes.cpp diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index 0649358661..68ad84efe7 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -28,7 +28,7 @@ #include "Indexes/Index.h" #include "Indexes/IndexIterator.h" #include "RestServer/DatabaseFeature.h" -#include "RocksDBEngine/RocksDBPrimaryIndex.h" +#include "RocksDBEngine/RocksDBPrimaryMockIndex.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" #include "VocBase/LogicalCollection.h" @@ -423,7 +423,7 @@ void RocksDBCollection::addIndexCoordinator( int RocksDBCollection::saveIndex(transaction::Methods* trx, std::shared_ptr idx) { TRI_ASSERT(!ServerState::instance()->isCoordinator()); - // we cannot persist PrimaryIndex + // we cannot persist PrimaryMockIndex TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); std::vector> indexListLocal; indexListLocal.emplace_back(idx); @@ -450,7 +450,7 @@ int RocksDBCollection::saveIndex(transaction::Methods* trx, std::shared_ptrlookupIndex(0); TRI_ASSERT(primary != nullptr); @@ -467,5 +467,5 @@ arangodb::RocksDBPrimaryIndex* RocksDBCollection::primaryIndex() const { #endif TRI_ASSERT(primary->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); // the primary index must be the index at position #0 - return static_cast(primary.get()); + return static_cast(primary.get()); } diff --git a/arangod/RocksDBEngine/RocksDBCollection.h b/arangod/RocksDBEngine/RocksDBCollection.h index 791db70587..22f34d7014 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.h +++ b/arangod/RocksDBEngine/RocksDBCollection.h @@ -36,7 +36,7 @@ namespace arangodb { class LogicalCollection; class ManagedDocumentResult; class Result; -class RocksDBPrimaryIndex; +class RocksDBPrimaryMockIndex; class RocksDBCollection final : public PhysicalCollection { friend class RocksDBEngine; @@ -180,7 +180,7 @@ class RocksDBCollection final : public PhysicalCollection { void addIndex(std::shared_ptr idx); void addIndexCoordinator(std::shared_ptr idx); int saveIndex(transaction::Methods* trx, std::shared_ptr idx); - arangodb::RocksDBPrimaryIndex* primaryIndex() const; + arangodb::RocksDBPrimaryMockIndex* primaryIndex() const; private: uint64_t _objectId; // rocksdb-specific object id for collection diff --git a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp index 64d47364a9..2f09c51261 100644 --- a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp +++ b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp @@ -27,7 +27,7 @@ #include "Basics/VelocyPackHelper.h" #include "Indexes/Index.h" #include "RocksDBEngine/RocksDBEdgeIndex.h" -#include "RocksDBEngine/RocksDBPrimaryIndex.h" +#include "RocksDBEngine/RocksDBPrimaryMockIndex.h" #include "VocBase/voc-types.h" #include @@ -330,7 +330,7 @@ std::shared_ptr RocksDBIndexFactory::prepareIndexFromSlice( THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create primary index"); } - newIdx.reset(new arangodb::RocksDBPrimaryIndex(col)); + newIdx.reset(new arangodb::RocksDBPrimaryMockIndex(col)); break; } case arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX: { @@ -363,7 +363,7 @@ void RocksDBIndexFactory::fillSystemIndexes( std::vector>& systemIndexes) const { // create primary index systemIndexes.emplace_back( - std::make_shared(col)); + std::make_shared(col)); // create edges index if (col->type() == TRI_COL_TYPE_EDGE) { diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp new file mode 100644 index 0000000000..aa40ac85b8 --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp @@ -0,0 +1,205 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +//////////////////////////////////////////////////////////////////////////////// + +#include "RocksDBPrimaryMockIndex.h" +#include "Aql/AstNode.h" +#include "Basics/Exceptions.h" +#include "Basics/StaticStrings.h" +#include "Indexes/SimpleAttributeEqualityMatcher.h" +#include "Transaction/Helpers.h" +#include "Transaction/Methods.h" +#include "Transaction/Context.h" +#include "VocBase/LogicalCollection.h" + +#include +#include +#include +#include + +using namespace arangodb; + +/// @brief hard-coded vector of the index attributes +/// note that the attribute names must be hard-coded here to avoid an init-order +/// fiasco with StaticStrings::FromString etc. +static std::vector> const IndexAttributes + {{arangodb::basics::AttributeName("_id", false)}, + {arangodb::basics::AttributeName("_key", false)}}; + +RocksDBPrimaryMockIndexIterator::RocksDBPrimaryMockIndexIterator(LogicalCollection* collection, + transaction::Methods* trx, + ManagedDocumentResult* mmdr, + RocksDBPrimaryMockIndex const* index, + std::unique_ptr& keys) + : IndexIterator(collection, trx, mmdr, index), + _index(index), + _keys(keys.get()), + _iterator(_keys->slice()) { + + keys.release(); // now we have ownership for _keys + TRI_ASSERT(_keys->slice().isArray()); +} + +RocksDBPrimaryMockIndexIterator::~RocksDBPrimaryMockIndexIterator() { + if (_keys != nullptr) { + // return the VPackBuilder to the transaction context + _trx->transactionContextPtr()->returnBuilder(_keys.release()); + } +} + +bool RocksDBPrimaryMockIndexIterator::next(TokenCallback const& cb, size_t limit) { + THROW_ARANGO_NOT_YET_IMPLEMENTED(); + return false; +} + +void RocksDBPrimaryMockIndexIterator::reset() { _iterator.reset(); } + +RocksDBAllIndexIterator::RocksDBAllIndexIterator(LogicalCollection* collection, + transaction::Methods* trx, + ManagedDocumentResult* mmdr, + RocksDBPrimaryMockIndex const* index, + bool reverse) + : IndexIterator(collection, trx, mmdr, index), _reverse(reverse), _total(0) {} + +bool RocksDBAllIndexIterator::next(TokenCallback const& cb, size_t limit) { + // TODO + return false; +} + +void RocksDBAllIndexIterator::reset() { + // TODO +} + +RocksDBAnyIndexIterator::RocksDBAnyIndexIterator(LogicalCollection* collection, transaction::Methods* trx, + ManagedDocumentResult* mmdr, + RocksDBPrimaryMockIndex const* index) + : IndexIterator(collection, trx, mmdr, index) {} + +bool RocksDBAnyIndexIterator::next(TokenCallback const& cb, size_t limit) { + THROW_ARANGO_NOT_YET_IMPLEMENTED(); + return true; +} + +void RocksDBAnyIndexIterator::reset() { + THROW_ARANGO_NOT_YET_IMPLEMENTED(); +} + +RocksDBPrimaryMockIndex::RocksDBPrimaryMockIndex(arangodb::LogicalCollection* collection) + : Index(0, collection, + std::vector>( + {{arangodb::basics::AttributeName(StaticStrings::KeyString, false)}}), + true, false) { +} + +RocksDBPrimaryMockIndex::~RocksDBPrimaryMockIndex() {} + +/// @brief return the number of documents from the index +size_t RocksDBPrimaryMockIndex::size() const { + // TODO + return 0; +} + +/// @brief return the memory usage of the index +size_t RocksDBPrimaryMockIndex::memory() const { + return 0; // TODO +} + +/// @brief return a VelocyPack representation of the index +void RocksDBPrimaryMockIndex::toVelocyPack(VPackBuilder& builder, bool withFigures) const { + Index::toVelocyPack(builder, withFigures); + // hard-coded + builder.add("unique", VPackValue(true)); + builder.add("sparse", VPackValue(false)); +} + +/// @brief return a VelocyPack representation of the index figures +void RocksDBPrimaryMockIndex::toVelocyPackFigures(VPackBuilder& builder) const { + Index::toVelocyPackFigures(builder); + // TODO: implement +} + +int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t, VPackSlice const&, bool) { +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "insert() called for primary index"; +#endif + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "insert() called for primary index"); +} + +int RocksDBPrimaryMockIndex::remove(transaction::Methods*, TRI_voc_rid_t, VPackSlice const&, bool) { +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "remove() called for primary index"; +#endif + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "remove() called for primary index"); +} + +/// @brief unload the index data from memory +int RocksDBPrimaryMockIndex::unload() { + // nothing to do + return TRI_ERROR_NO_ERROR; +} + +/// @brief checks whether the index supports the condition +bool RocksDBPrimaryMockIndex::supportsFilterCondition( + arangodb::aql::AstNode const* node, + arangodb::aql::Variable const* reference, size_t itemsInIndex, + size_t& estimatedItems, double& estimatedCost) const { + + SimpleAttributeEqualityMatcher matcher(IndexAttributes); + return matcher.matchOne(this, node, reference, itemsInIndex, estimatedItems, + estimatedCost); +} + +/// @brief creates an IndexIterator for the given Condition +IndexIterator* RocksDBPrimaryMockIndex::iteratorForCondition( + transaction::Methods* trx, + ManagedDocumentResult* mmdr, + arangodb::aql::AstNode const* node, + arangodb::aql::Variable const* reference, bool reverse) const { + + THROW_ARANGO_NOT_YET_IMPLEMENTED(); + return nullptr; +} + +/// @brief specializes the condition for use with the index +arangodb::aql::AstNode* RocksDBPrimaryMockIndex::specializeCondition( + arangodb::aql::AstNode* node, + arangodb::aql::Variable const* reference) const { + + SimpleAttributeEqualityMatcher matcher(IndexAttributes); + return matcher.specializeOne(this, node, reference); +} + +/// @brief request an iterator over all elements in the index in +/// a sequential order. +IndexIterator* RocksDBPrimaryMockIndex::allIterator(transaction::Methods* trx, + ManagedDocumentResult* mmdr, + bool reverse) const { + return new RocksDBAllIndexIterator(_collection, trx, mmdr, this, reverse); +} + +/// @brief request an iterator over all elements in the index in +/// a random order. It is guaranteed that each element is found +/// exactly once unless the collection is modified. +IndexIterator* RocksDBPrimaryMockIndex::anyIterator(transaction::Methods* trx, + ManagedDocumentResult* mmdr) const { + return new RocksDBAnyIndexIterator(_collection, trx, mmdr, this); +} diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h new file mode 100644 index 0000000000..08b7ca4e75 --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h @@ -0,0 +1,166 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_ROCKSDB_ENGINE_ROCKSDB_PRIMARY_INDEX_H +#define ARANGOD_ROCKSDB_ENGINE_ROCKSDB_PRIMARY_INDEX_H 1 + +#include "Basics/Common.h" +#include "Indexes/Index.h" +#include "Indexes/IndexIterator.h" +#include "VocBase/vocbase.h" +#include "VocBase/voc-types.h" + +#include +#include +#include + +namespace arangodb { + +class RocksDBPrimaryMockIndex; +namespace transaction { +class Methods; +} + +class RocksDBPrimaryMockIndexIterator final : public IndexIterator { + public: + RocksDBPrimaryMockIndexIterator(LogicalCollection* collection, + transaction::Methods* trx, + ManagedDocumentResult* mmdr, + RocksDBPrimaryMockIndex const* index, + std::unique_ptr& keys); + + ~RocksDBPrimaryMockIndexIterator(); + + char const* typeName() const override { return "primary-index-iterator"; } + + bool next(TokenCallback const& cb, size_t limit) override; + + void reset() override; + + private: + RocksDBPrimaryMockIndex const* _index; + std::unique_ptr _keys; + arangodb::velocypack::ArrayIterator _iterator; +}; + +class RocksDBAllIndexIterator final : public IndexIterator { + public: + RocksDBAllIndexIterator(LogicalCollection* collection, + transaction::Methods* trx, + ManagedDocumentResult* mmdr, + RocksDBPrimaryMockIndex const* index, + bool reverse); + + ~RocksDBAllIndexIterator() {} + + char const* typeName() const override { return "all-index-iterator"; } + + bool next(TokenCallback const& cb, size_t limit) override; + + void reset() override; + + private: + bool const _reverse; + uint64_t _total; +}; + +class RocksDBAnyIndexIterator final : public IndexIterator { + public: + RocksDBAnyIndexIterator(LogicalCollection* collection, transaction::Methods* trx, + ManagedDocumentResult* mmdr, + RocksDBPrimaryMockIndex const* index); + + ~RocksDBAnyIndexIterator() {} + + char const* typeName() const override { return "any-index-iterator"; } + + bool next(TokenCallback const& cb, size_t limit) override; + + void reset() override; +}; + +class RocksDBPrimaryMockIndex final : public Index { + friend class RocksDBPrimaryMockIndexIterator; + + public: + RocksDBPrimaryMockIndex() = delete; + + explicit RocksDBPrimaryMockIndex(arangodb::LogicalCollection*); + + ~RocksDBPrimaryMockIndex(); + + public: + IndexType type() const override { + return Index::TRI_IDX_TYPE_PRIMARY_INDEX; + } + + char const* typeName() const override { return "primary"; } + + bool allowExpansion() const override { return false; } + + bool canBeDropped() const override { return false; } + + bool isSorted() const override { return false; } + + bool hasSelectivityEstimate() const override { return true; } + + double selectivityEstimate(arangodb::StringRef const* = nullptr) const override { return 1.0; } + + size_t size() const; + + size_t memory() const override; + + void toVelocyPack(VPackBuilder&, bool) const override; + void toVelocyPackFigures(VPackBuilder&) const override; + + int insert(transaction::Methods*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; + + int remove(transaction::Methods*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; + + int unload() override; + + bool supportsFilterCondition(arangodb::aql::AstNode const*, + arangodb::aql::Variable const*, size_t, size_t&, + double&) const override; + + IndexIterator* iteratorForCondition(transaction::Methods*, + ManagedDocumentResult*, + arangodb::aql::AstNode const*, + arangodb::aql::Variable const*, + bool) const override; + + arangodb::aql::AstNode* specializeCondition( + arangodb::aql::AstNode*, arangodb::aql::Variable const*) const override; + + /// @brief request an iterator over all elements in the index in + /// a sequential order. + IndexIterator* allIterator(transaction::Methods*, ManagedDocumentResult*, bool reverse) const; + + /// @brief request an iterator over all elements in the index in + /// a random order. It is guaranteed that each element is found + /// exactly once unless the collection is modified. + IndexIterator* anyIterator(transaction::Methods*, ManagedDocumentResult*) const; +}; +} + +#endif From 6ca2b237d153451541a58a423f12c699df6124cd Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 11:01:09 +0200 Subject: [PATCH 13/27] add method to augment index creation data --- arangod/RocksDBEngine/RocksDBEngine.cpp | 6 ++++++ arangod/RocksDBEngine/RocksDBEngine.h | 1 + arangod/StorageEngine/StorageEngine.h | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index d18e86fa81..9e90bd268c 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -155,6 +155,12 @@ void RocksDBEngine::addParametersForNewCollection(VPackBuilder& builder, VPackSl } } +void RocksDBEngine::addParametersForNewIndex(VPackBuilder& builder, VPackSlice info) { + if (!info.hasKey("objectId")) { + builder.add("objectId", VPackValue(std::to_string(TRI_NewTickServer()))); + } +} + // create storage-engine specific collection PhysicalCollection* RocksDBEngine::createPhysicalCollection(LogicalCollection* collection, VPackSlice const& info) { diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index 303cc88098..d004d6f17d 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -226,6 +226,7 @@ class RocksDBEngine final : public StorageEngine { void addRestHandlers(rest::RestHandlerFactory*) override; void addParametersForNewCollection(arangodb::velocypack::Builder& builder, arangodb::velocypack::Slice info) override; + void addParametersForNewIndex(arangodb::velocypack::Builder& builder, arangodb::velocypack::Slice info) override; rocksdb::TransactionDB* db() const { return _db; } diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h index 3e3f04fa31..3c303fcc25 100644 --- a/arangod/StorageEngine/StorageEngine.h +++ b/arangod/StorageEngine/StorageEngine.h @@ -92,6 +92,10 @@ class StorageEngine : public application_features::ApplicationFeature { // when a new collection is created, this method is called to augment the collection // creation data with engine-specific information virtual void addParametersForNewCollection(VPackBuilder& builder, VPackSlice info) {} + + // when a new index is created, this method is called to augment the index + // creation data with engine-specific information + virtual void addParametersForNewIndex(VPackBuilder& builder, VPackSlice info) {} // create storage-engine specific collection virtual PhysicalCollection* createPhysicalCollection(LogicalCollection*, VPackSlice const&) = 0; From 3487097180ab3df1aeb5d755a296929e9b2a7d0b Mon Sep 17 00:00:00 2001 From: Frank Celler Date: Mon, 27 Mar 2017 10:47:56 +0200 Subject: [PATCH 14/27] added ulimit example --- arangod/RestServer/FileDescriptorsFeature.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arangod/RestServer/FileDescriptorsFeature.cpp b/arangod/RestServer/FileDescriptorsFeature.cpp index 2e4e23c2cc..06f54b7583 100644 --- a/arangod/RestServer/FileDescriptorsFeature.cpp +++ b/arangod/RestServer/FileDescriptorsFeature.cpp @@ -82,8 +82,8 @@ void FileDescriptorsFeature::start() { if (rlim.rlim_cur < RECOMMENDED) { LOG_TOPIC(WARN, arangodb::Logger::SYSCALL) << "file-descriptors limit is too low, currently " - << StringifyLimitValue(rlim.rlim_cur) << ", raise to at least " - << RECOMMENDED; + << StringifyLimitValue(rlim.rlim_cur) << ", please raise to at least " + << RECOMMENDED << " (e.g. ulimit -n " << RECOMMENDED << ")"; } #endif } From 78d586e670d57433e3435856b5ac2f17fbac0218 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 11:01:56 +0200 Subject: [PATCH 15/27] fix typo --- arangod/MMFiles/MMFilesCollection.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index 96d0b14db6..a8ebd5607d 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -2588,7 +2588,6 @@ int MMFilesCollection::insert(transaction::Methods* trx, } } - transaction::BuilderLeaser builder(trx); VPackSlice newSlice; int res = TRI_ERROR_NO_ERROR; @@ -2825,7 +2824,7 @@ int MMFilesCollection::insertSecondaryIndexes(arangodb::transaction::Methods* tr TRI_voc_rid_t revisionId, VPackSlice const& doc, bool isRollback) { - // Coordintor doesn't know index internals + // Coordinator doesn't know index internals TRI_ASSERT(!ServerState::instance()->isCoordinator()); TRI_IF_FAILURE("InsertSecondaryIndexes") { return TRI_ERROR_DEBUG; } From 00499c6a7a2175168a4bbad6e6f9debc151b4101 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 11:15:42 +0200 Subject: [PATCH 16/27] stub for collection::insert() --- arangod/RocksDBEngine/RocksDBCollection.cpp | 119 +++++++++++++++++++- arangod/RocksDBEngine/RocksDBCollection.h | 6 +- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index 68ad84efe7..1be1dacb5e 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -23,17 +23,23 @@ #include "RocksDBCollection.h" #include "Basics/Result.h" +#include "Basics/StaticStrings.h" #include "Aql/PlanCache.h" #include "Basics/VelocyPackHelper.h" #include "Indexes/Index.h" #include "Indexes/IndexIterator.h" #include "RestServer/DatabaseFeature.h" +#include "RocksDBEngine/RocksDBEngine.h" +#include "RocksDBEngine/RocksDBEntry.h" #include "RocksDBEngine/RocksDBPrimaryMockIndex.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" +#include "Transaction/Helpers.h" +#include "Utils/OperationOptions.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" +#include #include #include @@ -320,12 +326,72 @@ bool RocksDBCollection::readDocumentConditional( } int RocksDBCollection::insert(arangodb::transaction::Methods* trx, - arangodb::velocypack::Slice const newSlice, + arangodb::velocypack::Slice const slice, arangodb::ManagedDocumentResult& result, OperationOptions& options, - TRI_voc_tick_t& resultMarkerTick, bool lock) { - THROW_ARANGO_NOT_YET_IMPLEMENTED(); - return 0; + TRI_voc_tick_t& resultMarkerTick, bool /*lock*/) { + VPackSlice fromSlice; + VPackSlice toSlice; + + bool const isEdgeCollection = + (_logicalCollection->type() == TRI_COL_TYPE_EDGE); + + if (isEdgeCollection) { + // _from: + fromSlice = slice.get(StaticStrings::FromString); + if (!fromSlice.isString()) { + return TRI_ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE; + } + VPackValueLength len; + char const* docId = fromSlice.getString(len); + size_t split; + if (!TRI_ValidateDocumentIdKeyGenerator(docId, static_cast(len), + &split)) { + return TRI_ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE; + } + // _to: + toSlice = slice.get(StaticStrings::ToString); + if (!toSlice.isString()) { + return TRI_ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE; + } + docId = toSlice.getString(len); + if (!TRI_ValidateDocumentIdKeyGenerator(docId, static_cast(len), + &split)) { + return TRI_ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE; + } + } + + transaction::BuilderLeaser builder(trx); + VPackSlice newSlice; + int res = TRI_ERROR_NO_ERROR; + if (options.recoveryData == nullptr) { + res = newObjectForInsert(trx, slice, fromSlice, toSlice, isEdgeCollection, + *builder.get(), options.isRestore); + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + newSlice = builder->slice(); + } else { + TRI_ASSERT(slice.isObject()); + // we can get away with the fast hash function here, as key values are + // restricted to strings + newSlice = slice; + } + + TRI_voc_rid_t revisionId = transaction::helpers::extractRevFromDocument(newSlice); + + res = insertDocument(trx, revisionId, newSlice, options.waitForSync); + + if (res == TRI_ERROR_NO_ERROR) { +// uint8_t const* vpack = lookupRevisionVPack(revisionId); +// if (vpack != nullptr) { +// result.addExisting(vpack, revisionId); +// } + + // store the tick that was used for writing the document +// resultMarkerTick = operation.tick(); + } + return res; } int RocksDBCollection::update(arangodb::transaction::Methods* trx, @@ -469,3 +535,48 @@ arangodb::RocksDBPrimaryMockIndex* RocksDBCollection::primaryIndex() const { // the primary index must be the index at position #0 return static_cast(primary.get()); } + +int RocksDBCollection::insertDocument(arangodb::transaction::Methods* trx, + TRI_voc_rid_t revisionId, + VPackSlice const& doc, + bool& waitForSync) { + // Coordinator doesn't know index internals + TRI_ASSERT(!ServerState::instance()->isCoordinator()); + + RocksDBEntry entry(RocksDBEntry::Document(_objectId, revisionId, doc)); + + rocksdb::WriteBatch writeBatch; + writeBatch.Put(entry.key(), entry.value()); + + auto indexes = _indexes; + size_t const n = indexes.size(); + + int result = TRI_ERROR_NO_ERROR; + + for (size_t i = 0; i < n; ++i) { + auto idx = indexes[i]; + + int res = idx->insert(trx, revisionId, doc, false); + + // in case of no-memory, return immediately + if (res == TRI_ERROR_OUT_OF_MEMORY) { + return res; + } + if (res != TRI_ERROR_NO_ERROR) { + if (res == TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED || + result == TRI_ERROR_NO_ERROR) { + // "prefer" unique constraint violated + result = res; + } + } + } + + // TODO: handle waitForSync here? + if (result != TRI_ERROR_NO_ERROR) { + StorageEngine* engine = EngineSelectorFeature::ENGINE; + rocksdb::TransactionDB* db = static_cast(engine)->db(); + db->Write(rocksdb::WriteOptions(), &writeBatch); + } + + return result; +} diff --git a/arangod/RocksDBEngine/RocksDBCollection.h b/arangod/RocksDBEngine/RocksDBCollection.h index 22f34d7014..c61cf640e4 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.h +++ b/arangod/RocksDBEngine/RocksDBCollection.h @@ -31,7 +31,6 @@ #include "VocBase/ManagedDocumentResult.h" #include "VocBase/PhysicalCollection.h" - namespace arangodb { class LogicalCollection; class ManagedDocumentResult; @@ -182,6 +181,11 @@ class RocksDBCollection final : public PhysicalCollection { int saveIndex(transaction::Methods* trx, std::shared_ptr idx); arangodb::RocksDBPrimaryMockIndex* primaryIndex() const; + int insertDocument(arangodb::transaction::Methods* trx, + TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const& doc, + bool& waitForSync); + private: uint64_t _objectId; // rocksdb-specific object id for collection }; From 9276d8257852e58d88c99742d1b3190e09ade14e Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 11:20:43 +0200 Subject: [PATCH 17/27] remove debug log messages --- arangod/RocksDBEngine/RocksDBTransactionCollection.cpp | 10 ++-------- arangod/RocksDBEngine/RocksDBTransactionState.cpp | 10 ++-------- arangod/StorageEngine/TransactionState.cpp | 5 ----- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp b/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp index d2bd1e7039..5980a2891c 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp @@ -40,13 +40,9 @@ RocksDBTransactionCollection::RocksDBTransactionCollection(TransactionState* trx : TransactionCollection(trx, cid), _waitForSync(false), _accessType(accessType), - _numOperations(0) { - LOG_TOPIC(ERR, Logger::FIXME) << "ctor rocksdb transaction collection: " << cid; -} + _numOperations(0) {} -RocksDBTransactionCollection::~RocksDBTransactionCollection() { - LOG_TOPIC(ERR, Logger::FIXME) << "dtor rocksdb transaction collection: " << _cid; -} +RocksDBTransactionCollection::~RocksDBTransactionCollection() {} /// @brief request a main-level lock for a collection int RocksDBTransactionCollection::lock() { return TRI_ERROR_NO_ERROR; } @@ -131,7 +127,6 @@ int RocksDBTransactionCollection::use(int nestingLevel) { TRI_vocbase_col_status_e status; LOG_TRX(_transaction, nestingLevel) << "using collection " << _cid; _collection = _transaction->vocbase()->useCollection(_cid, status); - LOG_TOPIC(ERR, Logger::FIXME) << "using collection " << _cid << ": " << _collection; if (_collection == nullptr) { return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND; @@ -155,7 +150,6 @@ void RocksDBTransactionCollection::unuse(int /*nestingLevel*/) {} void RocksDBTransactionCollection::release() { // the top level transaction releases all collections if (_collection != nullptr) { - LOG_TOPIC(ERR, Logger::FIXME) << "releasing collection " << _cid << ": " << _collection; // unuse collection, remove usage-lock LOG_TRX(_transaction, 0) << "unusing collection " << _cid; diff --git a/arangod/RocksDBEngine/RocksDBTransactionState.cpp b/arangod/RocksDBEngine/RocksDBTransactionState.cpp index 3951330f47..a19648d52f 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionState.cpp +++ b/arangod/RocksDBEngine/RocksDBTransactionState.cpp @@ -51,14 +51,10 @@ struct RocksDBTransactionData final : public TransactionData { RocksDBTransactionState::RocksDBTransactionState(TRI_vocbase_t* vocbase) : TransactionState(vocbase), _beginWritten(false), - _hasOperations(false) { - LOG_TOPIC(ERR, Logger::FIXME) << "ctor rocksdb transaction state: " << this; -} + _hasOperations(false) {} /// @brief free a transaction container -RocksDBTransactionState::~RocksDBTransactionState() { - LOG_TOPIC(ERR, Logger::FIXME) << "dtor rocksdb transaction state: " << this; -} +RocksDBTransactionState::~RocksDBTransactionState() {} /// @brief start a transaction int RocksDBTransactionState::beginTransaction(transaction::Hints hints) { @@ -85,8 +81,6 @@ int RocksDBTransactionState::beginTransaction(transaction::Hints hints) { int res = useCollections(_nestingLevel); - LOG_TOPIC(ERR, Logger::FIXME) << "USE COLLECTIONS RETURNED: " << res << ", NESTING: " << _nestingLevel; - if (res == TRI_ERROR_NO_ERROR) { // all valid if (_nestingLevel == 0) { diff --git a/arangod/StorageEngine/TransactionState.cpp b/arangod/StorageEngine/TransactionState.cpp index 7a56859362..670ffd1ab0 100644 --- a/arangod/StorageEngine/TransactionState.cpp +++ b/arangod/StorageEngine/TransactionState.cpp @@ -93,7 +93,6 @@ TransactionCollection* TransactionState::collection(TRI_voc_cid_t cid, AccessMod int TransactionState::addCollection(TRI_voc_cid_t cid, AccessMode::Type accessType, int nestingLevel, bool force) { - LOG_TOPIC(ERR, Logger::FIXME) << "add collection: " << cid << ", " << this; LOG_TRX(this, nestingLevel) << "adding collection " << cid; // LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "cid: " << cid @@ -139,9 +138,7 @@ int TransactionState::addCollection(TRI_voc_cid_t cid, TRI_ASSERT(trxCollection == nullptr); StorageEngine* engine = EngineSelectorFeature::ENGINE; - LOG_TOPIC(ERR, Logger::FIXME) << "creating trx collection: " << cid << ", " << this; trxCollection = engine->createTransactionCollection(this, cid, accessType, nestingLevel); - LOG_TOPIC(ERR, Logger::FIXME) << "created trx collection: " << cid << ", " << this << "; " << trxCollection; TRI_ASSERT(trxCollection != nullptr); @@ -166,8 +163,6 @@ int TransactionState::ensureCollections(int nestingLevel) { int TransactionState::useCollections(int nestingLevel) { int res = TRI_ERROR_NO_ERROR; - LOG_TOPIC(ERR, Logger::FIXME) << "use collections " << this; - // process collections in forward order for (auto& trxCollection : _collections) { res = trxCollection->use(nestingLevel); From 0d38d2c287d9297f5b95c8f75af4374e84ee1938 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 11:30:30 +0200 Subject: [PATCH 18/27] finalize insert() --- arangod/RocksDBEngine/RocksDBCollection.cpp | 24 +++++++++++++++---- .../RocksDBTransactionCollection.cpp | 4 ---- .../RocksDBTransactionCollection.h | 2 -- arangod/Transaction/Methods.cpp | 1 - 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index 1be1dacb5e..01f0650ff1 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -34,6 +34,7 @@ #include "RocksDBEngine/RocksDBPrimaryMockIndex.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" +#include "StorageEngine/TransactionState.h" #include "Transaction/Helpers.h" #include "Utils/OperationOptions.h" #include "VocBase/LogicalCollection.h" @@ -389,7 +390,8 @@ int RocksDBCollection::insert(arangodb::transaction::Methods* trx, // } // store the tick that was used for writing the document -// resultMarkerTick = operation.tick(); + // note that we don't need it for this engine + resultMarkerTick = 0; } return res; } @@ -571,11 +573,25 @@ int RocksDBCollection::insertDocument(arangodb::transaction::Methods* trx, } } - // TODO: handle waitForSync here? - if (result != TRI_ERROR_NO_ERROR) { + if (result != TRI_ERROR_NO_ERROR) { + rocksdb::WriteOptions writeOptions; + + if (_logicalCollection->waitForSync()) { + waitForSync = true; + } + + if (waitForSync) { + trx->state()->waitForSync(true); + + // handle waitForSync for single operations here + if (trx->state()->isSingleOperation()) { + writeOptions.sync = true; + } + } + StorageEngine* engine = EngineSelectorFeature::ENGINE; rocksdb::TransactionDB* db = static_cast(engine)->db(); - db->Write(rocksdb::WriteOptions(), &writeBatch); + db->Write(writeOptions, &writeBatch); } return result; diff --git a/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp b/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp index 5980a2891c..04ee463332 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBTransactionCollection.cpp @@ -38,7 +38,6 @@ RocksDBTransactionCollection::RocksDBTransactionCollection(TransactionState* trx TRI_voc_cid_t cid, AccessMode::Type accessType) : TransactionCollection(trx, cid), - _waitForSync(false), _accessType(accessType), _numOperations(0) {} @@ -137,9 +136,6 @@ int RocksDBTransactionCollection::use(int nestingLevel) { !LogicalCollection::IsSystemName(_collection->name())) { return TRI_ERROR_ARANGO_READ_ONLY; } - - // store the waitForSync property - _waitForSync = _collection->waitForSync(); } return TRI_ERROR_NO_ERROR; diff --git a/arangod/RocksDBEngine/RocksDBTransactionCollection.h b/arangod/RocksDBEngine/RocksDBTransactionCollection.h index 4c32ed71c4..610cdcb0d6 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionCollection.h +++ b/arangod/RocksDBEngine/RocksDBTransactionCollection.h @@ -70,8 +70,6 @@ class RocksDBTransactionCollection final : public TransactionCollection { void release() override; private: - bool _waitForSync; // whether or not the collection has waitForSync - AccessMode::Type _accessType; // access type (read|write) uint64_t _numOperations; }; diff --git a/arangod/Transaction/Methods.cpp b/arangod/Transaction/Methods.cpp index 7c75c30c22..de4f3673eb 100644 --- a/arangod/Transaction/Methods.cpp +++ b/arangod/Transaction/Methods.cpp @@ -2578,7 +2578,6 @@ arangodb::LogicalCollection* transaction::Methods::documentCollection( TRI_ASSERT(trxCollection != nullptr); TRI_ASSERT(_state->status() == transaction::Status::RUNNING); - LOG_TOPIC(ERR, Logger::FIXME) << "accessing collection " << trxCollection->id() << ": " << trxCollection; TRI_ASSERT(trxCollection->collection() != nullptr); return trxCollection->collection(); From 91b868a01db644fdd6064536c0e680eaba9f2066 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Mon, 27 Mar 2017 11:32:26 +0200 Subject: [PATCH 19/27] add some code for creating the fake rocksdb index --- arangod/RocksDBEngine/RocksDBIndexFactory.cpp | 7 ++++-- .../RocksDBEngine/RocksDBPrimaryMockIndex.cpp | 24 ++++++++++++------- .../RocksDBEngine/RocksDBPrimaryMockIndex.h | 9 ++++++- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp index 2f09c51261..f311feb807 100644 --- a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp +++ b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp @@ -330,7 +330,7 @@ std::shared_ptr RocksDBIndexFactory::prepareIndexFromSlice( THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create primary index"); } - newIdx.reset(new arangodb::RocksDBPrimaryMockIndex(col)); + newIdx.reset(new arangodb::RocksDBPrimaryMockIndex(col,info)); break; } case arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX: { @@ -362,8 +362,11 @@ void RocksDBIndexFactory::fillSystemIndexes( arangodb::LogicalCollection* col, std::vector>& systemIndexes) const { // create primary index + + VPackBuilder builder; systemIndexes.emplace_back( - std::make_shared(col)); + std::make_shared(col, builder.slice())); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid index type"); // create edges index if (col->type() == TRI_COL_TYPE_EDGE) { diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp index aa40ac85b8..7f7c94f1a1 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp @@ -22,9 +22,11 @@ //////////////////////////////////////////////////////////////////////////////// #include "RocksDBPrimaryMockIndex.h" +#include "RocksDBEntry.h" #include "Aql/AstNode.h" #include "Basics/Exceptions.h" #include "Basics/StaticStrings.h" +#include "Basics/VelocyPackHelper.h" #include "Indexes/SimpleAttributeEqualityMatcher.h" #include "Transaction/Helpers.h" #include "Transaction/Methods.h" @@ -36,6 +38,8 @@ #include #include +#include + using namespace arangodb; /// @brief hard-coded vector of the index attributes @@ -103,11 +107,12 @@ void RocksDBAnyIndexIterator::reset() { THROW_ARANGO_NOT_YET_IMPLEMENTED(); } -RocksDBPrimaryMockIndex::RocksDBPrimaryMockIndex(arangodb::LogicalCollection* collection) - : Index(0, collection, +RocksDBPrimaryMockIndex::RocksDBPrimaryMockIndex(arangodb::LogicalCollection* collection, VPackSlice const& info) + : Index(basics::VelocyPackHelper::stringUInt64(info, "objectId"), collection, std::vector>( {{arangodb::basics::AttributeName(StaticStrings::KeyString, false)}}), - true, false) { + true, false), + _objectId(basics::VelocyPackHelper::stringUInt64(info, "objectId")){ } RocksDBPrimaryMockIndex::~RocksDBPrimaryMockIndex() {} @@ -137,11 +142,14 @@ void RocksDBPrimaryMockIndex::toVelocyPackFigures(VPackBuilder& builder) const { // TODO: implement } -int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t, VPackSlice const&, bool) { -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "insert() called for primary index"; -#endif - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "insert() called for primary index"); +int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revision_id, VPackSlice const& slice, bool) { +// #ifdef ARANGODB_ENABLE_MAINTAINER_MODE +// LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "insert() called for primary index"; +// #endif +// // THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "insert() called for primary index"); + auto value = RocksDBEntry::IndexValue(objectId(), revision_id, slice); + std::lock_guard lock(_keyRevMutex); + _keyRevMap.emplace(value.key(),value.revisionId()); } int RocksDBPrimaryMockIndex::remove(transaction::Methods*, TRI_voc_rid_t, VPackSlice const&, bool) { diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h index 08b7ca4e75..e2bf197750 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h @@ -34,6 +34,8 @@ #include #include +#include + namespace arangodb { class RocksDBPrimaryMockIndex; @@ -105,7 +107,7 @@ class RocksDBPrimaryMockIndex final : public Index { public: RocksDBPrimaryMockIndex() = delete; - explicit RocksDBPrimaryMockIndex(arangodb::LogicalCollection*); + explicit RocksDBPrimaryMockIndex(arangodb::LogicalCollection*, VPackSlice const& info); ~RocksDBPrimaryMockIndex(); @@ -160,6 +162,11 @@ class RocksDBPrimaryMockIndex final : public Index { /// a random order. It is guaranteed that each element is found /// exactly once unless the collection is modified. IndexIterator* anyIterator(transaction::Methods*, ManagedDocumentResult*) const; + uint64_t objectId() const { return _objectId; } + private: + uint64_t _objectId; + std::map _keyRevMap; + std::mutex _keyRevMutex; }; } From 87e1a0f7c2bca24c52b274358ebd247061cf89b5 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Mon, 27 Mar 2017 11:53:13 +0200 Subject: [PATCH 20/27] try to build insert / remove for index --- .../RocksDBEngine/RocksDBPrimaryMockIndex.cpp | 28 ++++++++++++------- .../RocksDBEngine/RocksDBPrimaryMockIndex.h | 1 + 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp index 7f7c94f1a1..800850bc43 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp @@ -38,8 +38,6 @@ #include #include -#include - using namespace arangodb; /// @brief hard-coded vector of the index attributes @@ -142,21 +140,31 @@ void RocksDBPrimaryMockIndex::toVelocyPackFigures(VPackBuilder& builder) const { // TODO: implement } -int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revision_id, VPackSlice const& slice, bool) { -// #ifdef ARANGODB_ENABLE_MAINTAINER_MODE -// LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "insert() called for primary index"; -// #endif +int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revisionId, VPackSlice const& slice, bool) { +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "insert() called for primary index"; +#endif // // THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "insert() called for primary index"); - auto value = RocksDBEntry::IndexValue(objectId(), revision_id, slice); + auto value = RocksDBEntry::IndexValue(objectId(), revisionId, slice); std::lock_guard lock(_keyRevMutex); - _keyRevMap.emplace(value.key(),value.revisionId()); + auto result = _keyRevMap.emplace(value.key(),value.revisionId()); + if(result.second){ + return TRI_ERROR_NO_ERROR; + } + return TRI_ERROR_INTERNAL; } -int RocksDBPrimaryMockIndex::remove(transaction::Methods*, TRI_voc_rid_t, VPackSlice const&, bool) { +int RocksDBPrimaryMockIndex::remove(transaction::Methods*, TRI_voc_rid_t revisionId, VPackSlice const& slice, bool) { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "remove() called for primary index"; #endif - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "remove() called for primary index"); + auto value = RocksDBEntry::IndexValue(objectId(), revisionId, slice); + std::lock_guard lock(_keyRevMutex); + auto result = _keyRevMap.erase(value.key()); //result number of deleted elements + if(result){ + return TRI_ERROR_NO_ERROR; + } + return TRI_ERROR_INTERNAL; } /// @brief unload the index data from memory diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h index e2bf197750..c131ca20d6 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h @@ -35,6 +35,7 @@ #include #include +#include namespace arangodb { From b03f823db05a9f89d7ba901b0237f0fd41031257 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 12:00:36 +0200 Subject: [PATCH 21/27] added lookupKey --- arangod/MMFiles/MMFilesCollection.cpp | 2 +- arangod/MMFiles/MMFilesCollection.h | 2 +- arangod/RocksDBEngine/RocksDBCollection.cpp | 132 +++++++++++++++++- arangod/RocksDBEngine/RocksDBCollection.h | 9 ++ .../RocksDBEngine/RocksDBPrimaryMockIndex.cpp | 18 +-- .../RocksDBEngine/RocksDBPrimaryMockIndex.h | 5 +- arangod/RocksDBEngine/RocksDBToken.h | 48 +++++++ 7 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 arangod/RocksDBEngine/RocksDBToken.h diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index a8ebd5607d..8a498b2b19 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -3530,7 +3530,7 @@ int MMFilesCollection::removeFastPath(arangodb::transaction::Methods* trx, /// the caller must make sure the read lock on the collection is held /// the key must be a string slice, no revision check is performed int MMFilesCollection::lookupDocument(transaction::Methods* trx, - VPackSlice const key, + VPackSlice key, ManagedDocumentResult& result) { if (!key.isString()) { return TRI_ERROR_ARANGO_DOCUMENT_KEY_BAD; diff --git a/arangod/MMFiles/MMFilesCollection.h b/arangod/MMFiles/MMFilesCollection.h index 3e7e67c1ec..a591b6a8f4 100644 --- a/arangod/MMFiles/MMFilesCollection.h +++ b/arangod/MMFiles/MMFilesCollection.h @@ -490,7 +490,7 @@ class MMFilesCollection final : public PhysicalCollection { int deleteSecondaryIndexes(transaction::Methods*, TRI_voc_rid_t revisionId, velocypack::Slice const&, bool isRollback); - int lookupDocument(transaction::Methods*, velocypack::Slice const, + int lookupDocument(transaction::Methods*, velocypack::Slice, ManagedDocumentResult& result); int updateDocument(transaction::Methods*, TRI_voc_rid_t oldRevisionId, diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index 01f0650ff1..8f1fb43c9e 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -32,6 +32,7 @@ #include "RocksDBEngine/RocksDBEngine.h" #include "RocksDBEngine/RocksDBEntry.h" #include "RocksDBEngine/RocksDBPrimaryMockIndex.h" +#include "RocksDBEngine/RocksDBToken.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" #include "StorageEngine/TransactionState.h" @@ -331,6 +332,10 @@ int RocksDBCollection::insert(arangodb::transaction::Methods* trx, arangodb::ManagedDocumentResult& result, OperationOptions& options, TRI_voc_tick_t& resultMarkerTick, bool /*lock*/) { + // store the tick that was used for writing the document + // note that we don't need it for this engine + resultMarkerTick = 0; + VPackSlice fromSlice; VPackSlice toSlice; @@ -384,14 +389,13 @@ int RocksDBCollection::insert(arangodb::transaction::Methods* trx, res = insertDocument(trx, revisionId, newSlice, options.waitForSync); if (res == TRI_ERROR_NO_ERROR) { + // TODO: handle returning of result value! + // uint8_t const* vpack = lookupRevisionVPack(revisionId); // if (vpack != nullptr) { // result.addExisting(vpack, revisionId); // } - // store the tick that was used for writing the document - // note that we don't need it for this engine - resultMarkerTick = 0; } return res; } @@ -424,11 +428,50 @@ int RocksDBCollection::remove(arangodb::transaction::Methods* trx, arangodb::velocypack::Slice const slice, arangodb::ManagedDocumentResult& previous, OperationOptions& options, - TRI_voc_tick_t& resultMarkerTick, bool lock, + TRI_voc_tick_t& resultMarkerTick, bool /*lock*/, TRI_voc_rid_t const& revisionId, TRI_voc_rid_t& prevRev) { - THROW_ARANGO_NOT_YET_IMPLEMENTED(); - return 0; + // store the tick that was used for writing the document + // note that we don't need it for this engine + resultMarkerTick = 0; + prevRev = 0; + + transaction::BuilderLeaser builder(trx); + newObjectForRemove(trx, slice, TRI_RidToString(revisionId), *builder.get()); + + VPackSlice key; + if (slice.isString()) { + key = slice; + } else { + key = slice.get(StaticStrings::KeyString); + } + TRI_ASSERT(!key.isNone()); + + // get the previous revision + int res = lookupDocument(trx, key, previous); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + uint8_t const* vpack = previous.vpack(); + VPackSlice oldDoc(vpack); + TRI_voc_rid_t oldRevisionId = arangodb::transaction::helpers::extractRevFromDocument(oldDoc); + prevRev = oldRevisionId; + + // Check old revision: + if (!options.ignoreRevs && slice.isObject()) { + TRI_voc_rid_t expectedRevisionId = TRI_ExtractRevisionId(slice); + int res = checkRevision(trx, expectedRevisionId, oldRevisionId); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + } + + res = removeDocument(trx, oldRevisionId, oldDoc, options.waitForSync); + + return res; } void RocksDBCollection::deferDropCollection( @@ -596,3 +639,80 @@ int RocksDBCollection::insertDocument(arangodb::transaction::Methods* trx, return result; } + +int RocksDBCollection::removeDocument(arangodb::transaction::Methods* trx, + TRI_voc_rid_t revisionId, + VPackSlice const& doc, + bool& waitForSync) { + // Coordinator doesn't know index internals + TRI_ASSERT(!ServerState::instance()->isCoordinator()); + + RocksDBEntry entry(RocksDBEntry::Document(_objectId, revisionId, basics::VelocyPackHelper::EmptyObjectValue())); + + rocksdb::WriteBatch writeBatch; + writeBatch.Delete(entry.key()); + + auto indexes = _indexes; + size_t const n = indexes.size(); + + int result = TRI_ERROR_NO_ERROR; + + for (size_t i = 0; i < n; ++i) { + auto idx = indexes[i]; + + int res = idx->remove(trx, revisionId, doc, false); + + // in case of no-memory, return immediately + if (res == TRI_ERROR_OUT_OF_MEMORY) { + return res; + } + } + + if (result != TRI_ERROR_NO_ERROR) { + rocksdb::WriteOptions writeOptions; + + if (_logicalCollection->waitForSync()) { + waitForSync = true; + } + + if (waitForSync) { + trx->state()->waitForSync(true); + + // handle waitForSync for single operations here + if (trx->state()->isSingleOperation()) { + writeOptions.sync = true; + } + } + + StorageEngine* engine = EngineSelectorFeature::ENGINE; + rocksdb::TransactionDB* db = static_cast(engine)->db(); + db->Write(writeOptions, &writeBatch); + } + + return result; +} + +/// @brief looks up a document by key, low level worker +/// the key must be a string slice, no revision check is performed +int RocksDBCollection::lookupDocument(transaction::Methods* trx, + VPackSlice key, + ManagedDocumentResult& result) { + if (!key.isString()) { + return TRI_ERROR_ARANGO_DOCUMENT_KEY_BAD; + } + + RocksDBToken token = primaryIndex()->lookupKey(trx, key, result); + TRI_voc_rid_t revisionId = token.revisionId(); + + if (revisionId > 0) { + // TODO: add result handling! +/* uint8_t const* vpack = lookupRevisionVPack(revisionId); + if (vpack != nullptr) { + result.addExisting(vpack, revisionId); + } +*/ + return TRI_ERROR_NO_ERROR; + } + + return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND; +} diff --git a/arangod/RocksDBEngine/RocksDBCollection.h b/arangod/RocksDBEngine/RocksDBCollection.h index c61cf640e4..c332efd31e 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.h +++ b/arangod/RocksDBEngine/RocksDBCollection.h @@ -186,6 +186,15 @@ class RocksDBCollection final : public PhysicalCollection { arangodb::velocypack::Slice const& doc, bool& waitForSync); + int removeDocument(arangodb::transaction::Methods* trx, + TRI_voc_rid_t revisionId, + arangodb::velocypack::Slice const& doc, + bool& waitForSync); + + int lookupDocument(transaction::Methods* trx, + arangodb::velocypack::Slice key, + ManagedDocumentResult& result); + private: uint64_t _objectId; // rocksdb-specific object id for collection }; diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp index 800850bc43..59f61bbc04 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp @@ -22,12 +22,12 @@ //////////////////////////////////////////////////////////////////////////////// #include "RocksDBPrimaryMockIndex.h" -#include "RocksDBEntry.h" #include "Aql/AstNode.h" #include "Basics/Exceptions.h" #include "Basics/StaticStrings.h" #include "Basics/VelocyPackHelper.h" #include "Indexes/SimpleAttributeEqualityMatcher.h" +#include "RocksDBEngine/RocksDBEntry.h" #include "Transaction/Helpers.h" #include "Transaction/Methods.h" #include "Transaction/Context.h" @@ -140,11 +140,16 @@ void RocksDBPrimaryMockIndex::toVelocyPackFigures(VPackBuilder& builder) const { // TODO: implement } +RocksDBToken RocksDBPrimaryMockIndex::lookupKey(transaction::Methods* trx, VPackSlice key, ManagedDocumentResult& result) { + std::lock_guard lock(_keyRevMutex); + auto it = _keyRevMap.find(key.copyString()); + if (it == _keyRevMap.end()) { + return RocksDBToken(); + } + return RocksDBToken((*it).second); +} + int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revisionId, VPackSlice const& slice, bool) { -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "insert() called for primary index"; -#endif -// // THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "insert() called for primary index"); auto value = RocksDBEntry::IndexValue(objectId(), revisionId, slice); std::lock_guard lock(_keyRevMutex); auto result = _keyRevMap.emplace(value.key(),value.revisionId()); @@ -155,9 +160,6 @@ int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revisio } int RocksDBPrimaryMockIndex::remove(transaction::Methods*, TRI_voc_rid_t revisionId, VPackSlice const& slice, bool) { -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "remove() called for primary index"; -#endif auto value = RocksDBEntry::IndexValue(objectId(), revisionId, slice); std::lock_guard lock(_keyRevMutex); auto result = _keyRevMap.erase(value.key()); //result number of deleted elements diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h index c131ca20d6..f12b5cb463 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h @@ -27,6 +27,7 @@ #include "Basics/Common.h" #include "Indexes/Index.h" #include "Indexes/IndexIterator.h" +#include "RocksDBEngine/RocksDBToken.h" #include "VocBase/vocbase.h" #include "VocBase/voc-types.h" @@ -38,7 +39,7 @@ #include namespace arangodb { - +class ManagedDocumentResult; class RocksDBPrimaryMockIndex; namespace transaction { class Methods; @@ -136,6 +137,8 @@ class RocksDBPrimaryMockIndex final : public Index { void toVelocyPack(VPackBuilder&, bool) const override; void toVelocyPackFigures(VPackBuilder&) const override; + RocksDBToken lookupKey(transaction::Methods* trx, arangodb::velocypack::Slice key, ManagedDocumentResult& result); + int insert(transaction::Methods*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; int remove(transaction::Methods*, TRI_voc_rid_t, arangodb::velocypack::Slice const&, bool isRollback) override; diff --git a/arangod/RocksDBEngine/RocksDBToken.h b/arangod/RocksDBEngine/RocksDBToken.h new file mode 100644 index 0000000000..c00d33dd36 --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBToken.h @@ -0,0 +1,48 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_ROCKSDB_ENGINE_ROCKSDB_TOKEN_H +#define ARANGOD_ROCKSDB_ENGINE_ROCKSDB_TOKEN_H 1 + +#include "StorageEngine/DocumentIdentifierToken.h" + +namespace arangodb { + +struct RocksDBToken : public DocumentIdentifierToken { + public: + RocksDBToken() : DocumentIdentifierToken() {} + explicit RocksDBToken(TRI_voc_rid_t revisionId) + : DocumentIdentifierToken(revisionId) {} + RocksDBToken(RocksDBToken const& other) + : DocumentIdentifierToken(other._data) {} + + inline TRI_voc_rid_t revisionId() const { + return static_cast(_data); + } +}; + +static_assert(sizeof(RocksDBToken) == sizeof(uint64_t), "invalid RocksDBToken size"); + +} + +#endif From cdf1c1d8942a66a46f65a6d33dd3557c93aa476f Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 12:37:23 +0200 Subject: [PATCH 22/27] primary index --- arangod/RocksDBEngine/RocksDBIndexFactory.cpp | 8 +++++--- .../RocksDBEngine/RocksDBPrimaryMockIndex.cpp | 16 ++++++++++------ arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h | 3 +-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp index f311feb807..15307134d9 100644 --- a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp +++ b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp @@ -330,7 +330,7 @@ std::shared_ptr RocksDBIndexFactory::prepareIndexFromSlice( THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create primary index"); } - newIdx.reset(new arangodb::RocksDBPrimaryMockIndex(col,info)); + newIdx.reset(new arangodb::RocksDBPrimaryMockIndex(col, info)); break; } case arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX: { @@ -361,12 +361,14 @@ std::shared_ptr RocksDBIndexFactory::prepareIndexFromSlice( void RocksDBIndexFactory::fillSystemIndexes( arangodb::LogicalCollection* col, std::vector>& systemIndexes) const { + // create primary index - VPackBuilder builder; + builder.openObject(); + builder.close(); + systemIndexes.emplace_back( std::make_shared(col, builder.slice())); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid index type"); // create edges index if (col->type() == TRI_COL_TYPE_EDGE) { diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp index 59f61bbc04..d76698cac2 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.cpp @@ -140,9 +140,11 @@ void RocksDBPrimaryMockIndex::toVelocyPackFigures(VPackBuilder& builder) const { // TODO: implement } -RocksDBToken RocksDBPrimaryMockIndex::lookupKey(transaction::Methods* trx, VPackSlice key, ManagedDocumentResult& result) { +RocksDBToken RocksDBPrimaryMockIndex::lookupKey(transaction::Methods* trx, VPackSlice slice, ManagedDocumentResult& result) { + std::string key = slice.copyString(); std::lock_guard lock(_keyRevMutex); - auto it = _keyRevMap.find(key.copyString()); + LOG_TOPIC(ERR, Logger::FIXME) << "LOOKUP. THE KEY IS: " << key; + auto it = _keyRevMap.find(key); if (it == _keyRevMap.end()) { return RocksDBToken(); } @@ -150,9 +152,10 @@ RocksDBToken RocksDBPrimaryMockIndex::lookupKey(transaction::Methods* trx, VPack } int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revisionId, VPackSlice const& slice, bool) { - auto value = RocksDBEntry::IndexValue(objectId(), revisionId, slice); + std::string key = slice.get("_key").copyString(); std::lock_guard lock(_keyRevMutex); - auto result = _keyRevMap.emplace(value.key(),value.revisionId()); + LOG_TOPIC(ERR, Logger::FIXME) << "INSERT. THE KEY IS: " << key << "; THE REVISION IS: " << revisionId; + auto result = _keyRevMap.emplace(key, revisionId); if(result.second){ return TRI_ERROR_NO_ERROR; } @@ -160,9 +163,10 @@ int RocksDBPrimaryMockIndex::insert(transaction::Methods*, TRI_voc_rid_t revisio } int RocksDBPrimaryMockIndex::remove(transaction::Methods*, TRI_voc_rid_t revisionId, VPackSlice const& slice, bool) { - auto value = RocksDBEntry::IndexValue(objectId(), revisionId, slice); + std::string key = slice.get("_key").copyString(); std::lock_guard lock(_keyRevMutex); - auto result = _keyRevMap.erase(value.key()); //result number of deleted elements + LOG_TOPIC(ERR, Logger::FIXME) << "REMOVE. THE KEY IS: " << key; + auto result = _keyRevMap.erase(key); //result number of deleted elements if(result){ return TRI_ERROR_NO_ERROR; } diff --git a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h index f12b5cb463..eee3d4d2d4 100644 --- a/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h +++ b/arangod/RocksDBEngine/RocksDBPrimaryMockIndex.h @@ -36,7 +36,6 @@ #include #include -#include namespace arangodb { class ManagedDocumentResult; @@ -169,7 +168,7 @@ class RocksDBPrimaryMockIndex final : public Index { uint64_t objectId() const { return _objectId; } private: uint64_t _objectId; - std::map _keyRevMap; + std::unordered_map _keyRevMap; std::mutex _keyRevMutex; }; } From 922a8af392ef955fa0187c819143b7871f77ec2f Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 14:22:00 +0200 Subject: [PATCH 23/27] turn off all MMFiles features in case RocksDB engine is selected --- arangod/MMFiles/MMFilesLogfileManager.cpp | 9 ++- .../MMFiles/MMFilesPersistentIndexFeature.cpp | 3 +- arangod/MMFiles/MMFilesWalRecoveryFeature.cpp | 5 +- lib/ApplicationFeatures/ApplicationFeature.h | 24 ++++--- lib/ApplicationFeatures/ApplicationServer.cpp | 63 +++++++++++-------- lib/ApplicationFeatures/ApplicationServer.h | 7 ++- 6 files changed, 63 insertions(+), 48 deletions(-) diff --git a/arangod/MMFiles/MMFilesLogfileManager.cpp b/arangod/MMFiles/MMFilesLogfileManager.cpp index f371f87c92..0114cdbdcc 100644 --- a/arangod/MMFiles/MMFilesLogfileManager.cpp +++ b/arangod/MMFiles/MMFilesLogfileManager.cpp @@ -109,15 +109,14 @@ MMFilesLogfileManager::MMFilesLogfileManager(ApplicationServer* server) LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "creating WAL logfile manager"; TRI_ASSERT(!_allowWrites); - setOptional(false); + setOptional(true); requiresElevatedPrivileges(false); startsAfter("DatabasePath"); startsAfter("EngineSelector"); startsAfter("FeatureCache"); - - for (auto const& it : EngineSelectorFeature::availableEngines()) { - startsAfter(it.second); - } + startsAfter("MMFilesEngine"); + + onlyEnabledWith("MMFilesEngine"); } // destroy the logfile manager diff --git a/arangod/MMFiles/MMFilesPersistentIndexFeature.cpp b/arangod/MMFiles/MMFilesPersistentIndexFeature.cpp index 33918583a7..daf99617fc 100644 --- a/arangod/MMFiles/MMFilesPersistentIndexFeature.cpp +++ b/arangod/MMFiles/MMFilesPersistentIndexFeature.cpp @@ -65,8 +65,9 @@ MMFilesPersistentIndexFeature::MMFilesPersistentIndexFeature( _keepLogFileNum(1000), _logFileTimeToRoll(0), _compactionReadaheadSize(0) { setOptional(true); requiresElevatedPrivileges(false); - // startsAfter("MMFilesLogfileManager"); startsAfter("DatabasePath"); + + onlyEnabledWith("MMFilesEngine"); } MMFilesPersistentIndexFeature::~MMFilesPersistentIndexFeature() { diff --git a/arangod/MMFiles/MMFilesWalRecoveryFeature.cpp b/arangod/MMFiles/MMFilesWalRecoveryFeature.cpp index cc94e1f00e..4005dc3289 100644 --- a/arangod/MMFiles/MMFilesWalRecoveryFeature.cpp +++ b/arangod/MMFiles/MMFilesWalRecoveryFeature.cpp @@ -37,11 +37,14 @@ using namespace arangodb::options; MMFilesWalRecoveryFeature::MMFilesWalRecoveryFeature(ApplicationServer* server) : ApplicationFeature(server, "MMFilesWalRecovery") { - setOptional(false); + setOptional(true); requiresElevatedPrivileges(false); startsAfter("Database"); startsAfter("MMFilesLogfileManager"); startsAfter("MMFilesPersistentIndex"); + + onlyEnabledWith("MMFilesEngine"); + onlyEnabledWith("MMFilesLogfileManager"); } /// @brief run the recovery procedure diff --git a/lib/ApplicationFeatures/ApplicationFeature.h b/lib/ApplicationFeatures/ApplicationFeature.h index 26a3caa6ab..92670a966b 100644 --- a/lib/ApplicationFeatures/ApplicationFeature.h +++ b/lib/ApplicationFeatures/ApplicationFeature.h @@ -68,7 +68,7 @@ class ApplicationFeature { // enable or disable a feature void setEnabled(bool value) { - if (!value && !isOptional() && _enableWith.empty()) { + if (!value && !isOptional()) { THROW_ARANGO_EXCEPTION_MESSAGE( TRI_ERROR_BAD_PARAMETER, "cannot disable non-optional feature '" + name() + "'"); @@ -76,9 +76,6 @@ class ApplicationFeature { _enabled = value; } - // return whether a feature is automatically enabled with another feature - std::string enableWith() const { return _enableWith; } - // names of features required to be enabled for this feature to be enabled std::vector const& requires() const { return _requires; } @@ -141,12 +138,6 @@ class ApplicationFeature { // make the feature optional (or not) void setOptional(bool value) { _optional = value; } - // enable this feature automatically when another is enabled - void enableWith(std::string const& other) { - _enableWith = other; - _requires.emplace_back(other); - } - // note that this feature requires another to be present void requires(std::string const& other) { _requires.emplace_back(other); } @@ -161,6 +152,13 @@ class ApplicationFeature { // determine all direct and indirect ancestors of a feature std::unordered_set ancestors() const; + void onlyEnabledWith(std::string const& other) { _onlyEnabledWith.emplace(other); } + + // return the list of other features that this feature depends on + std::unordered_set const& onlyEnabledWith() const { + return _onlyEnabledWith; + } + private: // set a feature's state. this method should be called by the // application server only @@ -182,14 +180,14 @@ class ApplicationFeature { // is enabled std::vector _requires; - // name of other feature that will enable or disable this feature - std::string _enableWith; - // a list of start dependencies for the feature std::unordered_set _startsAfter; // list of direct and indirect ancestors of the feature std::unordered_set _ancestors; + + // enable this feature only if the following other features are enabled + std::unordered_set _onlyEnabledWith; // state of feature ApplicationServer::FeatureState _state; diff --git a/lib/ApplicationFeatures/ApplicationServer.cpp b/lib/ApplicationFeatures/ApplicationServer.cpp index e28e80ad08..95636d6819 100644 --- a/lib/ApplicationFeatures/ApplicationServer.cpp +++ b/lib/ApplicationFeatures/ApplicationServer.cpp @@ -181,12 +181,13 @@ void ApplicationServer::run(int argc, char* argv[]) { reportServerProgress(_state); validateOptions(); - // enable automatic features - enableAutomaticFeatures(); - // setup and validate all feature dependencies setupDependencies(true); + // turn off all features that depend on other features that have been + // turned off + disableDependentFeatures(); + // allows process control daemonize(); @@ -198,6 +199,11 @@ void ApplicationServer::run(int argc, char* argv[]) { _state = ServerState::IN_PREPARE; reportServerProgress(_state); prepare(); + + // turn off all features that depend on other features that have been + // turned off. we repeat this to allow features to turn other features + // off even in the prepare phase + disableDependentFeatures(); // permanently drop the privileges dropPrivilegesPermanently(); @@ -346,28 +352,6 @@ void ApplicationServer::validateOptions() { } } -void ApplicationServer::enableAutomaticFeatures() { - bool changed; - do { - changed = false; - for (auto& it : _features) { - auto other = it.second->enableWith(); - if (other.empty()) { - continue; - } - if (!this->exists(other)) { - fail("feature '" + it.second->name() + - "' depends on unknown feature '" + other + "'"); - } - bool otherIsEnabled = this->feature(other)->isEnabled(); - if (otherIsEnabled != it.second->isEnabled()) { - it.second->setEnabled(otherIsEnabled); - changed = true; - } - } - } while (changed); -} - // setup and validate all feature dependencies, determine feature order void ApplicationServer::setupDependencies(bool failOnMissing) { LOG_TOPIC(TRACE, Logger::STARTUP) @@ -469,6 +453,35 @@ void ApplicationServer::daemonize() { } } +void ApplicationServer::disableDependentFeatures() { + LOG_TOPIC(TRACE, Logger::STARTUP) << "ApplicationServer::disableDependentFeatures"; + + for (auto feature : _orderedFeatures) { + auto const& onlyEnabledWith = feature->onlyEnabledWith(); + + if (!feature->isEnabled() || onlyEnabledWith.empty()) { + continue; + } + + for (auto const& other : onlyEnabledWith) { + ApplicationFeature* f = lookupFeature(other); + if (f == nullptr) { + LOG_TOPIC(TRACE, Logger::STARTUP) << "turning off feature '" << feature->name() + << "' because it is enabled only in conjunction with non-existing feature '" + << f->name() << "'"; + feature->disable(); + break; + } else if (!f->isEnabled()) { + LOG_TOPIC(TRACE, Logger::STARTUP) << "turning off feature '" << feature->name() + << "' because it is enabled only in conjunction with disabled feature '" + << f->name() << "'"; + feature->disable(); + break; + } + } + } +} + void ApplicationServer::prepare() { LOG_TOPIC(TRACE, Logger::STARTUP) << "ApplicationServer::prepare"; diff --git a/lib/ApplicationFeatures/ApplicationServer.h b/lib/ApplicationFeatures/ApplicationServer.h index 0ce43654ae..0154f54928 100644 --- a/lib/ApplicationFeatures/ApplicationServer.h +++ b/lib/ApplicationFeatures/ApplicationServer.h @@ -247,15 +247,16 @@ class ApplicationServer { // allows features to cross-validate their program options void validateOptions(); - // enable automatic features - void enableAutomaticFeatures(); - // setup and validate all feature dependencies, determine feature order void setupDependencies(bool failOnMissing); // allows process control void daemonize(); + // disables all features that depend on other features, which, themselves + // are disabled + void disableDependentFeatures(); + // allows features to prepare themselves void prepare(); From 544007a5c84a3fa54455ab96a1c868dc39bad8f9 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Mon, 27 Mar 2017 14:30:06 +0200 Subject: [PATCH 24/27] Squashed commit of the following: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 873bc5950028248bf209d0c53fc9eb93938e3610 Merge: 8e4457a 3487097 Author: Michael Hackstein Date: Mon Mar 27 14:00:27 2017 +0200 Merge branch 'devel' of github.com:arangodb/arangodb into devel-feature/traversal-cache commit 8e4457a96754bea3437e8d44c0ef9db2366ee86d Author: Michael Hackstein Date: Mon Mar 27 13:59:44 2017 +0200 Removed old VPackSlice based getVertex API in traversers. commit 2e0716eeeec1ec56b96571a18074539f5e27cb40 Author: Michael Hackstein Date: Fri Mar 24 09:09:02 2017 +0100 Fixed logic error in SingleServerEdgeCursor readAll. The cursorId was not propageted properly, which cased filter-evaluations to fail commit cea85c2ac1163ed0a989c492b8039ed4f7622325 Merge: 538f216 45a4fac Author: Michael Hackstein Date: Fri Mar 24 08:17:31 2017 +0100 Merge branch 'devel' of github.com:arangodb/arangodb into devel-feature/traversal-cache commit 538f216cbefcdfb4d06fe16a42f223cecae8598b Merge: 5f4afee 8bfcb49 Author: Simon Grätzer Date: Thu Mar 23 16:33:44 2017 +0100 Merge branch 'devel-feature/traversal-cache' of https://github.com/arangodb/arangodb into devel-feature/traversal-cache commit 5f4afee54094ea6d2a0cfec1abfb75451f228a23 Author: Simon Grätzer Date: Thu Mar 23 16:33:38 2017 +0100 VertexGetter StringRef methods commit 8bfcb490d7a907ebd7dcc9e6f38a8d29baa7c7e5 Author: Michael Hackstein Date: Thu Mar 23 16:32:38 2017 +0100 Fixed AqlValues created by traverser-cache. They are now not using externals anymore which cased everything to crash commit 614cdedb7e30e35f65f289b310c38b2585eb03e4 Merge: 773b0a5 416777b Author: Michael Hackstein Date: Thu Mar 23 12:52:19 2017 +0100 Merge branch 'devel-feature/traversal-cache' of github.com:arangodb/arangodb into devel-feature/traversal-cache This state is still red commit 773b0a5e51f87875fa9bf1440dd002b2e1fcfd2a Author: Michael Hackstein Date: Thu Mar 23 12:48:53 2017 +0100 Added Implementation or readAll in Cluster EdgeCursor. This commit is still red. commit 82ebbd6ec84918d43bbc64174bc251efa548a4de Merge: e52b210 35dffc4 Author: Michael Hackstein Date: Thu Mar 23 11:37:31 2017 +0100 Merge branch 'devel-feature/traversal-cache' of github.com:arangodb/arangodb into devel-feature/traversal-cache This commit is not green. commit 416777b60a8a0c741d11e9c654745866d0dfa15d Author: Simon Grätzer Date: Thu Mar 23 11:25:35 2017 +0100 Fixed errornous ClusterEdgeCursor implementation of `readAll commit e52b21025a5f56e2cfc03e3c129da83793f19846 Author: Michael Hackstein Date: Thu Mar 23 11:01:17 2017 +0100 Fixed API for Traversals: All functions now use StringRefs to identify vertices and edges which are presisted within the traverser-cache. commit 35dffc439c156366b1110c6770c1565021934632 Author: Simon Grätzer Date: Wed Mar 22 20:26:41 2017 +0100 Fixed a few cluster issues commit 73cf4fe41c363f6c034a1daeacee1630d7e827a5 Author: Simon Grätzer Date: Wed Mar 22 17:26:12 2017 +0100 Fixed EdgeCursor bug commit 2f7329b4e9711740b423cf2963c6cf3f880c4bdf Merge: bb9b129 8e5edf5 Author: Michael Hackstein Date: Wed Mar 22 14:32:07 2017 +0100 Merge branch 'devel-feature/traversal-cache' of github.com:arangodb/arangodb into devel-feature/traversal-cache commit 8e5edf5fbe93f34d3340099619baa9a56a738887 Author: Simon Grätzer Date: Wed Mar 22 14:09:43 2017 +0100 Fixing compile errors commit bb9b1294c9ff91f62bc27775a4c7a69c5daca2fd Merge: 048a4c6 8e51e3b Author: Michael Hackstein Date: Wed Mar 22 11:39:17 2017 +0100 Merge branch 'devel' of github.com:arangodb/arangodb into devel-feature/traversal-cache commit 048a4c65e11d6fd9cb25a58ee97ed756a419a067 Merge: 693607e c6b177d Author: Michael Hackstein Date: Wed Mar 22 11:34:57 2017 +0100 Solved merge conflicts. Build is still red. commit c6b177dfc348df0f49f07cd67e83b873871fe6c5 Author: Simon Grätzer Date: Tue Mar 21 23:04:28 2017 +0100 Fixed off by one error, and crash commit 5dbc9eb58f1d21e5a530806d7ad7e52058234b83 Merge: 463e352 db8c255 Author: Simon Grätzer Date: Tue Mar 21 17:46:01 2017 +0100 Merge branch 'devel-feature/traversal-cache' of https://github.com/arangodb/arangodb into devel-feature/traversal-cache # Conflicts: # arangod/Graph/BreadthFirstEnumerator.cpp commit 463e3520636c19ae7a7c1d445099f336c5bde31e Merge: 4e75f29 b8cabbe Author: Simon Grätzer Date: Tue Mar 21 17:00:17 2017 +0100 Merge branch 'devel-feature/traversal-cache' of https://github.com/arangodb/arangodb into devel-feature/traversal-cache # Conflicts: # arangod/Graph/BreadthFirstEnumerator.cpp commit 693607ea92ab3116a628c08c59a983a6fdf8b5f4 Author: Michael Hackstein Date: Tue Mar 21 16:26:57 2017 +0100 Moved specialized NeighborsEnumerator into its own files. commit db8c25587ebcdeff8b32d08cf2d97d723bf2b3fe Author: Michael Hackstein Date: Tue Mar 21 15:16:39 2017 +0100 Replaced SingleServerBreadthFirst with using Callbacks in readAll method of EdgeCursor commit 4e75f29de95edeea35573b128a0a13e04c9e65d2 Merge: 8716936 6de86c5 Author: Simon Grätzer Date: Tue Mar 21 14:32:12 2017 +0100 Merge branch 'devel-feature/traversal-cache' of https://github.com/arangodb/arangodb into devel-feature/traversal-cache commit b8cabbedebb16c09c8168ec8b5a4499e98c705f4 Author: Michael Hackstein Date: Tue Mar 21 11:17:23 2017 +0100 Removed _enumertedPath from BreadthFirstEnumerator. Less string copying expected now. commit 8716936bd480f4b7c85ece6fc1a51335f1942258 Author: Simon Grätzer Date: Tue Mar 21 11:05:04 2017 +0100 Traverser: changed method definitions commit 14af02fbc08b7e2ed6b0cbb918ff509ffaef76bf Author: Michael Hackstein Date: Tue Mar 21 10:45:13 2017 +0100 Removed usage of _enumeratedPath in lastEdge/Vertex to AQL. Goal is to delete this struct in the class, as it does unneccessary copies commit 06a20ab0549d97b47eda98d3428e3390619ca5ec Author: Simon Grätzer Date: Tue Mar 21 10:39:21 2017 +0100 Intermediate changes commit 3a9a5f7c7ad186c669c3986f6f89c5a26b835ee0 Author: Michael Hackstein Date: Tue Mar 21 10:34:43 2017 +0100 Logic of BreadthFirstEnumerator now points to the last path returned instead of the next path to return. This will allow to get rid of enumeratedPath in there. commit 6de86c548220a3ad21c53fb1de691070774130ec Author: Michael Hackstein Date: Tue Mar 21 10:17:25 2017 +0100 Extracted BreadthFirstEnumerator out of PathEnumerator files. Only moving of code. commit acd1d9eba5d7ea23e0d6beab831a42dc61a385a5 Merge: a99ad3c 1ab8c44 Author: Michael Hackstein Date: Mon Mar 20 13:13:16 2017 +0100 Merge branch 'devel' of github.com:arangodb/arangodb into devel-feature/traversal-cache commit a99ad3c624d7ae4815776f165d65f19e43167e56 Author: Simon Grätzer Date: Fri Mar 17 13:25:45 2017 +0100 Added TraverserCache to SingleServerTraverser commit e88ae1e53d99b55546329413e7b4a5edc48c4875 Author: Michael Hackstein Date: Wed Mar 15 13:36:05 2017 +0100 Fixed compiler Issues in TraverserCache commit 9d89cdc0297813f24064b46a67a43cde9a3d0297 Merge: 9d41804 f18ad19 Author: Michael Hackstein Date: Wed Mar 15 09:43:06 2017 +0100 Merge branch 'devel' of github.com:arangodb/arangodb into devel-feature/traversal-cache commit 9d4180480c7bc37f1f64b6e3141dae611f8d399c Author: Michael Hackstein Date: Wed Mar 15 09:42:53 2017 +0100 Started implementing a Traverser Cache Abstraction that is used to store already fetched documents. --- arangod/CMakeLists.txt | 3 + arangod/Cluster/ClusterEdgeCursor.cpp | 37 ++- arangod/Cluster/ClusterEdgeCursor.h | 11 +- arangod/Cluster/ClusterMethods.cpp | 22 +- arangod/Cluster/ClusterMethods.h | 12 +- arangod/Cluster/ClusterTraverser.cpp | 55 ++-- arangod/Cluster/ClusterTraverser.h | 25 +- arangod/Cluster/TraverserEngine.cpp | 36 +-- arangod/Graph/BreadthFirstEnumerator.cpp | 221 ++++++++++++++ arangod/Graph/BreadthFirstEnumerator.h | 166 ++++++++++ arangod/Graph/NeighborsEnumerator.cpp | 105 +++++++ arangod/Graph/NeighborsEnumerator.h | 76 +++++ arangod/VocBase/PathEnumerator.cpp | 354 ++++------------------ arangod/VocBase/PathEnumerator.h | 202 +----------- arangod/VocBase/SingleServerTraverser.cpp | 180 +++++------ arangod/VocBase/SingleServerTraverser.h | 36 ++- arangod/VocBase/Traverser.cpp | 90 +++--- arangod/VocBase/Traverser.h | 74 +++-- arangod/VocBase/TraverserCache.cpp | 192 ++++++++++++ arangod/VocBase/TraverserCache.h | 161 ++++++++++ arangod/VocBase/TraverserOptions.cpp | 49 ++- arangod/VocBase/TraverserOptions.h | 37 ++- 22 files changed, 1322 insertions(+), 822 deletions(-) create mode 100644 arangod/Graph/BreadthFirstEnumerator.cpp create mode 100644 arangod/Graph/BreadthFirstEnumerator.h create mode 100644 arangod/Graph/NeighborsEnumerator.cpp create mode 100644 arangod/Graph/NeighborsEnumerator.h create mode 100644 arangod/VocBase/TraverserCache.cpp create mode 100644 arangod/VocBase/TraverserCache.h diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index b6079257a9..50334fef1f 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -218,6 +218,8 @@ SET(ARANGOD_SOURCES GeneralServer/RestHandlerFactory.cpp GeneralServer/RestStatus.cpp GeneralServer/VppCommTask.cpp + Graph/BreadthFirstEnumerator.cpp + Graph/NeighborsEnumerator.cpp Indexes/Index.cpp Indexes/IndexIterator.cpp Indexes/SimpleAttributeEqualityMatcher.cpp @@ -337,6 +339,7 @@ SET(ARANGOD_SOURCES VocBase/SingleServerTraverser.cpp VocBase/TransactionManager.cpp VocBase/Traverser.cpp + VocBase/TraverserCache.cpp VocBase/TraverserOptions.cpp VocBase/modes.cpp VocBase/replication-applier.cpp diff --git a/arangod/Cluster/ClusterEdgeCursor.cpp b/arangod/Cluster/ClusterEdgeCursor.cpp index 0d22cdc76c..fe1ea1f7ee 100644 --- a/arangod/Cluster/ClusterEdgeCursor.cpp +++ b/arangod/Cluster/ClusterEdgeCursor.cpp @@ -26,39 +26,48 @@ #include "Cluster/ClusterMethods.h" #include "Cluster/ClusterTraverser.h" #include "Transaction/Helpers.h" +#include "Transaction/Methods.h" +#include "VocBase/TraverserCache.h" #include #include using ClusterEdgeCursor = arangodb::traverser::ClusterEdgeCursor; -ClusterEdgeCursor::ClusterEdgeCursor(VPackSlice v, uint64_t depth, +ClusterEdgeCursor::ClusterEdgeCursor(StringRef vertexId, uint64_t depth, arangodb::traverser::ClusterTraverser* traverser) - : _position(0) { + : _position(0), _resolver(traverser->_trx->resolver()), _traverser(traverser) { transaction::BuilderLeaser leased(traverser->_trx); - fetchEdgesFromEngines(traverser->_dbname, traverser->_engines, v, depth, + + transaction::BuilderLeaser b(traverser->_trx); + b->add(VPackValuePair(vertexId.data(), vertexId.length(), VPackValueType::String)); + + + fetchEdgesFromEngines(traverser->_dbname, traverser->_engines, b->slice(), depth, traverser->_edges, _edgeList, traverser->_datalake, *(leased.get()), traverser->_filteredPaths, traverser->_readDocuments); + } - -bool ClusterEdgeCursor::next(std::vector& result, size_t& cursorId) { +bool ClusterEdgeCursor::next(std::function callback) { if (_position < _edgeList.size()) { - result.emplace_back(_edgeList[_position]); + VPackSlice edge = _edgeList[_position]; + std::string eid = transaction::helpers::extractIdString(_resolver, edge, VPackSlice()); + StringRef persId = _traverser->traverserCache()->persistString(StringRef(eid)); + callback(persId, edge, _position); ++_position; return true; } return false; } -bool ClusterEdgeCursor::readAll(std::unordered_set& result, size_t& cursorId) { - if (_position == 0) { - // We have not yet returned anything. So we simply return everything at once. - std::copy(_edgeList.begin(), _edgeList.end(), std::inserter(result, result.end())); - _position++; - return true; +void ClusterEdgeCursor::readAll(std::function callback) { + for (auto const& edge : _edgeList) { + std::string eid = transaction::helpers::extractIdString(_resolver, edge, VPackSlice()); + StringRef persId = _traverser->traverserCache()->persistString(StringRef(eid)); + callback(persId, edge, _position); } - // We have already returned everything last time. - return false; } diff --git a/arangod/Cluster/ClusterEdgeCursor.h b/arangod/Cluster/ClusterEdgeCursor.h index 4aaea7744f..116427e535 100644 --- a/arangod/Cluster/ClusterEdgeCursor.h +++ b/arangod/Cluster/ClusterEdgeCursor.h @@ -27,25 +27,30 @@ #include "VocBase/TraverserOptions.h" namespace arangodb { +class CollectionNameResolver; namespace traverser { +class Traverser; + class ClusterEdgeCursor : public EdgeCursor { public: - ClusterEdgeCursor(arangodb::velocypack::Slice, uint64_t, ClusterTraverser*); + ClusterEdgeCursor(StringRef vid, uint64_t, ClusterTraverser*); ~ClusterEdgeCursor() { } - bool next(std::vector&, size_t&) override; + bool next(std::function callback) override; - bool readAll(std::unordered_set&, size_t&) override; + void readAll(std::function callback) override; private: std::vector _edgeList; size_t _position; + CollectionNameResolver const* _resolver; + arangodb::traverser::Traverser* _traverser; }; } } diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index cc700d2eea..2de8520c17 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -1486,7 +1486,7 @@ int fetchEdgesFromEngines( std::unordered_map const* engines, VPackSlice const vertexId, size_t depth, - std::unordered_map& cache, + std::unordered_map& cache, std::vector& result, std::vector>& datalake, VPackBuilder& builder, @@ -1547,11 +1547,12 @@ int fetchEdgesFromEngines( VPackSlice edges = resSlice.get("edges"); for (auto const& e : VPackArrayIterator(edges)) { VPackSlice id = e.get(StaticStrings::IdString); - auto resE = cache.find(id); + StringRef idRef(id); + auto resE = cache.find(idRef); if (resE == cache.end()) { // This edge is not yet cached. allCached = false; - cache.emplace(id, e); + cache.emplace(idRef, e); result.emplace_back(e); } else { result.emplace_back(resE->second); @@ -1576,8 +1577,8 @@ int fetchEdgesFromEngines( void fetchVerticesFromEngines( std::string const& dbname, std::unordered_map const* engines, - std::unordered_set& vertexIds, - std::unordered_map>>& + std::unordered_set& vertexIds, + std::unordered_map>>& result, VPackBuilder& builder) { auto cc = ClusterComm::instance(); @@ -1594,8 +1595,8 @@ void fetchVerticesFromEngines( builder.add(VPackValue("keys")); builder.openArray(); for (auto const& v : vertexIds) { - TRI_ASSERT(v.isString()); - builder.add(v); + //TRI_ASSERT(v.isString()); + builder.add(VPackValuePair(v.data(), v.length(), VPackValueType::String)); } builder.close(); // 'keys' Array builder.close(); // base object @@ -1638,17 +1639,18 @@ void fetchVerticesFromEngines( resSlice, "errorMessage", TRI_errno_string(code))); } for (auto const& pair : VPackObjectIterator(resSlice)) { - if (vertexIds.erase(pair.key) == 0) { + StringRef key(pair.key); + if (vertexIds.erase(key) == 0) { // We either found the same vertex twice, // or found a vertex we did not request. // Anyways something somewhere went seriously wrong THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_GOT_CONTRADICTING_ANSWERS); } - TRI_ASSERT(result.find(pair.key) == result.end()); + TRI_ASSERT(result.find(key) == result.end()); auto val = VPackBuilder::clone(pair.value); VPackSlice id = val.slice().get(StaticStrings::IdString); TRI_ASSERT(id.isString()); - result.emplace(id, val.steal()); + result.emplace(StringRef(id), val.steal()); } } diff --git a/arangod/Cluster/ClusterMethods.h b/arangod/Cluster/ClusterMethods.h index 917624bd8a..beb7cd7be0 100644 --- a/arangod/Cluster/ClusterMethods.h +++ b/arangod/Cluster/ClusterMethods.h @@ -25,7 +25,7 @@ #define ARANGOD_CLUSTER_CLUSTER_METHODS_H 1 #include "Basics/Common.h" - +#include "Basics/StringRef.h" #include #include @@ -130,9 +130,8 @@ int getDocumentOnCoordinator( int fetchEdgesFromEngines( std::string const&, std::unordered_map const*, - arangodb::velocypack::Slice const, size_t, - std::unordered_map&, + arangodb::velocypack::Slice vertexId, size_t, + std::unordered_map&, std::vector&, std::vector>&, arangodb::velocypack::Builder&, size_t&, size_t&); @@ -149,9 +148,8 @@ int fetchEdgesFromEngines( void fetchVerticesFromEngines( std::string const&, std::unordered_map const*, - std::unordered_set&, - std::unordered_map>>&, + std::unordered_set&, + std::unordered_map>>&, arangodb::velocypack::Builder&); //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Cluster/ClusterTraverser.cpp b/arangod/Cluster/ClusterTraverser.cpp index 9222e2176e..2cdab1cb49 100644 --- a/arangod/Cluster/ClusterTraverser.cpp +++ b/arangod/Cluster/ClusterTraverser.cpp @@ -25,7 +25,9 @@ #include "Basics/StaticStrings.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ClusterMethods.h" +#include "Graph/BreadthFirstEnumerator.h" #include "Transaction/Helpers.h" +#include "VocBase/TraverserCache.h" #include #include @@ -43,17 +45,17 @@ ClusterTraverser::ClusterTraverser( _opts->linkTraverser(this); } -void ClusterTraverser::setStartVertex(std::string const& id) { +void ClusterTraverser::setStartVertex(std::string const& vid) { _verticesToFetch.clear(); _startIdBuilder->clear(); - _startIdBuilder->add(VPackValue(id)); + _startIdBuilder->add(VPackValue(vid)); VPackSlice idSlice = _startIdBuilder->slice(); - auto it = _vertices.find(idSlice); + auto it = _vertices.find(StringRef(vid)); if (it == _vertices.end()) { - size_t firstSlash = id.find("/"); + size_t firstSlash = vid.find("/"); if (firstSlash == std::string::npos || - id.find("/", firstSlash + 1) != std::string::npos) { + vid.find("/", firstSlash + 1) != std::string::npos) { // We can stop here. The start vertex is not a valid _id ++_filteredPaths; _done = true; @@ -66,23 +68,24 @@ void ClusterTraverser::setStartVertex(std::string const& id) { _done = true; return; } + StringRef persId = traverserCache()->persistString(StringRef(vid)); - _vertexGetter->reset(idSlice); + _vertexGetter->reset(persId); if (_opts->useBreadthFirst) { _enumerator.reset( - new arangodb::traverser::BreadthFirstEnumerator(this, idSlice, _opts)); + new arangodb::graph::BreadthFirstEnumerator(this, idSlice, _opts)); } else { _enumerator.reset( - new arangodb::traverser::DepthFirstEnumerator(this, idSlice, _opts)); + new arangodb::traverser::DepthFirstEnumerator(this, vid, _opts)); } _done = false; } bool ClusterTraverser::getVertex(VPackSlice edge, - std::vector& result) { + std::vector& result) { bool res = _vertexGetter->getVertex(edge, result); if (res) { - VPackSlice other = result.back(); + StringRef const& other = result.back(); if (_vertices.find(other) == _vertices.end()) { // Vertex not yet cached. Prepare it. _verticesToFetch.emplace(other); @@ -91,13 +94,13 @@ bool ClusterTraverser::getVertex(VPackSlice edge, return res; } -bool ClusterTraverser::getSingleVertex(VPackSlice edge, VPackSlice comp, - uint64_t depth, VPackSlice& result) { - bool res = _vertexGetter->getSingleVertex(edge, comp, depth, result); +bool ClusterTraverser::getSingleVertex(arangodb::velocypack::Slice edge, StringRef const sourceVertexId, + uint64_t depth, StringRef& targetVertexId) { + bool res = _vertexGetter->getSingleVertex(edge, sourceVertexId, depth, targetVertexId); if (res) { - if (_vertices.find(result) == _vertices.end()) { + if (_vertices.find(targetVertexId) == _vertices.end()) { // Vertex not yet cached. Prepare it. - _verticesToFetch.emplace(result); + _verticesToFetch.emplace(targetVertexId); } } return res; @@ -111,8 +114,8 @@ void ClusterTraverser::fetchVertices() { _verticesToFetch.clear(); } -aql::AqlValue ClusterTraverser::fetchVertexData(VPackSlice idString) { - TRI_ASSERT(idString.isString()); +aql::AqlValue ClusterTraverser::fetchVertexData(StringRef idString) { + //TRI_ASSERT(idString.isString()); auto cached = _vertices.find(idString); if (cached == _vertices.end()) { // Vertex not yet cached. Prepare for load. @@ -125,23 +128,23 @@ aql::AqlValue ClusterTraverser::fetchVertexData(VPackSlice idString) { return aql::AqlValue((*cached).second->data()); } -aql::AqlValue ClusterTraverser::fetchEdgeData(VPackSlice edge) { - return aql::AqlValue(edge); +aql::AqlValue ClusterTraverser::fetchEdgeData(StringRef eid) { + return aql::AqlValue(_edges[eid]);//this->_cache->fetchAqlResult(edge); } ////////////////////////////////////////////////////////////////////////////// /// @brief Function to add the real data of a vertex into a velocypack builder ////////////////////////////////////////////////////////////////////////////// -void ClusterTraverser::addVertexToVelocyPack(VPackSlice id, +void ClusterTraverser::addVertexToVelocyPack(StringRef vid, VPackBuilder& result) { - TRI_ASSERT(id.isString()); - auto cached = _vertices.find(id); + //TRI_ASSERT(id.isString()); + auto cached = _vertices.find(vid); if (cached == _vertices.end()) { // Vertex not yet cached. Prepare for load. - _verticesToFetch.emplace(id); + _verticesToFetch.emplace(vid); fetchVertices(); - cached = _vertices.find(id); + cached = _vertices.find(vid); } // Now all vertices are cached!! TRI_ASSERT(cached != _vertices.end()); @@ -152,7 +155,7 @@ void ClusterTraverser::addVertexToVelocyPack(VPackSlice id, /// @brief Function to add the real data of an edge into a velocypack builder ////////////////////////////////////////////////////////////////////////////// -void ClusterTraverser::addEdgeToVelocyPack(arangodb::velocypack::Slice edge, +void ClusterTraverser::addEdgeToVelocyPack(StringRef eid, arangodb::velocypack::Builder& result) { - result.add(edge); + result.add(_edges[eid]); } diff --git a/arangod/Cluster/ClusterTraverser.h b/arangod/Cluster/ClusterTraverser.h index 34f7e707e3..89e3266cf7 100644 --- a/arangod/Cluster/ClusterTraverser.h +++ b/arangod/Cluster/ClusterTraverser.h @@ -61,57 +61,56 @@ class ClusterTraverser final : public Traverser { /// Returns true if the vertex passes filtering conditions /// Also apppends the _id value of the vertex in the given vector - bool getVertex(arangodb::velocypack::Slice, - std::vector&) override; + bool getVertex(arangodb::velocypack::Slice, std::vector&) override; /// @brief Function to load the other sides vertex of an edge /// Returns true if the vertex passes filtering conditions - - bool getSingleVertex(arangodb::velocypack::Slice, arangodb::velocypack::Slice, - uint64_t, arangodb::velocypack::Slice&) override; + bool getSingleVertex(arangodb::velocypack::Slice edge, + StringRef const sourceVertexId, + uint64_t depth, + StringRef& targetVertexId) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to fetch the real data of a vertex into an AQLValue ////////////////////////////////////////////////////////////////////////////// - aql::AqlValue fetchVertexData(arangodb::velocypack::Slice) override; + aql::AqlValue fetchVertexData(StringRef) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to fetch the real data of an edge into an AQLValue ////////////////////////////////////////////////////////////////////////////// - aql::AqlValue fetchEdgeData(arangodb::velocypack::Slice) override; + aql::AqlValue fetchEdgeData(StringRef) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to add the real data of a vertex into a velocypack builder ////////////////////////////////////////////////////////////////////////////// - void addVertexToVelocyPack(arangodb::velocypack::Slice, + void addVertexToVelocyPack(StringRef, arangodb::velocypack::Builder&) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to add the real data of an edge into a velocypack builder ////////////////////////////////////////////////////////////////////////////// - void addEdgeToVelocyPack(arangodb::velocypack::Slice, + void addEdgeToVelocyPack(StringRef, arangodb::velocypack::Builder&) override; private: void fetchVertices(); - std::unordered_map + std::unordered_map _edges; - std::unordered_map>> + std::unordered_map>> _vertices; std::string _dbname; std::unordered_map const* _engines; - std::unordered_set _verticesToFetch; + std::unordered_set _verticesToFetch; std::vector> _datalake; diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index f86df70a24..1effe6b6c0 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -136,42 +136,38 @@ void BaseTraverserEngine::getEdges(VPackSlice vertex, size_t depth, VPackBuilder // We just hope someone has locked the shards properly. We have no clue... Thanks locking TRI_ASSERT(vertex.isString() || vertex.isArray()); - size_t cursorId = 0; size_t read = 0; size_t filtered = 0; ManagedDocumentResult mmdr; - std::vector result; + //std::vector result; builder.openObject(); builder.add(VPackValue("edges")); builder.openArray(); if (vertex.isArray()) { for (VPackSlice v : VPackArrayIterator(vertex)) { TRI_ASSERT(v.isString()); - result.clear(); - auto edgeCursor = _opts->nextCursor(&mmdr, v, depth); - while (edgeCursor->next(result, cursorId)) { - if (!_opts->evaluateEdgeExpression(result.back(), v, depth, cursorId)) { + //result.clear(); + StringRef vertexId(v); + auto edgeCursor = _opts->nextCursor(&mmdr, vertexId, depth); + + edgeCursor->readAll([&] (StringRef const& documentId, VPackSlice edge, size_t cursorId) { + if (!_opts->evaluateEdgeExpression(edge, StringRef(v), depth, cursorId)) { filtered++; - result.pop_back(); + } else { + builder.add(edge); } - } - for (auto const& it : result) { - builder.add(it); - } + }); // Result now contains all valid edges, probably multiples. } } else if (vertex.isString()) { - std::unique_ptr edgeCursor(_opts->nextCursor(&mmdr, vertex, depth)); - - while (edgeCursor->next(result, cursorId)) { - if (!_opts->evaluateEdgeExpression(result.back(), vertex, depth, cursorId)) { + std::unique_ptr edgeCursor(_opts->nextCursor(&mmdr, StringRef(vertex), depth)); + edgeCursor->readAll([&] (StringRef const& documentId, VPackSlice edge, size_t cursorId) { + if (!_opts->evaluateEdgeExpression(edge, StringRef(vertex), depth, cursorId)) { filtered++; - result.pop_back(); + } else { + builder.add(edge); } - } - for (auto const& it : result) { - builder.add(it); - } + }); // Result now contains all valid edges, probably multiples. } else { THROW_ARANGO_EXCEPTION(TRI_ERROR_BAD_PARAMETER); diff --git a/arangod/Graph/BreadthFirstEnumerator.cpp b/arangod/Graph/BreadthFirstEnumerator.cpp new file mode 100644 index 0000000000..1f6396ece3 --- /dev/null +++ b/arangod/Graph/BreadthFirstEnumerator.cpp @@ -0,0 +1,221 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "BreadthFirstEnumerator.h" + +#include "VocBase/Traverser.h" +#include "VocBase/TraverserCache.h" +#include "VocBase/TraverserOptions.h" + +#include +#include + +using namespace arangodb; +using namespace arangodb::traverser; + +using BreadthFirstEnumerator = arangodb::graph::BreadthFirstEnumerator; + +BreadthFirstEnumerator::PathStep::PathStep(StringRef const vertex) + : sourceIdx(0), + vertex(vertex) { +} + + +BreadthFirstEnumerator::PathStep::PathStep(size_t sourceIdx, + StringRef const edge, + StringRef const vertex) : + sourceIdx(sourceIdx), edge(edge), vertex(vertex) { +} + +BreadthFirstEnumerator::BreadthFirstEnumerator(Traverser* traverser, + VPackSlice startVertex, + TraverserOptions* opts) + : PathEnumerator(traverser, startVertex.copyString(), opts), + _schreierIndex(1), + _lastReturned(0), + _currentDepth(0), + _toSearchPos(0) { + _schreier.reserve(32); + StringRef startVId = _opts->cache()->persistString(StringRef(startVertex)); + + _schreier.emplace_back(startVId); + _toSearch.emplace_back(NextStep(0)); +} + +bool BreadthFirstEnumerator::next() { + if (_isFirst) { + _isFirst = false; + if (_opts->minDepth == 0) { + return true; + } + } + _lastReturned++; + + if (_lastReturned < _schreierIndex) { + // We still have something on our stack. + // Paths have been read but not returned. + return true; + } + + if (_opts->maxDepth == 0) { + // Short circuit. + // We cannot find any path of length 0 or less + return false; + } + // Avoid large call stacks. + // Loop will be left if we are either finished + // with searching. + // Or we found vertices in the next depth for + // a vertex. + while (true) { + if (_toSearchPos >= _toSearch.size()) { + // This depth is done. GoTo next + if (_nextDepth.empty()) { + // That's it. we are done. + return false; + } + // Save copies: + // We clear current + // we swap current and next. + // So now current is filled + // and next is empty. + _toSearch.clear(); + _toSearchPos = 0; + _toSearch.swap(_nextDepth); + _currentDepth++; + TRI_ASSERT(_toSearchPos < _toSearch.size()); + TRI_ASSERT(_nextDepth.empty()); + TRI_ASSERT(_currentDepth < _opts->maxDepth); + } + // This access is always safe. + // If not it should have bailed out before. + TRI_ASSERT(_toSearchPos < _toSearch.size()); + + _tmpEdges.clear(); + auto const nextIdx = _toSearch[_toSearchPos++].sourceIdx; + auto const nextVertex = _schreier[nextIdx].vertex; + StringRef vId; + + std::unique_ptr cursor(_opts->nextCursor(_traverser->mmdr(), nextVertex, _currentDepth)); + if (cursor != nullptr) { + bool shouldReturnPath = _currentDepth + 1 >= _opts->minDepth; + bool didInsert = false; + + auto callback = [&] (arangodb::StringRef const& eid, VPackSlice e, size_t cursorIdx) -> void { + if (_opts->uniqueEdges == + TraverserOptions::UniquenessLevel::GLOBAL) { + if (_returnedEdges.find(eid) == _returnedEdges.end()) { + // Edge not yet visited. Mark and continue. + // TODO FIXME the edge will run out of scope + _returnedEdges.emplace(eid); + } else { + // Edge filtered due to unique_constraint + _traverser->_filteredPaths++; + return; + } + } + + if (!_traverser->edgeMatchesConditions(e, nextVertex, + _currentDepth, + cursorIdx)) { + return; + } + + if (_traverser->getSingleVertex(e, nextVertex, _currentDepth, vId)) { + _schreier.emplace_back(nextIdx, eid, vId); + if (_currentDepth < _opts->maxDepth - 1) { + _nextDepth.emplace_back(NextStep(_schreierIndex)); + } + _schreierIndex++; + didInsert = true; + } + }; + + cursor->readAll(callback); + + if (!shouldReturnPath) { + _lastReturned = _schreierIndex; + didInsert = false; + } + if (didInsert) { + // We exit the loop here. + // _schreierIndex is moved forward + break; + } + } + // Nothing found for this vertex. + // _toSearchPos is increased so + // we are not stuck in an endless loop + } + + // _lastReturned points to the last used + // entry. We compute the path to it. + return true; +} + +arangodb::aql::AqlValue BreadthFirstEnumerator::lastVertexToAqlValue() { + TRI_ASSERT(_lastReturned < _schreier.size()); + PathStep const& current = _schreier[_lastReturned]; + return _traverser->fetchVertexData(StringRef(current.vertex)); +} + +arangodb::aql::AqlValue BreadthFirstEnumerator::lastEdgeToAqlValue() { + TRI_ASSERT(_lastReturned < _schreier.size()); + if (_lastReturned == 0) { + // This is the first Vertex. No Edge Pointing to it + return arangodb::aql::AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); + } + PathStep const& current = _schreier[_lastReturned]; + return _traverser->fetchEdgeData(StringRef(current.edge)); +} + +arangodb::aql::AqlValue BreadthFirstEnumerator::pathToAqlValue( + arangodb::velocypack::Builder& result) { + // TODO make deque class variable + std::deque fullPath; + size_t cur = _lastReturned; + while (cur != 0) { + // Walk backwards through the path and push everything found on the local stack + fullPath.emplace_front(cur); + cur = _schreier[cur].sourceIdx; + } + + result.clear(); + result.openObject(); + result.add(VPackValue("edges")); + result.openArray(); + for (auto const& idx : fullPath) { + _traverser->addEdgeToVelocyPack(StringRef(_schreier[idx].edge), result); + } + result.close(); // edges + result.add(VPackValue("vertices")); + result.openArray(); + // Always add the start vertex + _traverser->addVertexToVelocyPack(_schreier[0].vertex, result); + for (auto const& idx : fullPath) { + _traverser->addVertexToVelocyPack(_schreier[idx].vertex, result); + } + result.close(); // vertices + result.close(); + return arangodb::aql::AqlValue(result.slice()); +} diff --git a/arangod/Graph/BreadthFirstEnumerator.h b/arangod/Graph/BreadthFirstEnumerator.h new file mode 100644 index 0000000000..35c80c7e5b --- /dev/null +++ b/arangod/Graph/BreadthFirstEnumerator.h @@ -0,0 +1,166 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_GRAPH_BREADTHFIRSTENUMERATOR_H +#define ARANGODB_GRAPH_BREADTHFIRSTENUMERATOR_H 1 + +#include "Basics/Common.h" +#include "VocBase/PathEnumerator.h" + +namespace arangodb { + +namespace traverser { +class Traverser; +struct TraverserOptions; +} + +namespace graph { + +class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator { + private: + + ////////////////////////////////////////////////////////////////////////////// + /// @brief One entry in the schreier vector + ////////////////////////////////////////////////////////////////////////////// + + struct PathStep { + size_t sourceIdx; + arangodb::StringRef const edge; + arangodb::StringRef const vertex; + + private: + PathStep() {} + + public: + explicit PathStep(arangodb::StringRef const vertex); + + PathStep(size_t sourceIdx, arangodb::StringRef const edge, + arangodb::StringRef const vertex); + }; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Struct to hold all information required to get the list of + /// connected edges + ////////////////////////////////////////////////////////////////////////////// + + struct NextStep { + size_t sourceIdx; + + private: + NextStep() = delete; + + public: + explicit NextStep(size_t sourceIdx) + : sourceIdx(sourceIdx) {} + }; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief schreier vector to store the visited vertices + ////////////////////////////////////////////////////////////////////////////// + + std::vector _schreier; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Next free index in schreier vector. + ////////////////////////////////////////////////////////////////////////////// + + size_t _schreierIndex; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Position of the last returned value in the schreier vector + ////////////////////////////////////////////////////////////////////////////// + + size_t _lastReturned; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Vector to store where to continue search on next depth + ////////////////////////////////////////////////////////////////////////////// + + std::vector _nextDepth; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Vector storing the position at current search depth + ////////////////////////////////////////////////////////////////////////////// + + std::vector _toSearch; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Vector storing the position at current search depth + ////////////////////////////////////////////////////////////////////////////// + + std::unordered_set _tmpEdges; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Marker for the search depth. Used to abort searching. + ////////////////////////////////////////////////////////////////////////////// + + uint64_t _currentDepth; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief position in _toSearch. If this is >= _toSearch.size() we are done + /// with this depth. + ////////////////////////////////////////////////////////////////////////////// + + size_t _toSearchPos; + + public: + BreadthFirstEnumerator(arangodb::traverser::Traverser* traverser, + arangodb::velocypack::Slice startVertex, + arangodb::traverser::TraverserOptions* opts); + + ~BreadthFirstEnumerator() {} + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Get the next Path element from the traversal. + ////////////////////////////////////////////////////////////////////////////// + + bool next() override; + + aql::AqlValue lastVertexToAqlValue() override; + + aql::AqlValue lastEdgeToAqlValue() override; + + aql::AqlValue pathToAqlValue(arangodb::velocypack::Builder& result) override; + + private: + + inline size_t getDepth(size_t index) const { + size_t depth = 0; + while (index != 0) { + ++depth; + index = _schreier[index].sourceIdx; + } + return depth; + } + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Build the enumerated path for the given index in the schreier + /// vector. + ////////////////////////////////////////////////////////////////////////////// + + void computeEnumeratedPath(size_t index); +}; +} +} + +#endif diff --git a/arangod/Graph/NeighborsEnumerator.cpp b/arangod/Graph/NeighborsEnumerator.cpp new file mode 100644 index 0000000000..c681190287 --- /dev/null +++ b/arangod/Graph/NeighborsEnumerator.cpp @@ -0,0 +1,105 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "NeighborsEnumerator.h" + +#include "Basics/VelocyPackHelper.h" +#include "VocBase/Traverser.h" +#include "VocBase/TraverserCache.h" + +using namespace arangodb; +using namespace arangodb::traverser; +using namespace arangodb::graph; + +NeighborsEnumerator::NeighborsEnumerator(Traverser* traverser, + VPackSlice const& startVertex, + TraverserOptions* opts) + : PathEnumerator(traverser, startVertex.copyString(), opts), + _searchDepth(0) { + StringRef vId = _traverser->traverserCache()->persistString(StringRef(startVertex)); + _allFound.insert(vId); + _currentDepth.insert(vId); + _iterator = _currentDepth.begin(); +} + +bool NeighborsEnumerator::next() { + if (_isFirst) { + _isFirst = false; + if (_opts->minDepth == 0) { + return true; + } + } + + if (_iterator == _currentDepth.end() || ++_iterator == _currentDepth.end()) { + do { + // This depth is done. Get next + if (_opts->maxDepth == _searchDepth) { + // We are finished. + return false; + } + + _lastDepth.swap(_currentDepth); + _currentDepth.clear(); + StringRef v; + for (auto const& nextVertex : _lastDepth) { + auto callback = [&](StringRef const& edgeId, VPackSlice e, size_t& cursorId) { + // Counting should be done in readAll + _traverser->_readDocuments++; + if (_traverser->getSingleVertex(e, nextVertex, _searchDepth, v)) { + StringRef otherId = _traverser->traverserCache()->persistString(v); + if (_allFound.find(otherId) == _allFound.end()) { + _currentDepth.emplace(otherId); + _allFound.emplace(otherId); + } + } + }; + std::unique_ptr cursor( + _opts->nextCursor(_traverser->mmdr(), nextVertex, _searchDepth)); + cursor->readAll(callback); + } + if (_currentDepth.empty()) { + // Nothing found. Cannot do anything more. + return false; + } + ++_searchDepth; + } while (_searchDepth < _opts->minDepth); + _iterator = _currentDepth.begin(); + } + TRI_ASSERT(_iterator != _currentDepth.end()); + return true; +} + +arangodb::aql::AqlValue NeighborsEnumerator::lastVertexToAqlValue() { + TRI_ASSERT(_iterator != _currentDepth.end()); + return _traverser->fetchVertexData(*_iterator); +} + +arangodb::aql::AqlValue NeighborsEnumerator::lastEdgeToAqlValue() { + // TODO should return Optimizer failed + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +} + +arangodb::aql::AqlValue NeighborsEnumerator::pathToAqlValue(arangodb::velocypack::Builder& result) { + // TODO should return Optimizer failed + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +} diff --git a/arangod/Graph/NeighborsEnumerator.h b/arangod/Graph/NeighborsEnumerator.h new file mode 100644 index 0000000000..63c994ce71 --- /dev/null +++ b/arangod/Graph/NeighborsEnumerator.h @@ -0,0 +1,76 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_GRAPH_NEIGHBORSENUMERATOR_H +#define ARANGODB_GRAPH_NEIGHBORSENUMERATOR_H 1 + +#include "Basics/Common.h" +#include "VocBase/PathEnumerator.h" + +#include + +namespace arangodb { +namespace graph { +// @brief Enumerator optimized for neighbors. Does not allow edge access + +class NeighborsEnumerator final : public arangodb::traverser::PathEnumerator { + std::unordered_set _allFound; + std::unordered_set _currentDepth; + std::unordered_set _lastDepth; + std::unordered_set::iterator _iterator; + + uint64_t _searchDepth; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Vector storing the position at current search depth + ////////////////////////////////////////////////////////////////////////////// + + std::unordered_set _tmpEdges; + + + public: + NeighborsEnumerator(arangodb::traverser::Traverser* traverser, + arangodb::velocypack::Slice const& startVertex, + arangodb::traverser::TraverserOptions* opts); + + ~NeighborsEnumerator() { + } + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Get the next Path element from the traversal. + ////////////////////////////////////////////////////////////////////////////// + + bool next() override; + + aql::AqlValue lastVertexToAqlValue() override; + + aql::AqlValue lastEdgeToAqlValue() override; + + aql::AqlValue pathToAqlValue(arangodb::velocypack::Builder& result) override; + +}; + +} // namespace graph +} // namespace arangodb + +#endif diff --git a/arangod/VocBase/PathEnumerator.cpp b/arangod/VocBase/PathEnumerator.cpp index 0ef556fa55..f645b9a89e 100644 --- a/arangod/VocBase/PathEnumerator.cpp +++ b/arangod/VocBase/PathEnumerator.cpp @@ -24,13 +24,22 @@ #include "PathEnumerator.h" #include "Basics/VelocyPackHelper.h" #include "VocBase/Traverser.h" +#include "VocBase/TraverserCache.h" +using PathEnumerator = arangodb::traverser::PathEnumerator; using DepthFirstEnumerator = arangodb::traverser::DepthFirstEnumerator; -using BreadthFirstEnumerator = arangodb::traverser::BreadthFirstEnumerator; -using NeighborsEnumerator = arangodb::traverser::NeighborsEnumerator; using Traverser = arangodb::traverser::Traverser; using TraverserOptions = arangodb::traverser::TraverserOptions; +PathEnumerator::PathEnumerator(Traverser* traverser, std::string const& startVertex, + TraverserOptions* opts) + : _traverser(traverser), _isFirst(true), _opts(opts) { + StringRef svId = _opts->cache()->persistString(StringRef(startVertex)); + // Guarantee that this vertex _id does not run away + _enumeratedPath.vertices.push_back(svId); + TRI_ASSERT(_enumeratedPath.vertices.size() == 1); +} + bool DepthFirstEnumerator::next() { if (_isFirst) { _isFirst = false; @@ -43,13 +52,11 @@ bool DepthFirstEnumerator::next() { return false; } - size_t cursorId = 0; - while (true) { if (_enumeratedPath.edges.size() < _opts->maxDepth) { // We are not done with this path, so // we reserve the cursor for next depth - auto cursor = _opts->nextCursor(_traverser->mmdr(), _enumeratedPath.vertices.back(), + auto cursor = _opts->nextCursor(_traverser->mmdr(), StringRef(_enumeratedPath.vertices.back()), _enumeratedPath.edges.size()); if (cursor != nullptr) { _edgeCursors.emplace(cursor); @@ -65,34 +72,39 @@ bool DepthFirstEnumerator::next() { while (!_edgeCursors.empty()) { TRI_ASSERT(_edgeCursors.size() == _enumeratedPath.edges.size() + 1); auto& cursor = _edgeCursors.top(); - if (cursor->next(_enumeratedPath.edges, cursorId)) { + + bool foundPath = false; + bool exitInnerLoop = false; + auto callback = [&] (StringRef const& eid, VPackSlice const& edge, size_t cursorId) { ++_traverser->_readDocuments; + _enumeratedPath.edges.push_back(eid); + _opts->cache()->insertDocument(StringRef(eid), edge); // TODO handle in cursor directly? + if (_opts->uniqueEdges == TraverserOptions::UniquenessLevel::GLOBAL) { - if (_returnedEdges.find(_enumeratedPath.edges.back()) == - _returnedEdges.end()) { + if (_returnedEdges.find(eid) == _returnedEdges.end()) { // Edge not yet visited. Mark and continue. - _returnedEdges.emplace(_enumeratedPath.edges.back()); + _returnedEdges.emplace(eid); } else { _traverser->_filteredPaths++; TRI_ASSERT(!_enumeratedPath.edges.empty()); _enumeratedPath.edges.pop_back(); - continue; + return; } } - if (!_traverser->edgeMatchesConditions(_enumeratedPath.edges.back(), - _enumeratedPath.vertices.back(), + if (!_traverser->edgeMatchesConditions(edge, + StringRef(_enumeratedPath.vertices.back()), _enumeratedPath.edges.size() - 1, cursorId)) { - // This edge does not pass the filtering - TRI_ASSERT(!_enumeratedPath.edges.empty()); - _enumeratedPath.edges.pop_back(); - continue; + // This edge does not pass the filtering + TRI_ASSERT(!_enumeratedPath.edges.empty()); + _enumeratedPath.edges.pop_back(); + return; } - + if (_opts->uniqueEdges == TraverserOptions::UniquenessLevel::PATH) { - auto& e = _enumeratedPath.edges.back(); + StringRef const& e = _enumeratedPath.edges.back(); bool foundOnce = false; - for (auto const& it : _enumeratedPath.edges) { + for (StringRef const& it : _enumeratedPath.edges) { if (foundOnce) { foundOnce = false; // if we leave with foundOnce == false we found the edge earlier break; @@ -106,13 +118,12 @@ bool DepthFirstEnumerator::next() { // This edge is allready on the path TRI_ASSERT(!_enumeratedPath.edges.empty()); _enumeratedPath.edges.pop_back(); - continue; + return; } } - + // We have to check if edge and vertex is valid - if (_traverser->getVertex(_enumeratedPath.edges.back(), - _enumeratedPath.vertices)) { + if (_traverser->getVertex(edge, _enumeratedPath.vertices)) { // case both are valid. if (_opts->uniqueVertices == TraverserOptions::UniquenessLevel::PATH) { auto& e = _enumeratedPath.vertices.back(); @@ -120,7 +131,7 @@ bool DepthFirstEnumerator::next() { for (auto const& it : _enumeratedPath.vertices) { if (foundOnce) { foundOnce = false; // if we leave with foundOnce == false we - // found the edge earlier + // found the edge earlier break; } if (it == e) { @@ -133,20 +144,28 @@ bool DepthFirstEnumerator::next() { TRI_ASSERT(!_enumeratedPath.edges.empty()); _enumeratedPath.vertices.pop_back(); _enumeratedPath.edges.pop_back(); - continue; + return; } } if (_enumeratedPath.edges.size() < _opts->minDepth) { // Do not return, but leave this loop. Continue with the outer. - break; + exitInnerLoop = true; + return; } - - return true; + + foundPath = true; + return; } // Vertex Invalid. Revoke edge TRI_ASSERT(!_enumeratedPath.edges.empty()); _enumeratedPath.edges.pop_back(); - continue; + }; + if (cursor->next(callback)) { + if (foundPath) { + return true; + } else if(exitInnerLoop) { + break; + } } else { // cursor is empty. _edgeCursors.pop(); @@ -155,25 +174,25 @@ bool DepthFirstEnumerator::next() { _enumeratedPath.vertices.pop_back(); } } - } + }// while (!_edgeCursors.empty()) if (_edgeCursors.empty()) { // If we get here all cursors are exhausted. _enumeratedPath.edges.clear(); _enumeratedPath.vertices.clear(); return false; } - } + }// while (true) } arangodb::aql::AqlValue DepthFirstEnumerator::lastVertexToAqlValue() { - return _traverser->fetchVertexData(_enumeratedPath.vertices.back()); + return _traverser->fetchVertexData(StringRef(_enumeratedPath.vertices.back())); } arangodb::aql::AqlValue DepthFirstEnumerator::lastEdgeToAqlValue() { if (_enumeratedPath.edges.empty()) { return arangodb::aql::AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); } - return _traverser->fetchEdgeData(_enumeratedPath.edges.back()); + return _traverser->fetchEdgeData(StringRef(_enumeratedPath.edges.back())); } arangodb::aql::AqlValue DepthFirstEnumerator::pathToAqlValue(arangodb::velocypack::Builder& result) { @@ -182,280 +201,15 @@ arangodb::aql::AqlValue DepthFirstEnumerator::pathToAqlValue(arangodb::velocypac result.add(VPackValue("edges")); result.openArray(); for (auto const& it : _enumeratedPath.edges) { - _traverser->addEdgeToVelocyPack(it, result); + _traverser->addEdgeToVelocyPack(StringRef(it), result); } result.close(); result.add(VPackValue("vertices")); result.openArray(); for (auto const& it : _enumeratedPath.vertices) { - _traverser->addVertexToVelocyPack(it, result); + _traverser->addVertexToVelocyPack(StringRef(it), result); } result.close(); result.close(); return arangodb::aql::AqlValue(result.slice()); } - -BreadthFirstEnumerator::BreadthFirstEnumerator(Traverser* traverser, - VPackSlice startVertex, - TraverserOptions const* opts) - : PathEnumerator(traverser, startVertex, opts), - _schreierIndex(1), - _lastReturned(0), - _currentDepth(0), - _toSearchPos(0) { - _schreier.reserve(32); - _schreier.emplace_back(startVertex); - - _toSearch.emplace_back(NextStep(0)); -} - -bool BreadthFirstEnumerator::next() { - if (_isFirst) { - _isFirst = false; - if (_opts->minDepth == 0) { - computeEnumeratedPath(_lastReturned++); - return true; - } - _lastReturned++; - } - - if (_lastReturned < _schreierIndex) { - // We still have something on our stack. - // Paths have been read but not returned. - computeEnumeratedPath(_lastReturned++); - return true; - } - - if (_opts->maxDepth == 0) { - // Short circuit. - // We cannot find any path of length 0 or less - return false; - } - // Avoid large call stacks. - // Loop will be left if we are either finished - // with searching. - // Or we found vertices in the next depth for - // a vertex. - while (true) { - if (_toSearchPos >= _toSearch.size()) { - // This depth is done. GoTo next - if (_nextDepth.empty()) { - // That's it. we are done. - _enumeratedPath.edges.clear(); - _enumeratedPath.vertices.clear(); - return false; - } - // Save copies: - // We clear current - // we swap current and next. - // So now current is filled - // and next is empty. - _toSearch.clear(); - _toSearchPos = 0; - _toSearch.swap(_nextDepth); - _currentDepth++; - TRI_ASSERT(_toSearchPos < _toSearch.size()); - TRI_ASSERT(_nextDepth.empty()); - TRI_ASSERT(_currentDepth < _opts->maxDepth); - } - // This access is always safe. - // If not it should have bailed out before. - TRI_ASSERT(_toSearchPos < _toSearch.size()); - - _tmpEdges.clear(); - auto const nextIdx = _toSearch[_toSearchPos++].sourceIdx; - auto const nextVertex = _schreier[nextIdx].vertex; - - std::unique_ptr cursor(_opts->nextCursor(_traverser->mmdr(), nextVertex, _currentDepth)); - if (cursor != nullptr) { - size_t cursorIdx; - bool shouldReturnPath = _currentDepth + 1 >= _opts->minDepth; - bool didInsert = false; - while (cursor->readAll(_tmpEdges, cursorIdx)) { - if (!_tmpEdges.empty()) { - _traverser->_readDocuments += _tmpEdges.size(); - VPackSlice v; - for (auto const& e : _tmpEdges) { - if (_opts->uniqueEdges == - TraverserOptions::UniquenessLevel::GLOBAL) { - if (_returnedEdges.find(e) == _returnedEdges.end()) { - // Edge not yet visited. Mark and continue. - _returnedEdges.emplace(e); - } else { - _traverser->_filteredPaths++; - continue; - } - } - - if (!_traverser->edgeMatchesConditions(e, nextVertex, - _currentDepth, - cursorIdx)) { - continue; - } - if (_traverser->getSingleVertex(e, nextVertex, _currentDepth, v)) { - _schreier.emplace_back(nextIdx, e, v); - if (_currentDepth < _opts->maxDepth - 1) { - _nextDepth.emplace_back(NextStep(_schreierIndex)); - } - _schreierIndex++; - didInsert = true; - } - } - _tmpEdges.clear(); - } - } - if (!shouldReturnPath) { - _lastReturned = _schreierIndex; - didInsert = false; - } - if (didInsert) { - // We exit the loop here. - // _schreierIndex is moved forward - break; - } - } - // Nothing found for this vertex. - // _toSearchPos is increased so - // we are not stuck in an endless loop - } - - // _lastReturned points to the last used - // entry. We compute the path to it - // and increase the schreierIndex to point - // to the next free position. - computeEnumeratedPath(_lastReturned++); - return true; -} - -// TODO Optimize this. Remove enumeratedPath -// All can be read from schreier vector directly -arangodb::aql::AqlValue BreadthFirstEnumerator::lastVertexToAqlValue() { - return _traverser->fetchVertexData( - _enumeratedPath.vertices.back()); -} - -arangodb::aql::AqlValue BreadthFirstEnumerator::lastEdgeToAqlValue() { - if (_enumeratedPath.edges.empty()) { - return arangodb::aql::AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); - } - return _traverser->fetchEdgeData(_enumeratedPath.edges.back()); -} - -arangodb::aql::AqlValue BreadthFirstEnumerator::pathToAqlValue( - arangodb::velocypack::Builder& result) { - result.clear(); - result.openObject(); - result.add(VPackValue("edges")); - result.openArray(); - for (auto const& it : _enumeratedPath.edges) { - _traverser->addEdgeToVelocyPack(it, result); - } - result.close(); - result.add(VPackValue("vertices")); - result.openArray(); - for (auto const& it : _enumeratedPath.vertices) { - _traverser->addVertexToVelocyPack(it, result); - } - result.close(); - result.close(); - return arangodb::aql::AqlValue(result.slice()); -} - -void BreadthFirstEnumerator::computeEnumeratedPath(size_t index) { - TRI_ASSERT(index < _schreier.size()); - - size_t depth = getDepth(index); - _enumeratedPath.edges.clear(); - _enumeratedPath.vertices.clear(); - _enumeratedPath.edges.resize(depth); - _enumeratedPath.vertices.resize(depth + 1); - - // Computed path. Insert it into the path enumerator. - while (index != 0) { - TRI_ASSERT(depth > 0); - PathStep const& current = _schreier[index]; - _enumeratedPath.vertices[depth] = current.vertex; - _enumeratedPath.edges[depth - 1] = current.edge; - - index = current.sourceIdx; - --depth; - } - - _enumeratedPath.vertices[0] = _schreier[0].vertex; -} - -NeighborsEnumerator::NeighborsEnumerator(Traverser* traverser, - VPackSlice startVertex, - TraverserOptions const* opts) - : PathEnumerator(traverser, startVertex, opts), - _searchDepth(0) { - _allFound.insert(arangodb::basics::VPackHashedSlice(startVertex)); - _currentDepth.insert(arangodb::basics::VPackHashedSlice(startVertex)); - _iterator = _currentDepth.begin(); -} - -bool NeighborsEnumerator::next() { - if (_isFirst) { - _isFirst = false; - if (_opts->minDepth == 0) { - return true; - } - } - - if (_iterator == _currentDepth.end() || ++_iterator == _currentDepth.end()) { - do { - // This depth is done. Get next - if (_opts->maxDepth == _searchDepth) { - // We are finished. - return false; - } - - _lastDepth.swap(_currentDepth); - _currentDepth.clear(); - for (auto const& nextVertex : _lastDepth) { - size_t cursorIdx = 0; - std::unique_ptr cursor( - _opts->nextCursor(_traverser->mmdr(), nextVertex.slice, _searchDepth)); - while (cursor->readAll(_tmpEdges, cursorIdx)) { - if (!_tmpEdges.empty()) { - _traverser->_readDocuments += _tmpEdges.size(); - VPackSlice v; - for (auto const& e : _tmpEdges) { - if (_traverser->getSingleVertex(e, nextVertex.slice, _searchDepth, v)) { - arangodb::basics::VPackHashedSlice hashed(v); - if (_allFound.find(hashed) == _allFound.end()) { - _currentDepth.emplace(hashed); - _allFound.emplace(hashed); - } - } - } - _tmpEdges.clear(); - } - } - } - if (_currentDepth.empty()) { - // Nothing found. Cannot do anything more. - return false; - } - ++_searchDepth; - } while (_searchDepth < _opts->minDepth); - _iterator = _currentDepth.begin(); - } - TRI_ASSERT(_iterator != _currentDepth.end()); - return true; -} - -arangodb::aql::AqlValue NeighborsEnumerator::lastVertexToAqlValue() { - TRI_ASSERT(_iterator != _currentDepth.end()); - return _traverser->fetchVertexData((*_iterator).slice); -} - -arangodb::aql::AqlValue NeighborsEnumerator::lastEdgeToAqlValue() { - // TODO should return Optimizer failed - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -arangodb::aql::AqlValue NeighborsEnumerator::pathToAqlValue(arangodb::velocypack::Builder& result) { - // TODO should return Optimizer failed - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} diff --git a/arangod/VocBase/PathEnumerator.h b/arangod/VocBase/PathEnumerator.h index 64712d94ad..391de44f3f 100644 --- a/arangod/VocBase/PathEnumerator.h +++ b/arangod/VocBase/PathEnumerator.h @@ -25,6 +25,7 @@ #define ARANGODB_VOCBASE_PATHENUMERATOR_H 1 #include "Basics/Common.h" +#include "VocBase/Traverser.h" #include "VocBase/TraverserOptions.h" #include #include @@ -43,8 +44,8 @@ class Traverser; struct TraverserOptions; struct EnumeratedPath { - std::vector edges; - std::vector vertices; + std::vector edges; + std::vector vertices; EnumeratedPath() {} }; @@ -69,10 +70,10 @@ class PathEnumerator { bool _isFirst; ////////////////////////////////////////////////////////////////////////////// - /// @brief Maximal path length which should be enumerated. + /// @brief Options used in the traversal ////////////////////////////////////////////////////////////////////////////// - TraverserOptions const* _opts; + TraverserOptions* _opts; ////////////////////////////////////////////////////////////////////////////// /// @brief List of the last path is used to @@ -81,16 +82,11 @@ class PathEnumerator { EnumeratedPath _enumeratedPath; /// @brief List which edges have been visited already. - std::unordered_set _returnedEdges; + std::unordered_set _returnedEdges; public: - PathEnumerator(Traverser* traverser, arangodb::velocypack::Slice startVertex, - TraverserOptions const* opts) - : _traverser(traverser), _isFirst(true), _opts(opts) { - TRI_ASSERT(startVertex.isString()); - _enumeratedPath.vertices.push_back(startVertex); - TRI_ASSERT(_enumeratedPath.vertices.size() == 1); - } + PathEnumerator(Traverser* traverser, std::string const& startVertex, + TraverserOptions* opts); virtual ~PathEnumerator() {} @@ -117,8 +113,8 @@ class DepthFirstEnumerator final : public PathEnumerator { public: DepthFirstEnumerator(Traverser* traverser, - arangodb::velocypack::Slice startVertex, - TraverserOptions const* opts) + std::string const& startVertex, + TraverserOptions* opts) : PathEnumerator(traverser, startVertex, opts) {} ~DepthFirstEnumerator() { @@ -142,184 +138,6 @@ class DepthFirstEnumerator final : public PathEnumerator { aql::AqlValue pathToAqlValue(arangodb::velocypack::Builder& result) override; }; - -class BreadthFirstEnumerator final : public PathEnumerator { - private: - - ////////////////////////////////////////////////////////////////////////////// - /// @brief One entry in the schreier vector - ////////////////////////////////////////////////////////////////////////////// - - struct PathStep { - size_t sourceIdx; - arangodb::velocypack::Slice edge; - arangodb::velocypack::Slice vertex; - - private: - PathStep() {} - - public: - explicit PathStep(arangodb::velocypack::Slice vertex) : sourceIdx(0), vertex(vertex) {} - - PathStep(size_t sourceIdx, arangodb::velocypack::Slice edge, - arangodb::velocypack::Slice vertex) - : sourceIdx(sourceIdx), edge(edge), vertex(vertex) {} - }; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Struct to hold all information required to get the list of - /// connected edges - ////////////////////////////////////////////////////////////////////////////// - - struct NextStep { - size_t sourceIdx; - - private: - NextStep() = delete; - - public: - explicit NextStep(size_t sourceIdx) - : sourceIdx(sourceIdx) {} - }; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief schreier vector to store the visited vertices - ////////////////////////////////////////////////////////////////////////////// - - std::vector _schreier; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Next free index in schreier vector. - ////////////////////////////////////////////////////////////////////////////// - - size_t _schreierIndex; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Position of the last returned value in the schreier vector - ////////////////////////////////////////////////////////////////////////////// - - size_t _lastReturned; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Vector to store where to continue search on next depth - ////////////////////////////////////////////////////////////////////////////// - - std::vector _nextDepth; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Vector storing the position at current search depth - ////////////////////////////////////////////////////////////////////////////// - - std::vector _toSearch; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Vector storing the position at current search depth - ////////////////////////////////////////////////////////////////////////////// - - std::unordered_set _tmpEdges; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Marker for the search depth. Used to abort searching. - ////////////////////////////////////////////////////////////////////////////// - - uint64_t _currentDepth; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief position in _toSearch. If this is >= _toSearch.size() we are done - /// with this depth. - ////////////////////////////////////////////////////////////////////////////// - - size_t _toSearchPos; - - public: - BreadthFirstEnumerator(Traverser* traverser, - arangodb::velocypack::Slice startVertex, - TraverserOptions const* opts); - - ~BreadthFirstEnumerator() {} - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Get the next Path element from the traversal. - ////////////////////////////////////////////////////////////////////////////// - - bool next() override; - - aql::AqlValue lastVertexToAqlValue() override; - - aql::AqlValue lastEdgeToAqlValue() override; - - aql::AqlValue pathToAqlValue(arangodb::velocypack::Builder& result) override; - - private: - - inline size_t getDepth(size_t index) const { - size_t depth = 0; - while (index != 0) { - ++depth; - index = _schreier[index].sourceIdx; - } - return depth; - } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Build the enumerated path for the given index in the schreier - /// vector. - ////////////////////////////////////////////////////////////////////////////// - - void computeEnumeratedPath(size_t index); -}; - -// @brief Enumerator optimized for neighbors. Does not allow edge access - -class NeighborsEnumerator final : public PathEnumerator { - std::unordered_set - _allFound; - - std::unordered_set - _currentDepth; - - std::unordered_set - _lastDepth; - - std::unordered_set::iterator _iterator; - uint64_t _searchDepth; - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Vector storing the position at current search depth - ////////////////////////////////////////////////////////////////////////////// - - std::unordered_set _tmpEdges; - - - public: - NeighborsEnumerator(Traverser* traverser, - arangodb::velocypack::Slice startVertex, - TraverserOptions const* opts); - - ~NeighborsEnumerator() { - } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief Get the next Path element from the traversal. - ////////////////////////////////////////////////////////////////////////////// - - bool next() override; - - aql::AqlValue lastVertexToAqlValue() override; - - aql::AqlValue lastEdgeToAqlValue() override; - - aql::AqlValue pathToAqlValue(arangodb::velocypack::Builder& result) override; - -}; - - } // namespace traverser } // namespace arangodb diff --git a/arangod/VocBase/SingleServerTraverser.cpp b/arangod/VocBase/SingleServerTraverser.cpp index 71b2719481..578a8f9202 100644 --- a/arangod/VocBase/SingleServerTraverser.cpp +++ b/arangod/VocBase/SingleServerTraverser.cpp @@ -23,12 +23,18 @@ #include "SingleServerTraverser.h" #include "Basics/StringRef.h" + +#include "Aql/AqlValue.h" +#include "Graph/BreadthFirstEnumerator.h" +#include "Graph/NeighborsEnumerator.h" #include "Transaction/Methods.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ManagedDocumentResult.h" +#include "VocBase/TraverserCache.h" using namespace arangodb; using namespace arangodb::traverser; +using namespace arangodb::graph; //////////////////////////////////////////////////////////////////////////////// /// @brief Get a document by it's ID. Also lazy locks the collection. @@ -37,28 +43,11 @@ using namespace arangodb::traverser; /// On all other cases this function throws. //////////////////////////////////////////////////////////////////////////////// -static int FetchDocumentById(transaction::Methods* trx, - StringRef const& id, - ManagedDocumentResult& result) { - size_t pos = id.find('/'); - if (pos == std::string::npos) { - TRI_ASSERT(false); - return TRI_ERROR_INTERNAL; - } - - int res = trx->documentFastPathLocal(id.substr(0, pos).toString(), - id.substr(pos + 1).toString(), result); - - if (res != TRI_ERROR_NO_ERROR && res != TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND) { - THROW_ARANGO_EXCEPTION(res); - } - return res; -} - SingleServerEdgeCursor::SingleServerEdgeCursor(ManagedDocumentResult* mmdr, - transaction::Methods* trx, + TraverserOptions* opts, size_t nrCursors, std::vector const* mapping) - : _trx(trx), + : _opts(opts), + _trx(opts->_trx), _mmdr(mmdr), _cursors(), _currentCursor(0), @@ -70,22 +59,24 @@ SingleServerEdgeCursor::SingleServerEdgeCursor(ManagedDocumentResult* mmdr, _cache.reserve(1000); }; -bool SingleServerEdgeCursor::next(std::vector& result, - size_t& cursorId) { +bool SingleServerEdgeCursor::next(std::function callback) { if (_currentCursor == _cursors.size()) { return false; } if (_cachePos < _cache.size()) { LogicalCollection* collection = _cursors[_currentCursor][_currentSubCursor]->collection(); if (collection->readDocument(_trx, _cache[_cachePos++], *_mmdr)) { - result.emplace_back(_mmdr->vpack()); - } - if (_internalCursorMapping != nullptr) { - TRI_ASSERT(_currentCursor < _internalCursorMapping->size()); - cursorId = _internalCursorMapping->at(_currentCursor); - } else { - cursorId = _currentCursor; + VPackSlice edgeDocument(_mmdr->vpack()); + std::string eid = _trx->extractIdString(edgeDocument); + StringRef persId = _opts->cache()->persistString(StringRef(eid)); + if (_internalCursorMapping != nullptr) { + TRI_ASSERT(_currentCursor < _internalCursorMapping->size()); + callback(persId, edgeDocument, _internalCursorMapping->at(_currentCursor)); + } else { + callback(persId, edgeDocument, _currentCursor); + } } + return true; } // We need to refill the cache. @@ -132,105 +123,73 @@ bool SingleServerEdgeCursor::next(std::vector& result, TRI_ASSERT(_cachePos < _cache.size()); LogicalCollection* collection = cursor->collection(); if (collection->readDocument(_trx, _cache[_cachePos++], *_mmdr)) { - result.emplace_back(_mmdr->vpack()); - } - if (_internalCursorMapping != nullptr) { - TRI_ASSERT(_currentCursor < _internalCursorMapping->size()); - cursorId = _internalCursorMapping->at(_currentCursor); - } else { - cursorId = _currentCursor; + VPackSlice edgeDocument(_mmdr->vpack()); + std::string eid = _trx->extractIdString(edgeDocument); + StringRef persId = _opts->cache()->persistString(StringRef(eid)); + if (_internalCursorMapping != nullptr) { + TRI_ASSERT(_currentCursor < _internalCursorMapping->size()); + callback(persId, edgeDocument, _internalCursorMapping->at(_currentCursor)); + } else { + callback(persId, edgeDocument, _currentCursor); + } } return true; } -bool SingleServerEdgeCursor::readAll(std::unordered_set& result, - size_t& cursorId) { - if (_currentCursor >= _cursors.size()) { - return false; - } - - if (_internalCursorMapping != nullptr) { - TRI_ASSERT(_currentCursor < _internalCursorMapping->size()); - cursorId = _internalCursorMapping->at(_currentCursor); - } else { - cursorId = _currentCursor; - } - - auto& cursorSet = _cursors[_currentCursor]; - for (auto& cursor : cursorSet) { - LogicalCollection* collection = cursor->collection(); - auto cb = [&] (DocumentIdentifierToken const& token) { - if (collection->readDocument(_trx, token, *_mmdr)) { - result.emplace(_mmdr->vpack()); +void SingleServerEdgeCursor::readAll(std::function callback) { + size_t cursorId = 0; + for (_currentCursor = 0; _currentCursor < _cursors.size(); ++_currentCursor) { + if (_internalCursorMapping != nullptr) { + TRI_ASSERT(_currentCursor < _internalCursorMapping->size()); + cursorId = _internalCursorMapping->at(_currentCursor); + } else { + cursorId = _currentCursor; + } + auto& cursorSet = _cursors[_currentCursor]; + for (auto& cursor : cursorSet) { + LogicalCollection* collection = cursor->collection(); + auto cb = [&] (DocumentIdentifierToken const& token) { + if (collection->readDocument(_trx, token, *_mmdr)) { + VPackSlice doc(_mmdr->vpack()); + std::string tmpId = _trx->extractIdString(doc); + StringRef edgeId = _opts->cache()->persistString(StringRef(tmpId)); + callback(edgeId, doc, cursorId); + } + }; + while (cursor->getMore(cb, 1000)) { } - }; - while (cursor->getMore(cb, 1000)) { } } - _currentCursor++; - return true; } SingleServerTraverser::SingleServerTraverser(TraverserOptions* opts, transaction::Methods* trx, ManagedDocumentResult* mmdr) - : Traverser(opts, trx, mmdr) {} + : Traverser(opts, trx, mmdr) {} SingleServerTraverser::~SingleServerTraverser() {} -aql::AqlValue SingleServerTraverser::fetchVertexData(VPackSlice id) { - TRI_ASSERT(id.isString()); - auto it = _vertices.find(id); - - if (it == _vertices.end()) { - StringRef ref(id); - int res = FetchDocumentById(_trx, ref, *_mmdr); - ++_readDocuments; - if (res != TRI_ERROR_NO_ERROR) { - return aql::AqlValue(basics::VelocyPackHelper::NullValue()); - } - - uint8_t const* p = _mmdr->vpack(); - _vertices.emplace(id, p); - return aql::AqlValue(p, aql::AqlValueFromManagedDocument()); - } - - return aql::AqlValue((*it).second, aql::AqlValueFromManagedDocument()); +aql::AqlValue SingleServerTraverser::fetchVertexData(StringRef vid) { + return _opts->cache()->fetchAqlResult(vid); } -aql::AqlValue SingleServerTraverser::fetchEdgeData(VPackSlice edge) { - return aql::AqlValue(edge); +aql::AqlValue SingleServerTraverser::fetchEdgeData(StringRef edge) { + return _opts->cache()->fetchAqlResult(edge); } -void SingleServerTraverser::addVertexToVelocyPack(VPackSlice id, +void SingleServerTraverser::addVertexToVelocyPack(StringRef vid, VPackBuilder& result) { - TRI_ASSERT(id.isString()); - auto it = _vertices.find(id); - - if (it == _vertices.end()) { - StringRef ref(id); - int res = FetchDocumentById(_trx, ref, *_mmdr); - ++_readDocuments; - if (res != TRI_ERROR_NO_ERROR) { - result.add(basics::VelocyPackHelper::NullValue()); - } else { - uint8_t const* p = _mmdr->vpack(); - _vertices.emplace(id, p); - result.addExternal(p); - } - } else { - result.addExternal((*it).second); - } + _opts->cache()->insertIntoResult(vid, result); } -void SingleServerTraverser::addEdgeToVelocyPack(VPackSlice edge, +void SingleServerTraverser::addEdgeToVelocyPack(StringRef edge, VPackBuilder& result) { - result.addExternal(edge.begin()); + _opts->cache()->insertIntoResult(edge, result); } -void SingleServerTraverser::setStartVertex(std::string const& v) { +void SingleServerTraverser::setStartVertex(std::string const& vid) { _startIdBuilder->clear(); - _startIdBuilder->add(VPackValue(v)); + _startIdBuilder->add(VPackValue(vid)); VPackSlice idSlice = _startIdBuilder->slice(); if (!vertexMatchesConditions(idSlice, 0)) { @@ -239,7 +198,8 @@ void SingleServerTraverser::setStartVertex(std::string const& v) { return; } - _vertexGetter->reset(idSlice); + StringRef persId = _opts->cache()->persistString(StringRef(vid)); + _vertexGetter->reset(persId); if (_opts->useBreadthFirst) { if (_canUseOptimizedNeighbors) { @@ -248,17 +208,21 @@ void SingleServerTraverser::setStartVertex(std::string const& v) { _enumerator.reset(new BreadthFirstEnumerator(this, idSlice, _opts)); } } else { - _enumerator.reset(new DepthFirstEnumerator(this, idSlice, _opts)); + _enumerator.reset(new DepthFirstEnumerator(this, vid, _opts)); } _done = false; } +size_t SingleServerTraverser::getAndResetReadDocuments() { + return _opts->cache()->getAndResetInsertedDocuments(); +} + bool SingleServerTraverser::getVertex(VPackSlice edge, - std::vector& result) { + std::vector& result) { return _vertexGetter->getVertex(edge, result); } -bool SingleServerTraverser::getSingleVertex(VPackSlice edge, VPackSlice vertex, - uint64_t depth, VPackSlice& result) { - return _vertexGetter->getSingleVertex(edge, vertex, depth, result); +bool SingleServerTraverser::getSingleVertex(VPackSlice edge, StringRef const sourceVertexId, + uint64_t depth, StringRef& targetVertexId) { + return _vertexGetter->getSingleVertex(edge, sourceVertexId, depth, targetVertexId); } diff --git a/arangod/VocBase/SingleServerTraverser.h b/arangod/VocBase/SingleServerTraverser.h index 0e42b9998f..dc1c8dfebb 100644 --- a/arangod/VocBase/SingleServerTraverser.h +++ b/arangod/VocBase/SingleServerTraverser.h @@ -38,9 +38,10 @@ class ManagedDocumentResult; namespace traverser { class PathEnumerator; - + class SingleServerEdgeCursor : public EdgeCursor { private: + TraverserOptions* _opts; transaction::Methods* _trx; ManagedDocumentResult* _mmdr; std::vector> _cursors; @@ -51,7 +52,7 @@ class SingleServerEdgeCursor : public EdgeCursor { std::vector const* _internalCursorMapping; public: - SingleServerEdgeCursor(ManagedDocumentResult* mmdr, transaction::Methods* trx, size_t, std::vector const* mapping = nullptr); + SingleServerEdgeCursor(ManagedDocumentResult* mmdr, TraverserOptions* options, size_t, std::vector const* mapping = nullptr); ~SingleServerEdgeCursor() { for (auto& it : _cursors) { @@ -61,9 +62,9 @@ class SingleServerEdgeCursor : public EdgeCursor { } } - bool next(std::vector&, size_t&) override; + bool next(std::function callback) override; - bool readAll(std::unordered_set&, size_t&) override; + void readAll(std::function) override; std::vector>& getCursors() { return _cursors; @@ -82,62 +83,65 @@ class SingleServerTraverser final : public Traverser { ////////////////////////////////////////////////////////////////////////////// void setStartVertex(std::string const& v) override; + + size_t getAndResetReadDocuments() override; protected: /// @brief Function to load the other sides vertex of an edge /// Returns true if the vertex passes filtering conditions /// Adds the _id of the vertex into the given vector - bool getVertex(arangodb::velocypack::Slice, - std::vector&) override; + bool getVertex(arangodb::velocypack::Slice edge, + std::vector&) override; /// @brief Function to load the other sides vertex of an edge /// Returns true if the vertex passes filtering conditions - - bool getSingleVertex(arangodb::velocypack::Slice, arangodb::velocypack::Slice, - uint64_t depth, arangodb::velocypack::Slice&) override; + bool getSingleVertex(arangodb::velocypack::Slice edge, + arangodb::StringRef const sourceVertexId, + uint64_t depth, + arangodb::StringRef& targetVertexId) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to fetch the real data of a vertex into an AQLValue ////////////////////////////////////////////////////////////////////////////// - aql::AqlValue fetchVertexData(arangodb::velocypack::Slice) override; + aql::AqlValue fetchVertexData(StringRef) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to fetch the real data of an edge into an AQLValue ////////////////////////////////////////////////////////////////////////////// - aql::AqlValue fetchEdgeData(arangodb::velocypack::Slice) override; + aql::AqlValue fetchEdgeData(StringRef) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to add the real data of a vertex into a velocypack builder ////////////////////////////////////////////////////////////////////////////// - void addVertexToVelocyPack(arangodb::velocypack::Slice, + void addVertexToVelocyPack(StringRef, arangodb::velocypack::Builder&) override; ////////////////////////////////////////////////////////////////////////////// /// @brief Function to add the real data of an edge into a velocypack builder ////////////////////////////////////////////////////////////////////////////// - void addEdgeToVelocyPack(arangodb::velocypack::Slice, + void addEdgeToVelocyPack(StringRef, arangodb::velocypack::Builder&) override; private: - + ////////////////////////////////////////////////////////////////////////////// /// @brief Cache for vertex documents, points from _id to start of /// document VPack value (in datafiles) ////////////////////////////////////////////////////////////////////////////// - std::unordered_map _vertices; + //std::unordered_map _vertices; ////////////////////////////////////////////////////////////////////////////// /// @brief Cache for edge documents, points from _id to start of edge /// VPack value (in datafiles) ////////////////////////////////////////////////////////////////////////////// - std::unordered_map _edges; + //std::unordered_map _edges; }; } // namespace traverser diff --git a/arangod/VocBase/Traverser.cpp b/arangod/VocBase/Traverser.cpp index 044784a0db..7920a3f92a 100644 --- a/arangod/VocBase/Traverser.cpp +++ b/arangod/VocBase/Traverser.cpp @@ -28,13 +28,14 @@ #include "Transaction/Context.h" #include "VocBase/KeyGenerator.h" #include "VocBase/TraverserOptions.h" +#include "VocBase/TraverserCache.h" #include #include using namespace arangodb; +using namespace arangodb::traverser; -using Traverser = arangodb::traverser::Traverser; /// @brief Class Shortest Path /// @brief Clears the path @@ -75,96 +76,89 @@ void arangodb::traverser::ShortestPath::vertexToVelocyPack(transaction::Methods* } } -bool Traverser::VertexGetter::getVertex( - VPackSlice edge, std::vector& result) { - VPackSlice cmp = result.back(); +bool Traverser::VertexGetter::getVertex(VPackSlice edge, std::vector& result) { VPackSlice res = transaction::helpers::extractFromFromDocument(edge); - if (cmp == res) { + if (result.back() == StringRef(res)) { res = transaction::helpers::extractToFromDocument(edge); } if (!_traverser->vertexMatchesConditions(res, result.size())) { return false; } - result.emplace_back(res); + result.emplace_back(_traverser->traverserCache()->persistString(StringRef(res))); return true; } -bool Traverser::VertexGetter::getSingleVertex(VPackSlice edge, - VPackSlice cmp, - uint64_t depth, - VPackSlice& result) { +bool Traverser::VertexGetter::getSingleVertex(arangodb::velocypack::Slice edge, StringRef cmp, + uint64_t depth, StringRef& result) { + VPackSlice resSlice; VPackSlice from = transaction::helpers::extractFromFromDocument(edge); - if (from != cmp) { - result = from; + if (from.compareString(cmp.data(), cmp.length()) != 0) { + resSlice = from; } else { - result = transaction::helpers::extractToFromDocument(edge); + resSlice = transaction::helpers::extractToFromDocument(edge); } - return _traverser->vertexMatchesConditions(result, depth); + result = _traverser->traverserCache()->persistString(StringRef(resSlice)); + return _traverser->vertexMatchesConditions(resSlice, depth); } -void Traverser::VertexGetter::reset(arangodb::velocypack::Slice) { +void Traverser::VertexGetter::reset(StringRef const&) { } -bool Traverser::UniqueVertexGetter::getVertex( - VPackSlice edge, std::vector& result) { +bool Traverser::UniqueVertexGetter::getVertex(VPackSlice edge, std::vector& result) { VPackSlice toAdd = transaction::helpers::extractFromFromDocument(edge); - VPackSlice cmp = result.back(); - - if (toAdd == cmp) { + StringRef const& cmp = result.back(); + TRI_ASSERT(toAdd.isString()); + if (cmp == StringRef(toAdd)) { toAdd = transaction::helpers::extractToFromDocument(edge); } - - arangodb::basics::VPackHashedSlice hashed(toAdd); - + StringRef toAddStr = _traverser->traverserCache()->persistString(StringRef(toAdd)); // First check if we visited it. If not, then mark - if (_returnedVertices.find(hashed) != _returnedVertices.end()) { + if (_returnedVertices.find(toAddStr) != _returnedVertices.end()) { // This vertex is not unique. ++_traverser->_filteredPaths; return false; } else { - _returnedVertices.emplace(hashed); + _returnedVertices.emplace(toAddStr); } if (!_traverser->vertexMatchesConditions(toAdd, result.size())) { return false; } - result.emplace_back(toAdd); + result.emplace_back(toAddStr); return true; } -bool Traverser::UniqueVertexGetter::getSingleVertex( - VPackSlice edge, VPackSlice cmp, uint64_t depth, VPackSlice& result) { - result = transaction::helpers::extractFromFromDocument(edge); - - if (cmp == result) { - result = transaction::helpers::extractToFromDocument(edge); +bool Traverser::UniqueVertexGetter::getSingleVertex(arangodb::velocypack::Slice edge, StringRef cmp, + uint64_t depth, StringRef& result) { + VPackSlice resSlice = transaction::helpers::extractFromFromDocument(edge); + + if (resSlice.compareString(cmp.data(), cmp.length()) == 0) { + resSlice = transaction::helpers::extractToFromDocument(edge); } + TRI_ASSERT(resSlice.isString()); - arangodb::basics::VPackHashedSlice hashed(result); - + result = _traverser->traverserCache()->persistString(StringRef(resSlice)); // First check if we visited it. If not, then mark - if (_returnedVertices.find(hashed) != _returnedVertices.end()) { + if (_returnedVertices.find(result) != _returnedVertices.end()) { // This vertex is not unique. ++_traverser->_filteredPaths; return false; } else { - _returnedVertices.emplace(hashed); + _returnedVertices.emplace(result); } - - return _traverser->vertexMatchesConditions(result, depth); + return _traverser->vertexMatchesConditions(resSlice, depth); } -void Traverser::UniqueVertexGetter::reset(VPackSlice startVertex) { +void Traverser::UniqueVertexGetter::reset(arangodb::StringRef const& startVertex) { _returnedVertices.clear(); - - arangodb::basics::VPackHashedSlice hashed(startVertex); // The startVertex always counts as visited! - _returnedVertices.emplace(hashed); + _returnedVertices.emplace(startVertex); } -Traverser::Traverser(arangodb::traverser::TraverserOptions* opts, transaction::Methods* trx, +Traverser::Traverser(arangodb::traverser::TraverserOptions* opts, + transaction::Methods* trx, arangodb::ManagedDocumentResult* mmdr) : _trx(trx), _mmdr(mmdr), @@ -182,8 +176,10 @@ Traverser::Traverser(arangodb::traverser::TraverserOptions* opts, transaction::M } } +Traverser::~Traverser() {} + bool arangodb::traverser::Traverser::edgeMatchesConditions(VPackSlice e, - VPackSlice vid, + StringRef vid, uint64_t depth, size_t cursorId) { if (!_opts->evaluateEdgeExpression(e, vid, depth, cursorId)) { @@ -196,7 +192,7 @@ bool arangodb::traverser::Traverser::edgeMatchesConditions(VPackSlice e, bool arangodb::traverser::Traverser::vertexMatchesConditions(VPackSlice v, uint64_t depth) { TRI_ASSERT(v.isString()); if (_opts->vertexHasFilter(depth)) { - aql::AqlValue vertex = fetchVertexData(v); + aql::AqlValue vertex = fetchVertexData(StringRef(v)); if (!_opts->evaluateVertexExpression(vertex.slice(), depth)) { ++_filteredPaths; return false; @@ -214,6 +210,10 @@ bool arangodb::traverser::Traverser::next() { return res; } +TraverserCache* arangodb::traverser::Traverser::traverserCache() { + return _opts->cache(); +} + arangodb::aql::AqlValue arangodb::traverser::Traverser::lastVertexToAqlValue() { return _enumerator->lastVertexToAqlValue(); } diff --git a/arangod/VocBase/Traverser.h b/arangod/VocBase/Traverser.h index 40baf7f1fc..15051f6053 100644 --- a/arangod/VocBase/Traverser.h +++ b/arangod/VocBase/Traverser.h @@ -27,6 +27,7 @@ #include "Basics/Common.h" #include "Basics/hashes.h" #include "Basics/ShortestPathFinder.h" +#include "Basics/StringRef.h" #include "Basics/VelocyPackHelper.h" #include "Aql/AqlValue.h" #include "Aql/AstNode.h" @@ -52,9 +53,17 @@ struct AstNode; class Expression; class Query; } + +namespace graph { +class BreadthFirstEnumerator; +class NeighborsEnumerator; +} + namespace traverser { +class PathEnumerator; struct TraverserOptions; +class TraverserCache; class ShortestPath { friend class arangodb::basics::DynamicDistanceFinder< @@ -163,9 +172,9 @@ class TraversalPath { class Traverser { - friend class BreadthFirstEnumerator; + friend class arangodb::graph::BreadthFirstEnumerator; friend class DepthFirstEnumerator; - friend class NeighborsEnumerator; + friend class arangodb::graph::NeighborsEnumerator; #ifdef USE_ENTERPRISE friend class SmartDepthFirstPathEnumerator; friend class SmartBreadthFirstPathEnumerator; @@ -185,13 +194,12 @@ class Traverser { virtual ~VertexGetter() = default; virtual bool getVertex(arangodb::velocypack::Slice, - std::vector&); + std::vector&); - virtual bool getSingleVertex(arangodb::velocypack::Slice, - arangodb::velocypack::Slice, uint64_t, - arangodb::velocypack::Slice&); + virtual bool getSingleVertex(arangodb::velocypack::Slice, StringRef, + uint64_t, StringRef&); - virtual void reset(arangodb::velocypack::Slice); + virtual void reset(arangodb::StringRef const&); protected: Traverser* _traverser; @@ -209,16 +217,15 @@ class Traverser { ~UniqueVertexGetter() = default; bool getVertex(arangodb::velocypack::Slice, - std::vector&) override; + std::vector&) override; - bool getSingleVertex(arangodb::velocypack::Slice, - arangodb::velocypack::Slice, uint64_t, - arangodb::velocypack::Slice&) override; + bool getSingleVertex(arangodb::velocypack::Slice, StringRef, + uint64_t, StringRef&) override; - void reset(arangodb::velocypack::Slice) override; + void reset(arangodb::StringRef const&) override; private: - std::unordered_set _returnedVertices; + std::unordered_set _returnedVertices; }; @@ -233,8 +240,8 @@ class Traverser { /// @brief Destructor ////////////////////////////////////////////////////////////////////////////// - virtual ~Traverser() {} - + virtual ~Traverser(); + ////////////////////////////////////////////////////////////////////////////// /// @brief Reset the traverser to use another start vertex ////////////////////////////////////////////////////////////////////////////// @@ -260,20 +267,23 @@ class Traverser { /// @brief Get the next possible path in the graph. bool next(); + TraverserCache* traverserCache(); + + protected: + /// @brief Function to load the other sides vertex of an edge /// Returns true if the vertex passes filtering conditions /// Also appends the _id value of the vertex in the given vector - - protected: virtual bool getVertex(arangodb::velocypack::Slice, - std::vector&) = 0; + std::vector&) = 0; /// @brief Function to load the other sides vertex of an edge /// Returns true if the vertex passes filtering conditions - - virtual bool getSingleVertex(arangodb::velocypack::Slice, - arangodb::velocypack::Slice, uint64_t, - arangodb::velocypack::Slice&) = 0; + virtual bool getSingleVertex(arangodb::velocypack::Slice edge, + arangodb::StringRef const sourceVertexId, + uint64_t depth, + arangodb::StringRef& targetVertexId) = 0; + public: ////////////////////////////////////////////////////////////////////////////// @@ -314,13 +324,13 @@ class Traverser { /// @brief Get the number of documents loaded ////////////////////////////////////////////////////////////////////////////// - size_t getAndResetReadDocuments() { + virtual size_t getAndResetReadDocuments() { size_t tmp = _readDocuments; _readDocuments = 0; return tmp; } - TraverserOptions const* options() { return _opts; } + TraverserOptions* options() { return _opts; } ManagedDocumentResult* mmdr() const { return _mmdr; } @@ -332,13 +342,13 @@ class Traverser { bool hasMore() { return !_done; } - bool edgeMatchesConditions(arangodb::velocypack::Slice, - arangodb::velocypack::Slice, uint64_t, size_t); + bool edgeMatchesConditions(arangodb::velocypack::Slice edge, StringRef vid, + uint64_t depth, size_t cursorId); bool vertexMatchesConditions(arangodb::velocypack::Slice, uint64_t); void allowOptimizedNeighbors(); - + protected: /// @brief Outer top level transaction @@ -369,21 +379,21 @@ class Traverser { /// @brief options for traversal TraverserOptions* _opts; - + bool _canUseOptimizedNeighbors; /// @brief Function to fetch the real data of a vertex into an AQLValue - virtual aql::AqlValue fetchVertexData(arangodb::velocypack::Slice) = 0; + virtual aql::AqlValue fetchVertexData(StringRef vid) = 0; /// @brief Function to fetch the real data of an edge into an AQLValue - virtual aql::AqlValue fetchEdgeData(arangodb::velocypack::Slice) = 0; + virtual aql::AqlValue fetchEdgeData(StringRef eid) = 0; /// @brief Function to add the real data of a vertex into a velocypack builder - virtual void addVertexToVelocyPack(arangodb::velocypack::Slice, + virtual void addVertexToVelocyPack(StringRef vid, arangodb::velocypack::Builder&) = 0; /// @brief Function to add the real data of an edge into a velocypack builder - virtual void addEdgeToVelocyPack(arangodb::velocypack::Slice, + virtual void addEdgeToVelocyPack(StringRef eid, arangodb::velocypack::Builder&) = 0; }; diff --git a/arangod/VocBase/TraverserCache.cpp b/arangod/VocBase/TraverserCache.cpp new file mode 100644 index 0000000000..534fca75d1 --- /dev/null +++ b/arangod/VocBase/TraverserCache.cpp @@ -0,0 +1,192 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2017-2017 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "TraverserCache.h" + +#include "Basics/StringHeap.h" +#include "Basics/StringRef.h" +#include "Basics/VelocyPackHelper.h" + +#include "Cache/Common.h" +#include "Cache/Cache.h" +#include "Cache/CacheManagerFeature.h" +#include "Cache/Finding.h" + +#include "Logger/Logger.h" +#include "Transaction/Methods.h" +#include "VocBase/ManagedDocumentResult.h" +#include "Aql/AqlValue.h" + +#include +#include +#include + +using namespace arangodb; +using namespace arangodb::traverser; + +TraverserCache::TraverserCache(transaction::Methods* trx) + : _cache(nullptr), _mmdr(new ManagedDocumentResult{}), + _trx(trx), _insertedDocuments(0), + _stringHeap(new StringHeap{4096}) /* arbitrary block-size may be adjusted for perforamnce */ { + auto cacheManager = CacheManagerFeature::MANAGER; + TRI_ASSERT(cacheManager != nullptr); + _cache = cacheManager->createCache(cache::CacheType::Plain); +} + +TraverserCache::~TraverserCache() { + auto cacheManager = CacheManagerFeature::MANAGER; + // TODO REMOVE ME + LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "Traverser-Cache used in total " + << _cache->size(); + cacheManager->destroyCache(_cache); +} + +// @brief Only for internal use, Cache::Finding prevents +// the cache from removing this specific object. Should not be retained +// for a longer period of time. +// DO NOT give it to a caller. +cache::Finding TraverserCache::lookup(StringRef idString) { + VPackValueLength keySize = idString.length(); + void const* key = idString.data(); + //uint32_t keySize = static_cast(idString.byteSize()); + return _cache->find(key, keySize); +} + +VPackSlice TraverserCache::lookupInCollection(StringRef id) { + size_t pos = id.find('/'); + if (pos == std::string::npos) { + // Invalid input. If we get here somehow we managed to store invalid _from/_to + // values or the traverser did a let an illegal start through + TRI_ASSERT(false); + return basics::VelocyPackHelper::NullValue(); + } + int res = _trx->documentFastPathLocal(id.substr(0, pos).toString(), + id.substr(pos + 1).toString(), *_mmdr); + + if (res != TRI_ERROR_NO_ERROR && res != TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND) { + // ok we are in a rather bad state. Better throw and abort. + THROW_ARANGO_EXCEPTION(res); + } + + VPackSlice result; + if (res == TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND) { + // This is expected, we may have dangling edges. Interpret as NULL + result = basics::VelocyPackHelper::NullValue(); + } else { + result = VPackSlice(_mmdr->vpack()); + } + + void const* key = id.begin(); + VPackValueLength keySize = id.length(); + + void const* resVal = result.begin(); + uint64_t resValSize = static_cast(result.byteSize()); + std::unique_ptr value( + cache::CachedValue::construct(key, keySize, resVal, resValSize)); + + if (value) { + bool success = _cache->insert(value.get()); + if (!success) { + LOG_TOPIC(DEBUG, Logger::GRAPHS) << "Insert failed"; + } + // Cache is responsible. + // If this failed, well we do not store it and read it again next time. + value.release(); + } + ++_insertedDocuments; + return result; +} + +void TraverserCache::insertIntoResult(StringRef idString, + VPackBuilder& builder) { + auto finding = lookup(idString); + if (finding.found()) { + auto val = finding.value(); + VPackSlice slice(val->value()); + // finding makes sure that slice contant stays valid. + builder.add(slice); + } else { + // Not in cache. Fetch and insert. + builder.add(lookupInCollection(idString)); + } +} + +aql::AqlValue TraverserCache::fetchAqlResult(StringRef idString) { + auto finding = lookup(idString); + if (finding.found()) { + auto val = finding.value(); + // finding makes sure that slice contant stays valid. + return aql::AqlValue(val->value()); + } + // Not in cache. Fetch and insert. + return aql::AqlValue(lookupInCollection(idString)); +} + +void TraverserCache::insertDocument(StringRef idString, arangodb::velocypack::Slice const& document) { + auto finding = lookup(idString); + if (!finding.found()) { + VPackValueLength keySize = idString.length(); + void const* key = idString.data(); + + void const* resVal = document.begin(); + uint64_t resValSize = static_cast(document.byteSize()); + std::unique_ptr value( + cache::CachedValue::construct(key, keySize, resVal, resValSize)); + + if (value) { + bool success = _cache->insert(value.get()); + if (!success) { + LOG_TOPIC(ERR, Logger::GRAPHS) << "Insert document into cache failed"; + } + // Cache is responsible. + // If this failed, well we do not store it and read it again next time. + value.release(); + } + ++_insertedDocuments; + } +} + +bool TraverserCache::validateFilter( + StringRef idString, + std::function filterFunc) { + auto finding = lookup(idString); + if (finding.found()) { + auto val = finding.value(); + VPackSlice slice(val->value()); + // finding makes sure that slice contant stays valid. + return filterFunc(slice); + } + // Not in cache. Fetch and insert. + VPackSlice slice = lookupInCollection(idString); + return filterFunc(slice); +} + +StringRef TraverserCache::persistString( + StringRef const idString) { + auto it = _persistedStrings.find(idString); + if (it != _persistedStrings.end()) { + return *it; + } + StringRef res = _stringHeap->registerString(idString.begin(), idString.length()); + _persistedStrings.emplace(res); + return res; +} diff --git a/arangod/VocBase/TraverserCache.h b/arangod/VocBase/TraverserCache.h new file mode 100644 index 0000000000..5533e2c6e6 --- /dev/null +++ b/arangod/VocBase/TraverserCache.h @@ -0,0 +1,161 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2017-2017 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_VOC_BASE_TRAVERSER_CACHE_H +#define ARANGOD_VOC_BASE_TRAVERSER_CACHE_H 1 + +#include "Basics/Common.h" +#include "Basics/StringRef.h" + +namespace arangodb { +class ManagedDocumentResult; +class StringHeap; + +namespace cache { +class Cache; +class Finding; +} + +namespace transaction { +class Methods; +} + +namespace velocypack { +class Builder; +class Slice; +} + +namespace aql { + struct AqlValue; +} + +namespace traverser { +class TraverserCache { + + public: + explicit TraverserCache(transaction::Methods* trx); + + ~TraverserCache(); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Inserts the real document stored within the token + /// into the given builder. + /// The document will be taken from the hash-cache. + /// If it is not cached it will be looked up in the StorageEngine + ////////////////////////////////////////////////////////////////////////////// + + void insertIntoResult(StringRef idString, + arangodb::velocypack::Builder& builder); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Return AQL value containing the result + /// The document will be taken from the hash-cache. + /// If it is not cached it will be looked up in the StorageEngine + ////////////////////////////////////////////////////////////////////////////// + + aql::AqlValue fetchAqlResult(StringRef idString); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Insert value into store + ////////////////////////////////////////////////////////////////////////////// + void insertDocument(StringRef idString, + arangodb::velocypack::Slice const& document); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Throws the document referenced by the token into the filter + /// function and returns it result. + /// The document will be taken from the hash-cache. + /// If it is not cached it will be looked up in the StorageEngine + ////////////////////////////////////////////////////////////////////////////// + + bool validateFilter(StringRef idString, + std::function filterFunc); + + size_t getAndResetInsertedDocuments() { + size_t tmp = _insertedDocuments; + _insertedDocuments = 0; + return tmp; + } + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Persist the given id string. The return value is guaranteed to + /// stay valid as long as this cache is valid + ////////////////////////////////////////////////////////////////////////////// + StringRef persistString(StringRef const idString); + + private: + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Lookup a document by token in the cache. + /// As long as finding is retained it is guaranteed that the result + /// stays valid. Finding should not be retained very long, if it is + /// needed for longer, copy the value. + ////////////////////////////////////////////////////////////////////////////// + cache::Finding lookup(StringRef idString); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Lookup a document from the database and insert it into the cache. + /// The Slice returned here is only valid until the NEXT call of this + /// function. + ////////////////////////////////////////////////////////////////////////////// + + arangodb::velocypack::Slice lookupInCollection( + StringRef idString); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief The hash-cache that saves documents found in the Database + ////////////////////////////////////////////////////////////////////////////// + std::shared_ptr _cache; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Reusable ManagedDocumentResult that temporarily takes + /// responsibility for one document. + ////////////////////////////////////////////////////////////////////////////// + std::unique_ptr _mmdr; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Transaction to access data, This class is NOT responsible for it. + ////////////////////////////////////////////////////////////////////////////// + arangodb::transaction::Methods* _trx; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Documents inserted in this cache + ////////////////////////////////////////////////////////////////////////////// + size_t _insertedDocuments; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Stringheap to take care of _id strings, s.t. they stay valid + /// during the entire traversal. + ////////////////////////////////////////////////////////////////////////////// + std::unique_ptr _stringHeap; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Set of all strings persisted in the stringHeap. So we can save some + /// memory by not storing them twice. + ////////////////////////////////////////////////////////////////////////////// + std::unordered_set _persistedStrings; +}; + +} +} + +#endif diff --git a/arangod/VocBase/TraverserOptions.cpp b/arangod/VocBase/TraverserOptions.cpp index ba3f301d4b..8fb0a7cda2 100644 --- a/arangod/VocBase/TraverserOptions.cpp +++ b/arangod/VocBase/TraverserOptions.cpp @@ -30,6 +30,7 @@ #include "Cluster/ClusterEdgeCursor.h" #include "Indexes/Index.h" #include "VocBase/SingleServerTraverser.h" +#include "VocBase/TraverserCache.h" #include #include @@ -153,6 +154,24 @@ double arangodb::traverser::TraverserOptions::LookupInfo::estimateCost(size_t& n return 1000.0; } +arangodb::traverser::TraverserCache* arangodb::traverser::TraverserOptions::cache() const { + return _cache.get(); +} + +arangodb::traverser::TraverserOptions::TraverserOptions(transaction::Methods* trx) + : _trx(trx), + _baseVertexExpression(nullptr), + _tmpVar(nullptr), + _ctx(new aql::FixedVarExpressionContext()), + _traverser(nullptr), + _isCoordinator(trx->state()->isCoordinator()), + _cache(new TraverserCache(trx)), + minDepth(1), + maxDepth(1), + useBreadthFirst(false), + uniqueVertices(UniquenessLevel::NONE), + uniqueEdges(UniquenessLevel::PATH) {} + arangodb::traverser::TraverserOptions::TraverserOptions( transaction::Methods* trx, VPackSlice const& slice) : _trx(trx), @@ -161,6 +180,7 @@ arangodb::traverser::TraverserOptions::TraverserOptions( _ctx(new aql::FixedVarExpressionContext()), _traverser(nullptr), _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), + _cache(new TraverserCache(trx)), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -214,6 +234,7 @@ arangodb::traverser::TraverserOptions::TraverserOptions( _ctx(new aql::FixedVarExpressionContext()), _traverser(nullptr), _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), + _cache(new TraverserCache(_trx)), minDepth(1), maxDepth(1), useBreadthFirst(false), @@ -372,6 +393,7 @@ arangodb::traverser::TraverserOptions::TraverserOptions( _ctx(new aql::FixedVarExpressionContext()), _traverser(nullptr), _isCoordinator(arangodb::ServerState::instance()->isCoordinator()), + _cache(new TraverserCache(_trx)), minDepth(other.minDepth), maxDepth(other.maxDepth), useBreadthFirst(other.useBreadthFirst), @@ -552,7 +574,7 @@ bool arangodb::traverser::TraverserOptions::vertexHasFilter( } bool arangodb::traverser::TraverserOptions::evaluateEdgeExpression( - arangodb::velocypack::Slice edge, arangodb::velocypack::Slice vertex, + arangodb::velocypack::Slice edge, StringRef vertexId, uint64_t depth, size_t cursorId) const { if (_isCoordinator) { // The Coordinator never checks conditions. The DBServer is responsible! @@ -574,9 +596,6 @@ bool arangodb::traverser::TraverserOptions::evaluateEdgeExpression( if (expression != nullptr) { TRI_ASSERT(!expression->isV8()); expression->setVariable(_tmpVar, edge); - - VPackValueLength vidLength; - char const* vid = vertex.getString(vidLength); // inject _from/_to value auto node = expression->nodeForModification(); @@ -590,7 +609,7 @@ bool arangodb::traverser::TraverserOptions::evaluateEdgeExpression( TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); idNode->stealComputedValue(); - idNode->setStringValue(vid, vidLength); + idNode->setStringValue(vertexId.data(), vertexId.length()); bool mustDestroy = false; aql::AqlValue res = expression->execute(_trx, _ctx, mustDestroy); @@ -635,10 +654,10 @@ bool arangodb::traverser::TraverserOptions::evaluateVertexExpression( arangodb::traverser::EdgeCursor* arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, - VPackSlice vertex, - uint64_t depth) const { + StringRef vid, + uint64_t depth) { if (_isCoordinator) { - return nextCursorCoordinator(vertex, depth); + return nextCursorCoordinator(vid, depth); } TRI_ASSERT(mmdr != nullptr); auto specific = _depthLookupInfo.find(depth); @@ -648,17 +667,15 @@ arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, } else { list = _baseLookupInfos; } - return nextCursorLocal(mmdr, vertex, depth, list); + return nextCursorLocal(mmdr, vid, depth, list); } arangodb::traverser::EdgeCursor* arangodb::traverser::TraverserOptions::nextCursorLocal(ManagedDocumentResult* mmdr, - VPackSlice vertex, uint64_t depth, std::vector& list) const { + StringRef vid, uint64_t depth, std::vector& list) { TRI_ASSERT(mmdr != nullptr); - auto allCursor = std::make_unique(mmdr, _trx, list.size()); + auto allCursor = std::make_unique(mmdr, this, list.size()); auto& opCursors = allCursor->getCursors(); - VPackValueLength vidLength; - char const* vid = vertex.getString(vidLength); for (auto& info : list) { auto& node = info.indexCondition; TRI_ASSERT(node->numMembers() > 0); @@ -671,7 +688,7 @@ arangodb::traverser::TraverserOptions::nextCursorLocal(ManagedDocumentResult* mm auto idNode = dirCmp->getMemberUnchecked(1); TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); - idNode->setStringValue(vid, vidLength); + idNode->setStringValue(vid.data(), vid.length()); } std::vector csrs; csrs.reserve(info.idxHandles.size()); @@ -686,9 +703,9 @@ arangodb::traverser::TraverserOptions::nextCursorLocal(ManagedDocumentResult* mm arangodb::traverser::EdgeCursor* arangodb::traverser::TraverserOptions::nextCursorCoordinator( - VPackSlice vertex, uint64_t depth) const { + StringRef vid, uint64_t depth) { TRI_ASSERT(_traverser != nullptr); - auto cursor = std::make_unique(vertex, depth, _traverser); + auto cursor = std::make_unique(vid, depth, _traverser); return cursor.release(); } diff --git a/arangod/VocBase/TraverserOptions.h b/arangod/VocBase/TraverserOptions.h index 3a0d685c9e..8cacf2f004 100644 --- a/arangod/VocBase/TraverserOptions.h +++ b/arangod/VocBase/TraverserOptions.h @@ -25,6 +25,7 @@ #define ARANGOD_VOC_BASE_TRAVERSER_OPTIONS_H 1 #include "Basics/Common.h" +#include "Basics/StringRef.h" #include "Aql/FixedVarExpressionContext.h" #include "StorageEngine/TransactionState.h" #include "Transaction/Methods.h" @@ -47,6 +48,7 @@ class TraversalNode; namespace traverser { class ClusterTraverser; +class TraverserCache; /// @brief Abstract class used in the traversals /// to abstract away access to indexes / DBServers. @@ -57,9 +59,10 @@ class EdgeCursor { EdgeCursor() {} virtual ~EdgeCursor() {} - virtual bool next(std::vector&, size_t&) = 0; - virtual bool readAll(std::unordered_set&, - size_t&) = 0; + virtual bool next(std::function callback) = 0; + + virtual void readAll(std::function) = 0; + }; @@ -110,6 +113,9 @@ struct TraverserOptions { arangodb::traverser::ClusterTraverser* _traverser; bool const _isCoordinator; + /// @brief the traverser cache + std::unique_ptr _cache; + public: uint64_t minDepth; @@ -121,18 +127,7 @@ struct TraverserOptions { UniquenessLevel uniqueEdges; - explicit TraverserOptions(transaction::Methods* trx) - : _trx(trx), - _baseVertexExpression(nullptr), - _tmpVar(nullptr), - _ctx(new aql::FixedVarExpressionContext()), - _traverser(nullptr), - _isCoordinator(trx->state()->isCoordinator()), - minDepth(1), - maxDepth(1), - useBreadthFirst(false), - uniqueVertices(UniquenessLevel::NONE), - uniqueEdges(UniquenessLevel::PATH) {} + explicit TraverserOptions(transaction::Methods* trx); TraverserOptions(transaction::Methods*, arangodb::velocypack::Slice const&); @@ -158,12 +153,12 @@ struct TraverserOptions { bool vertexHasFilter(uint64_t) const; bool evaluateEdgeExpression(arangodb::velocypack::Slice, - arangodb::velocypack::Slice, uint64_t, + StringRef vertexId, uint64_t, size_t) const; bool evaluateVertexExpression(arangodb::velocypack::Slice, uint64_t) const; - EdgeCursor* nextCursor(ManagedDocumentResult*, arangodb::velocypack::Slice, uint64_t) const; + EdgeCursor* nextCursor(ManagedDocumentResult*, StringRef vid, uint64_t); void clearVariableValues(); @@ -175,16 +170,18 @@ struct TraverserOptions { double estimateCost(size_t& nrItems) const; + TraverserCache* cache() const; + private: double costForLookupInfoList(std::vector const& list, size_t& createItems) const; EdgeCursor* nextCursorLocal(ManagedDocumentResult*, - arangodb::velocypack::Slice, uint64_t, - std::vector&) const; + StringRef vid, uint64_t, + std::vector&); - EdgeCursor* nextCursorCoordinator(arangodb::velocypack::Slice, uint64_t) const; + EdgeCursor* nextCursorCoordinator(StringRef vid, uint64_t); }; } From 2facc4b2b7826c6f2cf3305d84e2e326c1a7dccb Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Mon, 27 Mar 2017 14:31:10 +0200 Subject: [PATCH 25/27] Removed unused log-output --- arangod/VocBase/TraverserCache.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/arangod/VocBase/TraverserCache.cpp b/arangod/VocBase/TraverserCache.cpp index 534fca75d1..fa4c210526 100644 --- a/arangod/VocBase/TraverserCache.cpp +++ b/arangod/VocBase/TraverserCache.cpp @@ -54,9 +54,6 @@ TraverserCache::TraverserCache(transaction::Methods* trx) TraverserCache::~TraverserCache() { auto cacheManager = CacheManagerFeature::MANAGER; - // TODO REMOVE ME - LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "Traverser-Cache used in total " - << _cache->size(); cacheManager->destroyCache(_cache); } From 53742d87904c122f292bfcb49d909de1d4505126 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Mon, 27 Mar 2017 14:40:56 +0200 Subject: [PATCH 26/27] Removed dead code --- arangod/MMFiles/MMFilesEdgeIndex.cpp | 124 --------------------------- arangod/MMFiles/MMFilesEdgeIndex.h | 13 --- 2 files changed, 137 deletions(-) diff --git a/arangod/MMFiles/MMFilesEdgeIndex.cpp b/arangod/MMFiles/MMFilesEdgeIndex.cpp index d89171e019..f632bf1ba4 100644 --- a/arangod/MMFiles/MMFilesEdgeIndex.cpp +++ b/arangod/MMFiles/MMFilesEdgeIndex.cpp @@ -214,130 +214,6 @@ MMFilesEdgeIndex::~MMFilesEdgeIndex() { delete _edgesTo; } -void MMFilesEdgeIndex::buildSearchValue(TRI_edge_direction_e dir, - std::string const& id, VPackBuilder& builder) { - builder.openArray(); - switch (dir) { - case TRI_EDGE_OUT: - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, VPackValue(id)); - builder.close(); - builder.close(); - builder.add(VPackValue(VPackValueType::Null)); - break; - case TRI_EDGE_IN: - builder.add(VPackValue(VPackValueType::Null)); - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, VPackValue(id)); - builder.close(); - builder.close(); - break; - case TRI_EDGE_ANY: - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, VPackValue(id)); - builder.close(); - builder.close(); - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, VPackValue(id)); - builder.close(); - builder.close(); - } - builder.close(); -} - -void MMFilesEdgeIndex::buildSearchValue(TRI_edge_direction_e dir, - VPackSlice const& id, VPackBuilder& builder) { - TRI_ASSERT(id.isString()); - builder.openArray(); - switch (dir) { - case TRI_EDGE_OUT: - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - builder.close(); - builder.add(VPackValue(VPackValueType::Null)); - break; - case TRI_EDGE_IN: - builder.add(VPackValue(VPackValueType::Null)); - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - builder.close(); - break; - case TRI_EDGE_ANY: - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - builder.close(); - builder.openArray(); - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - builder.close(); - } - builder.close(); -} - -void MMFilesEdgeIndex::buildSearchValueFromArray(TRI_edge_direction_e dir, - VPackSlice const ids, - VPackBuilder& builder) { - TRI_ASSERT(ids.isArray()); - builder.openArray(); - switch (dir) { - case TRI_EDGE_OUT: - builder.openArray(); - for (auto const& id : VPackArrayIterator(ids)) { - if (id.isString()) { - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - } - } - builder.close(); - builder.add(VPackValue(VPackValueType::Null)); - break; - case TRI_EDGE_IN: - builder.add(VPackValue(VPackValueType::Null)); - builder.openArray(); - for (auto const& id : VPackArrayIterator(ids)) { - if (id.isString()) { - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - } - } - builder.close(); - break; - case TRI_EDGE_ANY: - builder.openArray(); - for (auto const& id : VPackArrayIterator(ids)) { - if (id.isString()) { - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - } - } - builder.close(); - builder.openArray(); - for (auto const& id : VPackArrayIterator(ids)) { - if (id.isString()) { - builder.openObject(); - builder.add(StaticStrings::IndexEq, id); - builder.close(); - } - } - builder.close(); - } - builder.close(); -} - /// @brief return a selectivity estimate for the index double MMFilesEdgeIndex::selectivityEstimate(arangodb::StringRef const* attribute) const { if (_edgesFrom == nullptr || diff --git a/arangod/MMFiles/MMFilesEdgeIndex.h b/arangod/MMFiles/MMFilesEdgeIndex.h index 57c1d1ea1d..50cc1e8c44 100644 --- a/arangod/MMFiles/MMFilesEdgeIndex.h +++ b/arangod/MMFiles/MMFilesEdgeIndex.h @@ -79,19 +79,6 @@ class MMFilesEdgeIndex final : public Index { ~MMFilesEdgeIndex(); - static void buildSearchValue(TRI_edge_direction_e, std::string const&, - arangodb::velocypack::Builder&); - - static void buildSearchValue(TRI_edge_direction_e, - arangodb::velocypack::Slice const&, - arangodb::velocypack::Builder&); - - static void buildSearchValueFromArray(TRI_edge_direction_e, - arangodb::velocypack::Slice const, - arangodb::velocypack::Builder&); - - public: - /// @brief typedef for hash tables public: IndexType type() const override { return Index::TRI_IDX_TYPE_EDGE_INDEX; } From 3a2462532dee33978a6f74d015f6be0605ea7e25 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 27 Mar 2017 16:36:34 +0200 Subject: [PATCH 27/27] move engine-specific includes out of standard file --- arangod/MMFiles/MMFilesEngine.cpp | 5 ++ arangod/RestServer/arangod.cpp | 15 ++--- arangod/RocksDBEngine/RocksDBCollection.cpp | 74 ++++++++++++++++++++- arangod/RocksDBEngine/RocksDBCollection.h | 4 ++ 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp index 33b63f1648..20426e9b79 100644 --- a/arangod/MMFiles/MMFilesEngine.cpp +++ b/arangod/MMFiles/MMFilesEngine.cpp @@ -48,6 +48,7 @@ #include "MMFiles/MMFilesTransactionState.h" #include "MMFiles/MMFilesV8Functions.h" #include "MMFiles/MMFilesView.h" +#include "MMFiles/MMFilesWalRecoveryFeature.h" #include "Random/RandomGenerator.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/DatabasePathFeature.h" @@ -142,6 +143,10 @@ MMFilesEngine::MMFilesEngine(application_features::ApplicationServer* server) _isUpgrade(false), _maxTick(0) { startsAfter("MMFilesPersistentIndex"); + + server->addFeature(new MMFilesWalRecoveryFeature(server)); + server->addFeature(new MMFilesLogfileManager(server)); + server->addFeature(new MMFilesPersistentIndexFeature(server)); } MMFilesEngine::~MMFilesEngine() {} diff --git a/arangod/RestServer/arangod.cpp b/arangod/RestServer/arangod.cpp index 50d21ada4e..ecf3f9bd61 100644 --- a/arangod/RestServer/arangod.cpp +++ b/arangod/RestServer/arangod.cpp @@ -80,14 +80,6 @@ #include "Statistics/StatisticsFeature.h" #include "StorageEngine/EngineSelectorFeature.h" -// TODO - move the following MMFiles includes to the storage engine -#include "MMFiles/MMFilesLogfileManager.h" -#include "MMFiles/MMFilesPersistentIndexFeature.h" -#include "MMFiles/MMFilesWalRecoveryFeature.h" -#include "MMFiles/MMFilesEngine.h" - -#include "RocksDBEngine/RocksDBEngine.h" - #include "V8Server/FoxxQueuesFeature.h" #include "V8Server/V8DealerFeature.h" @@ -99,6 +91,10 @@ #include "Enterprise/RestServer/arangodEE.h" #endif +// storage engine +#include "MMFiles/MMFilesEngine.h" +#include "RocksDBEngine/RocksDBEngine.h" + using namespace arangodb; static int runServer(int argc, char** argv) { @@ -195,9 +191,6 @@ static int runServer(int argc, char** argv) { // storage engines server.addFeature(new MMFilesEngine(&server)); - server.addFeature(new MMFilesWalRecoveryFeature(&server)); - server.addFeature(new MMFilesLogfileManager(&server)); - server.addFeature(new MMFilesPersistentIndexFeature(&server)); server.addFeature(new RocksDBEngine(&server)); try { diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index 8f1fb43c9e..e3624dfa2e 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -26,6 +26,7 @@ #include "Basics/StaticStrings.h" #include "Aql/PlanCache.h" #include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterMethods.h" #include "Indexes/Index.h" #include "Indexes/IndexIterator.h" #include "RestServer/DatabaseFeature.h" @@ -37,6 +38,7 @@ #include "StorageEngine/StorageEngine.h" #include "StorageEngine/TransactionState.h" #include "Transaction/Helpers.h" +#include "Utils/CollectionNameResolver.h" #include "Utils/OperationOptions.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" @@ -416,12 +418,70 @@ int RocksDBCollection::update(arangodb::transaction::Methods* trx, int RocksDBCollection::replace( transaction::Methods* trx, arangodb::velocypack::Slice const newSlice, ManagedDocumentResult& result, OperationOptions& options, - TRI_voc_tick_t& resultMarkerTick, bool lock, TRI_voc_rid_t& prevRev, + TRI_voc_tick_t& resultMarkerTick, bool /*lock*/, TRI_voc_rid_t& prevRev, ManagedDocumentResult& previous, TRI_voc_rid_t const revisionId, arangodb::velocypack::Slice const fromSlice, arangodb::velocypack::Slice const toSlice) { - THROW_ARANGO_NOT_YET_IMPLEMENTED(); - return 0; + + resultMarkerTick = 0; + + bool const isEdgeCollection = (_logicalCollection->type() == TRI_COL_TYPE_EDGE); + + // get the previous revision + VPackSlice key = newSlice.get(StaticStrings::KeyString); + if (key.isNone()) { + return TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD; + } + + // get the previous revision + int res = lookupDocument(trx, key, previous); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + uint8_t const* vpack = previous.vpack(); + VPackSlice oldDoc(vpack); + TRI_voc_rid_t oldRevisionId = transaction::helpers::extractRevFromDocument(oldDoc); + prevRev = oldRevisionId; + + // Check old revision: + if (!options.ignoreRevs) { + TRI_voc_rid_t expectedRev = 0; + if (newSlice.isObject()) { + expectedRev = TRI_ExtractRevisionId(newSlice); + } + int res = checkRevision(trx, expectedRev, prevRev); + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + } + + // merge old and new values + transaction::BuilderLeaser builder(trx); + newObjectForReplace(trx, oldDoc, newSlice, fromSlice, toSlice, + isEdgeCollection, TRI_RidToString(revisionId), + *builder.get()); + + if (trx->state()->isDBServer()) { + // Need to check that no sharding keys have changed: + if (arangodb::shardKeysChanged(_logicalCollection->dbName(), + trx->resolver()->getCollectionNameCluster( + _logicalCollection->planId()), + oldDoc, builder->slice(), false)) { + return TRI_ERROR_CLUSTER_MUST_NOT_CHANGE_SHARDING_ATTRIBUTES; + } + } + + res = updateDocument(trx, oldRevisionId, oldDoc, revisionId, VPackSlice(builder->slice()), options.waitForSync); + /* TODO: handle result handling + uint8_t const* vpack = lookupRevisionVPack(revisionId); + if (vpack != nullptr) { + result.addExisting(vpack, revisionId); + } + */ + + return res; } int RocksDBCollection::remove(arangodb::transaction::Methods* trx, @@ -716,3 +776,11 @@ int RocksDBCollection::lookupDocument(transaction::Methods* trx, return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND; } + +int RocksDBCollection::updateDocument( + transaction::Methods* trx, TRI_voc_rid_t oldRevisionId, + VPackSlice const& oldDoc, TRI_voc_rid_t newRevisionId, + VPackSlice const& newDoc, bool& waitForSync) { + // TODO + return TRI_ERROR_NO_ERROR; +} diff --git a/arangod/RocksDBEngine/RocksDBCollection.h b/arangod/RocksDBEngine/RocksDBCollection.h index c332efd31e..e401141368 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.h +++ b/arangod/RocksDBEngine/RocksDBCollection.h @@ -195,6 +195,10 @@ class RocksDBCollection final : public PhysicalCollection { arangodb::velocypack::Slice key, ManagedDocumentResult& result); + int updateDocument(transaction::Methods* trx, TRI_voc_rid_t oldRevisionId, + arangodb::velocypack::Slice const& oldDoc, TRI_voc_rid_t newRevisionId, + arangodb::velocypack::Slice const& newDoc, bool& waitForSync); + private: uint64_t _objectId; // rocksdb-specific object id for collection };