From c2be40b4ab57a35e2d6122f40094e9eafa9f8c4f Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 11 Apr 2017 18:03:56 +0200 Subject: [PATCH] Moved some Edge Lookup logic around and moved the EdgeCursor to it's own class. This is still ongoing work. Expect shortest-path to fail. --- arangod/Cluster/ClusterEdgeCursor.h | 3 +- arangod/Cluster/TraverserEngine.cpp | 3 +- arangod/Graph/BaseOptions.h | 2 + arangod/Graph/BreadthFirstEnumerator.cpp | 3 +- arangod/Graph/EdgeCursor.h | 59 ++++++++++++++++++++++ arangod/Graph/NeighborsEnumerator.cpp | 3 +- arangod/Graph/ShortestPathOptions.cpp | 33 ++++++++++++- arangod/Graph/ShortestPathOptions.h | 12 ++++- arangod/VocBase/PathEnumerator.cpp | 60 ++++++++++++++--------- arangod/VocBase/PathEnumerator.h | 13 ++--- arangod/VocBase/SingleServerTraverser.cpp | 2 +- arangod/VocBase/SingleServerTraverser.h | 12 +++-- arangod/VocBase/TraverserOptions.cpp | 6 +-- arangod/VocBase/TraverserOptions.h | 24 ++------- 14 files changed, 172 insertions(+), 63 deletions(-) create mode 100644 arangod/Graph/EdgeCursor.h diff --git a/arangod/Cluster/ClusterEdgeCursor.h b/arangod/Cluster/ClusterEdgeCursor.h index 116427e535..bdbe09b454 100644 --- a/arangod/Cluster/ClusterEdgeCursor.h +++ b/arangod/Cluster/ClusterEdgeCursor.h @@ -24,6 +24,7 @@ #ifndef ARANGOD_CLUSTER_CLUSTER_EDGE_CURSOR_H #define ARANGOD_CLUSTER_CLUSTER_EDGE_CURSOR_H 1 +#include "Graph/EdgeCursor.h" #include "VocBase/TraverserOptions.h" namespace arangodb { @@ -32,7 +33,7 @@ namespace traverser { class Traverser; -class ClusterEdgeCursor : public EdgeCursor { +class ClusterEdgeCursor : public graph::EdgeCursor { public: ClusterEdgeCursor(StringRef vid, uint64_t, ClusterTraverser*); diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index d3c5460f4b..0eee6681cc 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -26,6 +26,7 @@ #include "Aql/AqlTransaction.h" #include "Aql/Ast.h" #include "Aql/Query.h" +#include "Graph/EdgeCursor.h" #include "Utils/CollectionNameResolver.h" #include "Transaction/Context.h" #include "VocBase/ManagedDocumentResult.h" @@ -160,7 +161,7 @@ void BaseTraverserEngine::getEdges(VPackSlice vertex, size_t depth, VPackBuilder // Result now contains all valid edges, probably multiples. } } else if (vertex.isString()) { - std::unique_ptr edgeCursor(_opts->nextCursor(&mmdr, StringRef(vertex), depth)); + std::unique_ptr edgeCursor(_opts->nextCursor(&mmdr, StringRef(vertex), depth)); edgeCursor->readAll([&] (StringRef const& documentId, VPackSlice edge, size_t cursorId) { if (!_opts->evaluateEdgeExpression(edge, StringRef(vertex), depth, cursorId)) { filtered++; diff --git a/arangod/Graph/BaseOptions.h b/arangod/Graph/BaseOptions.h index f922e10e79..72bbbdcdbb 100644 --- a/arangod/Graph/BaseOptions.h +++ b/arangod/Graph/BaseOptions.h @@ -48,6 +48,8 @@ class Slice; namespace graph { +class EdgeCursor; + struct BaseOptions { protected: struct LookupInfo { diff --git a/arangod/Graph/BreadthFirstEnumerator.cpp b/arangod/Graph/BreadthFirstEnumerator.cpp index 1f6396ece3..40e7158f6a 100644 --- a/arangod/Graph/BreadthFirstEnumerator.cpp +++ b/arangod/Graph/BreadthFirstEnumerator.cpp @@ -23,6 +23,7 @@ #include "BreadthFirstEnumerator.h" +#include "Graph/EdgeCursor.h" #include "VocBase/Traverser.h" #include "VocBase/TraverserCache.h" #include "VocBase/TraverserOptions.h" @@ -116,7 +117,7 @@ bool BreadthFirstEnumerator::next() { auto const nextVertex = _schreier[nextIdx].vertex; StringRef vId; - std::unique_ptr cursor(_opts->nextCursor(_traverser->mmdr(), nextVertex, _currentDepth)); + std::unique_ptr cursor(_opts->nextCursor(_traverser->mmdr(), nextVertex, _currentDepth)); if (cursor != nullptr) { bool shouldReturnPath = _currentDepth + 1 >= _opts->minDepth; bool didInsert = false; diff --git a/arangod/Graph/EdgeCursor.h b/arangod/Graph/EdgeCursor.h new file mode 100644 index 0000000000..1b3df26459 --- /dev/null +++ b/arangod/Graph/EdgeCursor.h @@ -0,0 +1,59 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_GRAPH_EDGECURSOR_H +#define ARANGOD_GRAPH_EDGECURSOR_H 1 + +#include "Basics/Common.h" + +namespace arangodb { + +class StringRef; + +namespace velocypack { +class Slice; +} + +namespace graph { + +/// @brief Abstract class used in the traversals +/// to abstract away access to indexes / DBServers. +/// Returns edges as VelocyPack. +class EdgeCursor { + public: + EdgeCursor() {} + virtual ~EdgeCursor() {} + + virtual bool next(std::function + callback) = 0; + + virtual void readAll( + std::function) = 0; +}; + +} // namespace graph +} // namespace arangodb + +#endif diff --git a/arangod/Graph/NeighborsEnumerator.cpp b/arangod/Graph/NeighborsEnumerator.cpp index c681190287..23a05635b2 100644 --- a/arangod/Graph/NeighborsEnumerator.cpp +++ b/arangod/Graph/NeighborsEnumerator.cpp @@ -24,6 +24,7 @@ #include "NeighborsEnumerator.h" #include "Basics/VelocyPackHelper.h" +#include "Graph/EdgeCursor.h" #include "VocBase/Traverser.h" #include "VocBase/TraverserCache.h" @@ -73,7 +74,7 @@ bool NeighborsEnumerator::next() { } } }; - std::unique_ptr cursor( + std::unique_ptr cursor( _opts->nextCursor(_traverser->mmdr(), nextVertex, _searchDepth)); cursor->readAll(callback); } diff --git a/arangod/Graph/ShortestPathOptions.cpp b/arangod/Graph/ShortestPathOptions.cpp index 1cecf29e67..fbee3b0068 100644 --- a/arangod/Graph/ShortestPathOptions.cpp +++ b/arangod/Graph/ShortestPathOptions.cpp @@ -32,7 +32,6 @@ using namespace arangodb; using namespace arangodb::basics; using namespace arangodb::graph; - ShortestPathOptions::ShortestPathOptions(transaction::Methods* trx) : BaseOptions(trx), direction("outbound"), @@ -100,3 +99,35 @@ double ShortestPathOptions::estimateCost(size_t& nrItems) const { // TODO Implement me return 0; } + +void ShortestPathOptions::addReverseLookupInfo( + aql::Ast* ast, std::string const& collectionName, + std::string const& attributeName, aql::AstNode* condition) { + injectLookupInfoInList(_reverseLookupInfos, ast, collectionName, + attributeName, condition); +} + +EdgeCursor* ShortestPathOptions::nextCursor(ManagedDocumentResult* mmdr, + StringRef vid, uint64_t depth) { + if (_isCoordinator) { + return nextCursorCoordinator(vid, depth); + } + TRI_ASSERT(mmdr != nullptr); + return nextCursorLocal(mmdr, vid, depth, _baseLookupInfos); +} + +EdgeCursor* ShortestPathOptions::nextCursorLocal(ManagedDocumentResult* mmdr, + StringRef vid, uint64_t depth, + std::vector&) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +} + +EdgeCursor* ShortestPathOptions::nextCursorCoordinator(StringRef vid, + uint64_t depth) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +} + +EdgeCursor* ShortestPathOptions::nextReverseCursorCoordinator(StringRef vid, + uint64_t depth) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +} diff --git a/arangod/Graph/ShortestPathOptions.h b/arangod/Graph/ShortestPathOptions.h index 8dcdc0a50d..8b5f4e3cb0 100644 --- a/arangod/Graph/ShortestPathOptions.h +++ b/arangod/Graph/ShortestPathOptions.h @@ -84,8 +84,18 @@ struct ShortestPathOptions : public BaseOptions { std::string const& attributeName, aql::AstNode* condition); - private: + EdgeCursor* nextCursor(ManagedDocumentResult*, StringRef vid, uint64_t); + EdgeCursor* nextReverseCursor(ManagedDocumentResult*, StringRef vid, uint64_t); + + private: + EdgeCursor* nextCursorLocal(ManagedDocumentResult*, StringRef vid, uint64_t, + std::vector&); + + EdgeCursor* nextCursorCoordinator(StringRef vid, uint64_t); + EdgeCursor* nextReverseCursorCoordinator(StringRef vid, uint64_t); + + private: /// @brief Lookup info to find all reverse edges. std::vector _reverseLookupInfos; }; diff --git a/arangod/VocBase/PathEnumerator.cpp b/arangod/VocBase/PathEnumerator.cpp index f645b9a89e..ac591187ec 100644 --- a/arangod/VocBase/PathEnumerator.cpp +++ b/arangod/VocBase/PathEnumerator.cpp @@ -23,6 +23,7 @@ #include "PathEnumerator.h" #include "Basics/VelocyPackHelper.h" +#include "Graph/EdgeCursor.h" #include "VocBase/Traverser.h" #include "VocBase/TraverserCache.h" @@ -31,7 +32,8 @@ using DepthFirstEnumerator = arangodb::traverser::DepthFirstEnumerator; using Traverser = arangodb::traverser::Traverser; using TraverserOptions = arangodb::traverser::TraverserOptions; -PathEnumerator::PathEnumerator(Traverser* traverser, std::string const& startVertex, +PathEnumerator::PathEnumerator(Traverser* traverser, + std::string const& startVertex, TraverserOptions* opts) : _traverser(traverser), _isFirst(true), _opts(opts) { StringRef svId = _opts->cache()->persistString(StringRef(startVertex)); @@ -40,6 +42,13 @@ PathEnumerator::PathEnumerator(Traverser* traverser, std::string const& startVer TRI_ASSERT(_enumeratedPath.vertices.size() == 1); } +DepthFirstEnumerator::DepthFirstEnumerator(Traverser* traverser, + std::string const& startVertex, + TraverserOptions* opts) + : PathEnumerator(traverser, startVertex, opts) {} + +DepthFirstEnumerator::~DepthFirstEnumerator() {} + bool DepthFirstEnumerator::next() { if (_isFirst) { _isFirst = false; @@ -56,8 +65,9 @@ bool DepthFirstEnumerator::next() { if (_enumeratedPath.edges.size() < _opts->maxDepth) { // We are not done with this path, so // we reserve the cursor for next depth - auto cursor = _opts->nextCursor(_traverser->mmdr(), StringRef(_enumeratedPath.vertices.back()), - _enumeratedPath.edges.size()); + auto cursor = _opts->nextCursor( + _traverser->mmdr(), StringRef(_enumeratedPath.vertices.back()), + _enumeratedPath.edges.size()); if (cursor != nullptr) { _edgeCursors.emplace(cursor); } @@ -72,14 +82,16 @@ bool DepthFirstEnumerator::next() { while (!_edgeCursors.empty()) { TRI_ASSERT(_edgeCursors.size() == _enumeratedPath.edges.size() + 1); auto& cursor = _edgeCursors.top(); - + bool foundPath = false; bool exitInnerLoop = false; - auto callback = [&] (StringRef const& eid, VPackSlice const& edge, size_t cursorId) { + auto callback = [&](StringRef const& eid, VPackSlice const& edge, + size_t cursorId) { ++_traverser->_readDocuments; _enumeratedPath.edges.push_back(eid); - _opts->cache()->insertDocument(StringRef(eid), edge); // TODO handle in cursor directly? - + _opts->cache()->insertDocument( + StringRef(eid), edge); // TODO handle in cursor directly? + if (_opts->uniqueEdges == TraverserOptions::UniquenessLevel::GLOBAL) { if (_returnedEdges.find(eid) == _returnedEdges.end()) { // Edge not yet visited. Mark and continue. @@ -91,22 +103,22 @@ bool DepthFirstEnumerator::next() { return; } } - if (!_traverser->edgeMatchesConditions(edge, - StringRef(_enumeratedPath.vertices.back()), - _enumeratedPath.edges.size() - 1, - cursorId)) { + if (!_traverser->edgeMatchesConditions( + edge, StringRef(_enumeratedPath.vertices.back()), + _enumeratedPath.edges.size() - 1, cursorId)) { // This edge does not pass the filtering TRI_ASSERT(!_enumeratedPath.edges.empty()); _enumeratedPath.edges.pop_back(); return; } - + if (_opts->uniqueEdges == TraverserOptions::UniquenessLevel::PATH) { StringRef const& e = _enumeratedPath.edges.back(); bool foundOnce = false; for (StringRef const& it : _enumeratedPath.edges) { if (foundOnce) { - foundOnce = false; // if we leave with foundOnce == false we found the edge earlier + foundOnce = false; // if we leave with foundOnce == false we + // found the edge earlier break; } if (it == e) { @@ -121,11 +133,12 @@ bool DepthFirstEnumerator::next() { return; } } - + // We have to check if edge and vertex is valid if (_traverser->getVertex(edge, _enumeratedPath.vertices)) { // case both are valid. - if (_opts->uniqueVertices == TraverserOptions::UniquenessLevel::PATH) { + if (_opts->uniqueVertices == + TraverserOptions::UniquenessLevel::PATH) { auto& e = _enumeratedPath.vertices.back(); bool foundOnce = false; for (auto const& it : _enumeratedPath.vertices) { @@ -152,7 +165,7 @@ bool DepthFirstEnumerator::next() { exitInnerLoop = true; return; } - + foundPath = true; return; } @@ -163,7 +176,7 @@ bool DepthFirstEnumerator::next() { if (cursor->next(callback)) { if (foundPath) { return true; - } else if(exitInnerLoop) { + } else if (exitInnerLoop) { break; } } else { @@ -174,28 +187,31 @@ bool DepthFirstEnumerator::next() { _enumeratedPath.vertices.pop_back(); } } - }// while (!_edgeCursors.empty()) + } // while (!_edgeCursors.empty()) if (_edgeCursors.empty()) { // If we get here all cursors are exhausted. _enumeratedPath.edges.clear(); _enumeratedPath.vertices.clear(); return false; } - }// while (true) + } // while (true) } arangodb::aql::AqlValue DepthFirstEnumerator::lastVertexToAqlValue() { - return _traverser->fetchVertexData(StringRef(_enumeratedPath.vertices.back())); + return _traverser->fetchVertexData( + StringRef(_enumeratedPath.vertices.back())); } arangodb::aql::AqlValue DepthFirstEnumerator::lastEdgeToAqlValue() { if (_enumeratedPath.edges.empty()) { - return arangodb::aql::AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); + return arangodb::aql::AqlValue( + arangodb::basics::VelocyPackHelper::NullValue()); } return _traverser->fetchEdgeData(StringRef(_enumeratedPath.edges.back())); } -arangodb::aql::AqlValue DepthFirstEnumerator::pathToAqlValue(arangodb::velocypack::Builder& result) { +arangodb::aql::AqlValue DepthFirstEnumerator::pathToAqlValue( + arangodb::velocypack::Builder& result) { result.clear(); result.openObject(); result.add(VPackValue("edges")); diff --git a/arangod/VocBase/PathEnumerator.h b/arangod/VocBase/PathEnumerator.h index 391de44f3f..f2a9c1990b 100644 --- a/arangod/VocBase/PathEnumerator.h +++ b/arangod/VocBase/PathEnumerator.h @@ -39,6 +39,10 @@ namespace velocypack { class Builder; } +namespace graph { +class EdgeCursor; +} + namespace traverser { class Traverser; struct TraverserOptions; @@ -49,7 +53,6 @@ struct EnumeratedPath { EnumeratedPath() {} }; - class PathEnumerator { protected: @@ -109,16 +112,14 @@ class DepthFirstEnumerator final : public PathEnumerator { /// @brief The stack of EdgeCursors to walk through. ////////////////////////////////////////////////////////////////////////////// - std::stack> _edgeCursors; + std::stack> _edgeCursors; public: DepthFirstEnumerator(Traverser* traverser, std::string const& startVertex, - TraverserOptions* opts) - : PathEnumerator(traverser, startVertex, opts) {} + TraverserOptions* opts); - ~DepthFirstEnumerator() { - } + ~DepthFirstEnumerator(); ////////////////////////////////////////////////////////////////////////////// /// @brief Get the next Path element from the traversal. diff --git a/arangod/VocBase/SingleServerTraverser.cpp b/arangod/VocBase/SingleServerTraverser.cpp index 59b383d6eb..b91085381c 100644 --- a/arangod/VocBase/SingleServerTraverser.cpp +++ b/arangod/VocBase/SingleServerTraverser.cpp @@ -44,7 +44,7 @@ using namespace arangodb::graph; //////////////////////////////////////////////////////////////////////////////// SingleServerEdgeCursor::SingleServerEdgeCursor(ManagedDocumentResult* mmdr, - TraverserOptions* opts, + BaseOptions* opts, size_t nrCursors, std::vector const* mapping) : _opts(opts), _trx(opts->trx()), diff --git a/arangod/VocBase/SingleServerTraverser.h b/arangod/VocBase/SingleServerTraverser.h index dc1c8dfebb..7e5e45fb4d 100644 --- a/arangod/VocBase/SingleServerTraverser.h +++ b/arangod/VocBase/SingleServerTraverser.h @@ -25,9 +25,9 @@ #define ARANGOD_SINGLE_SERVER_TRAVERSER_H 1 #include "Aql/AqlValue.h" +#include "Graph/EdgeCursor.h" #include "Utils/OperationCursor.h" #include "VocBase/Traverser.h" -#include "VocBase/TraverserOptions.h" #include "VocBase/voc-types.h" namespace arangodb { @@ -35,13 +35,17 @@ namespace arangodb { class LogicalCollection; class ManagedDocumentResult; +namespace graph { +class BaseOptions; +} + namespace traverser { class PathEnumerator; -class SingleServerEdgeCursor : public EdgeCursor { +class SingleServerEdgeCursor : public graph::EdgeCursor { private: - TraverserOptions* _opts; + graph::BaseOptions* _opts; transaction::Methods* _trx; ManagedDocumentResult* _mmdr; std::vector> _cursors; @@ -52,7 +56,7 @@ class SingleServerEdgeCursor : public EdgeCursor { std::vector const* _internalCursorMapping; public: - SingleServerEdgeCursor(ManagedDocumentResult* mmdr, TraverserOptions* options, size_t, std::vector const* mapping = nullptr); + SingleServerEdgeCursor(ManagedDocumentResult* mmdr, graph::BaseOptions* options, size_t, std::vector const* mapping = nullptr); ~SingleServerEdgeCursor() { for (auto& it : _cursors) { diff --git a/arangod/VocBase/TraverserOptions.cpp b/arangod/VocBase/TraverserOptions.cpp index 8b8e68c2af..c30901c20f 100644 --- a/arangod/VocBase/TraverserOptions.cpp +++ b/arangod/VocBase/TraverserOptions.cpp @@ -507,7 +507,7 @@ bool arangodb::traverser::TraverserOptions::evaluateVertexExpression( return evaluateExpression(expression, vertex); } -arangodb::traverser::EdgeCursor* +EdgeCursor* arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, StringRef vid, uint64_t depth) { @@ -525,7 +525,7 @@ arangodb::traverser::TraverserOptions::nextCursor(ManagedDocumentResult* mmdr, return nextCursorLocal(mmdr, vid, depth, list); } -arangodb::traverser::EdgeCursor* +EdgeCursor* arangodb::traverser::TraverserOptions::nextCursorLocal( ManagedDocumentResult* mmdr, StringRef vid, uint64_t depth, std::vector& list) { @@ -558,7 +558,7 @@ arangodb::traverser::TraverserOptions::nextCursorLocal( return allCursor.release(); } -arangodb::traverser::EdgeCursor* +EdgeCursor* arangodb::traverser::TraverserOptions::nextCursorCoordinator(StringRef vid, uint64_t depth) { TRI_ASSERT(_traverser != nullptr); diff --git a/arangod/VocBase/TraverserOptions.h b/arangod/VocBase/TraverserOptions.h index b5ad4ac95a..a047c28049 100644 --- a/arangod/VocBase/TraverserOptions.h +++ b/arangod/VocBase/TraverserOptions.h @@ -51,24 +51,6 @@ namespace traverser { class ClusterTraverser; class TraverserCache; -/// @brief Abstract class used in the traversals -/// to abstract away access to indexes / DBServers. -/// Returns edges as VelocyPack. - -class EdgeCursor { - public: - EdgeCursor() {} - virtual ~EdgeCursor() {} - - virtual bool next( - std::function - callback) = 0; - - virtual void readAll( - std::function) = 0; -}; - struct TraverserOptions : public graph::BaseOptions { friend class arangodb::aql::TraversalNode; @@ -127,17 +109,17 @@ struct TraverserOptions : public graph::BaseOptions { bool evaluateVertexExpression(arangodb::velocypack::Slice, uint64_t) const; - EdgeCursor* nextCursor(ManagedDocumentResult*, StringRef vid, uint64_t); + graph::EdgeCursor* nextCursor(ManagedDocumentResult*, StringRef vid, uint64_t); void linkTraverser(arangodb::traverser::ClusterTraverser*); double estimateCost(size_t& nrItems) const; private: - EdgeCursor* nextCursorLocal(ManagedDocumentResult*, StringRef vid, uint64_t, + graph::EdgeCursor* nextCursorLocal(ManagedDocumentResult*, StringRef vid, uint64_t, std::vector&); - EdgeCursor* nextCursorCoordinator(StringRef vid, uint64_t); + graph::EdgeCursor* nextCursorCoordinator(StringRef vid, uint64_t); }; } }