From 70ba155083a586f20ab8e23670faada78eabb557 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 22 Aug 2014 09:24:38 +0200 Subject: [PATCH 1/7] Teach unlinkNode to deal with nodes with multiple parents. --- arangod/Aql/ExecutionPlan.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index 33003b0878..4fcda543e4 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -1016,23 +1016,27 @@ void ExecutionPlan::unlinkNodes (std::unordered_set& toRemove) { //////////////////////////////////////////////////////////////////////////////// void ExecutionPlan::unlinkNode (ExecutionNode* node) { + checkLinkage(); + std::cout << toJson(TRI_UNKNOWN_MEM_ZONE, true).toString() << std::endl; auto parents = node->getParents(); if (parents.empty()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "Cannot unlink root node of plan."); } - else if (parents.size() > 1) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "Cannot remove node with more than one parent."); - } else { auto dep = node->getDependencies(); - parents[0]->removeDependency(node); + for (auto p : parents) { + p->removeDependency(node); + for (auto x : dep) { + p->addDependency(x); + } + } for (auto x : dep) { node->removeDependency(x); - parents[0]->addDependency(x); } } + std::cout << toJson(TRI_UNKNOWN_MEM_ZONE, true).toString() << std::endl; + checkLinkage(); } //////////////////////////////////////////////////////////////////////////////// From 821ce84b4b15c4a42ab12346020b9c9db94e709a Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 22 Aug 2014 09:28:41 +0200 Subject: [PATCH 2/7] Disable debugging output. --- arangod/Aql/ExecutionPlan.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index 4fcda543e4..46da2cb43f 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -1016,8 +1016,6 @@ void ExecutionPlan::unlinkNodes (std::unordered_set& toRemove) { //////////////////////////////////////////////////////////////////////////////// void ExecutionPlan::unlinkNode (ExecutionNode* node) { - checkLinkage(); - std::cout << toJson(TRI_UNKNOWN_MEM_ZONE, true).toString() << std::endl; auto parents = node->getParents(); if (parents.empty()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, @@ -1035,8 +1033,6 @@ void ExecutionPlan::unlinkNode (ExecutionNode* node) { node->removeDependency(x); } } - std::cout << toJson(TRI_UNKNOWN_MEM_ZONE, true).toString() << std::endl; - checkLinkage(); } //////////////////////////////////////////////////////////////////////////////// From 5d15b4fba58d22ea46fea9745696d61586858da3 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 22 Aug 2014 09:33:26 +0200 Subject: [PATCH 3/7] Silence an (unjustified) compiler warning. --- lib/ProgramOptions/program-options.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ProgramOptions/program-options.cpp b/lib/ProgramOptions/program-options.cpp index 2b556260e8..f18dfa9a69 100644 --- a/lib/ProgramOptions/program-options.cpp +++ b/lib/ProgramOptions/program-options.cpp @@ -1410,7 +1410,7 @@ bool TRI_ParseArgumentsProgramOptions (TRI_program_options_t * options, extern int optind; TRI_string_buffer_t buffer; - TRI_PO_item_t * item; + TRI_PO_item_t* item = nullptr; const char* shortOptions; size_t i; int idx; From c4918e4fb7dd408f7c555c56be504b5344cf4a0f Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 22 Aug 2014 09:36:31 +0200 Subject: [PATCH 4/7] Silence another compiler warning. --- arangod/VocBase/transaction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/VocBase/transaction.cpp b/arangod/VocBase/transaction.cpp index 962f656947..6aa8700bb2 100644 --- a/arangod/VocBase/transaction.cpp +++ b/arangod/VocBase/transaction.cpp @@ -865,7 +865,7 @@ int TRI_AddCollectionTransaction (TRI_transaction_t* trx, } // check if we already have got this collection in the _collections vector - size_t position; + size_t position = 0; TRI_transaction_collection_t* trxCollection = FindCollection(trx, cid, &position); if (trxCollection != nullptr) { From 2b27957546531eae1a354098415ada59156fdf5a Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Fri, 22 Aug 2014 09:51:48 +0200 Subject: [PATCH 5/7] added AQL_EXPLAIN() --- arangod/Aql/ExecutionNode.cpp | 8 +++-- arangod/Aql/Query.cpp | 59 +++++++++++++++++++++++++++++++-- arangod/Aql/Query.h | 4 +-- arangod/V8Server/v8-vocbase.cpp | 55 +++++++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 8 deletions(-) diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 0021f1bcc6..59e26f0122 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -329,11 +329,13 @@ Json ExecutionNode::toJsonHelperGeneric (triagens::basics::Json& nodes, for (size_t i = 0; i < _parents.size(); i++) { parents(Json(static_cast(_parents[i]->id()))); } - json("parents", parents); + if (verbose) { + json("parents", parents); + } json("id", Json(static_cast(id()))); - if (this->_estimatedCost != 0.0){ - json("estimatedCost", Json(this->_estimatedCost)); + if (_estimatedCost != 0.0){ + json("estimatedCost", Json(_estimatedCost)); } return json; } diff --git a/arangod/Aql/Query.cpp b/arangod/Aql/Query.cpp index 356e0e1c72..f98f018e88 100644 --- a/arangod/Aql/Query.cpp +++ b/arangod/Aql/Query.cpp @@ -323,10 +323,65 @@ QueryResult Query::parse () { } //////////////////////////////////////////////////////////////////////////////// -/// @brief explain an AQL query - TODO: implement and determine return type +/// @brief explain an AQL query //////////////////////////////////////////////////////////////////////////////// -void Query::explain () { +QueryResult Query::explain () { + try { + ExecutionPlan* plan; + Parser parser(this); + + parser.parse(); + // put in bind parameters + parser.ast()->injectBindParameters(_bindParameters); + // optimize the ast + parser.ast()->optimize(); + // std::cout << "AST: " << triagens::basics::JsonHelper::toString(parser.ast()->toJson(TRI_UNKNOWN_MEM_ZONE)) << "\n"; + + // create the transaction object, but do not start it yet + AQL_TRANSACTION_V8 trx(_vocbase, _collections.collections()); + + // we have an AST + int res = trx.begin(); + + if (res != TRI_ERROR_NO_ERROR) { + return QueryResult(res, TRI_errno_string(res)); + } + + plan = ExecutionPlan::instanciateFromAst(parser.ast()); + if (plan == nullptr) { + // oops + return QueryResult(TRI_ERROR_INTERNAL); + } + + // Run the query optimiser: + triagens::aql::Optimizer opt; + opt.createPlans(plan); // Now plan and all derived plans belong to the + // optimizer + plan = opt.stealBest(); // Now we own the best one again + TRI_ASSERT(plan != nullptr); + + trx.commit(); + //triagens::basic::Json json(triagens::basic::Json::Array); + + QueryResult result(TRI_ERROR_NO_ERROR); + result.json = plan->toJson(TRI_UNKNOWN_MEM_ZONE, false).steal(); //json(); + delete plan; + + return result; + } + catch (triagens::arango::Exception const& ex) { + return QueryResult(ex.code(), ex.message()); + } + catch (std::bad_alloc const& ex) { + return QueryResult(TRI_ERROR_OUT_OF_MEMORY, TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY)); + } + catch (std::exception const& ex) { + return QueryResult(TRI_ERROR_INTERNAL, ex.what()); + } + catch (...) { + return QueryResult(TRI_ERROR_INTERNAL, TRI_errno_string(TRI_ERROR_INTERNAL)); + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Aql/Query.h b/arangod/Aql/Query.h index 1bcea290e7..33c01761eb 100644 --- a/arangod/Aql/Query.h +++ b/arangod/Aql/Query.h @@ -179,10 +179,10 @@ namespace triagens { QueryResult parse (); //////////////////////////////////////////////////////////////////////////////// -/// @brief explain an AQL query - TODO: implement and determine return type +/// @brief explain an AQL query //////////////////////////////////////////////////////////////////////////////// - void explain (); + QueryResult explain (); //////////////////////////////////////////////////////////////////////////////// /// @brief get v8 executor diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index 9938994a1d..3482e17c3e 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -857,6 +857,59 @@ static v8::Handle JS_ParseAql (v8::Arguments const& argv) { return scope.Close(result); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief explains an AQL query +//////////////////////////////////////////////////////////////////////////////// + +static v8::Handle JS_ExplainAql (v8::Arguments const& argv) { + v8::HandleScope scope; + + TRI_vocbase_t* vocbase = GetContextVocBase(); + + if (vocbase == nullptr) { + TRI_V8_EXCEPTION(scope, TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); + } + + if (argv.Length() < 1 || argv.Length() > 2) { + TRI_V8_EXCEPTION_USAGE(scope, "AQL_EXPLAIN(, )"); + } + + // get the query string + if (! argv[0]->IsString()) { + TRI_V8_TYPE_ERROR(scope, "expecting string for "); + } + + string const&& queryString = TRI_ObjectToString(argv[0]); + + // bind parameters + TRI_json_t* parameters = nullptr; + + if (argv.Length() > 1) { + if (! argv[1]->IsUndefined() && ! argv[1]->IsNull() && ! argv[1]->IsObject()) { + TRI_V8_TYPE_ERROR(scope, "expecting object for "); + } + if (argv[1]->IsObject()) { + parameters = TRI_ObjectToJson(argv[1]); + } + } + + // bind parameters will be freed by the query later + triagens::aql::Query query(vocbase, queryString.c_str(), queryString.size(), parameters); + + auto queryResult = query.explain(); + + if (queryResult.code != TRI_ERROR_NO_ERROR) { + TRI_V8_EXCEPTION_FULL(scope, queryResult.code, queryResult.details); + } + + v8::Handle result = v8::Object::New(); + if (queryResult.json != nullptr) { + result->Set(TRI_V8_STRING("plan"), TRI_ObjectJson(queryResult.json)); + } + + return scope.Close(result); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief executes an AQL query //////////////////////////////////////////////////////////////////////////////// @@ -1098,7 +1151,6 @@ static v8::Handle JS_ExecuteAql (v8::Arguments const& argv) { return scope.Close(cursorObject); } - // ----------------------------------------------------------------------------- // --SECTION-- AHUACATL // ----------------------------------------------------------------------------- @@ -2679,6 +2731,7 @@ void TRI_InitV8VocBridge (v8::Handle context, // new AQL functions. not intended to be used directly by end users TRI_AddGlobalFunctionVocbase(context, "AQL_EXECUTE", JS_ExecuteAql, true); TRI_AddGlobalFunctionVocbase(context, "AQL_EXECUTEJSON", JS_ExecuteAqlJson, true); + TRI_AddGlobalFunctionVocbase(context, "AQL_EXPLAIN", JS_ExplainAql, true); TRI_AddGlobalFunctionVocbase(context, "AQL_PARSE", JS_ParseAql, true); TRI_InitV8replication(context, server, vocbase, loader, threadNumber, v8g); From 871351f803c534db320df7cd208cb6514898594f Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Fri, 22 Aug 2014 09:56:39 +0200 Subject: [PATCH 6/7] make count() const --- arangod/Aql/ExecutionBlock.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index e40c7bd36b..5bf90276e6 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -310,7 +310,7 @@ namespace triagens { virtual bool hasMore (); - virtual int64_t count () { + virtual int64_t count () const { return _dependencies[0]->count(); } @@ -443,7 +443,7 @@ namespace triagens { return ! _done; } - int64_t count () { + int64_t count () const { return 1; } @@ -804,7 +804,7 @@ namespace triagens { bool hasMore (); - int64_t count () { + int64_t count () const { return -1; // refuse to work } @@ -1297,7 +1297,7 @@ namespace triagens { return false; } - int64_t count () { + int64_t count () const { return 0; } From f932e9277cc7fc3d5f3cc358d551ae8da04cda0f Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Fri, 22 Aug 2014 10:16:32 +0200 Subject: [PATCH 7/7] cloned EnumerateCollectionBlock into IndexRangeBlock --- arangod/Aql/ExecutionBlock.cpp | 174 ++++++++++++++++++++++++++++++++ arangod/Aql/ExecutionBlock.h | 90 +++++++++++++++++ arangod/Aql/ExecutionEngine.cpp | 5 + 3 files changed, 269 insertions(+) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index 0cbe32d60a..7ce9374850 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -704,6 +704,180 @@ size_t EnumerateCollectionBlock::skipSome (size_t atLeast, size_t atMost) { return skipped; } +// ----------------------------------------------------------------------------- +// --SECTION-- class IndexRangeBlock +// ----------------------------------------------------------------------------- + +IndexRangeBlock::IndexRangeBlock (ExecutionEngine* engine, + IndexRangeNode const* ep) + : ExecutionBlock(engine, ep), + _collection(ep->_collection), + _posInAllDocs(0) { +} + +IndexRangeBlock::~IndexRangeBlock () { +} + +bool IndexRangeBlock::moreDocuments () { + if (_documents.empty()) { + _documents.reserve(DefaultBatchSize); + } + + _documents.clear(); + + int res = _trx->readIncremental(_trx->trxCollection(_collection->cid()), + _documents, + _internalSkip, + static_cast(DefaultBatchSize), + 0, + TRI_QRY_NO_LIMIT, + &_totalCount); + + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + + return (! _documents.empty()); +} + +int IndexRangeBlock::initialize () { + return ExecutionBlock::initialize(); +} + +int IndexRangeBlock::initCursor (AqlItemBlock* items, size_t pos) { + int res = ExecutionBlock::initCursor(items, pos); + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + initDocuments(); + + if (_totalCount == 0) { + _done = true; + } + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief getSome +//////////////////////////////////////////////////////////////////////////////// + +AqlItemBlock* IndexRangeBlock::getSome (size_t atLeast, + size_t atMost) { + if (_done) { + return nullptr; + } + + if (_buffer.empty()) { + if (! ExecutionBlock::getBlock(DefaultBatchSize, DefaultBatchSize)) { + _done = true; + return nullptr; + } + _pos = 0; // this is in the first block + _posInAllDocs = 0; // Note that we know _allDocs.size() > 0, + // otherwise _done would be true already + } + + // If we get here, we do have _buffer.front() + AqlItemBlock* cur = _buffer.front(); + size_t const curRegs = cur->getNrRegs(); + + size_t available = _documents.size() - _posInAllDocs; + size_t toSend = std::min(atMost, available); + + unique_ptr res(new AqlItemBlock(toSend, _varOverview->nrRegs[_depth])); + // automatically freed if we throw + TRI_ASSERT(curRegs <= res->getNrRegs()); + + // only copy 1st row of registers inherited from previous frame(s) + inheritRegisters(cur, res.get(), _pos); + + // set our collection for our output register + res->setDocumentCollection(curRegs, _trx->documentCollection(_collection->cid())); + + for (size_t j = 0; j < toSend; j++) { + if (j > 0) { + // re-use already copied aqlvalues + for (RegisterId i = 0; i < curRegs; i++) { + res->setValue(j, i, res->getValue(0, i)); + // Note: if this throws, then all values will be deleted + // properly since the first one is. + } + } + + // The result is in the first variable of this depth, + // we do not need to do a lookup in _varOverview->varInfo, + // but can just take cur->getNrRegs() as registerId: + res->setValue(j, curRegs, + AqlValue(reinterpret_cast(_documents[_posInAllDocs++].getDataPtr()))); + // No harm done, if the setValue throws! + } + + // Advance read position: + if (_posInAllDocs >= _documents.size()) { + // we have exhausted our local documents buffer + _posInAllDocs = 0; + + // fetch more documents into our buffer + if (! moreDocuments()) { + // nothing more to read, re-initialize fetching of documents + initDocuments(); + if (++_pos >= cur->size()) { + _buffer.pop_front(); // does not throw + delete cur; + _pos = 0; + } + } + } + return res.release(); +} + +size_t IndexRangeBlock::skipSome (size_t atLeast, size_t atMost) { + + size_t skipped = 0; + + if (_done) { + return skipped; + } + + while (skipped < atLeast) { + if (_buffer.empty()) { + if (! getBlock(DefaultBatchSize, DefaultBatchSize)) { + _done = true; + return skipped; + } + _pos = 0; // this is in the first block + _posInAllDocs = 0; // Note that we know _allDocs.size() > 0, + // otherwise _done would be true already + } + + // if we get here, then _buffer.front() exists + AqlItemBlock* cur = _buffer.front(); + + if (atMost >= skipped + _documents.size() - _posInAllDocs) { + skipped += _documents.size() - _posInAllDocs; + _posInAllDocs = 0; + + // fetch more documents into our buffer + if (! moreDocuments()) { + // nothing more to read, re-initialize fetching of documents + initDocuments(); + if (++_pos >= cur->size()) { + _buffer.pop_front(); // does not throw + delete cur; + _pos = 0; + } + } + } + else { + _posInAllDocs += atMost - skipped; + skipped = atMost; + } + } + return skipped; +} // ----------------------------------------------------------------------------- // --SECTION-- class EnumerateListBlock diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index 5bf90276e6..c870b90744 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -561,6 +561,96 @@ namespace triagens { }; +// ----------------------------------------------------------------------------- +// --SECTION-- IndexRangeBlock +// ----------------------------------------------------------------------------- + + class IndexRangeBlock : public ExecutionBlock { + + public: + + IndexRangeBlock (ExecutionEngine* engine, + IndexRangeNode const* ep); + + ~IndexRangeBlock (); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initialize fetching of documents +//////////////////////////////////////////////////////////////////////////////// + + void initDocuments () { + _internalSkip = 0; + if (! moreDocuments()) { + _done = true; + } + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief continue fetching of documents +//////////////////////////////////////////////////////////////////////////////// + + bool moreDocuments (); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initialize, here we fetch all docs from the database +//////////////////////////////////////////////////////////////////////////////// + + int initialize (); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initCursor, here we release our docs from this collection +//////////////////////////////////////////////////////////////////////////////// + + int initCursor (AqlItemBlock* items, size_t pos); + + AqlItemBlock* getSome (size_t atLeast, size_t atMost); + +//////////////////////////////////////////////////////////////////////////////// +// skip between atLeast and atMost, returns the number actually skipped . . . +// will only return less than atLeast if there aren't atLeast many +// things to skip overall. +//////////////////////////////////////////////////////////////////////////////// + + size_t skipSome (size_t atLeast, size_t atMost); + +// ----------------------------------------------------------------------------- +// --SECTION-- private variables +// ----------------------------------------------------------------------------- + + private: + +//////////////////////////////////////////////////////////////////////////////// +/// @brief collection +//////////////////////////////////////////////////////////////////////////////// + + Collection* _collection; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief total number of documents in the collection +//////////////////////////////////////////////////////////////////////////////// + + uint32_t _totalCount; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief internal skip value +//////////////////////////////////////////////////////////////////////////////// + + TRI_voc_size_t _internalSkip; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief document buffer +//////////////////////////////////////////////////////////////////////////////// + + std::vector _documents; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief current position in _allDocs +//////////////////////////////////////////////////////////////////////////////// + + size_t _posInAllDocs; + + }; + // ----------------------------------------------------------------------------- // --SECTION-- EnumerateListBlock // ----------------------------------------------------------------------------- diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index befeddea1c..444c2897bc 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -93,6 +93,11 @@ struct Instanciator : public WalkerWorker { static_cast(en)); break; } + case ExecutionNode::INDEX_RANGE: { + eb = new IndexRangeBlock(engine, + static_cast(en)); + break; + } case ExecutionNode::ENUMERATE_COLLECTION: { eb = new EnumerateCollectionBlock(engine, static_cast(en));