From 197be68f7d73db79d6f04d9bb54876cb1d09b9ef Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 13 Oct 2015 15:12:32 +0200 Subject: [PATCH] removed IndexRange* --- arangod/Aql/AggregateNode.cpp | 1 - arangod/Aql/ConditionFinder.cpp | 1 - arangod/Aql/ExecutionEngine.cpp | 7 - arangod/Aql/ExecutionNode.cpp | 24 +- arangod/Aql/ExecutionNode.h | 2 +- arangod/Aql/ExecutionPlan.cpp | 3 +- arangod/Aql/IndexBlock.cpp | 4 +- arangod/Aql/IndexBlock.h | 2 +- arangod/Aql/IndexRangeBlock.cpp | 1480 ----------------- arangod/Aql/IndexRangeBlock.h | 357 ---- arangod/Aql/IndexRangeNode.cpp | 371 ----- arangod/Aql/IndexRangeNode.h | 258 --- arangod/Aql/Optimizer.h | 5 +- arangod/Aql/OptimizerRules.cpp | 27 +- arangod/CMakeLists.txt | 2 - .../SimpleAttributeEqualityMatcher.cpp | 25 - arangod/Makefile.files | 2 - 17 files changed, 18 insertions(+), 2553 deletions(-) delete mode 100644 arangod/Aql/IndexRangeBlock.cpp delete mode 100644 arangod/Aql/IndexRangeBlock.h delete mode 100644 arangod/Aql/IndexRangeNode.cpp delete mode 100644 arangod/Aql/IndexRangeNode.h diff --git a/arangod/Aql/AggregateNode.cpp b/arangod/Aql/AggregateNode.cpp index 237db1efb0..5a9438cd2f 100644 --- a/arangod/Aql/AggregateNode.cpp +++ b/arangod/Aql/AggregateNode.cpp @@ -193,7 +193,6 @@ struct UserVarFinder final : public WalkerWorker { } else if (en->getType() == ExecutionNode::ENUMERATE_COLLECTION || en->getType() == ExecutionNode::INDEX || - en->getType() == ExecutionNode::INDEX_RANGE || en->getType() == ExecutionNode::ENUMERATE_LIST || en->getType() == ExecutionNode::AGGREGATE) { depth += 1; diff --git a/arangod/Aql/ConditionFinder.cpp b/arangod/Aql/ConditionFinder.cpp index 0b3e128a61..556bf73061 100644 --- a/arangod/Aql/ConditionFinder.cpp +++ b/arangod/Aql/ConditionFinder.cpp @@ -52,7 +52,6 @@ bool ConditionFinder::before (ExecutionNode* en) { case EN::REMOTE: case EN::SUBQUERY: case EN::INDEX: - case EN::INDEX_RANGE: case EN::INSERT: case EN::REMOVE: case EN::REPLACE: diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index 61eb6a53fd..bbdfa7aa34 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -40,7 +40,6 @@ #include "Aql/ExecutionNode.h" #include "Aql/ExecutionPlan.h" #include "Aql/IndexBlock.h" -#include "Aql/IndexRangeBlock.h" #include "Aql/ModificationBlocks.h" #include "Aql/QueryRegistry.h" #include "Aql/SortBlock.h" @@ -66,9 +65,6 @@ static ExecutionBlock* CreateBlock (ExecutionEngine* engine, case ExecutionNode::SINGLETON: { return new SingletonBlock(engine, static_cast(en)); } - case ExecutionNode::INDEX_RANGE: { - return new IndexRangeBlock(engine, static_cast(en)); - } case ExecutionNode::INDEX: { return new IndexBlock(engine, static_cast(en)); } @@ -393,9 +389,6 @@ struct CoordinatorInstanciator : public WalkerWorker { if ((*en)->getType() == ExecutionNode::ENUMERATE_COLLECTION) { collection = const_cast(static_cast((*en))->collection()); } - else if ((*en)->getType() == ExecutionNode::INDEX_RANGE) { - collection = const_cast(static_cast((*en))->collection()); - } else if ((*en)->getType() == ExecutionNode::INDEX) { collection = const_cast(static_cast((*en))->collection()); } diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index e48e154ffc..fb228a6bee 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -32,7 +32,6 @@ #include "Aql/Collection.h" #include "Aql/ExecutionPlan.h" #include "Aql/IndexNode.h" -#include "Aql/IndexRangeNode.h" #include "Aql/ModificationNodes.h" #include "Aql/SortNode.h" #include "Aql/WalkerWorker.h" @@ -65,7 +64,6 @@ std::unordered_map const ExecutionNode::TypeNames{ { static_cast(ENUMERATE_COLLECTION), "EnumerateCollectionNode" }, { static_cast(ENUMERATE_LIST), "EnumerateListNode" }, { static_cast(INDEX), "IndexNode" }, - { static_cast(INDEX_RANGE), "IndexRangeNode" }, { static_cast(LIMIT), "LimitNode" }, { static_cast(CALCULATION), "CalculationNode" }, { static_cast(SUBQUERY), "SubqueryNode" }, @@ -234,8 +232,6 @@ ExecutionNode* ExecutionNode::fromJsonFactory (ExecutionPlan* plan, return new ReturnNode(plan, oneNode); case NORESULTS: return new NoResultsNode(plan, oneNode); - case INDEX_RANGE: - return new IndexRangeNode(plan, oneNode); case INDEX: return new IndexNode(plan, oneNode); case REMOTE: @@ -548,7 +544,6 @@ bool ExecutionNode::isInInnerLoop () const { auto type = node->getType(); if (type == ENUMERATE_COLLECTION || - type == INDEX_RANGE || type == INDEX || type == ENUMERATE_LIST) { // we are contained in an outer loop @@ -836,21 +831,6 @@ void ExecutionNode::RegisterPlan::after (ExecutionNode* en) { break; } - case ExecutionNode::INDEX_RANGE: { - depth++; - nrRegsHere.emplace_back(1); - // create a copy of the last value here - // this is requried because back returns a reference and emplace/push_back may invalidate all references - RegisterId registerId = 1 + nrRegs.back(); - nrRegs.emplace_back(registerId); - - auto ep = static_cast(en); - TRI_ASSERT(ep != nullptr); - varInfo.emplace(ep->outVariable()->id, VarInfo(depth, totalNrRegs)); - totalNrRegs++; - break; - } - case ExecutionNode::INDEX: { depth++; nrRegsHere.emplace_back(1); @@ -1642,11 +1622,11 @@ double FilterNode::estimateCost (size_t& nrItems) const { // worst case the filter does not reduce the items at all. Furthermore, // no optimizer rule introduces FilterNodes, thus it is not important // that they appear to lower the costs. Note that contrary to this, - // an IndexRangeNode does lower the costs, it also has a better idea + // an IndexNode does lower the costs, it also has a better idea // to what extent the number of items is reduced. On the other hand it // is important that a FilterNode produces additional costs, otherwise // the rule throwing away a FilterNode that is already covered by an - // IndexRangeNode cannot reduce the costs. + // IndexNode cannot reduce the costs. return depCost + nrItems; } diff --git a/arangod/Aql/ExecutionNode.h b/arangod/Aql/ExecutionNode.h index 41e89a7899..fa6bc6e310 100644 --- a/arangod/Aql/ExecutionNode.h +++ b/arangod/Aql/ExecutionNode.h @@ -71,7 +71,7 @@ namespace triagens { ILLEGAL = 0, SINGLETON = 1, ENUMERATE_COLLECTION = 2, - INDEX_RANGE = 3, + // INDEX_RANGE = 3, // not used anymore ENUMERATE_LIST = 4, FILTER = 5, LIMIT = 6, diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index b6a1036c43..3bb4e9b5f6 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -1849,8 +1849,7 @@ bool ExecutionPlan::isDeadSimple () const { if (nodeType == ExecutionNode::SUBQUERY || nodeType == ExecutionNode::ENUMERATE_COLLECTION || nodeType == ExecutionNode::ENUMERATE_LIST || - nodeType == ExecutionNode::INDEX || - nodeType == ExecutionNode::INDEX_RANGE) { + nodeType == ExecutionNode::INDEX) { // these node types are not simple return false; } diff --git a/arangod/Aql/IndexBlock.cpp b/arangod/Aql/IndexBlock.cpp index bf8c8bc5fb..d8f6fe056b 100644 --- a/arangod/Aql/IndexBlock.cpp +++ b/arangod/Aql/IndexBlock.cpp @@ -53,7 +53,7 @@ using Json = triagens::basics::Json; #endif // ----------------------------------------------------------------------------- -// --SECTION-- class IndexRangeBlock +// --SECTION-- class IndexBlock // ----------------------------------------------------------------------------- IndexBlock::IndexBlock (ExecutionEngine* engine, @@ -352,7 +352,7 @@ bool IndexBlock::readIndex (size_t atMost) { // entire index when we only want a small number of documents. if (_documents.empty()) { - TRI_IF_FAILURE("IndexRangeBlock::readIndex") { + TRI_IF_FAILURE("IndexBlock::readIndex") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } _documents.reserve(atMost); diff --git a/arangod/Aql/IndexBlock.h b/arangod/Aql/IndexBlock.h index a89e76efa7..f25028cdcc 100644 --- a/arangod/Aql/IndexBlock.h +++ b/arangod/Aql/IndexBlock.h @@ -81,7 +81,7 @@ namespace triagens { }; // ----------------------------------------------------------------------------- -// --SECTION-- IndexRangeBlock +// --SECTION-- IndexBlock // ----------------------------------------------------------------------------- class IndexBlock : public ExecutionBlock { diff --git a/arangod/Aql/IndexRangeBlock.cpp b/arangod/Aql/IndexRangeBlock.cpp deleted file mode 100644 index c20a536899..0000000000 --- a/arangod/Aql/IndexRangeBlock.cpp +++ /dev/null @@ -1,1480 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// @brief AQL IndexRangeBlock -/// -/// @file -/// -/// DISCLAIMER -/// -/// Copyright 2010-2014 triagens GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is triAGENS GmbH, Cologne, Germany -/// -/// @author Max Neunhoeffer -/// @author Copyright 2014, triagens GmbH, Cologne, Germany -//////////////////////////////////////////////////////////////////////////////// - -#include "IndexRangeBlock.h" -#include "Aql/ExecutionEngine.h" -#include "Aql/Functions.h" -#include "Basics/ScopeGuard.h" -#include "Basics/json-utilities.h" -#include "Basics/Exceptions.h" -#include "Indexes/EdgeIndex.h" -#include "Indexes/HashIndex.h" -#include "Indexes/SkiplistIndex.h" -#include "V8/v8-globals.h" -#include "VocBase/edge-collection.h" -#include "VocBase/vocbase.h" - -using namespace std; -using namespace triagens::arango; -using namespace triagens::aql; - -using Json = triagens::basics::Json; - -// uncomment the following to get some debugging information -#if 0 -#define ENTER_BLOCK try { (void) 0; -#define LEAVE_BLOCK } catch (...) { std::cout << "caught an exception in " << __FUNCTION__ << ", " << __FILE__ << ":" << __LINE__ << "!\n"; throw; } -#else -#define ENTER_BLOCK -#define LEAVE_BLOCK -#endif - -// ----------------------------------------------------------------------------- -// --SECTION-- class IndexRangeBlock -// ----------------------------------------------------------------------------- - -IndexRangeBlock::IndexRangeBlock (ExecutionEngine* engine, - IndexRangeNode const* en) - : ExecutionBlock(engine, en), - _collection(en->collection()), - _posInDocs(0), - _anyBoundVariable(false), - _skiplistIterator(nullptr), - _edgeIndexIterator(nullptr), - _hashIndexSearchValue(), - _hashNextElement(nullptr), - _condition(new IndexOrCondition()), - _posInRanges(0), - _sortCoords(), - _freeCondition(true), - _hasV8Expression(false) { - - auto trxCollection = _trx->trxCollection(_collection->cid()); - - if (trxCollection != nullptr) { - _trx->orderDitch(trxCollection); - } - - std::vector> const& orRanges = en->_ranges; - size_t const n = orRanges.size(); - - for (size_t i = 0; i < n; i++) { - _condition->emplace_back(IndexAndCondition()); - - for (auto const& ri : en->_ranges[i]) { - _condition->at(i).emplace_back(ri.clone()); - } - } - - if (_condition->size() > 1) { - removeOverlapsIndexOr(*_condition); - } - - TRI_ASSERT(en->_index != nullptr); - - _allBoundsConstant.clear(); - _allBoundsConstant.reserve(orRanges.size()); - - // Detect, whether all ranges are constant: - for (size_t i = 0; i < orRanges.size(); i++) { - bool isConstant = true; - - std::vector const& attrRanges = orRanges[i]; - for (auto const& r : attrRanges) { - isConstant &= r.isConstant(); - } - _anyBoundVariable |= ! isConstant; - _allBoundsConstant.push_back(isConstant); // note: emplace_back() is not supported in C++11 but only from C++14 - } -} - -IndexRangeBlock::~IndexRangeBlock () { - destroyHashIndexSearchValues(); - - for (auto& e : _allVariableBoundExpressions) { - delete e; - } - - if (_freeCondition && _condition != nullptr) { - delete _condition; - } - - delete _skiplistIterator; - delete _edgeIndexIterator; -} - -bool IndexRangeBlock::useHighBounds () const { - auto en = static_cast(getPlanNode()); - return (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_SKIPLIST_INDEX); -} - -bool IndexRangeBlock::hasV8Expression () const { - for (auto const& expression : _allVariableBoundExpressions) { - TRI_ASSERT(expression != nullptr); - - if (expression->isV8()) { - return true; - } - } - return false; -} - -void IndexRangeBlock::buildExpressions () { - bool const useHighBounds = this->useHighBounds(); - - size_t posInExpressions = 0; - - // The following are needed to evaluate expressions with local data from - // the current incoming item: - AqlItemBlock* cur = _buffer.front(); - - auto en = static_cast(getPlanNode()); - std::unique_ptr newCondition; - - for (size_t i = 0; i < en->_ranges.size(); i++) { - size_t const n = en->_ranges[i].size(); - // prefill with n default-constructed vectors - std::vector> collector(n); - - // collect the evaluated bounds here - for (size_t k = 0; k < n; k++) { - auto const& r = en->_ranges[i][k]; - - { - // First create a new RangeInfo containing only the constant - // low and high bound of r: - RangeInfo riConst(r._var, r._attr, r._lowConst, r._highConst, r.is1ValueRangeInfo()); - collector[k].emplace_back(std::move(riConst)); - } - - // Now work the actual values of the variable lows and highs into - // this constant range: - for (auto const& l : r._lows) { - Expression* e = _allVariableBoundExpressions[posInExpressions]; - TRI_ASSERT(e != nullptr); - TRI_document_collection_t const* myCollection = nullptr; - AqlValue a = e->execute(_trx, cur, _pos, _inVars[posInExpressions], _inRegs[posInExpressions], &myCollection); - posInExpressions++; - - Json bound; - if (a._type == AqlValue::JSON) { - bound = *(a._json); - a.destroy(); // the TRI_json_t* of a._json has been stolen - } - else if (a._type == AqlValue::SHAPED || a._type == AqlValue::DOCVEC) { - bound = a.toJson(_trx, myCollection, true); - a.destroy(); // the TRI_json_t* of a._json has been stolen - } - else { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "AQL: computed a variable bound and got non-JSON"); - } - - if (! bound.isArray()) { - if (useHighBounds) { - auto b(bound.copy()); - - RangeInfo ri(r._var, - r._attr, - RangeInfoBound(l.inclusive(), true, b), // will steal b's JSON - RangeInfoBound(), - false); - - for (size_t j = 0; j < collector[k].size(); j++) { - collector[k][j].fuse(ri); - } - } - else { - auto b1(bound.copy()); // first instance of bound - auto b2(bound.copy()); // second instance of same bound - - RangeInfo ri(r._var, - r._attr, - RangeInfoBound(l.inclusive(), true, b1), // will steal b1's JSON - RangeInfoBound(l.inclusive(), true, b2), // will steal b2's JSON - false); - - for (size_t j = 0; j < collector[k].size(); j++) { - collector[k][j].fuse(ri); - } - } - } - else { - std::vector riv; - riv.reserve(bound.size()); - - for (size_t j = 0; j < bound.size(); j++) { - auto b1(bound.at(static_cast(j)).copy()); // first instance of bound - auto b2(bound.at(static_cast(j)).copy()); // second instance of same bound - - riv.emplace_back(RangeInfo(r._var, - r._attr, - RangeInfoBound(l.inclusive(), true, b1), // will steal b1's JSON - RangeInfoBound(l.inclusive(), true, b2), // will steal b2's JSON - true)); - } - - collector[k] = std::move(andCombineRangeInfoVecs(collector[k], riv)); - } - } - - if (useHighBounds) { - for (auto const& h : r._highs) { - Expression* e = _allVariableBoundExpressions[posInExpressions]; - TRI_ASSERT(e != nullptr); - TRI_document_collection_t const* myCollection = nullptr; - AqlValue a = e->execute(_trx, cur, _pos, _inVars[posInExpressions], _inRegs[posInExpressions], &myCollection); - posInExpressions++; - - Json bound; - if (a._type == AqlValue::JSON) { - bound = *(a._json); - a.destroy(); // the TRI_json_t* of a._json has been stolen - } - else if (a._type == AqlValue::SHAPED || a._type == AqlValue::DOCVEC) { - bound = a.toJson(_trx, myCollection, true); - a.destroy(); // the TRI_json_t* of a._json has been stolen - } - else { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "AQL: computed a variable bound and got non-JSON"); - } - if (! bound.isArray()) { - auto b(bound.copy()); - RangeInfo ri(r._var, - r._attr, - RangeInfoBound(), - RangeInfoBound(h.inclusive(), true, b), // will steal b's JSON - false); - - for (size_t j = 0; j < collector[k].size(); j++) { - collector[k][j].fuse(ri); - } - } - else { - std::vector riv; - riv.reserve(bound.size()); - - for (size_t j = 0; j < bound.size(); j++) { - auto b1(bound.at(static_cast(j)).copy()); // first instance of bound - auto b2(bound.at(static_cast(j)).copy()); // second instance of same bound - - riv.emplace_back(RangeInfo(r._var, - r._attr, - RangeInfoBound(h.inclusive(), true, b1), // will steal b1's JSON - RangeInfoBound(h.inclusive(), true, b2), // will steal b2's JSON - true)); - } - - collector[k] = std::move(andCombineRangeInfoVecs(collector[k], riv)); - } - } - } - } - - - bool isEmpty = false; - - for (auto const& x : collector) { - if (x.empty()) { - isEmpty = true; - break; - } - } - - if (! isEmpty) { - // otherwise the condition is impossible to fulfill - // the elements of the direct product of the collector are and - // conditions which should be added to newCondition - - // create cartesian product - std::unique_ptr indexAnds(cartesian(collector)); - - if (newCondition == nullptr) { - newCondition.reset(indexAnds.release()); - } - else { - for (auto const& indexAnd : *indexAnds) { - newCondition->emplace_back(std::move(indexAnd)); - } - } - } - - } - - freeCondition(); - - if (newCondition != nullptr) { - _condition = newCondition.release(); - _freeCondition = true; - - // remove duplicates . . . - removeOverlapsIndexOr(*_condition); - } - else { - _condition = new IndexOrCondition; - _freeCondition = true; - } -} - -int IndexRangeBlock::initialize () { - ENTER_BLOCK - int res = ExecutionBlock::initialize(); - - if (res == TRI_ERROR_NO_ERROR) { - if (_trx->orderDitch(_trx->trxCollection(_collection->cid())) == nullptr) { - res = TRI_ERROR_OUT_OF_MEMORY; - } - } - - _allVariableBoundExpressions.clear(); - - // instantiate expressions: - auto instantiateExpression = [&] (RangeInfoBound const& b) -> void { - AstNode const* a = b.getExpressionAst(_engine->getQuery()->ast()); - Expression* expression = nullptr; - - { - // all new AstNodes are registered with the Ast in the Query - std::unique_ptr e(new Expression(_engine->getQuery()->ast(), a)); - - TRI_IF_FAILURE("IndexRangeBlock::initialize") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - _allVariableBoundExpressions.emplace_back(e.get()); - expression = e.release(); - } - - // Prepare _inVars and _inRegs: - _inVars.emplace_back(); - std::vector& inVarsCur = _inVars.back(); - _inRegs.emplace_back(); - std::vector& inRegsCur = _inRegs.back(); - - std::unordered_set inVars; - expression->variables(inVars); - - for (auto const& v : inVars) { - inVarsCur.emplace_back(v); - auto it = getPlanNode()->getRegisterPlan()->varInfo.find(v->id); - TRI_ASSERT(it != getPlanNode()->getRegisterPlan()->varInfo.end()); - TRI_ASSERT(it->second.registerId < ExecutionNode::MaxRegisterId); - inRegsCur.emplace_back(it->second.registerId); - } - }; - - // Get the ranges from the node: - auto en = static_cast(getPlanNode()); - std::vector> const& orRanges = en->_ranges; - - for (size_t i = 0; i < orRanges.size(); i++) { - if (! _allBoundsConstant[i]) { - try { - for (auto const& r : orRanges[i]) { - for (auto const& l : r._lows) { - instantiateExpression(l); - } - - if (useHighBounds()) { - for (auto const& h : r._highs) { - instantiateExpression(h); - } - } - } - TRI_IF_FAILURE("IndexRangeBlock::initializeExpressions") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - } - catch (...) { - for (auto& e : _allVariableBoundExpressions) { - delete e; - } - _allVariableBoundExpressions.clear(); - throw; - } - } - } - - _hasV8Expression = hasV8Expression(); - - return res; - LEAVE_BLOCK; -} - -// init the ranges for reading, this should be called once per new incoming -// block! -// -// This is either called every time we get a new incoming block. -// If all the bounds are constant, then in the case of hash, primary or edges -// indexes it does nothing. In the case of a skiplist index, it creates a -// skiplistIterator which is used by readIndex. If at least one bound is -// variable, then this this also evaluates the IndexOrCondition required to -// determine the values of the bounds. -// -// It is guaranteed that -// _buffer is not empty, in particular _buffer.front() is defined -// _pos points to a position in _buffer.front() -// Therefore, we can use the register values in _buffer.front() in row -// _pos to evaluate the variable bounds. - -bool IndexRangeBlock::initRanges () { - ENTER_BLOCK - _flag = true; - - // Find out about the actual values for the bounds in the variable bound case: - - if (_anyBoundVariable) { - if (_hasV8Expression) { - bool const isRunningInCluster = triagens::arango::ServerState::instance()->isRunningInCluster(); - - // must have a V8 context here to protect Expression::execute() - auto engine = _engine; - triagens::basics::ScopeGuard guard{ - [&engine]() -> void { - engine->getQuery()->enterContext(); - }, - [&]() -> void { - if (isRunningInCluster) { - // must invalidate the expression now as we might be called from - // different threads - if (triagens::arango::ServerState::instance()->isRunningInCluster()) { - for (auto const& e : _allVariableBoundExpressions) { - e->invalidate(); - } - } - - engine->getQuery()->exitContext(); - } - } - }; - - ISOLATE; - v8::HandleScope scope(isolate); // do not delete this! - - buildExpressions(); - } - else { - // no V8 context required! - - Functions::InitializeThreadContext(); - try { - buildExpressions(); - Functions::DestroyThreadContext(); - } - catch (...) { - Functions::DestroyThreadContext(); - throw; - } - } - } - - auto en = static_cast(getPlanNode()); - TRI_ASSERT(en->_index != nullptr); - - if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_PRIMARY_INDEX) { - return true; //no initialization here! - } - - if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_EDGE_INDEX) { - if (_condition == nullptr || _condition->empty()) { - return false; - } - - _posInRanges = 0; - getEdgeIndexIterator(_condition->at(_posInRanges)); - return (_edgeIndexIterator != nullptr); - } - - if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_HASH_INDEX) { - if (_condition == nullptr || _condition->empty()) { - return false; - } - - _posInRanges = 0; - getHashIndexIterator(_condition->at(_posInRanges)); - return (_hashIndexSearchValue._values != nullptr); - } - - if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_SKIPLIST_INDEX) { - if (_condition == nullptr || _condition->empty()) { - return false; - } - - sortConditions(); - _posInRanges = 0; - - getSkiplistIterator(_condition->at(_sortCoords[_posInRanges])); - return (_skiplistIterator != nullptr); - } - - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unexpected index type"); - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -// @brief: sorts the index range conditions and resets _posInRanges to 0 -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::sortConditions () { - size_t const n = _condition->size(); - - if (! _sortCoords.empty()) { - _sortCoords.clear(); - _sortCoords.reserve(n); - } - - if (n == 1) { - // nothing to do - _sortCoords.emplace_back(0); - TRI_IF_FAILURE("IndexRangeBlock::sortConditions") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - return; - } - - // first sort by the prefix of the index - std::vector> prefix; - prefix.reserve(n); - - auto en = static_cast(getPlanNode()); - size_t const numFields = en->_index->fields.size(); - - for (size_t s = 0; s < n; s++) { - _sortCoords.emplace_back(s); - - TRI_IF_FAILURE("IndexRangeBlock::sortConditions") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - { - std::vector next; - next.reserve(numFields); - prefix.emplace_back(std::move(next)); - } - - // prefix[s][t] = position in _condition[s] corresponding to the th index - // field - for (size_t t = 0; t < numFields; t++) { - for (size_t u = 0; u < _condition->at(s).size(); u++) { - auto const& ri = _condition->at(s)[u]; - std::string fieldString; - TRI_AttributeNamesToString(en->_index->fields[t], fieldString, true); - - if (fieldString.compare(ri._attr) == 0) { - - TRI_IF_FAILURE("IndexRangeBlock::sortConditionsInner") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - prefix.at(s).insert(prefix.at(s).begin() + t, u); - break; - } - } - } - } - - SortFunc sortFunc(prefix, _condition, en->_reverse); - - // then sort by the values of the bounds - std::sort(_sortCoords.begin(), _sortCoords.end(), sortFunc); - - _posInRanges = 0; -} - -//////////////////////////////////////////////////////////////////////////////// -// @brief: is _condition[i] < _condition[j]? these are IndexAndConditions. -//////////////////////////////////////////////////////////////////////////////// - -bool IndexRangeBlock::SortFunc::operator() (size_t const& i, size_t const& j) const { - size_t l, r; - - if (! _reverse) { - l = i; - r = j; - } - else { - l = j; - r = i; - } - - size_t shortest = (std::min)(_prefix[i].size(), _prefix[j].size()); - - for (size_t k = 0; k < shortest; k++) { - RangeInfo const& lhs = _condition->at(l).at(_prefix[l][k]); - RangeInfo const& rhs = _condition->at(r).at(_prefix[r][k]); - int cmp; - - if (lhs.is1ValueRangeInfo() && rhs.is1ValueRangeInfo()) { - cmp = TRI_CompareValuesJson(lhs._lowConst.bound().json(), - rhs._lowConst.bound().json()); - if (cmp != 0) { - return (cmp < 0); - } - } - else { - // assuming lhs and rhs are disjoint!! - TRI_ASSERT_EXPENSIVE(areDisjointRangeInfos(lhs, rhs)); - if (lhs._highConst.isDefined() && rhs._lowConst.isDefined()) { - cmp = (TRI_CompareValuesJson(lhs._highConst.bound().json(), - rhs._lowConst.bound().json())); - return (cmp == 0 || cmp < 0); - } - // lhs._lowConst.isDefined() && rhs._highConst.isDefined() - return false; - } - } - TRI_ASSERT(false); - // shouldn't get here since the IndexAndConditions in _condition should be - // disjoint! - return false; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief andCombineRangeInfoVecs: combine the arguments into a single vector, -/// by intersecting every pair of range infos and inserting them in the returned -/// value if the intersection is valid. -//////////////////////////////////////////////////////////////////////////////// - -std::vector IndexRangeBlock::andCombineRangeInfoVecs (std::vector const& riv1, - std::vector const& riv2) const { - std::vector out; - - std::unordered_set cache( - 16, - triagens::basics::JsonHash(), - triagens::basics::JsonEqual() - ); - - triagens::basics::ScopeGuard guard{ - []() -> void { }, - [&cache]() -> void { - // free the JSON values in the cache - for (auto& it : cache) { - TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, it); - } - } - }; - - for (RangeInfo const& ri1: riv1) { - for (RangeInfo const& ri2: riv2) { - RangeInfo x(ri1.clone()); - x.fuse(ri2); - - if (x.isValid()) { - TRI_IF_FAILURE("IndexRangeBlock::andCombineRangeInfoVecs") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - if (x.is1ValueRangeInfo()) { - // de-duplicate - auto lowBoundValue = x._lowConst.bound().json(); - - if (cache.find(lowBoundValue) != cache.end()) { - // already seen the same value - continue; - } - - std::unique_ptr copy(TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, lowBoundValue)); - - if (copy == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); - } - - // every JSON in the cache is a copy - cache.emplace(copy.get()); - copy.release(); - } - - out.emplace_back(std::move(x)); - } - } - } - - return out; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief cartesian: form the cartesian product of the inner vectors. This is -/// required in case a dynamic bound evaluates to a list, then we have an -/// "and" condition containing an "or" condition, which we must then distribute. -//////////////////////////////////////////////////////////////////////////////// - -IndexOrCondition* IndexRangeBlock::cartesian (std::vector> const& collector) const { - size_t const n = collector.size(); - - std::vector indexes; - indexes.reserve(n); - - for (size_t i = 0; i < n; i++) { - indexes.emplace_back(0); - } - - std::unique_ptr out(new IndexOrCondition()); - - while (true) { - IndexAndCondition next; - for (size_t i = 0; i < n; i++) { - next.emplace_back(collector[i][indexes[i]].clone()); - } - - TRI_IF_FAILURE("IndexRangeBlock::cartesian") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - out->emplace_back(next); - size_t j = n - 1; - - while (true) { - indexes[j]++; - if (indexes[j] < collector[j].size()) { - break; - } - - indexes[j] = 0; - if (j == 0) { - return out.release(); - } - j--; - } - } -} - -void IndexRangeBlock::freeCondition () { - if (_condition != nullptr && _freeCondition) { - delete _condition; - _condition = nullptr; - _freeCondition = false; - } -} - -// this is called every time everything in _documents has been passed on - -bool IndexRangeBlock::readIndex (size_t atMost) { - ENTER_BLOCK; - // this is called every time we want more in _documents. - // For the primary key index, this only reads the index once, and never - // again (although there might be multiple calls to this function). - // For the edge, hash or skiplists indexes, initRanges creates an iterator - // and read*Index just reads from the iterator until it is done. - // Then initRanges is read again and so on. This is to avoid reading the - // entire index when we only want a small number of documents. - - if (_documents.empty()) { - TRI_IF_FAILURE("IndexRangeBlock::readIndex") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - _documents.reserve(atMost); - } - else { - _documents.clear(); - } - - auto en = static_cast(getPlanNode()); - - if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_PRIMARY_INDEX) { - if (_flag && _condition != nullptr) { - readPrimaryIndex(*_condition); - } - } - else if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_EDGE_INDEX) { - readEdgeIndex(atMost); - } - else if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_HASH_INDEX) { - readHashIndex(atMost); - } - else if (en->_index->type == triagens::arango::Index::TRI_IDX_TYPE_SKIPLIST_INDEX) { - readSkiplistIndex(atMost); - } - else { - TRI_ASSERT(false); - } - _flag = false; - return (! _documents.empty()); - LEAVE_BLOCK; -} - -int IndexRangeBlock::initializeCursor (AqlItemBlock* items, size_t pos) { - ENTER_BLOCK; - int res = ExecutionBlock::initializeCursor(items, pos); - - if (res != TRI_ERROR_NO_ERROR) { - return res; - } - _pos = 0; - _posInDocs = 0; - - return TRI_ERROR_NO_ERROR; - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getSome -//////////////////////////////////////////////////////////////////////////////// - -AqlItemBlock* IndexRangeBlock::getSome (size_t atLeast, - size_t atMost) { - ENTER_BLOCK; - if (_done) { - return nullptr; - } - - std::unique_ptr res(nullptr); - - do { - // repeatedly try to get more stuff from upstream - // note that the value of the variable we have to loop over - // can contain zero entries, in which case we have to - // try again! - - if (_buffer.empty()) { - size_t toFetch = (std::min)(DefaultBatchSize, atMost); - if (! ExecutionBlock::getBlock(toFetch, toFetch) - || (! initRanges())) { - _done = true; - return nullptr; - } - _pos = 0; // this is in the first block - - // This is a new item, so let's read the index (it is already - // initialized). - readIndex(atMost); - _posInDocs = 0; // position in _documents . . . - } - else if (_posInDocs >= _documents.size()) { - // we have exhausted our local documents buffer, - - _posInDocs = 0; - AqlItemBlock* cur = _buffer.front(); - - if (! readIndex(atMost)) { //no more output from this version of the index - if (++_pos >= cur->size()) { - _buffer.pop_front(); // does not throw - delete cur; - _pos = 0; - } - if (_buffer.empty()) { - if (! ExecutionBlock::getBlock(DefaultBatchSize, DefaultBatchSize) ) { - _done = true; - return nullptr; - } - _pos = 0; // this is in the first block - } - - if (! initRanges()) { - _done = true; - return nullptr; - } - readIndex(atMost); - } - } - - // If we get here, we do have _buffer.front() and _pos points into it - AqlItemBlock* cur = _buffer.front(); - size_t const curRegs = cur->getNrRegs(); - - size_t available = _documents.size() - _posInDocs; - size_t toSend = (std::min)(atMost, available); - - if (toSend > 0) { - - res.reset(new AqlItemBlock(toSend, - getPlanNode()->getRegisterPlan()->nrRegs[getPlanNode()->getDepth()])); - - // automatically freed should we throw - TRI_ASSERT(curRegs <= res->getNrRegs()); - - // only copy 1st row of registers inherited from previous frame(s) - inheritRegisters(cur, res.get(), _pos); - - // set our collection for our output register - res->setDocumentCollection(static_cast(curRegs), - _trx->documentCollection(_collection->cid())); - - for (size_t j = 0; j < toSend; j++) { - if (j > 0) { - // re-use already copied aqlvalues - for (RegisterId i = 0; i < curRegs; i++) { - res->setValue(j, i, res->getValueReference(0, i)); - // Note: if this throws, then all values will be deleted - // properly since the first one is. - } - } - - // The result is in the first variable of this depth, - // we do not need to do a lookup in getPlanNode()->_registerPlan->varInfo, - // but can just take cur->getNrRegs() as registerId: - res->setValue(j, static_cast(curRegs), - AqlValue(reinterpret_cast(_documents[_posInDocs++].getDataPtr()))); - // No harm done, if the setValue throws! - } - } - - } - while (res.get() == nullptr); - - // Clear out registers no longer needed later: - clearRegisters(res.get()); - return res.release(); - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief skipSome -//////////////////////////////////////////////////////////////////////////////// - -size_t IndexRangeBlock::skipSome (size_t atLeast, - size_t atMost) { - - if (_done) { - return 0; - } - - size_t skipped = 0; - - while (skipped < atLeast) { - if (_buffer.empty()) { - size_t toFetch = (std::min)(DefaultBatchSize, atMost); - if (! ExecutionBlock::getBlock(toFetch, toFetch) - || (! initRanges())) { - _done = true; - return skipped; - } - _pos = 0; // this is in the first block - - // This is a new item, so let's read the index if bounds are variable: - readIndex(atMost); - _posInDocs = 0; // position in _documents . . . - } - - // If we get here, we do have _buffer.front() and _pos points into it - AqlItemBlock* cur = _buffer.front(); - - size_t available = _documents.size() - _posInDocs; - size_t toSkip = (std::min)(atMost - skipped, available); - - _posInDocs += toSkip; - skipped += toSkip; - - // Advance read position: - if (_posInDocs >= _documents.size()) { - // we have exhausted our local documents buffer, - - _posInDocs = 0; - if (! readIndex(atMost)) { - if (++_pos >= cur->size()) { - _buffer.pop_front(); // does not throw - delete cur; - _pos = 0; - } - - // let's read the index if bounds are variable: - if (! _buffer.empty()) { - if (! initRanges()) { - _done = true; - return skipped; - } - readIndex(atMost); - } - } - - // If _buffer is empty, then we will fetch a new block in the next round - // and then read the index. - } - } - - return skipped; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief read documents using the primary index -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::readPrimaryIndex (IndexOrCondition const& ranges) { - ENTER_BLOCK; - auto primaryIndex = _collection->documentCollection()->primaryIndex(); - - for (size_t i = 0; i < ranges.size(); i++) { - std::string key; - - for (auto const& x : ranges[i]) { - if (x._attr == std::string(TRI_VOC_ATTRIBUTE_ID)) { - // lookup by _id - - // we can use lower bound because only equality is supported - TRI_ASSERT(x.is1ValueRangeInfo()); - auto const json = x._lowConst.bound().json(); - - if (TRI_IsStringJson(json)) { - // _id must be a string - TRI_voc_cid_t documentCid; - std::string documentKey; - - // parse _id value - int errorCode = resolve(json->_value._string.data, documentCid, documentKey); - - if (errorCode == TRI_ERROR_NO_ERROR) { - bool const isCluster = triagens::arango::ServerState::instance()->isRunningInCluster(); - - if (! isCluster && documentCid == _collection->documentCollection()->_info._cid) { - // only continue lookup if the id value is syntactically correct and - // refers to "our" collection, using local collection id - key = documentKey; - } - else if (isCluster && documentCid == _collection->documentCollection()->_info._planId) { - // only continue lookup if the id value is syntactically correct and - // refers to "our" collection, using cluster collection id - key = documentKey; - } - } - } - /*if (! x._lows.empty() || ! x._highs.empty() || x._lowConst.isDefined() || x._highConst.isDefined()) { - break; - }*/ - } - else if (x._attr == std::string(TRI_VOC_ATTRIBUTE_KEY)) { - // lookup by _key - - // we can use lower bound because only equality is supported - TRI_ASSERT(x.is1ValueRangeInfo()); - auto const json = x._lowConst.bound().json(); - if (TRI_IsStringJson(json)) { - key = std::string(json->_value._string.data, json->_value._string.length - 1); - } - - /*if (! x._lows.empty() || ! x._highs.empty() || x._lowConst.isDefined() || x._highConst.isDefined()) { - break; - }*/ - } - } - - if (! key.empty()) { - ++_engine->_stats.scannedIndex; - - auto found = static_cast(primaryIndex->lookupKey(key.c_str())); - - if (found != nullptr) { - _documents.emplace_back(*found); - } - } - } - - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief build search values for edge index lookup -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::getEdgeIndexIterator (IndexAndCondition const& ranges) { - ENTER_BLOCK; - - _edgeNextElement = nullptr; - - if (_edgeIndexIterator != nullptr) { - delete _edgeIndexIterator; - _edgeIndexIterator = nullptr; - } - - auto buildIterator = [this] (TRI_edge_direction_e direction, TRI_json_t const* key) -> void { - TRI_ASSERT(_edgeIndexIterator == nullptr); - - TRI_voc_cid_t documentCid; - std::string documentKey; - - int errorCode = resolve(key->_value._string.data, documentCid, documentKey); - - if (errorCode == TRI_ERROR_NO_ERROR) { - _edgeIndexIterator = new TRI_edge_index_iterator_t(direction, documentCid, (TRI_voc_key_t) documentKey.c_str()); - } - }; - - for (auto const& x : ranges) { - if (x._attr == std::string(TRI_VOC_ATTRIBUTE_FROM)) { - // we can use lower bound because only equality is supported - TRI_ASSERT(x.is1ValueRangeInfo()); - auto const json = x._lowConst.bound().json(); - if (TRI_IsStringJson(json)) { - // no error will be thrown if _from is not a string - buildIterator(TRI_EDGE_OUT, json); - } - break; - } - else if (x._attr == std::string(TRI_VOC_ATTRIBUTE_TO)) { - // we can use lower bound because only equality is supported - TRI_ASSERT(x.is1ValueRangeInfo()); - auto const json = x._lowConst.bound().json(); - if (TRI_IsStringJson(json)) { - // no error will be thrown if _to is not a string - buildIterator(TRI_EDGE_IN, json); - } - break; - } - } - - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief actually read from the edge index -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::readEdgeIndex (size_t atMost) { - ENTER_BLOCK; - - if (_edgeIndexIterator == nullptr) { - return; - } - - auto en = static_cast(getPlanNode()); - auto idx = en->_index->getInternals(); - TRI_ASSERT(idx != nullptr); - - try { - size_t nrSent = 0; - while (nrSent < atMost && _edgeIndexIterator != nullptr) { - size_t const n = _documents.size(); - - TRI_IF_FAILURE("IndexRangeBlock::readEdgeIndex") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - static_cast(idx)->lookup(_edgeIndexIterator, _documents, _edgeNextElement, atMost); - - size_t const numRead = _documents.size() - n; - - _engine->_stats.scannedIndex += static_cast(numRead); - nrSent += numRead; - - if (_edgeNextElement == nullptr) { - delete _edgeIndexIterator; - _edgeIndexIterator = nullptr; - - if (++_posInRanges < _condition->size()) { - getEdgeIndexIterator(_condition->at(_posInRanges)); - } - } - } - } - catch (...) { - if (_edgeIndexIterator != nullptr) { - delete _edgeIndexIterator; - _edgeIndexIterator = nullptr; - } - throw; - } - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief destroy the search values for the hash index lookup -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::destroyHashIndexSearchValues () { - _hashIndexSearchValue.destroy(); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief set up search values for the hash index lookup -//////////////////////////////////////////////////////////////////////////////// - -bool IndexRangeBlock::setupHashIndexSearchValue (IndexAndCondition const& range) { - auto en = static_cast(getPlanNode()); - auto idx = en->_index->getInternals(); - TRI_ASSERT(idx != nullptr); - - auto hashIndex = static_cast(idx); - auto const& paths = hashIndex->paths(); - - auto shaper = _collection->documentCollection()->getShaper(); - - size_t const n = paths.size(); - - _hashIndexSearchValue.reserve(n); - - for (size_t i = 0; i < n; ++i) { - TRI_shape_pid_t pid = paths[i][0].first; - TRI_ASSERT(pid != 0); - - char const* name = shaper->attributeNameShapePid(pid); - std::string const lookFor(name); - - for (auto const& x : range) { - if (x._attr == lookFor) { //found attribute - if (x._lowConst.bound().json() == nullptr) { - // attribute is empty. this may be case if a function expression is used as a - // comparison value, and the function returns an empty list, e.g. x.a IN PASSTHRU([]) - return false; - } - - auto shaped = TRI_ShapedJsonJson(shaper, x._lowConst.bound().json(), false); - // here x->_low->_bound == x->_high->_bound - if (shaped == nullptr) { - return false; - } - - _hashIndexSearchValue._values[i] = *shaped; - // free only the pointer, but not the internals - TRI_Free(shaper->memoryZone(), shaped); - break; - } - } - } - - return true; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief build search values for hash index lookup -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::getHashIndexIterator (IndexAndCondition const& ranges) { - ENTER_BLOCK; - - _hashNextElement = nullptr; - - destroyHashIndexSearchValues(); - if (! setupHashIndexSearchValue(ranges)) { - destroyHashIndexSearchValues(); - } - - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief actually read from the hash index -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::readHashIndex (size_t atMost) { - ENTER_BLOCK; - - if (_hashIndexSearchValue._values == nullptr) { - return; - } - - auto en = static_cast(getPlanNode()); - auto idx = en->_index->getInternals(); - TRI_ASSERT(idx != nullptr); - - size_t nrSent = 0; - while (nrSent < atMost) { - size_t const n = _documents.size(); - - TRI_IF_FAILURE("IndexRangeBlock::readHashIndex") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - static_cast(idx)->lookup(&_hashIndexSearchValue, _documents, _hashNextElement, atMost); - size_t const numRead = _documents.size() - n; - - _engine->_stats.scannedIndex += static_cast(numRead); - nrSent += numRead; - - if (_hashNextElement == nullptr) { - destroyHashIndexSearchValues(); - - if (++_posInRanges < _condition->size()) { - getHashIndexIterator(_condition->at(_posInRanges)); - } - if (_hashIndexSearchValue._values == nullptr) { - _hashNextElement = nullptr; - break; - } - } - } - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief read documents using a skiplist index -//////////////////////////////////////////////////////////////////////////////// - -// it is only possible to query a skip list using more than one attribute if we -// only have equalities followed by a single arbitrary comparison (i.e x.a == 1 -// && x.b == 2 && x.c > 3 && x.c <= 4). Then we do: -// -// TRI_CreateIndexOperator(TRI_AND_INDEX_OPERATOR, left, right, nullptr, shaper, -// 2); -// -// where -// -// left = TRI_CreateIndexOperator(TRI_GT_INDEX_OPERATOR, nullptr, nullptr, [1,2,3], -// shaper, 3) -// -// right = TRI_CreateIndexOperator(TRI_LE_INDEX_OPERATOR, nullptr, nullptr, [1,2,4], -// shaper, 3) -// -// If the final comparison is an equality (x.a == 1 && x.b == 2 && x.c ==3), then -// we just do: -// -// TRI_CreateIndexOperator(TRI_EQ_INDEX_OPERATOR, nullptr, nullptr, [1,2,3], -// shaper, 3) -// -// It is necessary that values of the attributes are listed in the correct -// order (i.e. must be the first attribute indexed, and must be the -// second). If one of the attributes is not indexed, then it is ignored, -// provided we are querying all the previously indexed attributes (i.e. we -// cannot do (x.c == 1 && x.a == 2) if the index covers , , in this -// order but we can do (x.a == 2)). -// -// If the comparison is not equality, then the values of the parameters -// (i.e. the 1 in x.c >= 1) cannot be lists or arrays. -// - -void IndexRangeBlock::getSkiplistIterator (IndexAndCondition const& ranges) { - ENTER_BLOCK; - TRI_ASSERT(_skiplistIterator == nullptr); - - auto en = static_cast(getPlanNode()); - auto idx = en->_index->getInternals(); - TRI_ASSERT(idx != nullptr); - - auto shaper = _collection->documentCollection()->getShaper(); - TRI_ASSERT(shaper != nullptr); - - TRI_index_operator_t* skiplistOperator = nullptr; - - Json parameters(Json::Array); - size_t i = 0; - for (; i < ranges.size(); i++) { - auto const& range = ranges[i]; - // TRI_ASSERT(range.isConstant()); - if (range.is1ValueRangeInfo()) { // it's an equality . . . - parameters(range._lowConst.bound().copy()); - } - else { // it's not an equality and so the final comparison - if (parameters.size() != 0) { - skiplistOperator = TRI_CreateIndexOperator(TRI_EQ_INDEX_OPERATOR, nullptr, - nullptr, parameters.copy().steal(), shaper, i); - } - if (range._lowConst.isDefined()) { - auto op = range._lowConst.toIndexOperator(false, parameters.copy(), shaper); - - if (skiplistOperator != nullptr) { - skiplistOperator = TRI_CreateIndexOperator(TRI_AND_INDEX_OPERATOR, - skiplistOperator, op, nullptr, shaper, 2); - } - else { - skiplistOperator = op; - } - } - if (range._highConst.isDefined()) { - auto op = range._highConst.toIndexOperator(true, parameters.copy(), shaper); - - if (skiplistOperator != nullptr) { - skiplistOperator = TRI_CreateIndexOperator(TRI_AND_INDEX_OPERATOR, - skiplistOperator, op, nullptr, shaper, 2); - } - else { - skiplistOperator = op; - } - } - } - } - - if (skiplistOperator == nullptr) { // only have equalities . . . - if (parameters.size() == 0) { - // this creates the infinite range (i.e. >= null) - Json hass(Json::Array); - hass.add(Json(Json::Null)); - skiplistOperator = TRI_CreateIndexOperator(TRI_GE_INDEX_OPERATOR, nullptr, - nullptr, hass.steal(), shaper, 1); - } - else { - skiplistOperator = TRI_CreateIndexOperator(TRI_EQ_INDEX_OPERATOR, nullptr, - nullptr, parameters.steal(), shaper, i); - } - } - - delete _skiplistIterator; - - _skiplistIterator = static_cast(idx)->lookup(skiplistOperator, en->_reverse); - - if (skiplistOperator != nullptr) { - delete skiplistOperator; - } - - if (_skiplistIterator == nullptr) { - int res = TRI_errno(); - - if (res == TRI_RESULT_ELEMENT_NOT_FOUND) { - return; - } - - THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_NO_INDEX); - } - LEAVE_BLOCK; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief actually read from the skiplist index -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeBlock::readSkiplistIndex (size_t atMost) { - ENTER_BLOCK; - if (_skiplistIterator == nullptr) { - return; - } - - try { - size_t nrSent = 0; - while (nrSent < atMost && _skiplistIterator != nullptr) { - TRI_index_element_t* indexElement = _skiplistIterator->next(); - - if (indexElement == nullptr) { - delete _skiplistIterator; - _skiplistIterator = nullptr; - if (++_posInRanges < _condition->size()) { - getSkiplistIterator(_condition->at(_sortCoords[_posInRanges])); - } - } - else { - TRI_IF_FAILURE("IndexRangeBlock::readSkiplistIndex") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - _documents.emplace_back(*(indexElement->document())); - ++nrSent; - ++_engine->_stats.scannedIndex; - } - } - } - catch (...) { - if (_skiplistIterator != nullptr) { - delete _skiplistIterator; - _skiplistIterator = nullptr; - } - throw; - } - LEAVE_BLOCK; -} - -// Local Variables: -// mode: outline-minor -// outline-regexp: "^\\(/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|// --SECTION--\\|/// @\\}\\)" -// End: diff --git a/arangod/Aql/IndexRangeBlock.h b/arangod/Aql/IndexRangeBlock.h deleted file mode 100644 index 01a5f14561..0000000000 --- a/arangod/Aql/IndexRangeBlock.h +++ /dev/null @@ -1,357 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// @brief AQL IndexRangeBlock -/// -/// @file -/// -/// DISCLAIMER -/// -/// Copyright 2010-2014 triagens GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is triAGENS GmbH, Cologne, Germany -/// -/// @author Max Neunhoeffer -/// @author Copyright 2014, triagens GmbH, Cologne, Germany -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGODB_AQL_INDEX_RANGE_BLOCK_H -#define ARANGODB_AQL_INDEX_RANGE_BLOCK_H 1 - -#include "Aql/Collection.h" -#include "Aql/ExecutionBlock.h" -#include "Aql/IndexRangeNode.h" -#include "Indexes/HashIndex.h" -#include "Indexes/SkiplistIndex.h" -#include "Utils/AqlTransaction.h" -#include "VocBase/shaped-json.h" - -struct TRI_doc_mptr_copy_t; -struct TRI_edge_index_iterator_t; -struct TRI_hash_index_element_multi_s; - -namespace triagens { - namespace aql { - - class AqlItemBlock; - - class ExecutionEngine; - -// ----------------------------------------------------------------------------- -// --SECTION-- IndexRangeBlock -// ----------------------------------------------------------------------------- - - class IndexRangeBlock : public ExecutionBlock { - - public: - - IndexRangeBlock (ExecutionEngine* engine, - IndexRangeNode const* ep); - - ~IndexRangeBlock (); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief initialize, here we fetch all docs from the database -//////////////////////////////////////////////////////////////////////////////// - - int initialize () override; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief initializeCursor, here we release our docs from this collection -//////////////////////////////////////////////////////////////////////////////// - - int initializeCursor (AqlItemBlock* items, size_t pos) override; - - AqlItemBlock* getSome (size_t atLeast, size_t atMost) override final; - -//////////////////////////////////////////////////////////////////////////////// -// skip between atLeast and atMost, returns the number actually skipped . . . -// will only return less than atLeast if there aren't atLeast many -// things to skip overall. -//////////////////////////////////////////////////////////////////////////////// - - size_t skipSome (size_t atLeast, size_t atMost) override final; - -// ----------------------------------------------------------------------------- -// --SECTION-- private methods -// ----------------------------------------------------------------------------- - - private: - -//////////////////////////////////////////////////////////////////////////////// -/// @brief whether or not the high bound values should be taken into account -/// they can be ignored for indexes that only support equality conditions, -/// i.e. primary index, edge index and hash index -//////////////////////////////////////////////////////////////////////////////// - - bool useHighBounds () const; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief whether or not one of the bounds expressions requires V8 -//////////////////////////////////////////////////////////////////////////////// - - bool hasV8Expression () const; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief build the bounds expressions -//////////////////////////////////////////////////////////////////////////////// - - void buildExpressions (); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief free _condition if it belongs to us -//////////////////////////////////////////////////////////////////////////////// - - void freeCondition (); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief continue fetching of documents -//////////////////////////////////////////////////////////////////////////////// - - bool readIndex (size_t atMost); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief set up the index for reading. This should be called once per incoming -/// block. -//////////////////////////////////////////////////////////////////////////////// - - bool initRanges (); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief read using the primary index -//////////////////////////////////////////////////////////////////////////////// - - void readPrimaryIndex (IndexOrCondition const&); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief destroy the hash index search value -//////////////////////////////////////////////////////////////////////////////// - - void destroyHashIndexSearchValues (); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief set up a hash index search value -//////////////////////////////////////////////////////////////////////////////// - - bool setupHashIndexSearchValue (IndexAndCondition const&); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief produce a reentrant hash index iterator -//////////////////////////////////////////////////////////////////////////////// - - void getHashIndexIterator (IndexAndCondition const&); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief read using a hash index -//////////////////////////////////////////////////////////////////////////////// - - void readHashIndex (size_t); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief this tries to create an edge iterator to read from the index. -//////////////////////////////////////////////////////////////////////////////// - - void getEdgeIndexIterator (IndexAndCondition const&); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief read using an edge index -//////////////////////////////////////////////////////////////////////////////// - - void readEdgeIndex (size_t atMost); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief this tries to create a skiplistIterator to read from the index. -//////////////////////////////////////////////////////////////////////////////// - - void getSkiplistIterator (IndexAndCondition const&); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief read using a skiplist index -//////////////////////////////////////////////////////////////////////////////// - - void readSkiplistIndex (size_t atMost); - -//////////////////////////////////////////////////////////////////////////////// -// @brief: sorts the index range conditions and resets _posInRanges to 0 -//////////////////////////////////////////////////////////////////////////////// - - void sortConditions (); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief andCombineRangeInfoVecs: combine the arguments into a single vector, -/// by intersecting every pair of range infos and inserting them in the returned -/// value if the intersection is valid. -//////////////////////////////////////////////////////////////////////////////// - - std::vector andCombineRangeInfoVecs (std::vector const&, - std::vector const&) const; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief cartesian: form the cartesian product of the inner vectors. This is -/// required in case a dynamic bound evaluates to a list, then we have an -/// "and" condition containing an "or" condition, which we must then distribute. -//////////////////////////////////////////////////////////////////////////////// - - IndexOrCondition* cartesian (std::vector> const&) const; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief: subclass for comparing IndexAndConditions in _condition. Similar to -/// OurLessThan in the SortBlock -//////////////////////////////////////////////////////////////////////////////// - - class SortFunc { - public: - SortFunc (std::vector> const& prefix, - IndexOrCondition* condition, - bool reverse) - : _prefix(prefix), - _condition(condition), - _reverse(reverse) { - } - - bool operator() (size_t const&, - size_t const&) const; - - private: - std::vector> const& _prefix; - IndexOrCondition* _condition; - bool const _reverse; - }; - -// ----------------------------------------------------------------------------- -// --SECTION-- private variables -// ----------------------------------------------------------------------------- - - private: - -//////////////////////////////////////////////////////////////////////////////// -/// @brief collection -//////////////////////////////////////////////////////////////////////////////// - - Collection const* _collection; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief document buffer -//////////////////////////////////////////////////////////////////////////////// - - std::vector _documents; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief current position in _allDocs -//////////////////////////////////////////////////////////////////////////////// - - size_t _posInDocs; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _allBoundsConstant, this indicates whether all given bounds -/// are constant -//////////////////////////////////////////////////////////////////////////////// - - std::vector _allBoundsConstant; - bool _anyBoundVariable; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _allBoundsConstant, this indicates whether all given bounds -/// are constant -//////////////////////////////////////////////////////////////////////////////// - - std::vector _allVariableBoundExpressions; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _inVars, a vector containing for each expression above -/// a vector of Variable*, used to execute the expression -///////////////////////////////////////////////////////////////////////////////// - - std::vector> _inVars; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _inRegs, a vector containing for each expression above -/// a vector of RegisterId, used to execute the expression -//////////////////////////////////////////////////////////////////////////////// - - std::vector> _inRegs; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _skiplistIterator: holds the skiplist iterator found using -/// getSkiplistIterator (if any) so that it can be read in chunks and not -/// necessarily all at once. -//////////////////////////////////////////////////////////////////////////////// - - triagens::arango::SkiplistIterator* _skiplistIterator; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _edgeIterator: holds the edge iterator found using -/// getEdgeIndexIterator (if any) so that it can be read in chunks and not -/// necessarily all at once. -//////////////////////////////////////////////////////////////////////////////// - - struct TRI_edge_index_iterator_t* _edgeIndexIterator; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief current search value for hash index lookup -//////////////////////////////////////////////////////////////////////////////// - - TRI_hash_index_search_value_t _hashIndexSearchValue; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief reentrant hash index iterator state -//////////////////////////////////////////////////////////////////////////////// - - TRI_index_element_t* _hashNextElement; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief reentrant edge index iterator state -//////////////////////////////////////////////////////////////////////////////// - - TRI_doc_mptr_copy_t* _edgeNextElement; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _condition: holds the IndexAndCondition for the current incoming block, -/// this is just the _ranges[_rangesPos] member of the plan node if _allBoundsConstant -/// otherwise it is reevaluated every time initIndex is called, i.e. once per -/// incoming block. -//////////////////////////////////////////////////////////////////////////////// - - IndexOrCondition* _condition; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _flag: since readIndex for primary, hash, edges indexes reads the -/// whole index, this is if initIndex has been called but readIndex has -/// not been called, otherwise it is to avoid rereading the entire index -/// with successive calls to readIndex. -////////////////////////////////////////////////////////////////////////////////// - - bool _flag; - size_t _posInRanges; - std::vector _sortCoords; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief _freeCondition: whether or not the _condition is owned by the -/// IndexRangeBlock and must be freed -//////////////////////////////////////////////////////////////////////////////// - - bool _freeCondition; - - bool _hasV8Expression; - - }; - - } // namespace triagens::aql -} // namespace triagens - -#endif - -// Local Variables: -// mode: outline-minor -// outline-regexp: "^\\(/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|// --SECTION--\\|/// @\\}\\)" -// End: diff --git a/arangod/Aql/IndexRangeNode.cpp b/arangod/Aql/IndexRangeNode.cpp deleted file mode 100644 index cf68986675..0000000000 --- a/arangod/Aql/IndexRangeNode.cpp +++ /dev/null @@ -1,371 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// @brief IndexRangeNode -/// -/// @file -/// -/// DISCLAIMER -/// -/// Copyright 2010-2014 triagens GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is triAGENS GmbH, Cologne, Germany -/// -/// @author Max Neunhoeffer -/// @author Copyright 2014, triagens GmbH, Cologne, Germany -//////////////////////////////////////////////////////////////////////////////// - -#include "Aql/IndexRangeNode.h" -#include "Aql/Collection.h" -#include "Aql/ExecutionPlan.h" -#include "Aql/WalkerWorker.h" -#include "Aql/Ast.h" -#include "Basics/StringBuffer.h" - -using namespace std; -using namespace triagens::basics; -using namespace triagens::aql; - -// ----------------------------------------------------------------------------- -// --SECTION-- methods of IndexRangeNode -// ----------------------------------------------------------------------------- - -//////////////////////////////////////////////////////////////////////////////// -/// @brief toJson, for IndexRangeNode -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeNode::toJsonHelper (triagens::basics::Json& nodes, - TRI_memory_zone_t* zone, - bool verbose) const { - triagens::basics::Json json(ExecutionNode::toJsonHelperGeneric(nodes, zone, verbose)); - // call base class method - - if (json.isEmpty()) { - return; - } - - // put together the range info . . . - triagens::basics::Json ranges(triagens::basics::Json::Array, _ranges.size()); - - for (auto const& x : _ranges) { - triagens::basics::Json range(triagens::basics::Json::Array, x.size()); - for(auto const& y : x) { - range.add(y.toJson()); - } - ranges.add(range); - } - - // Now put info about vocbase and cid in there - json("database", triagens::basics::Json(_vocbase->_name)) - ("collection", triagens::basics::Json(_collection->getName())) - ("outVariable", _outVariable->toJson()) - ("ranges", ranges); - - json("index", _index->toJson()); - json("reverse", triagens::basics::Json(_reverse)); - - // And add it: - nodes(json); -} - -ExecutionNode* IndexRangeNode::clone (ExecutionPlan* plan, - bool withDependencies, - bool withProperties) const { - std::vector> ranges; - for (size_t i = 0; i < _ranges.size(); i++){ - ranges.emplace_back(std::vector()); - - for (auto const& x : _ranges.at(i)) { - ranges.at(i).emplace_back(x); - } - } - - auto outVariable = _outVariable; - - if (withProperties) { - outVariable = plan->getAst()->variables()->createVariable(outVariable); - } - - auto c = new IndexRangeNode(plan, _id, _vocbase, _collection, - outVariable, _index, ranges, _reverse); - - cloneHelper(c, plan, withDependencies, withProperties); - - return static_cast(c); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief constructor for IndexRangeNode from Json -//////////////////////////////////////////////////////////////////////////////// - -IndexRangeNode::IndexRangeNode (ExecutionPlan* plan, - triagens::basics::Json const& json) - : ExecutionNode(plan, json), - _vocbase(plan->getAst()->query()->vocbase()), - _collection(plan->getAst()->query()->collections()->get(JsonHelper::checkAndGetStringValue(json.json(), "collection"))), - _outVariable(varFromJson(plan->getAst(), json, "outVariable")), - _index(nullptr), - _ranges(), - _reverse(false) { - - triagens::basics::Json rangeArrayJson(TRI_UNKNOWN_MEM_ZONE, JsonHelper::checkAndGetArrayValue(json.json(), "ranges")); - - for (size_t i = 0; i < rangeArrayJson.size(); i++) { //loop over the ranges . . . - _ranges.emplace_back(); - - triagens::basics::Json rangeJson(rangeArrayJson.at(static_cast(i))); - for (size_t j = 0; j < rangeJson.size(); j++) { - _ranges.at(i).emplace_back(rangeJson.at(static_cast(j))); - } - } - - // now the index . . . - // TODO the following could be a constructor method for - // an Index object when these are actually used - auto index = JsonHelper::checkAndGetObjectValue(json.json(), "index"); - auto iid = JsonHelper::checkAndGetStringValue(index, "id"); - - _index = _collection->getIndex(iid); - _reverse = JsonHelper::checkAndGetBooleanValue(json.json(), "reverse"); - - if (_index == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "index not found"); - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief the cost of an index range node is a multiple of the cost of -/// its unique dependency -//////////////////////////////////////////////////////////////////////////////// - -double IndexRangeNode::estimateCost (size_t& nrItems) const { - static double const EqualityReductionFactor = 100.0; - - size_t incoming = 0; - double const dependencyCost = _dependencies.at(0)->getCost(incoming); - size_t docCount = _collection->count(); - - TRI_ASSERT(! _ranges.empty()); - - if (_index->type == triagens::arango::Index::TRI_IDX_TYPE_PRIMARY_INDEX) { - // always an equality lookup - - // selectivity of primary index is always 1 - nrItems = incoming * _ranges.size(); - return dependencyCost + nrItems; - } - - if (_index->type == triagens::arango::Index::TRI_IDX_TYPE_EDGE_INDEX) { - // always an equality lookup - - // check if the index can provide a selectivity estimate - if (! estimateItemsWithIndexSelectivity(incoming, nrItems)) { - // use hard-coded heuristic - nrItems = incoming * _ranges.size() * docCount / static_cast(EqualityReductionFactor); - } - - nrItems = (std::max)(nrItems, static_cast(1)); - - return dependencyCost + nrItems; - } - - if (_index->type == triagens::arango::Index::TRI_IDX_TYPE_HASH_INDEX) { - // always an equality lookup - - // check if the index can provide a selectivity estimate - if (! estimateItemsWithIndexSelectivity(incoming, nrItems)) { - // use hard-coded heuristic - if (_index->unique) { - nrItems = incoming * _ranges.size(); - } - else { - double cost = static_cast(docCount) * incoming * _ranges.size(); - // the more attributes are contained in the index, the more specific the lookup will be - for (size_t i = 0; i < _ranges.at(0).size(); ++i) { - cost /= EqualityReductionFactor; - } - - nrItems = static_cast(cost); - } - } - - nrItems = (std::max)(nrItems, static_cast(1)); - // the more attributes an index matches, the better it is - double matchLengthFactor = _ranges.at(0).size() * 0.01; - - // this is to prefer the hash index over skiplists if everything else is equal - return dependencyCost + ((static_cast(nrItems) - matchLengthFactor) * 0.9999995); - } - - if (_index->type == triagens::arango::Index::TRI_IDX_TYPE_SKIPLIST_INDEX) { - auto const count = _ranges.at(0).size(); - - if (count == 0) { - // no ranges? so this is unlimited -> has to be more expensive - nrItems = incoming * docCount; - return dependencyCost + nrItems; - } - - if (_index->unique) { - bool allEquality = true; - for (auto const& x : _ranges) { - // check if we are using all indexed attributes in the query - if (x.size() != _index->fields.size()) { - allEquality = false; - break; - } - - // check if this is an equality comparison - if (x.empty() || ! x.back().is1ValueRangeInfo()) { - allEquality = false; - break; - } - } - - if (allEquality) { - // unique index, all attributes compared using eq (==) operator - nrItems = incoming * _ranges.size(); - return dependencyCost + nrItems; - } - } - - // build a total cost for the index usage by peeking into all ranges - double totalCost = 0.0; - - for (auto const& x : _ranges) { - double cost = static_cast(docCount) * incoming; - - for (auto const& y : x) { //only doing the 1-d case so far - if (y.is1ValueRangeInfo()) { - // equality lookup - cost /= EqualityReductionFactor; - continue; - } - - bool hasLowerBound = false; - bool hasUpperBound = false; - - if (y._lowConst.isDefined() || y._lows.size() > 0) { - hasLowerBound = true; - } - if (y._highConst.isDefined() || y._highs.size() > 0) { - hasUpperBound = true; - } - - if (hasLowerBound && hasUpperBound) { - // both lower and upper bounds defined - cost /= 10.0; - } - else if (hasLowerBound || hasUpperBound) { - // either only low or high bound defined - cost /= 2.0; - } - - // each bound (const and dynamic) counts! - size_t const numBounds = y._lows.size() + - y._highs.size() + - (y._lowConst.isDefined() ? 1 : 0) + - (y._highConst.isDefined() ? 1 : 0); - - for (size_t j = 0; j < numBounds; ++j) { - // each dynamic bound again reduces the cost - cost *= 0.95; - } - } - - totalCost += cost; - } - - totalCost = static_cast((std::max)(static_cast(totalCost), static_cast(1))); - - nrItems = static_cast(totalCost); - - return dependencyCost + totalCost; - } - - // no index - nrItems = incoming * docCount; - return dependencyCost + nrItems; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getVariablesUsedHere, returning a vector -//////////////////////////////////////////////////////////////////////////////// - -std::vector IndexRangeNode::getVariablesUsedHere () const { - std::unordered_set s; - // actual work is done by that method - getVariablesUsedHere(s); - - // copy result into vector - std::vector v; - v.reserve(s.size()); - - for (auto const& vv : s) { - v.emplace_back(const_cast(vv)); - } - return v; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getVariablesUsedHere, modifying the set in-place -//////////////////////////////////////////////////////////////////////////////// - -void IndexRangeNode::getVariablesUsedHere (std::unordered_set& vars) const { - for (auto const& x : _ranges) { - for (RangeInfo const& y : x) { - for (RangeInfoBound const& z : y._lows) { - AstNode const* a = z.getExpressionAst(_plan->getAst()); - Ast::getReferencedVariables(a, vars); - } - for (RangeInfoBound const& z : y._highs) { - AstNode const* a = z.getExpressionAst(_plan->getAst()); - Ast::getReferencedVariables(a, vars); - } - } - } -} - -// ----------------------------------------------------------------------------- -// --SECTION-- private methods -// ----------------------------------------------------------------------------- - -//////////////////////////////////////////////////////////////////////////////// -/// @brief provide an estimate for the number of items, using the index -/// selectivity info (if present) -//////////////////////////////////////////////////////////////////////////////// - -bool IndexRangeNode::estimateItemsWithIndexSelectivity (size_t incoming, - size_t& nrItems) const { - // check if the index can provide a selectivity estimate - if (! _index->hasSelectivityEstimate()) { - return false; - } - - // use index selectivity estimate - double estimate = _index->selectivityEstimate(); - - if (estimate <= 0.0) { - // avoid DIV0 - return false; - } - - nrItems = static_cast(incoming * _ranges.size() * (1.0 / estimate)); - return true; -} - -// Local Variables: -// mode: outline-minor -// outline-regexp: "^\\(/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|// --SECTION--\\|/// @\\}\\)" -// End: - diff --git a/arangod/Aql/IndexRangeNode.h b/arangod/Aql/IndexRangeNode.h deleted file mode 100644 index a3aa4049d4..0000000000 --- a/arangod/Aql/IndexRangeNode.h +++ /dev/null @@ -1,258 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// @brief IndexRangeNode -/// -/// @file -/// -/// DISCLAIMER -/// -/// Copyright 2010-2014 triagens GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is triAGENS GmbH, Cologne, Germany -/// -/// @author Max Neunhoeffer -/// @author Copyright 2014, triagens GmbH, Cologne, Germany -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGODB_AQL_INDEX_RANGE_NODE_H -#define ARANGODB_AQL_INDEX_RANGE_NODE_H 1 - -#include "Basics/Common.h" -#include "Aql/Ast.h" -#include "Aql/ExecutionNode.h" -#include "Aql/RangeInfo.h" -#include "Aql/types.h" -#include "Aql/Variable.h" -#include "Basics/JsonHelper.h" -#include "VocBase/voc-types.h" -#include "VocBase/vocbase.h" - -namespace triagens { - namespace aql { - struct Collection; - class ExecutionBlock; - class ExecutionPlan; - struct Index; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief class IndexRangeNode -//////////////////////////////////////////////////////////////////////////////// - - class IndexRangeNode : public ExecutionNode { - friend class ExecutionBlock; - friend class IndexRangeBlock; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief constructor with a vocbase and a collection name -//////////////////////////////////////////////////////////////////////////////// - -// _ranges must correspond to a prefix of the fields of the index , i.e. -// _ranges.at(i) is a range of values for idx->_fields._buffer[i]. - - public: - - IndexRangeNode (ExecutionPlan* plan, - size_t id, - TRI_vocbase_t* vocbase, - Collection const* collection, - Variable const* outVariable, - Index const* index, - std::vector> const& ranges, - bool reverse) - : ExecutionNode(plan, id), - _vocbase(vocbase), - _collection(collection), - _outVariable(outVariable), - _index(index), - _ranges(ranges), - _reverse(reverse) { - TRI_ASSERT(false); - - TRI_ASSERT(_vocbase != nullptr); - TRI_ASSERT(_collection != nullptr); - TRI_ASSERT(_outVariable != nullptr); - TRI_ASSERT(_index != nullptr); - } - - IndexRangeNode (ExecutionPlan*, triagens::basics::Json const& base); - - ~IndexRangeNode () { - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief return the type of the node -//////////////////////////////////////////////////////////////////////////////// - - NodeType getType () const override final { - return INDEX_RANGE; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief return the database -//////////////////////////////////////////////////////////////////////////////// - - TRI_vocbase_t* vocbase () const { - return _vocbase; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief return the collection -//////////////////////////////////////////////////////////////////////////////// - - Collection const* collection () const { - return _collection; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief return out variable -//////////////////////////////////////////////////////////////////////////////// - - Variable const* outVariable () const { - return _outVariable; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief return the ranges -//////////////////////////////////////////////////////////////////////////////// - - std::vector> const& ranges () const { - return _ranges; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief export to JSON -//////////////////////////////////////////////////////////////////////////////// - - void toJsonHelper (triagens::basics::Json&, - TRI_memory_zone_t*, - bool) const override final; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief clone ExecutionNode recursively -//////////////////////////////////////////////////////////////////////////////// - - ExecutionNode* clone (ExecutionPlan* plan, - bool withDependencies, - bool withProperties) const override final; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getVariablesSetHere -//////////////////////////////////////////////////////////////////////////////// - - std::vector getVariablesSetHere () const override final { - return std::vector{ _outVariable }; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getVariablesUsedHere, returning a vector -//////////////////////////////////////////////////////////////////////////////// - - std::vector getVariablesUsedHere () const override final; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getVariablesUsedHere, modifying the set in-place -//////////////////////////////////////////////////////////////////////////////// - - void getVariablesUsedHere (std::unordered_set& vars) const override final; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief estimateCost -//////////////////////////////////////////////////////////////////////////////// - - double estimateCost (size_t&) const override final; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief whether or not a reverse index traversal is used -//////////////////////////////////////////////////////////////////////////////// - - void reverse (bool value) { - _reverse = value; - } - -//////////////////////////////////////////////////////////////////////////////// -/// @brief getIndex, hand out the index used -//////////////////////////////////////////////////////////////////////////////// - - Index const* getIndex () { - return _index; - } - -// ----------------------------------------------------------------------------- -// --SECTION-- private methods -// ----------------------------------------------------------------------------- - - private: - -//////////////////////////////////////////////////////////////////////////////// -/// @brief provide an estimate for the number of items, using the index -/// selectivity info (if present) -//////////////////////////////////////////////////////////////////////////////// - - bool estimateItemsWithIndexSelectivity (size_t, - size_t&) const; - -// ----------------------------------------------------------------------------- -// --SECTION-- private variables -// ----------------------------------------------------------------------------- - - private: - -//////////////////////////////////////////////////////////////////////////////// -/// @brief the database -//////////////////////////////////////////////////////////////////////////////// - - TRI_vocbase_t* _vocbase; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief collection -//////////////////////////////////////////////////////////////////////////////// - - Collection const* _collection; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief output variable -//////////////////////////////////////////////////////////////////////////////// - - Variable const* _outVariable; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief the index -//////////////////////////////////////////////////////////////////////////////// - - Index const* _index; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief the range info -//////////////////////////////////////////////////////////////////////////////// - - std::vector> _ranges; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief use a reverse index scan -//////////////////////////////////////////////////////////////////////////////// - - bool _reverse; - }; - - } // namespace triagens::aql -} // namespace triagens - -#endif - -// Local Variables: -// mode: outline-minor -// outline-regexp: "^\\(/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|// --SECTION--\\|/// @\\}\\)" -// End: - - diff --git a/arangod/Aql/Optimizer.h b/arangod/Aql/Optimizer.h index 89635f4aaf..c1f1ef87a0 100644 --- a/arangod/Aql/Optimizer.h +++ b/arangod/Aql/Optimizer.h @@ -183,11 +183,8 @@ namespace triagens { // remove redundant OR conditions removeRedundantOrRule_pass6 = 820, - useIndexesRule_pass6 = 825, + useIndexesRule_pass6 = 830, - // try to find a filter after an enumerate collection and find an index . . . - useIndexRangeRule_pass6 = 830, - // try to remove filters covered by index ranges removeFiltersCoveredByIndexRule_pass6 = 840, diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 3f84ad2f55..b27fad2cd0 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -34,11 +34,12 @@ #include "Aql/ExecutionNode.h" #include "Aql/Function.h" #include "Aql/IndexNode.h" -#include "Aql/IndexRangeNode.h" #include "Aql/ModificationNodes.h" +#include "Aql/RangeInfo.h" #include "Aql/SortNode.h" #include "Aql/Variable.h" #include "Aql/types.h" +#include "Basics/json-utilities.h" #include @@ -846,8 +847,7 @@ int triagens::aql::removeSortRandRule (Optimizer* opt, case EN::FILTER: case EN::SUBQUERY: case EN::ENUMERATE_LIST: - case EN::INDEX: - case EN::INDEX_RANGE: { + case EN::INDEX: { // if we found another SortNode, an AggregateNode, FilterNode, a SubqueryNode, // an EnumerateListNode or an IndexNode // this means we cannot apply our optimization @@ -1046,8 +1046,7 @@ int triagens::aql::moveCalculationsDownRule (Optimizer* opt, // we found something interesting that justifies moving our node down shouldMove = true; } - else if (currentType == EN::INDEX_RANGE || - currentType == EN::INDEX || + else if (currentType == EN::INDEX || currentType == EN::ENUMERATE_COLLECTION || currentType == EN::ENUMERATE_LIST || currentType == EN::AGGREGATE || @@ -1914,7 +1913,6 @@ struct SortToIndexNode final : public WalkerWorker { case EN::ENUMERATE_LIST: case EN::SUBQUERY: case EN::FILTER: - case EN::INDEX_RANGE: return false; // skip. we don't care. case EN::CALCULATION: { @@ -2165,6 +2163,9 @@ struct FilterCondition { int triagens::aql::removeFiltersCoveredByIndexRule (Optimizer* opt, ExecutionPlan* plan, Optimizer::Rule const* rule) { + opt->addPlan(plan, rule, false); + return TRI_ERROR_NO_ERROR; +/* std::unordered_set toUnlink; std::vector&& nodes= plan->findNodesOfType(EN::FILTER, true); @@ -2233,6 +2234,7 @@ int triagens::aql::removeFiltersCoveredByIndexRule (Optimizer* opt, opt->addPlan(plan, rule, ! toUnlink.empty()); return TRI_ERROR_NO_ERROR; + */ } //////////////////////////////////////////////////////////////////////////////// @@ -2407,10 +2409,9 @@ int triagens::aql::scatterInClusterRule (Optimizer* opt, if (triagens::arango::ServerState::instance()->isCoordinator()) { // we are a coordinator. now look in the plan for nodes of type - // EnumerateCollectionNode and IndexRangeNode + // EnumerateCollectionNode and IndexNode std::vector const types = { ExecutionNode::ENUMERATE_COLLECTION, - ExecutionNode::INDEX_RANGE, ExecutionNode::INDEX, ExecutionNode::INSERT, ExecutionNode::UPDATE, @@ -2444,10 +2445,6 @@ int triagens::aql::scatterInClusterRule (Optimizer* opt, vocbase = static_cast(node)->vocbase(); collection = static_cast(node)->collection(); } - else if (nodeType == ExecutionNode::INDEX_RANGE) { - vocbase = static_cast(node)->vocbase(); - collection = static_cast(node)->collection(); - } else if (nodeType == ExecutionNode::INDEX) { vocbase = static_cast(node)->vocbase(); collection = static_cast(node)->collection(); @@ -2777,7 +2774,6 @@ int triagens::aql::distributeFilternCalcToClusterRule (Optimizer* opt, case EN::LIMIT: case EN::SORT: case EN::INDEX: - case EN::INDEX_RANGE: case EN::ENUMERATE_COLLECTION: //do break stopSearching = true; @@ -2873,7 +2869,6 @@ int triagens::aql::distributeSortToClusterRule (Optimizer* opt, case EN::ILLEGAL: case EN::REMOTE: case EN::LIMIT: - case EN::INDEX_RANGE: case EN::INDEX: case EN::ENUMERATE_COLLECTION: // For all these, we do not want to pull a SortNode further down @@ -3148,8 +3143,7 @@ class RemoveToEnumCollFinder final : public WalkerWorker { case EN::ILLEGAL: case EN::LIMIT: case EN::SORT: - case EN::INDEX: - case EN::INDEX_RANGE: { + case EN::INDEX: { // if we meet any of the above, then we abort . . . } } @@ -3691,7 +3685,6 @@ int triagens::aql::patchUpdateStatementsRule (Optimizer* opt, if (type == EN::ENUMERATE_LIST || type == EN::INDEX || - type == EN::INDEX_RANGE || type == EN::SUBQUERY) { // not suitable modified = false; diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 95ab1b3a23..4ef1962e0f 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -91,8 +91,6 @@ add_executable( Aql/Index.cpp Aql/IndexBlock.cpp Aql/IndexNode.cpp - Aql/IndexRangeBlock.cpp - Aql/IndexRangeNode.cpp Aql/ModificationBlocks.cpp Aql/ModificationNodes.cpp Aql/NodeFinder.cpp diff --git a/arangod/Indexes/SimpleAttributeEqualityMatcher.cpp b/arangod/Indexes/SimpleAttributeEqualityMatcher.cpp index 07e8154218..d105938f02 100644 --- a/arangod/Indexes/SimpleAttributeEqualityMatcher.cpp +++ b/arangod/Indexes/SimpleAttributeEqualityMatcher.cpp @@ -34,8 +34,6 @@ #include "Indexes/Index.h" #include "VocBase/vocbase.h" -#include - using namespace triagens::arango; // ----------------------------------------------------------------------------- @@ -49,14 +47,6 @@ using namespace triagens::arango; SimpleAttributeEqualityMatcher::SimpleAttributeEqualityMatcher (std::vector> const& attributes) : _attributes(attributes), _found() { - - std::cout << "SIMPLE ATT CTOR. ATTR SIZE: " << _attributes.size() << "\n"; - for (auto& it : _attributes) { - std::cout << "OUTER ITER. INNER SIZE: " << it.size() << "\n"; - for (auto& it2 : it) { - std::cout << "INNER: " << it2 << "\n"; - } - } } // ----------------------------------------------------------------------------- @@ -433,14 +423,7 @@ bool SimpleAttributeEqualityMatcher::accessFitsIndex (triagens::arango::Index co triagens::aql::AstNode const* other, triagens::aql::AstNode const* op, triagens::aql::Variable const* reference) { - std::cout << "CHECKING CONDITION\n"; - std::cout << "INDEX: " << index << "\n"; - std::cout << "ACCESS: " << access << "\n"; - std::cout << "OTHER: " << other << "\n"; - std::cout << "OP: " << op << "\n"; - if (! index->canUseConditionPart(access, other, op, reference)) { - std::cout << "CHECKING CONDITION - EXIT 1\n"; return false; } @@ -459,26 +442,20 @@ bool SimpleAttributeEqualityMatcher::accessFitsIndex (triagens::arango::Index co if (! what->isAttributeAccessForVariable(attributeData) || attributeData.first != reference) { // this access is not referencing this collection - std::cout << "CHECKING CONDITION - EXIT 3\n"; return false; } std::vector const& fieldNames = attributeData.second; - std::cout << "INDEX _ATTRIBUTES ARE: " << _attributes << "\n"; - std::cout << "FIELDNAMES ARE: " << fieldNames << "\n"; - for (size_t i = 0; i < _attributes.size(); ++i) { if (_attributes[i].size() != fieldNames.size()) { // attribute path length differs - std::cout << "DIFFERENT LENGTHS: " << _attributes[i].size() << ", " << fieldNames.size() << "\n"; continue; } bool match = true; for (size_t j = 0; j < _attributes[i].size(); ++j) { if (_attributes[i][j] != fieldNames[j]) { - std::cout << "ATTRIBUTES DIFFER: " << _attributes[i][j] << " - " << fieldNames[j] << "\n"; match = false; break; } @@ -487,11 +464,9 @@ bool SimpleAttributeEqualityMatcher::accessFitsIndex (triagens::arango::Index co if (match) { // mark ith attribute as being covered _found.emplace(i); - std::cout << "IS A MATCH--------------------------\n"; return true; } } - std::cout << "CHECKING CONDITION - EXIT 2\n"; return false; } diff --git a/arangod/Makefile.files b/arangod/Makefile.files index 13bb84f6e2..9c1c787a54 100644 --- a/arangod/Makefile.files +++ b/arangod/Makefile.files @@ -50,8 +50,6 @@ arangod_libarangod_a_SOURCES = \ arangod/Aql/Index.cpp \ arangod/Aql/IndexBlock.cpp \ arangod/Aql/IndexNode.cpp \ - arangod/Aql/IndexRangeBlock.cpp \ - arangod/Aql/IndexRangeNode.cpp \ arangod/Aql/ModificationBlocks.cpp \ arangod/Aql/ModificationNodes.cpp \ arangod/Aql/NodeFinder.cpp \