diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index 88d3e6db0f..a6b6bd59da 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -1053,7 +1053,7 @@ void ExecutionPlan::replaceNode (ExecutionNode* oldNode, newNode->addDependency(x); } - auto&& oldNodeParents = oldNode->getParents(); + auto oldNodeParents = oldNode->getParents(); for (auto oldNodeParent : oldNodeParents) { if(! oldNodeParent->replaceDependency(oldNode, newNode)){ THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, @@ -1073,7 +1073,7 @@ void ExecutionPlan::insertDependency (ExecutionNode* oldNode, TRI_ASSERT(newNode->getDependencies().size() == 1); TRI_ASSERT(oldNode != _root); - auto&& oldDeps = oldNode->getDependencies(); + auto oldDeps = oldNode->getDependencies(); if (! oldNode->replaceDependency(oldDeps[0], newNode)) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "Could not replace dependencies of an old node."); diff --git a/arangod/Aql/Optimizer.cpp b/arangod/Aql/Optimizer.cpp index 9d3bbf57d8..9c3f342c09 100644 --- a/arangod/Aql/Optimizer.cpp +++ b/arangod/Aql/Optimizer.cpp @@ -50,7 +50,11 @@ Optimizer::Optimizer () { registerRule(removeUnnecessaryFiltersRule, 10000); // move calculations up the dependency chain (to pull them out of inner loops etc.) - // registerRule(moveCalculationsUpRule, 1000); + registerRule(moveCalculationsUpRule, 1001); + + // move filters up the dependency chain (to make result sets as small as possible + // as early as possible) + registerRule(moveFiltersUpRule, 1000); // remove calculations that are never necessary registerRule(removeUnnecessaryCalculationsRule, 999); diff --git a/arangod/Aql/Optimizer.h b/arangod/Aql/Optimizer.h index 276d5448f0..c2c095189c 100644 --- a/arangod/Aql/Optimizer.h +++ b/arangod/Aql/Optimizer.h @@ -116,7 +116,7 @@ namespace triagens { /// @brief appends all the plans to the target and clears *this at the same time //////////////////////////////////////////////////////////////////////////////// - void appendTo (vector& target) { + void appendTo (std::vector& target) { while (list.size() > 0) { auto p = list.front(); list.pop_front(); @@ -226,7 +226,7 @@ namespace triagens { /// @brief getPlans, ownership of the plans remains with the optimizer //////////////////////////////////////////////////////////////////////////////// - vector& getPlans () { + std::vector& getPlans () { return _plans; } @@ -252,8 +252,8 @@ namespace triagens { /// the optimizer will forget about them! //////////////////////////////////////////////////////////////////////////////// - vector stealPlans () { - vector res; + std::vector stealPlans () { + std::vector res; res.swap(_plans); return res; } @@ -292,13 +292,13 @@ namespace triagens { /// @brief the rules database //////////////////////////////////////////////////////////////////////////////// - vector _rules; + std::vector _rules; //////////////////////////////////////////////////////////////////////////////// /// @brief the current set of plans to be optimised //////////////////////////////////////////////////////////////////////////////// - vector _plans; + std::vector _plans; }; diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 09646e2350..980a3cc900 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -48,7 +48,6 @@ int triagens::aql::removeUnnecessaryFiltersRule (Optimizer* opt, ExecutionPlan* plan, Optimizer::PlanList& out, bool& keep) { - keep = true; // plan will always be kept std::unordered_set toUnlink; std::vector nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::FILTER, true); @@ -97,6 +96,167 @@ int triagens::aql::removeUnnecessaryFiltersRule (Optimizer* opt, if (! toUnlink.empty()) { plan->unlinkNodes(toUnlink); + plan->findVarUsage(); + } + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief move calculations up in the plan +/// this rule modifies the plan in place +/// it aims to move up calculations as far up in the plan as possible, to +/// avoid redundant calculations in inner loops +//////////////////////////////////////////////////////////////////////////////// + +int triagens::aql::moveCalculationsUpRule (Optimizer* opt, + ExecutionPlan* plan, + Optimizer::PlanList& out, + bool& keep) { + keep = true; // plan will always be kept + std::vector nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::CALCULATION, true); + bool modified = false; + + for (auto n : nodes) { + auto nn = static_cast(n); + if (nn->expression()->canThrow()) { + // we will only move expressions up that cannot throw + continue; + } + + auto neededVars = n->getVariablesUsedHere(); + // sort the list of variables that the expression needs as its input + // (sorting is needed for intersection later) + std::sort(neededVars.begin(), neededVars.end(), &Variable::Comparator); + + std::vector stack; + for (auto dep : n->getDependencies()) { + stack.push_back(dep); + } + + while (! stack.empty()) { + auto current = stack.back(); + stack.pop_back(); + + bool found = false; + + auto&& varsSet = current->getVariablesSetHere(); + for (auto v : varsSet) { + for (auto it = neededVars.begin(); it != neededVars.end(); ++it) { + if ((*it)->id == v->id) { + // shared variable, cannot move up any more + found = true; + break; + } + } + } + + if (found) { + // done with optimizing this calculation node + break; + } + + auto deps = current->getDependencies(); + if (deps.size() != 1) { + // node either has no or more than one dependency. we don't know what to do and must abort + // note: this will also handle Singleton nodes + break; + } + + for (auto dep : deps) { + stack.push_back(dep); + } + + // first, unlink the calculation from the plan + plan->unlinkNode(n); + // and re-insert into before the current node + plan->insertDependency(current, n); + modified = true; + } + + } + + if (modified) { + plan->findVarUsage(); + } + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief move filters up in the plan +/// this rule modifies the plan in place +/// filters are moved as far up in the plan as possible to make result sets +/// as small as possible as early as possible +/// filters are not pushed beyond limits +//////////////////////////////////////////////////////////////////////////////// + +int triagens::aql::moveFiltersUpRule (Optimizer* opt, + ExecutionPlan* plan, + Optimizer::PlanList& out, + bool& keep) { + keep = true; // plan will always be kept + std::vector nodes = plan->findNodesOfType(triagens::aql::ExecutionNode::FILTER, true); + bool modified = false; + + for (auto n : nodes) { + auto neededVars = n->getVariablesUsedHere(); + TRI_ASSERT(neededVars.size() == 1); + + std::vector stack; + for (auto dep : n->getDependencies()) { + stack.push_back(dep); + } + + while (! stack.empty()) { + auto current = stack.back(); + stack.pop_back(); + + if (current->getType() == triagens::aql::ExecutionNode::LIMIT) { + // cannot push a filter beyond a LIMIT node + break; + } + + bool found = false; + + auto&& varsSet = current->getVariablesSetHere(); + for (auto v : varsSet) { + for (auto it = neededVars.begin(); it != neededVars.end(); ++it) { + if ((*it)->id == v->id) { + // shared variable, cannot move up any more + found = true; + break; + } + } + } + + if (found) { + // done with optimizing this calculation node + break; + } + + auto deps = current->getDependencies(); + if (deps.size() != 1) { + // node either has no or more than one dependency. we don't know what to do and must abort + // note: this will also handle Singleton nodes + break; + } + + for (auto dep : deps) { + stack.push_back(dep); + } + + // first, unlink the filter from the plan + plan->unlinkNode(n); + // and re-insert into before the current node + plan->insertDependency(current, n); + modified = true; + } + + } + + if (modified) { + plan->findVarUsage(); } return TRI_ERROR_NO_ERROR; @@ -136,16 +296,15 @@ int triagens::aql::removeUnnecessaryCalculationsRule (Optimizer* opt, } if (! toUnlink.empty()) { - std::cout << "Removing " << toUnlink.size() << " unnecessary " - "CalculationNodes..." << std::endl; plan->unlinkNodes(toUnlink); + plan->findVarUsage(); } return TRI_ERROR_NO_ERROR; } //////////////////////////////////////////////////////////////////////////////// -/// @brief find nodes of a certain type +/// @brief prefer IndexRange nodes over EnumerateCollection nodes //////////////////////////////////////////////////////////////////////////////// class CalculationNodeFinder : public WalkerWorker { @@ -158,6 +317,7 @@ class CalculationNodeFinder : public WalkerWorker { //EnumerateCollectionNode const* _enumColl; public: + CalculationNodeFinder (ExecutionPlan* plan, Variable const * var, Optimizer::PlanList& out) : _plan(plan), _var(var), _out(out), _prev(nullptr){ _ranges = new RangesInfo(); diff --git a/arangod/Aql/OptimizerRules.h b/arangod/Aql/OptimizerRules.h index 2e761f8875..0aea58eafb 100644 --- a/arangod/Aql/OptimizerRules.h +++ b/arangod/Aql/OptimizerRules.h @@ -51,10 +51,23 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// /// @brief move calculations up in the plan +/// this rule modifies the plan in place +/// it aims to move up calculations as far up in the plan as possible, to +/// avoid redundant calculations in inner loops //////////////////////////////////////////////////////////////////////////////// int moveCalculationsUpRule (Optimizer*, ExecutionPlan*, Optimizer::PlanList&, bool&); +//////////////////////////////////////////////////////////////////////////////// +/// @brief move filters up in the plan +/// this rule modifies the plan in place +/// filters are moved as far up in the plan as possible to make result sets +/// as small as possible as early as possible +/// filters are not pushed beyond limits +//////////////////////////////////////////////////////////////////////////////// + + int moveFiltersUpRule (Optimizer*, ExecutionPlan*, Optimizer::PlanList&, bool&); + //////////////////////////////////////////////////////////////////////////////// /// @brief remove a CalculationNode that is never needed ////////////////////////////////////////////////////////////////////////////////