Merge branch 'aql2' of ssh://github.com/triAGENS/ArangoDB into aql2

arangod/Aql/ExecutionNode.cpp

@@ -187,6 +187,18 @@ Json ExecutionNode::toJsonHelperGeneric (std::map<ExecutionNode*, int>& indexTab
   if(this->_estimatedCost != 0){
     json("estimated cost", Json(this->_estimatedCost));
   }
+  if (_varUsageValid) {
+    Json varsValid(Json::List, _varsValid.size());
+    for (auto v : _varsValid) {
+      varsValid(Json(v->name));
+    }
+    json("varsValid", varsValid);
+    Json varsUsedLater(Json::List, _varsUsedLater.size());
+    for (auto v : _varsUsedLater) {
+      varsUsedLater(Json(v->name));
+    }
+    json("varsUsedLater", varsUsedLater);
+  }
 
   return json;
 }
@@ -333,8 +345,12 @@ void CalculationNode::toJsonHelper (std::map<ExecutionNode*, int>& indexTab,
     return;
   }
   
+  std::cout << "CANTHROW2" << _expression->canThrow() << std::endl;
+
   json("expression", _expression->toJson(TRI_UNKNOWN_MEM_ZONE))
-      ("outVariable", _outVariable->toJson());
+      ("outVariable", _outVariable->toJson())
+      ("canThrow", Json(_expression->canThrow()));
+
 
   // And add it:
   int len = static_cast<int>(nodes.size());

arangod/Aql/ExecutionNode.h

@@ -96,7 +96,7 @@ namespace triagens {
 /// @brief default constructor
 ////////////////////////////////////////////////////////////////////////////////
 
-        ExecutionNode () : _estimatedCost(0) {
+        ExecutionNode () : _estimatedCost(0), _varUsageValid(false) {
         }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -218,12 +218,6 @@ namespace triagens {
 
         triagens::basics::Json toJson (TRI_memory_zone_t* zone = TRI_UNKNOWN_MEM_ZONE);
 
-// -----------------------------------------------------------------------------
-// --SECTION--                                                 protected methods
-// -----------------------------------------------------------------------------
-
-      protected:
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief toJsonHelper, for a generic node
 ////////////////////////////////////////////////////////////////////////////////
@@ -241,6 +235,74 @@ namespace triagens {
                                    triagens::basics::Json& nodes,
                                    TRI_memory_zone_t* zone = TRI_UNKNOWN_MEM_ZONE) = 0;
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesUsedHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          return std::vector<Variable const*>();
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesSetHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesSetHere () {
+          return std::vector<Variable const*>();
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief setVarsUsedLater
+////////////////////////////////////////////////////////////////////////////////
+
+        void setVarsUsedLater (std::unordered_set<Variable const*>& v) {
+          _varsUsedLater = v;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVarsUsedLater
+////////////////////////////////////////////////////////////////////////////////
+
+        std::unordered_set<Variable const*>& getVarsUsedLater () {
+          TRI_ASSERT(_varUsageValid);
+          return _varsUsedLater;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief setVarsValid
+////////////////////////////////////////////////////////////////////////////////
+
+        void setVarsValid (std::unordered_set<Variable const*>& v) {
+          _varsValid = v;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVarsValid
+////////////////////////////////////////////////////////////////////////////////
+
+        std::unordered_set<Variable const*>& getVarsValid () {
+          TRI_ASSERT(_varUsageValid);
+          return _varsValid;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief setVarUsageValid
+////////////////////////////////////////////////////////////////////////////////
+
+        void setVarUsageValid () {
+          _varUsageValid = true;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief invalidateVarUsage
+////////////////////////////////////////////////////////////////////////////////
+
+        void invalidateVarUsage () {
+          _varsUsedLater.clear();
+          _varsValid.clear();
+          _varUsageValid = false;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                               protected variables
 // -----------------------------------------------------------------------------
@@ -265,6 +327,21 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         double _estimatedCost;
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief _varsUsedLater and _varsValid, the former contains those
+/// variables that are still needed further down in the chain. The
+/// latter contains the variables that are set from the dependent nodes
+/// when an item comes into the current node. Both are only valid if
+/// _varUsageValid is true. Use ExecutionPlan::findVarUsage to set
+/// this.
+////////////////////////////////////////////////////////////////////////////////
+
+        std::unordered_set<Variable const*> _varsUsedLater;
+        std::unordered_set<Variable const*> _varsValid;
+
+        bool _varUsageValid;
+
     };
 
 // -----------------------------------------------------------------------------
@@ -391,6 +468,16 @@ namespace triagens {
           //FIXME improve this estimate . . .
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesSetHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesSetHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_outVariable);
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -430,6 +517,7 @@ namespace triagens {
       
       friend class ExecutionBlock;
       friend class EnumerateListBlock;
+      friend struct VarUsageFinder;
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief constructor
@@ -480,6 +568,26 @@ namespace triagens {
           //FIXME improve this estimate . . .
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesUsedHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_inVariable);
+          return v;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesSetHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesSetHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_outVariable);
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -804,6 +912,29 @@ namespace triagens {
           //FIXME improve this estimate . . . 
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesUsedHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          std::unordered_set<Variable*> vars = _expression->variables();
+          std::vector<Variable const*> v;
+          for (auto vv : vars) {
+            v.push_back(vv);
+          }
+          return v;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesSetHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesSetHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_outVariable);
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -894,6 +1025,16 @@ namespace triagens {
           //FIXME improve this estimate . . .
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesSetHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesSetHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_outVariable);
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -976,9 +1117,15 @@ namespace triagens {
         }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief we need to know the offset and limit
+/// @brief getVariablesUsedHere
 ////////////////////////////////////////////////////////////////////////////////
 
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_inVariable);
+          return v;
+        }
+
       private:
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1047,6 +1194,18 @@ namespace triagens {
           return log(depCost) * depCost;
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesUsedHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          std::vector<Variable const*> v;
+          for (auto p : _elements) {
+            v.push_back(p.first);
+          }
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -1127,6 +1286,31 @@ namespace triagens {
           //FIXME improve this estimate . . .
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesUsedHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          std::vector<Variable const*> v;
+          for (auto p : _aggregateVariables) {
+            v.push_back(p.second);
+          }
+          return v;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesSetHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesSetHere () {
+          std::vector<Variable const*> v;
+          for (auto p : _aggregateVariables) {
+            v.push_back(p.first);
+          }
+          v.push_back(_outVariable);
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -1213,6 +1397,16 @@ namespace triagens {
           return _dependencies.at(0)->getCost();
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief getVariablesUsedHere
+////////////////////////////////////////////////////////////////////////////////
+
+        virtual std::vector<Variable const*> getVariablesUsedHere () {
+          std::vector<Variable const*> v;
+          v.push_back(_inVariable);
+          return v;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------

arangod/Aql/ExecutionPlan.cpp

@@ -83,6 +83,7 @@ ExecutionPlan* ExecutionPlan::instanciateFromAst (Ast const* ast) {
 
   try {
     plan->_root = plan->fromNode(ast, root);
+    plan->findVarUsage();
     
     std::cout << "ESTIMATED COST = €" << plan->getCost() << "\n";
     std::cout << plan->_root->toJson().toString() << "\n";
@@ -782,6 +783,55 @@ std::vector<ExecutionNode*> ExecutionPlan::findNodesOfType (
   return result;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief determine and set _varsUsedLater in all nodes
+////////////////////////////////////////////////////////////////////////////////
+
+struct triagens::aql::VarUsageFinder : public WalkerWorker<ExecutionNode> {
+
+    std::unordered_set<Variable const*> _usedLater;
+    std::unordered_set<Variable const*> _valid;
+
+    VarUsageFinder () {
+    };
+
+    ~VarUsageFinder () {
+    };
+
+    void before (ExecutionNode* en) {
+      en->invalidateVarUsage();
+      en->setVarsUsedLater(_usedLater);
+      // Add variables used here to _usedLater:
+      std::vector<Variable const*> usedHere = en->getVariablesUsedHere();
+      for (auto v : usedHere) {
+        _usedLater.insert(v);
+      }
+    }
+
+    void after (ExecutionNode* en) {
+      // Add variables set here to _valid:
+      std::vector<Variable const*> setHere = en->getVariablesSetHere();
+      for (auto v : setHere) {
+        _valid.insert(v);
+      }
+      en->setVarsValid(_valid);
+      en->setVarUsageValid();
+    }
+
+    bool enterSubquery (ExecutionNode* super, ExecutionNode* sub) {
+      VarUsageFinder subfinder;
+      subfinder._valid = _valid;  // need a copy for the subquery!
+      sub->walk(&subfinder);
+      return false;
+    }
+};
+
+void ExecutionPlan::findVarUsage () {
+  VarUsageFinder finder;
+  root()->walk(&finder);
+}
+
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                       END-OF-FILE
 // -----------------------------------------------------------------------------

arangod/Aql/ExecutionPlan.h

@@ -104,6 +104,12 @@ namespace triagens {
 
         std::vector<ExecutionNode*> findNodesOfType (ExecutionNode::NodeType);
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief determine and set _varsUsedLater in all nodes
+////////////////////////////////////////////////////////////////////////////////
+
+        void findVarUsage ();
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                   private methods
 // -----------------------------------------------------------------------------

arangod/Aql/Optimizer.cpp

@@ -51,8 +51,6 @@ Optimizer::Optimizer () {
 ////////////////////////////////////////////////////////////////////////////////
 
 int Optimizer::createPlans (ExecutionPlan* plan) {
-  // This vector holds the plans we have created in this pass:
-  PlanList newPlans;
   // This vector holds the plans we have created in the previous pass:
   PlanList oldPlans(plan);
 
@@ -66,13 +64,14 @@ int Optimizer::createPlans (ExecutionPlan* plan) {
   _plans.clear();
 
   for (int pass = 1; pass <= numberOfPasses; pass++) {
+    // This vector holds the plans we have created in this pass:
+    PlanList newPlans;
     for (auto r : _rules) {
-      PlanList nextNewPlans;
       PlanList nextOldPlans;
       while (oldPlans.size() > 0) {
         auto p = oldPlans.pop_front();
         try {
-          res = r.func(this, p, nextNewPlans, keep);
+          res = r.func(this, p, newPlans, keep);
           if (keep) {
             nextOldPlans.push_back(p);
           }
@@ -85,24 +84,7 @@ int Optimizer::createPlans (ExecutionPlan* plan) {
           return res;
         }
       }
-      while (newPlans.size() > 0) {
-        auto p = newPlans.pop_front();
-        try {
-          res = r.func(this, p, nextNewPlans, keep);
-          if (keep) {
-            nextNewPlans.push_back(p);
-          }
-        }
-        catch (...) {
-          delete p;
-          throw;
-        }
-        if (res != TRI_ERROR_NO_ERROR) {
-          return res;
-        }
-      }
       oldPlans.steal(nextOldPlans);
-      newPlans.steal(nextNewPlans);
     }
     // Now move the surviving old plans to the result:
     oldPlans.appendTo(_plans);

arangod/Aql/Optimizer.h

@@ -192,18 +192,27 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         ~Optimizer () {
+          for (auto p : _plans) {
+            delete p;
+          }
+          _plans.clear();
         }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief do the optimization, this does the optimization, the resulting
 /// plans are all estimated, sorted by that estimate and can then be got
-/// by getPlans, until the next initialize is called
+/// by getPlans, until the next initialize is called. Note that the optimizer
+/// object takes ownership of the execution plan and will delete it 
+/// automatically on destruction. It will also have ownership of all the
+/// newly created plans it recalls and will automatically delete them.
+/// If you need to extract the plans from the optimizer use stealBest or
+/// stealPlans.
 ////////////////////////////////////////////////////////////////////////////////
 
         int createPlans (ExecutionPlan* p);
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief getBest
+/// @brief getBest, ownership of the plan remains with the optimizer
 ////////////////////////////////////////////////////////////////////////////////
 
         ExecutionPlan* getBest () {
@@ -214,13 +223,41 @@ namespace triagens {
         }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief getPlans
+/// @brief getPlans, ownership of the plans remains with the optimizer
 ////////////////////////////////////////////////////////////////////////////////
 
         vector<ExecutionPlan*>& getPlans () {
           return _plans;
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief stealBest, ownership of the plan is handed over to the caller,
+/// all other plans are deleted
+////////////////////////////////////////////////////////////////////////////////
+
+        ExecutionPlan* stealBest () {
+          if (_plans.size() == 0) {
+            return nullptr;
+          }
+          auto res = _plans[0];
+          for (size_t i = 1; i < _plans.size(); i++) {
+            delete _plans[i];
+          }
+          _plans.clear();
+          return res;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief stealPlans, ownership of the plans is handed over to the caller,
+/// the optimizer will forget about them!
+////////////////////////////////////////////////////////////////////////////////
+
+        vector<ExecutionPlan*> stealPlans () {
+          vector<ExecutionPlan*> res;
+          res.swap(_plans);
+          return res;
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                   private methods
 // -----------------------------------------------------------------------------

arangod/Aql/Query.cpp

@@ -195,8 +195,9 @@ QueryResult Query::execute () {
 
     // Run the query optimiser:
     triagens::aql::Optimizer opt;
-    opt.createPlans(plan);
+    opt.createPlans(plan);  // Now plan and all derived plans belong to the
-    plan = opt.getBest();
+                            // optimizer
+    plan = opt.stealBest(); // Now we own the best one again
 
     triagens::basics::Json json(triagens::basics::Json::List);
 
@@ -235,7 +236,7 @@ QueryResult Query::execute () {
     trx.commit();
     
     QueryResult result(TRI_ERROR_NO_ERROR);
-    result.json = json; 
+    result.json = json.steal(); 
     return result;
   }
   catch (triagens::arango::Exception const& ex) {

lib/Basics/JsonHelper.h

@@ -530,14 +530,6 @@ namespace triagens {
           return res;
         }
    
-////////////////////////////////////////////////////////////////////////////////
-/// @brief type cast operator to TRI_json_t*, this steals the pointer
-////////////////////////////////////////////////////////////////////////////////
-
-        operator TRI_json_t* () throw() {
-          return steal();
-        }
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief assignment operator, note that, as the copy constructor, this 
 /// has steal semantics, which avoids deep copies in situations that 
@@ -588,11 +580,32 @@ namespace triagens {
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief set an attribute value in an array, an exception is thrown
-/// if *this is not a Json array. Note that you can call this with
+/// if *this is not a Json array. The pointer managed by sub is
-/// a Json as second argument because of the automatic type conversion
+/// stolen. The purpose of this method is that you can do
-/// to TRI_json_t* with steal semantics. Therefore
 ///   Json(Json::Array).set("a",Json(12)).set("b",Json(true))
-/// is both legal and efficient.
+/// and that this is both legal and efficient.
+////////////////////////////////////////////////////////////////////////////////
+
+        Json& set (char const* name, Json sub) {
+          if (! TRI_IsArrayJson(_json)) {
+            throw JsonException("Json is no array");
+          }
+          TRI_Insert3ArrayJson(_zone, _json, name, sub.steal());
+          return *this;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief this is a syntactic shortcut for the set method using operator()
+////////////////////////////////////////////////////////////////////////////////
+
+        Json& operator() (char const* name, Json sub) {
+          return set(name, sub);
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set an attribute value in an array, an exception is thrown if
+/// *this is not a Json array. The pointer sub is integrated into the
+/// list and will be freed if and only if the main thing is freed.
 ////////////////////////////////////////////////////////////////////////////////
 
         Json& set (char const* name, TRI_json_t* sub) {
@@ -608,21 +621,38 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         Json& operator() (char const* name, TRI_json_t* sub) {
-          if (! TRI_IsArrayJson(_json)) {
+          return set(name, sub);
-            throw JsonException("Json is no array");
-          }
-          TRI_Insert3ArrayJson(_zone, _json, name, sub);
-          return *this;
         }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief append a Json value to the end of a Json list, an exception
-/// is thrown if *this is not a Json list. Note that you can call this with
+/// is thrown if *this is not a Json list. The pointer managed by sub is
-/// a Json as argument because of the automatic type conversion
+/// stolen. The purpose of this method is that you can do
-/// to TRI_json_t* with steal semantics. Therefore
 ///   Json(Json::List).add(Json(12)).add(Json(13))
-/// is both legal and efficient.
+/// and that this is both legal and efficient.
+////////////////////////////////////////////////////////////////////////////////
 
+        Json& add (Json sub) {
+          if (! TRI_IsListJson(_json)) {
+            throw JsonException("Json is no list");
+          }
+          TRI_PushBack3ListJson(_zone, _json, sub.steal());
+          return *this;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief this is a syntactic shortcut for the add method using operator()
+////////////////////////////////////////////////////////////////////////////////
+
+        Json& operator() (Json sub) {
+          return add(sub);
+        }
+ 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief append a TRI_json_t value to the end of a Json list, an exception
+/// is thrown if *this is not a Json list. The pointer sub is integrated
+/// into the list and will be freed if and only if the main thing is
+/// freed.
 ////////////////////////////////////////////////////////////////////////////////
 
         Json& add (TRI_json_t* sub) {
@@ -638,11 +668,7 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         Json& operator() (TRI_json_t* sub) {
-          if (! TRI_IsListJson(_json)) {
+          return add(sub);
-            throw JsonException("Json is no list");
-          }
-          TRI_PushBack3ListJson(_zone, _json, sub);
-          return *this;
         }
  
 ////////////////////////////////////////////////////////////////////////////////