Merge branch 'devel' of https://github.com/triAGENS/ArangoDB into devel

arangod/Aql/ExecutionBlock.cpp

@@ -3638,6 +3638,36 @@ bool ScatterBlock::hasMore () {
   return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief remainingForShard: remaining for shard, sum of the number of row left
+/// in the buffer and _dependencies[0]->remaining()
+////////////////////////////////////////////////////////////////////////////////
+
+int64_t ScatterBlock::remainingForShard (std::string const& shardId) {
+  
+  size_t clientId = getClientId(shardId);
+
+  if (_doneForClient.at(clientId)) {
+    return 0;
+  }
+  
+  int64_t sum = _dependencies[0]->remaining();
+  if (sum == -1) {
+    return -1;
+  }
+
+  std::pair<size_t,size_t> pos = _posForClient.at(clientId);
+
+  if (pos.first <= _buffer.size()) {
+    sum += _buffer.at(pos.first)->size() - pos.second;
+    for (auto i = pos.first + 1; i < _buffer.size(); i++) {
+      sum += _buffer.at(i)->size();
+    }
+  }
+
+  return sum;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief hasMoreForShard: any more for shard <shardId>?
 ////////////////////////////////////////////////////////////////////////////////

arangod/Aql/ExecutionBlock.h

@@ -1574,7 +1574,7 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         int64_t remaining () {
-          return _dependencies[0]->remaining();
+          TRI_ASSERT(false);
         }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1606,6 +1606,12 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
         
         bool hasMoreForShard (std::string const& shardId);
+       
+////////////////////////////////////////////////////////////////////////////////
+/// @brief remainingForShard: remaining for shard <shardId>?
+////////////////////////////////////////////////////////////////////////////////
+        
+        int64_t remainingForShard (std::string const& shardId);
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief getOrSkipSomeForShard

arangod/Aql/ExecutionEngine.cpp

@@ -37,6 +37,7 @@
 #include "Utils/Exception.h"
 
 using namespace triagens::aql;
+using Json = triagens::basics::Json;
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief helper function to create a block
@@ -422,7 +423,7 @@ std::cout << "REGISTERING QUERY ON COORDINATOR WITH ID: " << id << "\n";
       while (current != nullptr) {
         bool stop = false;
 
-        auto clone = current->clone(&plan, false, false);
+        auto clone = current->clone(&plan, false, true);
         plan.registerNode(clone);
         
         if (current->getType() == ExecutionNode::REMOTE) {
@@ -456,28 +457,35 @@ std::cout << "REGISTERING QUERY ON COORDINATOR WITH ID: " << id << "\n";
         previous = clone;
         current = deps[0];
       }
-
+      
       // inject the current shard id into the collection
       collection->setCurrentShard(shardId);
-
-      plan.findVarUsage();
-      plan.planRegisters();
+      plan.setVarUsageComputed();
 
       // create a JSON representation of the plan
-      triagens::basics::Json result(triagens::basics::Json::Array);
-      triagens::basics::Json jsonNodesList(plan.root()->toJson(TRI_UNKNOWN_MEM_ZONE, true));
+      Json result(Json::Array);
+      Json jsonNodesList(plan.root()->toJson(TRI_UNKNOWN_MEM_ZONE, true));
     
       // add the collection
-      triagens::basics::Json jsonCollectionsList(triagens::basics::Json::List);
-      triagens::basics::Json json(triagens::basics::Json::Array);
-      jsonCollectionsList(json("name", triagens::basics::Json(collection->getName()))
-                              ("type", triagens::basics::Json(TRI_TransactionTypeGetStr(collection->accessType))));
+      Json jsonCollectionsList(Json::List);
+      Json json(Json::Array);
+      jsonCollectionsList(json("name", Json(collection->getName()))
+                              ("type", Json(TRI_TransactionTypeGetStr(collection->accessType))));
 
       jsonNodesList.set("collections", jsonCollectionsList);
+      jsonNodesList.set("variables", query->ast()->variables()->toJson(TRI_UNKNOWN_MEM_ZONE));
 
       result.set("plan", jsonNodesList);
-      result.set("part", triagens::basics::Json("main")); // TODO: set correct query type
+      result.set("part", Json("main")); // TODO: set correct query type
 
+      Json optimizerOptionsRules(Json::List);
+      Json optimizerOptions(Json::Array);
+
+      Json options(Json::Array);
+      optimizerOptionsRules.add(Json("-all"));
+      optimizerOptions.set("rules", optimizerOptionsRules);
+      options.set("optimizer", optimizerOptions);
+      result.set("options", options);
       std::unique_ptr<std::string> body(new std::string(triagens::basics::JsonHelper::toString(result.json())));
     
       std::cout << "GENERATED A PLAN FOR THE REMOTE SERVERS: " << *(body.get()) << "\n";

arangod/Aql/ExecutionPlan.h

@@ -230,6 +230,14 @@ namespace triagens {
 
         bool varUsageComputed ();
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief determine if the above are already set
+////////////////////////////////////////////////////////////////////////////////
+
+        void setVarUsageComputed () {
+          _varUsageComputed = true;
+        }
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief static analysis
 ////////////////////////////////////////////////////////////////////////////////

arangod/Aql/OptimizerRules.cpp

@@ -1998,6 +1998,7 @@ class RemoveToEnumCollFinder: public WalkerWorker<ExecutionNode> {
     }
 
     bool before (ExecutionNode* en) {
+      std::cout << "before!\n";
       switch (en->getType()) {
         case EN::REMOVE:{
           TRI_ASSERT(_remove == false);
@@ -2013,10 +2014,10 @@ class RemoveToEnumCollFinder: public WalkerWorker<ExecutionNode> {
           _variable = varsToRemove[0];    // the variable we'll remove
           
           auto _enumColl = static_cast<EnumerateCollectionNode*>(_plan->getVarSetBy(_variable->id));
-
+          
           if (_enumColl == nullptr 
-              || _enumColl->getType() != triagens::aql::ExecutionNode::ENUMERATE_COLLECTION 
-              || _enumColl->collection()->cid() != rn->collection()->cid() ) {
+              || _enumColl->getType() != triagens::aql::ExecutionNode::ENUMERATE_COLLECTION ) {
+         //     || _enumColl->collection()->cid() != rn->collection()->cid() ) {
             // remove variable was not introduced by an enumerate collection or 
             // it was but the collections differ
             break; // abort . . . 
@@ -2057,6 +2058,10 @@ class RemoveToEnumCollFinder: public WalkerWorker<ExecutionNode> {
           _toUnlink.insert(en); 
           return false; // continue . . .
         }
+        
+        case EN::CALCULATION: {//Check this is the calculation node of one of the filter nodes!
+                              
+        }
         case EN::ENUMERATE_COLLECTION: {
           // check that we are enumerating the variable we are to remove
           if (en->id() != _enumColl->id()) {
@@ -2067,7 +2072,6 @@ class RemoveToEnumCollFinder: public WalkerWorker<ExecutionNode> {
         }
         
         case EN::SINGLETON:
-        case EN::CALCULATION: 
         case EN::ENUMERATE_LIST:
         case EN::SUBQUERY:        
         case EN::AGGREGATE:

arangod/Aql/Query.cpp

@@ -387,6 +387,7 @@ QueryResult Query::prepare (QueryRegistry* registry) {
       enterState(PLAN_INSTANCIATION);
       ExecutionPlan::getCollectionsFromJson(parser->ast(), _queryJson);
 
+      parser->ast()->variables()->fromJson(_queryJson);
       // creating the plan may have produced some collections
       // we need to add them to the transaction now (otherwise the query will fail)
       int res = _trx->addCollectionList(_collections.collections());
@@ -411,7 +412,7 @@ QueryResult Query::prepare (QueryRegistry* registry) {
     enterState(PLAN_OPTIMIZATION);
     triagens::aql::Optimizer opt(maxNumberOfPlans());
     // get enabled/disabled rules
-    opt.createPlans(plan.release(), getRulesFromOptions());  
+    opt.createPlans(plan.release(), getRulesFromOptions());
 
     // Now plan and all derived plans belong to the optimizer
     plan.reset(opt.stealBest()); // Now we own the best one again
@@ -851,6 +852,10 @@ void Query::cleanupPlanAndEngine () {
     _engine = nullptr;
   }
 
+  if (_trx.get() != nullptr) {
+    // TODO: this doesn't unblock the collection on the coordinator. Y?
+    _trx->abort();
+  }
   _trx.reset();
 
   if (_parser != nullptr) {

arangod/Aql/VariableGenerator.cpp

@@ -27,9 +27,11 @@
 /// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany
 ////////////////////////////////////////////////////////////////////////////////
 
+#include "Utils/Exception.h"
 #include "Aql/VariableGenerator.h"
 
 using namespace triagens::aql;
+using Json = triagens::basics::Json;
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                        constructors / destructors
@@ -202,6 +204,36 @@ std::string VariableGenerator::nextName () const {
   return std::to_string(_id); // to_string: c++11
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief export to JSON, returns an AUTOFREE Json object
+////////////////////////////////////////////////////////////////////////////////
+
+triagens::basics::Json VariableGenerator::toJson (TRI_memory_zone_t* zone) const {
+  Json jsonAllVariablesList(Json::List, _variables.size());
+  for (auto oneVariable: _variables) {
+    jsonAllVariablesList(oneVariable.second->toJson());
+  }
+  return jsonAllVariablesList;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief import from JSON
+////////////////////////////////////////////////////////////////////////////////
+
+void VariableGenerator::fromJson (Json const& query) {
+  
+  Json jsonAllVariablesList = query.get("variables");
+  if (!jsonAllVariablesList.isList()) {
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "variables needs to be a list");
+  }
+  
+  auto len = jsonAllVariablesList.size();
+  _variables.reserve(len);
+  for (size_t i = 0; i < len; i++) {
+    createVariable(jsonAllVariablesList.at(i));
+  }
+}
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                       END-OF-FILE
 // -----------------------------------------------------------------------------

arangod/Aql/VariableGenerator.h

@@ -117,6 +117,18 @@ namespace triagens {
   
         std::string nextName () const;
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief export to JSON, returns an AUTOFREE Json object
+////////////////////////////////////////////////////////////////////////////////
+
+        triagens::basics::Json toJson (TRI_memory_zone_t*) const;
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief import from JSON
+////////////////////////////////////////////////////////////////////////////////
+
+        void fromJson (triagens::basics::Json const& jsonAllVariablesList);
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private functions
 // -----------------------------------------------------------------------------