Merge branch 'aql-jmmh-conditions' of github.com:arangodb/arangodb into aql-jmmh-conditions

Documentation/StyleGuide/.gitignore

@@ -0,0 +1,3 @@
+*.aux
+*.log
+*.toc

UnitTests/Makefile.unittests

@@ -744,7 +744,8 @@ SHELL_CLIENT_ONLY = \
 	       @top_srcdir@/js/client/tests/shell-endpoints.js \
 	       @top_srcdir@/js/client/tests/shell-client.js \
 	       @top_srcdir@/js/client/tests/shell-fm.js \
-	       @top_srcdir@/js/client/tests/shell-request.js 
+	       @top_srcdir@/js/client/tests/shell-request.js \
+	       @top_srcdir@/js/client/tests/shell-require-canceled.js 
 
 SHELL_CLIENT = $(SHELL_COMMON) $(SHELL_CLIENT_ONLY)
 

arangod/Aql/AqlItemBlock.cpp

@@ -231,7 +231,7 @@ void AqlItemBlock::shrink (size_t nrItems) {
   
   if (nrItems > _nrItems) {
     // cannot use shrink() to increase the size of the block
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannout use shrink() to increase block");
   }
 
   // erase all stored values in the region that we freed

arangod/Aql/AstNode.cpp

@@ -1210,7 +1210,7 @@ bool AstNode::isFalse () const {
 bool AstNode::isAttributeAccessForVariable (std::pair<Variable const*, std::vector<triagens::basics::AttributeName>>& result) const {
   if (type != NODE_TYPE_ATTRIBUTE_ACCESS &&
       type != NODE_TYPE_EXPANSION) {
-    return nullptr;
+    return false;
   }
 
   // initialize
@@ -1224,7 +1224,10 @@ bool AstNode::isAttributeAccessForVariable (std::pair<Variable const*, std::vect
   while (node->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
          node->type == NODE_TYPE_EXPANSION) {
     if (node->type == NODE_TYPE_ATTRIBUTE_ACCESS) {
-      result.second.insert(result.second.begin(), triagens::basics::AttributeName(std::string(node->getStringValue(), node->getStringLength()), expandNext));
+      result.second.insert(
+        result.second.begin(), 
+        triagens::basics::AttributeName(std::string(node->getStringValue(), node->getStringLength()), expandNext)
+      );
       node = node->getMember(0);
       expandNext = false;
     }

arangod/Aql/AstNode.h

@@ -485,10 +485,11 @@ namespace triagens {
               node = node->getMember(0);
             }
             else {
-              // expansion
+              // expansion, i.e. [*]
               TRI_ASSERT(node->type == NODE_TYPE_EXPANSION);
+              TRI_ASSERT(node->numMembers() >= 2);
 
-              if (node->getMember(1)->getMember(1)->getAttributeAccessForVariable() == nullptr) {
+              if (node->getMember(1)->getAttributeAccessForVariable() == nullptr) {
                 return nullptr;
               }
               

arangod/Aql/Condition.cpp

@@ -72,9 +72,11 @@ ConditionPart::~ConditionPart () {
 
 }
 
+#if 0
 void ConditionPart::dump () const {
   std::cout << "VARIABLE NAME: " << variable->name << "." << attributeName << " " << triagens::basics::JsonHelper::toString(valueNode->toJson(TRI_UNKNOWN_MEM_ZONE, false)) << "\n";
 }
+#endif
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                   class Condition
@@ -84,6 +86,7 @@ void ConditionPart::dump () const {
 // --SECTION--                                            static helper function
 // -----------------------------------------------------------------------------
 
+#if 0
 static void dumpNode (AstNode const* node, int indent) {
   if (node == nullptr) {
     return;
@@ -111,6 +114,7 @@ static void dumpNode (AstNode const* node, int indent) {
     dumpNode(node->getMember(i), indent + 1);
   }
 }
+#endif
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                        constructors / destructors
@@ -140,6 +144,24 @@ Condition::~Condition () {
 // --SECTION--                                                  public functions
 // -----------------------------------------------------------------------------
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create a condition from JSON
+////////////////////////////////////////////////////////////////////////////////
+        
+Condition* Condition::fromJson (ExecutionPlan* plan,
+                                triagens::basics::Json const& json) {
+  std::unique_ptr<Condition> condition(new Condition(plan->getAst()));
+
+  std::unique_ptr<AstNode> node(new AstNode(plan->getAst(), json)); 
+  condition->andCombine(node.get());
+  node.release();
+
+  condition->_isNormalized = true;
+  //condition->normalize(plan);
+
+  return condition.release();
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief clone the condition
 ////////////////////////////////////////////////////////////////////////////////
@@ -303,10 +325,12 @@ void Condition::normalize (ExecutionPlan* plan) {
 
   optimize(plan);
 
+#if 0
   std::cout << "\n";
   dump();
   std::cout << "\n";
   _isNormalized = true;
+#endif  
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -675,9 +699,11 @@ AstNode* Condition::mergeInOperations (AstNode const* lhs,
 /// @brief dump the condition for debug purposes
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
 void Condition::dump () const {
   dumpNode(_root, 0);
 }
+#endif
 
 AstNode* Condition::transformNode (AstNode* node) {
   if (node == nullptr) {
@@ -697,6 +723,8 @@ AstNode* Condition::transformNode (AstNode* node) {
 
   if (node->type == NODE_TYPE_OPERATOR_NARY_AND) {
     // first recurse into subnodes
+
+    // TODO: this will not work in the general case with an NARY_AND node having 0..n children
     node->changeMember(0, transformNode(node->getMember(0)));
     node->changeMember(1, transformNode(node->getMember(1)));
 
@@ -730,8 +758,10 @@ AstNode* Condition::transformNode (AstNode* node) {
   }
   else if (node->type == NODE_TYPE_OPERATOR_NARY_OR) {
     // recurse into subnodes
-    node->changeMember(0, transformNode(node->getMember(0)));
+    size_t const n = node->numMembers();
-    node->changeMember(1, transformNode(node->getMember(1)));
+    for (size_t i = 0; i < n; ++i) {
+      node->changeMember(i, transformNode(node->getMemberUnchecked(i)));
+    }
   }
   else if (node->type == NODE_TYPE_OPERATOR_UNARY_NOT) {
     // push down logical negations

arangod/Aql/Condition.h

@@ -103,11 +103,13 @@ namespace triagens {
         }
       }
 
+#if 0
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief dump the condition for debug purposes
 ////////////////////////////////////////////////////////////////////////////////
 
       void dump () const;
+#endif      
       
       Variable const*             variable;
       std::string const           attributeName;
@@ -175,14 +177,20 @@ namespace triagens {
 /// @brief return the condition as a Json object
 ////////////////////////////////////////////////////////////////////////////////
 
-        triagens::basics::Json toJson (TRI_memory_zone_t* zone) const {
+        triagens::basics::Json toJson (TRI_memory_zone_t* zone, bool verbose) const {
           if (_root == nullptr) {
             return triagens::basics::Json(triagens::basics::Json::Object);
           }
 
-          return triagens::basics::Json(zone, _root->toJson(zone, false));
+          return triagens::basics::Json(zone, _root->toJson(zone, verbose));
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create a condition from JSON
+////////////////////////////////////////////////////////////////////////////////
+        
+        static Condition* fromJson (ExecutionPlan*, triagens::basics::Json const&);
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief clone the condition
 ////////////////////////////////////////////////////////////////////////////////
@@ -221,7 +229,9 @@ namespace triagens {
 /// @brief dump the condition
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
         void dump () const;
+#endif        
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                   private methods

arangod/Aql/ConditionFinder.cpp

@@ -104,7 +104,7 @@ bool ConditionFinder::before (ExecutionNode* en) {
     case EN::ENUMERATE_COLLECTION: {
       auto node = static_cast<EnumerateCollectionNode const*>(en);
       if (_changes->find(node->id()) != _changes->end()) {
-        std::cout << "Already optimized " << node->id() << std::endl;
+        // already optimized this node
         break;
       }
 

arangod/Aql/ExecutionEngine.cpp

@@ -730,20 +730,12 @@ struct CoordinatorInstanciator : public WalkerWorker<ExecutionNode> {
 
         if (nodeType == ExecutionNode::GATHER) {
           // we found a gather node
-          TRI_ASSERT(remoteNode != nullptr);
+          if (remoteNode == nullptr) {
+            THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "expecting a remoteNode");
+          }
 
           // now we'll create a remote node for each shard and add it to the gather node
-          Collection const* collection = nullptr;
+          Collection const* collection = static_cast<GatherNode const*>((*en))->collection();
-          if (nodeType == ExecutionNode::GATHER) {
-            collection = static_cast<GatherNode const*>((*en))->collection();
-          }
-          else {
-            THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
-          }
-
-          if (remoteNode == nullptr) {
-            THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
-          }
 
           auto&& shardIds = collection->shardIds();
 

arangod/Aql/ExecutionNode.cpp

@@ -131,7 +131,7 @@ void ExecutionNode::getSortElements (SortElementVector& elements,
     triagens::basics::Json oneJsonElement = jsonElements.at(static_cast<int>(i));
     bool ascending = JsonHelper::checkAndGetBooleanValue(oneJsonElement.json(), "ascending");
     Variable *v = varFromJson(plan->getAst(), oneJsonElement, "inVariable");
-    elements.emplace_back(std::make_pair(v, ascending));
+    elements.emplace_back(v, ascending);
   }
 }
 
@@ -143,6 +143,7 @@ ExecutionNode* ExecutionNode::fromJsonFactory (ExecutionPlan* plan,
   validateType(nodeTypeID);
 
   NodeType nodeType = (NodeType) nodeTypeID;
+
   switch (nodeType) {
     case SINGLETON:
       return new SingletonNode(plan, oneNode);

arangod/Aql/ExecutionPlan.cpp

@@ -1773,7 +1773,6 @@ void ExecutionPlan::insertDependency (ExecutionNode* oldNode,
 ExecutionNode* ExecutionPlan::fromJson (triagens::basics::Json const& json) {
   ExecutionNode* ret = nullptr;
   triagens::basics::Json nodes = json.get("nodes");
-  //std::cout << nodes.toString() << "\n";
   
   if (! nodes.isArray()) {
     THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "nodes is not an array");
@@ -1789,8 +1788,8 @@ ExecutionNode* ExecutionPlan::fromJson (triagens::basics::Json const& json) {
     if (! oneJsonNode.isObject()) {
       THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "json node is not an object");
     }
-    ret = ExecutionNode::fromJsonFactory(this, oneJsonNode);
 
+    ret = ExecutionNode::fromJsonFactory(this, oneJsonNode);
     registerNode(ret);
 
     TRI_ASSERT(ret != nullptr);

arangod/Aql/Executor.cpp

@@ -583,11 +583,11 @@ void Executor::HandleV8Error (v8::TryCatch& tryCatch,
     }
     
     // we can't figure out what kind of error occurred and throw a generic error
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unknown error in scripting");
   }
   
   if (result.IsEmpty()) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unknown error in scripting");
   }
 
   // if we get here, no exception has been raised

arangod/Aql/Functions.cpp

@@ -1723,6 +1723,22 @@ static Json VertexIdToJson (triagens::arango::AqlTransaction* trx,
                             VertexId const& id) {
   TRI_doc_mptr_copy_t mptr;
   auto collection = trx->trxCollection(id.cid);
+  if (collection == nullptr) {
+    int res = TRI_AddCollectionTransaction(trx->getInternals(), 
+                                           id.cid,
+                                           TRI_TRANSACTION_READ,
+                                           trx->nestingLevel(),
+                                           true,
+                                           true);
+    if (res != TRI_ERROR_NO_ERROR) {
+      THROW_ARANGO_EXCEPTION(res);
+    }
+    TRI_EnsureCollectionsTransaction(trx->getInternals());
+    collection = trx->trxCollection(id.cid);
+    if (collection == nullptr) {
+      THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "collection is a nullptr");
+    }
+  }
   int res = trx->readSingle(collection, &mptr, id.key); 
 
   if (res != TRI_ERROR_NO_ERROR) {

arangod/Aql/IndexBlock.cpp

@@ -65,7 +65,6 @@ IndexBlock::IndexBlock (ExecutionEngine* engine,
     _context(nullptr),
     _iterator(nullptr),
     _condition(en->_condition->root()),
-    _freeCondition(false),
     _hasV8Expression(false) {
 
   _context = new IndexIteratorContext(en->_vocbase);
@@ -80,10 +79,6 @@ IndexBlock::IndexBlock (ExecutionEngine* engine,
 IndexBlock::~IndexBlock () {
   delete _iterator;
   delete _context;
-
-  if (_freeCondition && _condition != nullptr) {
-    delete _condition;
-  }
 }
 
 void IndexBlock::buildExpressions () {
@@ -106,208 +101,20 @@ void IndexBlock::buildExpressions () {
               ->getMember(toReplace.andMember)
               ->changeMember(toReplace.operatorMember, evaluatedNode);
   }
-
-  /*
-  bool const useHighBounds = this->useHighBounds(); 
-  // TODO
-  std::unique_ptr<IndexOrCondition> newCondition;
-
-  for (size_t i = 0; i < en->_ranges.size(); i++) {
-    size_t const n = en->_ranges[i].size(); 
-    // prefill with n default-constructed vectors
-    std::vector<std::vector<RangeInfo>> collector(n);   
-
-    // collect the evaluated bounds here
-    for (size_t k = 0; k < n; k++) {
-      auto const& r = en->_ranges[i][k];
-
-      {
-        // First create a new RangeInfo containing only the constant 
-        // low and high bound of r:
-        RangeInfo riConst(r._var, r._attr, r._lowConst, r._highConst, r.is1ValueRangeInfo());
-        collector[k].emplace_back(std::move(riConst));
-      }
-
-      // Now work the actual values of the variable lows and highs into 
-      // this constant range:
-      for (auto const& l : r._lows) {
-        Expression* e = _allVariableBoundExpressions[posInExpressions];
-        TRI_ASSERT(e != nullptr);
-        AqlValue a = e->execute(_trx, cur, _pos, _inVars[posInExpressions], _inRegs[posInExpressions], &myCollection);
-        posInExpressions++;
-
-        Json bound;
-        if (a._type == AqlValue::JSON) {
-          bound = *(a._json);
-          a.destroy();  // the TRI_json_t* of a._json has been stolen
-        } 
-        else if (a._type == AqlValue::SHAPED || a._type == AqlValue::DOCVEC) {
-          bound = a.toJson(_trx, myCollection, true);
-          a.destroy();  // the TRI_json_t* of a._json has been stolen
-        } 
-        else {
-          THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, 
-              "AQL: computed a variable bound and got non-JSON");
-        }
-
-        if (! bound.isArray()) {
-          if (useHighBounds) {
-            auto b(bound.copy());
-
-            RangeInfo ri(r._var, 
-                         r._attr, 
-                         RangeInfoBound(l.inclusive(), true, b), // will steal b's JSON
-                         RangeInfoBound(), 
-                         false);
-
-            for (size_t j = 0; j < collector[k].size(); j++) {
-              collector[k][j].fuse(ri);
-            }
-          }
-          else {
-            auto b1(bound.copy()); // first instance of bound
-            auto b2(bound.copy()); // second instance of same bound
-
-            RangeInfo ri(r._var, 
-                         r._attr, 
-                         RangeInfoBound(l.inclusive(), true, b1), // will steal b1's JSON
-                         RangeInfoBound(l.inclusive(), true, b2), // will steal b2's JSON
-                         false);
-
-            for (size_t j = 0; j < collector[k].size(); j++) {
-              collector[k][j].fuse(ri);
-            }
-          }
-        } 
-        else {
-          std::vector<RangeInfo> riv; 
-          riv.reserve(bound.size());
-
-          for (size_t j = 0; j < bound.size(); j++) {
-            auto b1(bound.at(static_cast<int>(j)).copy()); // first instance of bound
-            auto b2(bound.at(static_cast<int>(j)).copy()); // second instance of same bound
-
-            riv.emplace_back(RangeInfo(r._var, 
-                                       r._attr, 
-                                       RangeInfoBound(l.inclusive(), true, b1), // will steal b1's JSON
-                                       RangeInfoBound(l.inclusive(), true, b2), // will steal b2's JSON
-                                       true));
-          }
-
-          collector[k] = std::move(andCombineRangeInfoVecs(collector[k], riv));
-        } 
-      }
-
-      if (useHighBounds) {
-        for (auto const& h : r._highs) {
-          Expression* e = _allVariableBoundExpressions[posInExpressions];
-          TRI_ASSERT(e != nullptr);
-          TRI_document_collection_t const* myCollection = nullptr; 
-          AqlValue a = e->execute(_trx, cur, _pos, _inVars[posInExpressions], _inRegs[posInExpressions], &myCollection);
-          posInExpressions++;
-
-          Json bound;
-          if (a._type == AqlValue::JSON) {
-            bound = *(a._json);
-            a.destroy();  // the TRI_json_t* of a._json has been stolen
-          } 
-          else if (a._type == AqlValue::SHAPED || a._type == AqlValue::DOCVEC) {
-            bound = a.toJson(_trx, myCollection, true);
-            a.destroy();  // the TRI_json_t* of a._json has been stolen
-          } 
-          else {
-            THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, 
-                "AQL: computed a variable bound and got non-JSON");
-          }
-          if (! bound.isArray()) {
-            auto b(bound.copy());
-            RangeInfo ri(r._var, 
-                          r._attr, 
-                          RangeInfoBound(), 
-                          RangeInfoBound(h.inclusive(), true, b), // will steal b's JSON
-                          false);
-
-            for (size_t j = 0; j < collector[k].size(); j++) {
-              collector[k][j].fuse(ri);
-            }
-          } 
-          else {
-            std::vector<RangeInfo> riv; 
-            riv.reserve(bound.size());
-
-            for (size_t j = 0; j < bound.size(); j++) {
-              auto b1(bound.at(static_cast<int>(j)).copy()); // first instance of bound
-              auto b2(bound.at(static_cast<int>(j)).copy()); // second instance of same bound
-
-              riv.emplace_back(RangeInfo(r._var, 
-                                          r._attr, 
-                                          RangeInfoBound(h.inclusive(), true, b1), // will steal b1's JSON
-                                          RangeInfoBound(h.inclusive(), true, b2), // will steal b2's JSON
-                                          true));
-            } 
-
-            collector[k] = std::move(andCombineRangeInfoVecs(collector[k], riv));
-          } 
-        }
-      }
-    }
-
-
-    bool isEmpty = false;
-
-    for (auto const& x : collector) {
-      if (x.empty()) { 
-        isEmpty = true;
-        break;
-      }
-    }
-
-    if (! isEmpty) { 
-      // otherwise the condition is impossible to fulfill
-      // the elements of the direct product of the collector are and
-      // conditions which should be added to newCondition 
-
-      // create cartesian product
-      std::unique_ptr<IndexOrCondition> indexAnds(cartesian(collector));
-
-      if (newCondition == nullptr) {
-        newCondition.reset(indexAnds.release());
-      }
-      else {
-        for (auto const& indexAnd : *indexAnds) {
-          newCondition->emplace_back(std::move(indexAnd));
-        }
-      } 
-    }
-
-  }
-    
-  freeCondition(); 
-    
-  if (newCondition != nullptr) {
-    _condition = newCondition.release();
-    _freeCondition = true;
- 
-    // remove duplicates . . .
-    removeOverlapsIndexOr(*_condition);
-  }
-  else {
-    _condition = new IndexOrCondition;
-    _freeCondition = true;
-  }
-  */
 }
 
 int IndexBlock::initialize () {
   ENTER_BLOCK
   int res = ExecutionBlock::initialize();
 
+/*
+  // why is this called again here? it has already been called in the constructor...!?
   if (res == TRI_ERROR_NO_ERROR) {
     if (_trx->orderDitch(_trx->trxCollection(_collection->cid())) == nullptr) {
       res = TRI_ERROR_OUT_OF_MEMORY;
     }
   }
-
+*/
   _nonConstExpressions.clear();
   _alreadyReturned.clear();
 
@@ -349,9 +156,9 @@ int IndexBlock::initialize () {
   auto outVariable = static_cast<IndexNode const*>(getPlanNode())->outVariable();
 
   for (size_t i = 0; i < _condition->numMembers(); ++i) {
-    auto andCond = _condition->getMember(i);
+    auto andCond = _condition->getMemberUnchecked(i);
     for (size_t j = 0; j < andCond->numMembers(); ++j) {
-      auto leaf = andCond->getMember(j);
+      auto leaf = andCond->getMemberUnchecked(j);
 
       // We only support binary conditions
       TRI_ASSERT(leaf->numMembers() == 2);
@@ -366,7 +173,8 @@ int IndexBlock::initialize () {
             THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
           }
         }
-      } else {
+      } 
+      else {
         // Index is responsible for the right side, check if left side has to be evaluated
         if (! lhs->isConstant()) {
           instantiateExpression(i, j, 0, lhs);
@@ -400,8 +208,6 @@ int IndexBlock::initialize () {
 
 bool IndexBlock::initIndexes () {
   ENTER_BLOCK
-  _flag = true; 
-
 
   // We start with a different context. Return documents found in the previous context again.
   _alreadyReturned.clear();
@@ -472,14 +278,6 @@ bool IndexBlock::initIndexes () {
   LEAVE_BLOCK;
 }
 
-void IndexBlock::freeCondition () {
-  if (_condition != nullptr && _freeCondition) {
-    delete _condition;
-    _condition = nullptr;
-    _freeCondition = false;
-  }
-}
-
 void IndexBlock::startNextIterator () {
   delete _iterator;
   _iterator = nullptr;

arangod/Aql/IndexBlock.h

@@ -146,24 +146,12 @@ namespace triagens {
 
         void buildExpressions ();
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief free _condition if it belongs to us
-////////////////////////////////////////////////////////////////////////////////
-
-        void freeCondition ();
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief continue fetching of documents
 ////////////////////////////////////////////////////////////////////////////////
 
         bool readIndex (size_t atMost);
 
-////////////////////////////////////////////////////////////////////////////////
-// @brief: sorts the index range conditions and resets _posInRanges to 0
-////////////////////////////////////////////////////////////////////////////////
-
-        void sortConditions ();
-
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
@@ -247,29 +235,7 @@ namespace triagens {
         
         std::unordered_set<TRI_doc_mptr_t*> _alreadyReturned;
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief _condition: holds the complete condition this Block can serve for
-////////////////////////////////////////////////////////////////////////////////
-        
-        AstNode* _evaluatedCondition;
-        
-////////////////////////////////////////////////////////////////////////////////
-/// @brief _flag: since readIndex for primary, hash, edges indexes reads the
-/// whole index, this is <true> if initIndex has been called but readIndex has
-/// not been called, otherwise it is <false> to avoid rereading the entire index
-/// with successive calls to readIndex.
-//////////////////////////////////////////////////////////////////////////////////
-
-        bool _flag;
         size_t _posInRanges;
-        std::vector<size_t> _sortCoords;
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief _freeCondition: whether or not the _condition is owned by the
-/// IndexRangeBlock and must be freed
-////////////////////////////////////////////////////////////////////////////////
-
-        bool _freeCondition;
 
         bool _hasV8Expression;
 

arangod/Aql/IndexNode.cpp

@@ -65,9 +65,7 @@ void IndexNode::toJsonHelper (triagens::basics::Json& nodes,
   }
 
   json("indexes",   indexes); 
-
+  json("condition", _condition->toJson(TRI_UNKNOWN_MEM_ZONE, verbose)); 
-  json("condition", _condition->toJson(TRI_UNKNOWN_MEM_ZONE)); 
-  
   json("reverse",   triagens::basics::Json(_reverse));
 
   // And add it:
@@ -83,9 +81,8 @@ ExecutionNode* IndexNode::clone (ExecutionPlan* plan,
     outVariable = plan->getAst()->variables()->createVariable(outVariable);
   }
 
-  // TODO: check if we need to clone _condition or if we can reuse it here
   auto c = new IndexNode(plan, _id, _vocbase, _collection, 
-                         outVariable, _indexes, _condition, _reverse);
+                         outVariable, _indexes, _condition->clone(), _reverse);
 
   cloneHelper(c, plan, withDependencies, withProperties);
 
@@ -106,7 +103,7 @@ IndexNode::IndexNode (ExecutionPlan* plan,
     _condition(nullptr),
     _reverse(JsonHelper::checkAndGetBooleanValue(json.json(), "reverse")) { 
 
-  auto indexes = JsonHelper::checkAndGetObjectValue(json.json(), "indexes");
+  auto indexes = JsonHelper::checkAndGetArrayValue(json.json(), "indexes");
 
   TRI_ASSERT(TRI_IsArrayJson(indexes));
   size_t length = TRI_LengthArrayJson(indexes);
@@ -123,7 +120,12 @@ IndexNode::IndexNode (ExecutionPlan* plan,
     _indexes.emplace_back(index);
   }
 
-  // TODO: rebuild _condition here!!
+  TRI_json_t const* condition = JsonHelper::checkAndGetObjectValue(json.json(), "condition");
+
+  triagens::basics::Json conditionJson(TRI_UNKNOWN_MEM_ZONE, condition, triagens::basics::Json::NOFREE);
+  _condition = Condition::fromJson(plan, conditionJson);
+
+  TRI_ASSERT(_condition != nullptr);
 }
 
 ////////////////////////////////////////////////////////////////////////////////

arangod/Aql/OptimizerRules.cpp

@@ -2054,8 +2054,6 @@ int triagens::aql::useIndexesRule (Optimizer* opt,
     n->walk(&finder);
   }
 
-  std::cout << "Candidates to replace " << changes.size() << std::endl;
-  
   if (! changes.empty()) {
     for (auto& it : changes) {
       plan->registerNode(it.second); 

arangod/Aql/grammar.h

@@ -1,8 +1,8 @@
-/* A Bison parser, made by GNU Bison 3.0.2.  */
+/* A Bison parser, made by GNU Bison 3.0.4.  */
 
 /* Bison interface for Yacc-like parsers in C
 
-   Copyright (C) 1984, 1989-1990, 2000-2013 Free Software Foundation, Inc.
+   Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -110,10 +110,10 @@ extern int Aqldebug;
 
 /* Value type.  */
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef union YYSTYPE YYSTYPE;
+
 union YYSTYPE
 {
-#line 17 "arangod/Aql/grammar.y" /* yacc.c:1909  */
+#line 17 "arangod/Aql/grammar.y" /* yacc.c:1915  */
 
   triagens::aql::AstNode*  node;
   struct {
@@ -123,8 +123,10 @@ union YYSTYPE
   bool                     boolval;
   int64_t                  intval;
 
-#line 127 "arangod/Aql/grammar.hpp" /* yacc.c:1909  */
+#line 127 "arangod/Aql/grammar.hpp" /* yacc.c:1915  */
 };
+
+typedef union YYSTYPE YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define YYSTYPE_IS_DECLARED 1
 #endif

arangod/Dispatcher/Dispatcher.cpp

@@ -102,8 +102,7 @@ void Dispatcher::addStandardQueue (size_t nrThreads, size_t maxSize) {
 /// @brief adds the AQL queue (used for the cluster)
 ////////////////////////////////////////////////////////////////////////////////
 
-void Dispatcher::addAQLQueue (size_t nrThreads,
+void Dispatcher::addAQLQueue (size_t nrThreads, size_t maxSize) {
-                             size_t maxSize) {
   TRI_ASSERT(_queues[AQL_QUEUE] == nullptr);
 
   _queues[AQL_QUEUE] = new DispatcherQueue(
@@ -278,8 +277,3 @@ void Dispatcher::setProcessorAffinity (size_t id, std::vector<size_t> const& cor
 // -----------------------------------------------------------------------------
 // --SECTION--                                                       END-OF-FILE
 // -----------------------------------------------------------------------------
-
-// Local Variables:
-// mode: outline-minor
-// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
-// End:

arangod/Indexes/SimpleAttributeEqualityMatcher.h

@@ -262,11 +262,20 @@ namespace triagens {
           else if (index->hasSelectivityEstimate()) {
             // use index selectivity estimate
             estimatedCost = 1.0 - index->selectivityEstimate(); 
+          
+            // the more attributes are covered by an index, the more accurate it
+            // is considered to be
+            estimatedCost /= index->fields().size();
           }
           else {
             // set to highest possible cost by default
             estimatedCost = 1.0; 
+            
+            // the more attributes are covered by an index, the more accurate it
+            // is considered to be
+            estimatedCost /= index->fields().size();
           }
+
           // can use this index
           return true;
         }

arangod/Utils/Transaction.h

@@ -235,6 +235,10 @@ namespace triagens {
             return TRI_TRANSACTION_UNDEFINED;
           }
 
+          int nestingLevel () const {
+            return _nestingLevel;
+          }
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief begin the transaction
 ////////////////////////////////////////////////////////////////////////////////

arangod/V8Server/ApplicationV8.cpp

@@ -349,13 +349,61 @@ void ApplicationV8::setVocbase (TRI_vocbase_t* vocbase) {
 ////////////////////////////////////////////////////////////////////////////////
 
 ApplicationV8::V8Context* ApplicationV8::enterContext (TRI_vocbase_t* vocbase,
-                                                       bool allowUseDatabase) {
+                                                       bool allowUseDatabase,
+						       ssize_t forceContext) {
   v8::Isolate* isolate = nullptr;
   V8Context* context   = nullptr;
 
+  // this is for TESTING / DEBUGGING only
+  if (forceContext != -1) {
+    size_t id = (size_t) forceContext;
+
+    while (! _stopping) {
       {
 	CONDITION_LOCKER(guard, _contextCondition);
 
+	for (auto iter = _freeContexts.begin();  iter != _freeContexts.end();  ++iter) {
+	  if ((*iter)->_id == id) {
+	    context = *iter;
+	    _freeContexts.erase(iter);
+	    _busyContexts.emplace(context);
+	    break;
+	  }
+	}
+
+	if (context != nullptr) {
+	  break;
+	}
+
+	for (auto iter = _dirtyContexts.begin();  iter != _dirtyContexts.end();  ++iter) {
+	  if ((*iter)->_id == id) {
+	    context = *iter;
+	    _dirtyContexts.erase(iter);
+	    _busyContexts.emplace(context);
+	    break;
+	  }
+	}
+
+	if (context != nullptr) {
+	  break;
+	}
+      }
+
+      LOG_DEBUG("waiting for V8 context %d to become available", (int) id);
+      usleep(100 * 1000);
+    }
+
+    if (context == nullptr) {
+      return nullptr;
+    }
+
+    isolate = context->isolate;
+  }
+
+  // look for a free context
+  else {
+    CONDITION_LOCKER(guard, _contextCondition);
+
     while (_freeContexts.empty() && ! _stopping) {
       LOG_DEBUG("waiting for unused V8 context");
 
@@ -393,6 +441,7 @@ ApplicationV8::V8Context* ApplicationV8::enterContext (TRI_vocbase_t* vocbase,
     isolate = context->isolate;
 
     _freeContexts.pop_back();
+
     // should not fail because we reserved enough space beforehand
     _busyContexts.emplace(context);
   }
@@ -407,6 +456,7 @@ ApplicationV8::V8Context* ApplicationV8::enterContext (TRI_vocbase_t* vocbase,
   v8::HandleScope scope(isolate);
   auto localContext = v8::Local<v8::Context>::New(isolate, context->_context);
   localContext->Enter();
+
   {
     v8::Context::Scope contextScope(localContext);
 

arangod/V8Server/ApplicationV8.h

@@ -331,7 +331,8 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         V8Context* enterContext (TRI_vocbase_t*,
-                                 bool useDatabase);
+                                 bool useDatabase,
+				 ssize_t forceContext = -1);
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief exists an context

arangod/V8Server/v8-actions.cpp

@@ -140,9 +140,22 @@ class v8_action_t : public TRI_action_t {
         allowUseDatabaseInRestActions = true;
       }
 
+
+      // this is for TESTING / DEBUGGING only - do not document this feature
+      ssize_t forceContext = -1;
+      bool found;
+      
+      char const* c = request->header("x-arango-v8-context", found);
+
+      if (found && c != nullptr) {
+	forceContext = StringUtils::int32(c);
+      }
+
+      // get a V8 context
       ApplicationV8::V8Context* context = GlobalV8Dealer->enterContext(
         vocbase,
-        allowUseDatabaseInRestActions
+        allowUseDatabaseInRestActions,
+	forceContext
       );
 
       // note: the context might be 0 in case of shut-down

arangod/VocBase/transaction.cpp

@@ -1013,8 +1013,8 @@ int TRI_AddCollectionTransaction (TRI_transaction_t* trx,
 /// @brief make sure all declared collections are used & locked
 ////////////////////////////////////////////////////////////////////////////////
 
-int TRI_EnsureCollectionsTransaction (TRI_transaction_t* trx) {
+int TRI_EnsureCollectionsTransaction (TRI_transaction_t* trx, int nestingLevel) {
-  return UseCollections(trx, 0);
+  return UseCollections(trx, nestingLevel);
 }
 
 ////////////////////////////////////////////////////////////////////////////////

arangod/VocBase/transaction.h

@@ -263,7 +263,7 @@ int TRI_AddCollectionTransaction (TRI_transaction_t*,
 /// @brief make sure all declared collections are used & locked
 ////////////////////////////////////////////////////////////////////////////////
 
-int TRI_EnsureCollectionsTransaction (TRI_transaction_t*);
+int TRI_EnsureCollectionsTransaction (TRI_transaction_t*, int = 0);
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief request a lock for a collection

js/client/tests/shell-require-canceled.js

@@ -0,0 +1,104 @@
+/*jshint globalstrict:false, strict:false */
+/*global assertEqual, arango */
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test the require which is canceled
+///
+/// @file
+///
+/// DISCLAIMER
+///
+/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+var jsunity = require('jsunity');
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                        test suite
+// -----------------------------------------------------------------------------
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test suite for 'require-canceled'
+////////////////////////////////////////////////////////////////////////////////
+
+function RequireCanceledTestSuite () {
+  'use strict';
+
+  return {
+
+    setUp() {
+      arango.POST_RAW("/_admin/execute",
+		      "require('module').globalPaths.unshift(require('path').resolve('./js/common/test-data/modules'));",
+		      {'x-arango-v8-context': 0});
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief tear down
+////////////////////////////////////////////////////////////////////////////////
+
+    tearDown() {
+      arango.POST_RAW("/_admin/execute",
+		      "require('module').globalPaths.splice(0,1);",
+		      {'x-arango-v8-context': 0});
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test requiring JSON
+////////////////////////////////////////////////////////////////////////////////
+
+    testRequireJson() {
+      var internal = require("internal");
+      var a = arango.POST_RAW("/_admin/execute",
+			      'return Object.keys(require("a"));',
+			      {'x-arango-async': "store", 'x-arango-v8-context': 0});
+      internal.sleep(3);
+
+      var id = a.headers['x-arango-async-id'];
+      arango.PUT_RAW("/_api/job/" + id + "/cancel", '');
+
+      var c = arango.POST_RAW("/_admin/execute?returnAsJSON=true",
+			      'return Object.keys(require("a"));',
+			      {'x-arango-async': "false", 'x-arango-v8-context': 0});
+      var d;
+
+      try {
+        d = JSON.parse(c.body);
+      }
+      catch (err) {
+	require("internal").print(c.body);
+        throw err;
+      }
+
+      assertEqual(2, d.length);
+    },
+  };
+}
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                              main
+// -----------------------------------------------------------------------------
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief executes the test suite
+////////////////////////////////////////////////////////////////////////////////
+
+jsunity.run(RequireCanceledTestSuite);
+
+return jsunity.done();

js/common/test-data/modules/a.js

@@ -0,0 +1,11 @@
+'use strict';
+
+let internal = require("internal");
+
+var j;
+for (j = 0;  j < 5;  ++j) {
+  internal.sleep(1);
+}
+
+exports.c = 10;
+exports.a = require("b").b;

js/common/test-data/modules/b.js

@@ -0,0 +1,3 @@
+'use strict';
+
+exports.b = require("a").c;

js/server/modules/org/arangodb/aql-helper.js

@@ -403,7 +403,7 @@ function getQueryMultiplePlansAndExecutions (query, bindVars, testObject, debug)
 
   // first fetch the unmodified version
   if (debug) {
-    require("internal").print("Analysing Query unoptimized: " + query);
+    require("internal").print("Analyzing Query unoptimized: " + query);
   }
   plans[0] = AQL_EXPLAIN(query, bindVars, paramNone);
   // then all of the ones permuted by by the optimizer.

js/server/modules/org/arangodb/foxx/routing.js

@@ -676,7 +676,7 @@ var mountController = function (service, routes, mount, filename) {
       createMiddlewareMatchers(ri.middleware, routes, mount, ri.urlPrefix);
     }
     if (ri.hasOwnProperty('routes')) {
-      transformRoutes(ri.routes, routes, mount, ri.urlPrefix, service._isDevelopment);
+      transformRoutes(ri.routes, routes, mount, ri.urlPrefix, service.isDevelopment);
     }
   }
 };

js/server/modules/org/arangodb/foxx/service.js

@@ -179,9 +179,12 @@ class FoxxService {
       try {
         this.thumbnail = fs.read64(thumb);
       } catch (e) {
+        this.thumbnail = null;
+        /*
         console.warnLines(
           `Cannot read thumbnail "${thumb}" for app "${data.mount}": ${e.stack}`
         );
+        */
       }
     } else {
       this.thumbnail = null;