Bug fix/3007 (#6019)

arangod/Aql/EngineInfoContainerCoordinator.cpp

@@ -152,12 +152,12 @@ ExecutionEngineResult EngineInfoContainerCoordinator::buildEngines(
   TRI_ASSERT(_engineStack.top() == 0);
 
   std::vector<uint64_t> coordinatorQueryIds{};
-  auto cleanup = [&]() {
+  // destroy all query snippets in case of error
+  auto guard = scopeGuard([&dbname, &registry, &coordinatorQueryIds]() {
     for (auto const& it : coordinatorQueryIds) {
       registry->destroy(dbname, it, TRI_ERROR_INTERNAL);
     }
-  };
+  });
-  TRI_DEFER(cleanup());
 
   Query* localQuery = query;
   try {
@@ -166,9 +166,7 @@ ExecutionEngineResult EngineInfoContainerCoordinator::buildEngines(
       if (!first) {
         // need a new query instance on the coordinator
         localQuery = query->clone(PART_DEPENDENT, false);
-        if (localQuery == nullptr) {
+        TRI_ASSERT(localQuery != nullptr);
-          return ExecutionEngineResult(TRI_ERROR_INTERNAL, "cannot clone query");
-        }
       }
       try {
         auto res = info.buildEngine(localQuery, registry, dbname,
@@ -205,6 +203,6 @@ ExecutionEngineResult EngineInfoContainerCoordinator::buildEngines(
 
   // This deactivates the defered cleanup.
   // From here on we rely on the AQL shutdown mechanism.
-  coordinatorQueryIds.clear();
+  guard.cancel();
   return ExecutionEngineResult(query->engine());
 }

arangod/Aql/ExecutionEngine.cpp

@@ -484,11 +484,13 @@ Result ExecutionEngine::shutdownSync(int errorCode) noexcept {
   Result res{TRI_ERROR_INTERNAL};
   ExecutionState state = ExecutionState::WAITING;
   try {
-    _query->setContinueCallback([&]() { _query->tempSignalAsyncResponse(); });
+    std::shared_ptr<SharedQueryState> sharedState = _query->sharedState();
+    sharedState->setContinueCallback();
+    
     while (state == ExecutionState::WAITING) {
       std::tie(state, res) = shutdown(errorCode);
       if (state == ExecutionState::WAITING) {
-        _query->tempWaitForAsyncResponse();
+        sharedState->waitForAsyncResponse();
       }
     }
   } catch (...) {

arangod/Aql/Query.cpp

@@ -95,14 +95,8 @@ Query::Query(
       _killed(false),
       _isModificationQuery(false),
       _preparedV8Context(false),
-      _hasHandler(false),
+      _executionPhase(ExecutionPhase::INITIALIZE),
-      _executionPhase(ExecutionPhase::INITIALIZE) {
+      _sharedState(std::make_shared<SharedQueryState>()) {
-  AqlFeature* aql = AqlFeature::lease();
-
-  if (aql == nullptr) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
-  }
-
   if (_contextOwnedByExterior) {
     // copy transaction options from global state into our local query options
     TransactionState* state = transaction::V8Context::getParentState();
@@ -153,6 +147,12 @@ Query::Query(
   }
 
   _resourceMonitor.setMemoryLimit(_queryOptions.memoryLimit);
+  
+  AqlFeature* aql = AqlFeature::lease();
+
+  if (aql == nullptr) {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
+  }
 }
 
 /// @brief creates a query from VelocyPack
@@ -181,13 +181,8 @@ Query::Query(
       _killed(false),
       _isModificationQuery(false),
       _preparedV8Context(false),
-      _hasHandler(false),
+      _executionPhase(ExecutionPhase::INITIALIZE),
-      _executionPhase(ExecutionPhase::INITIALIZE) {
+      _sharedState(std::make_shared<SharedQueryState>()) {
-  AqlFeature* aql = AqlFeature::lease();
-
-  if (aql == nullptr) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
-  }
   
   // populate query options
   if (_options != nullptr) {
@@ -210,6 +205,12 @@ Query::Query(
   }
 
   _resourceMonitor.setMemoryLimit(_queryOptions.memoryLimit);
+  
+  AqlFeature* aql = AqlFeature::lease();
+
+  if (aql == nullptr) {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
+  }
 }
 
 /// @brief destroys a query
@@ -225,10 +226,8 @@ Query::~Query() {
   exitContext();
 
   _ast.reset();
+  _graphs.clear();
 
-  for (auto& it : _graphs) {
-    delete it.second;
-  }
   LOG_TOPIC(DEBUG, Logger::QUERIES)
       << TRI_microtime() - _startTime << " "
       << "Query::~Query this: " << (uintptr_t) this;
@@ -284,6 +283,16 @@ Query* Query::clone(QueryPart part, bool withPlan) {
   return clone.release();
 }
 
+/// @brief whether or not the query is killed
+bool Query::killed() const { 
+  return _killed; 
+}
+
+/// @brief set the query to killed
+void Query::kill() { 
+  _killed = true; 
+}
+
 void Query::setExecutionTime() {
   if (_engine != nullptr) {
     _engine->_stats.setExecutionTime(TRI_microtime() - _startTime);
@@ -745,15 +754,17 @@ ExecutionState Query::execute(QueryRegistry* registry, QueryResult& queryResult)
  * @return The result of this query. The result is always complete
  */
 QueryResult Query::executeSync(QueryRegistry* registry) {
+  std::shared_ptr<SharedQueryState> ss = sharedState();
+  ss->setContinueCallback();
+
   QueryResult queryResult;
-  setContinueCallback([&]() { tempSignalAsyncResponse(); });
   while(true) {
     auto state = execute(registry, queryResult);
     if (state != aql::ExecutionState::WAITING) {
       TRI_ASSERT(state == aql::ExecutionState::DONE);
       return queryResult;
     }
-    tempWaitForAsyncResponse();
+    ss->waitForAsyncResponse();
   }
 }
 
@@ -764,7 +775,9 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
                                     << " this: " << (uintptr_t) this;
   TRI_ASSERT(registry != nullptr);
 
-  setContinueCallback([&]() { tempSignalAsyncResponse(); });
+  std::shared_ptr<SharedQueryState> ss = sharedState();
+  ss->setContinueCallback();
+
   try {
     bool useQueryCache = canUseQueryCache();
     uint64_t queryHash = hash();
@@ -821,6 +834,8 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
     std::unique_ptr<AqlItemBlock> value;
 
     try {
+      std::shared_ptr<SharedQueryState> ss = sharedState();
+
       if (useQueryCache) {
         VPackOptions options = VPackOptions::Defaults;
         options.buildUnindexedArrays = true;
@@ -837,7 +852,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
           state = res.first;
           // TODO MAX: We need to let the thread sleep here instead of while loop
           while (state == ExecutionState::WAITING) {
-            tempWaitForAsyncResponse();
+            ss->waitForAsyncResponse();
             res = _engine->getSome(ExecutionBlock::DefaultBatchSize());
             state = res.first;
           }
@@ -882,7 +897,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
           state = res.first;
           // TODO MAX: We need to let the thread sleep here instead of while loop
           while (state == ExecutionState::WAITING) {
-            tempWaitForAsyncResponse();
+            ss->waitForAsyncResponse();
             res = _engine->getSome(ExecutionBlock::DefaultBatchSize());
             state = res.first;
           }
@@ -926,7 +941,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
     // will set warnings, stats, profile and cleanup plan and engine
     ExecutionState state = finalize(queryResult);
     while (state == ExecutionState::WAITING) {
-      tempWaitForAsyncResponse();
+      ss->waitForAsyncResponse();
       state = finalize(queryResult);
     }
   } catch (arangodb::basics::Exception const& ex) {
@@ -1147,10 +1162,6 @@ void Query::setEngine(ExecutionEngine* engine) {
   _engine.reset(engine);
 }
 
-void Query::releaseEngine() {
-  _engine.release();
-}
-
 /// @brief prepare a V8 context for execution for this expression
 /// this needs to be called once before executing any V8 function in this
 /// expression
@@ -1373,10 +1384,12 @@ void Query::enterState(QueryExecutionState::ValueType state) {
 
 void Query::cleanupPlanAndEngineSync(int errorCode, VPackBuilder* statsBuilder) noexcept {
   try {
-    setContinueCallback([&]() { tempSignalAsyncResponse(); });
+    std::shared_ptr<SharedQueryState> ss = sharedState();
+    ss->setContinueCallback();
+    
     ExecutionState state = cleanupPlanAndEngine(errorCode, statsBuilder);
     while (state == ExecutionState::WAITING) {
-      tempWaitForAsyncResponse();
+      ss->waitForAsyncResponse();
       state = cleanupPlanAndEngine(errorCode, statsBuilder);
     }
   } catch (...) {
@@ -1404,11 +1417,11 @@ ExecutionState Query::cleanupPlanAndEngine(int errorCode, VPackBuilder* statsBui
     _engine.reset();
   }
 
-  if (_trx != nullptr) {
+  _sharedState->invalidate();
+
   // If the transaction was not committed, it is automatically aborted
   delete _trx;
   _trx = nullptr;
-  }
 
   _plan.reset();
   return ExecutionState::DONE;
@@ -1432,10 +1445,7 @@ std::shared_ptr<transaction::Context> Query::createTransactionContext() {
 Graph const* Query::lookupGraphByName(std::string const& name) {
   auto it = _graphs.find(name);
 
-  if (it != _graphs.end()) {
+  if (it == _graphs.end()) {
-    return it->second;
-  }
-
     std::unique_ptr<arangodb::aql::Graph> g(
         arangodb::lookupGraphByName(createTransactionContext(), name));
  
@@ -1443,9 +1453,13 @@ Graph const* Query::lookupGraphByName(std::string const& name) {
       return nullptr;
     }
  
-  _graphs.emplace(name, g.get());
+    auto result = _graphs.emplace(name, std::move(g));
+    TRI_ASSERT(result.second);
+    it = result.first;
+  }
  
-  return g.release();
+  TRI_ASSERT((*it).second != nullptr);
+  return (*it).second.get();
 }
 
 /// @brief returns the next query id

arangod/Aql/Query.h

@@ -36,6 +36,7 @@
 #include "Aql/QueryString.h"
 #include "Aql/RegexCache.h"
 #include "Aql/ResourceUsage.h"
+#include "Aql/SharedQueryState.h"
 #include "Aql/types.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/ConditionVariable.h"
@@ -109,6 +110,14 @@ class Query {
 
  public:
   
+/// @brief whether or not the query is killed
+  bool killed() const;
+
+  /// @brief set the query to killed
+  void kill();
+
+  void setExecutionTime();
+
   QueryString const& queryString() const { return _queryString; }
 
   /// @brief Inject a transaction from outside. Use with care!
@@ -143,12 +152,6 @@ class Query {
   /// @brief return the current runtime of the query
   double runTime() const { return runTime(TRI_microtime()); }
 
-  /// @brief whether or not the query is killed
-  inline bool killed() const { return _killed; }
-
-  /// @brief set the query to killed
-  inline void killed(bool) { _killed = true; }
-
   /// @brief the part of the query
   inline QueryPart part() const { return _part; }
 
@@ -232,8 +235,6 @@ class Query {
   /// @brief inject the engine
   TEST_VIRTUAL void setEngine(ExecutionEngine* engine);
 
-  void releaseEngine();
-
   /// @brief return the transaction, if prepared
   TEST_VIRTUAL inline transaction::Methods* trx() { return _trx; }
 
@@ -288,45 +289,9 @@ class Query {
 
   QueryExecutionState::ValueType state() const { return _state; }
 
-  /// @brief continueAfterPause is to be called on the query object to
+  /// @brief return the query's shared state
-  /// continue execution in this query part, if the query got paused
+  std::shared_ptr<SharedQueryState> sharedState() const { 
-  /// because it is waiting for network responses. The idea is that a
+    return _sharedState;
-  /// RemoteBlock that does an asynchronous cluster-internal request can
-  /// register a callback with the asynchronous request and then return
-  /// with the result `ExecutionState::WAITING`, which will bubble up
-  /// the stack and eventually lead to a suspension of the work on the
-  /// RestHandler. In the callback function one can first store the
-  /// results in the RemoteBlock object and can then call this method on
-  /// the query.
-  /// This will lead to the following: The original request that lead to
-  /// the network communication will be rescheduled on the ioservice and
-  /// continues its execution where it left off.
-  void continueAfterPause() {
-    TRI_ASSERT(!hasHandler());
-    _continueCallback();
-  }
-
-  std::function<void()> continueHandler() {
-    TRI_ASSERT(hasHandler());
-    return _continueCallback;
-  }
-
-  bool hasHandler() {
-    return _hasHandler;
-  }
-
-  /// @brief setter for the continue callback:
-  ///        We can either have a handler or a callback
-  void setContinueCallback(std::function<void()> const& cb) {
-    _continueCallback = cb;
-    _hasHandler = false;
-  }
-
-  /// @brief setter for the continue handler:
-  ///        We can either have a handler or a callback
-  void setContinueHandler(std::function<void()> const& handler) {
-    _continueCallback = handler;
-    _hasHandler = true;
   }
 
  private:
@@ -339,8 +304,6 @@ class Query {
   /// QueryRegistry.
   ExecutionPlan* preparePlan();
 
-  void setExecutionTime();
-
   /// @brief log a query
   void log();
 
@@ -392,7 +355,7 @@ class Query {
   V8Context* _context;
 
   /// @brief graphs used in query, identified by name
-  std::unordered_map<std::string, Graph*> _graphs;
+  std::unordered_map<std::string, std::unique_ptr<Graph>> _graphs;
 
   /// @brief the actual query string
   QueryString _queryString;
@@ -460,49 +423,19 @@ class Query {
   /// it needs to be run once before any V8-based function is called
   bool _preparedV8Context;
 
-  /// @brief a callback function which is used to implement continueAfterPause.
-  /// Typically, the RestHandler using the Query object will put a closure
-  /// in here, which continueAfterPause simply calls.
-  std::function<void()> _continueCallback;
-
-  /// @brief decide if the _continueCallback needs to be pushed onto the ioservice
-  ///        or if it has to be executed in this thread.
-  bool _hasHandler;
-
   /// Create the result in this builder. It is also used to determine
   /// if we are continuing the query or of we called
   std::shared_ptr<arangodb::velocypack::Builder> _resultBuilder;
 
-  /// Options for _resultBuilder. Optimally, it's lifetime should be linked to
+  /// Options for _resultBuilder. Optimally, its lifetime should be linked to
   /// it, but this is hard to do.
   std::shared_ptr<arangodb::velocypack::Options> _resultBuilderOptions;
 
   /// Track in which phase of execution we are, in order to implement repeatability.
   ExecutionPhase _executionPhase;
 
-  /// Temporary Section only used during the development of
+  /// @brief shared state 
-  /// async AQL
+  std::shared_ptr<SharedQueryState> _sharedState;
-  /// TODO REMOVE
- private:
-  basics::ConditionVariable _tempWaitForAsyncResponse;
-  bool _wasNotified = false;
-
- public:
-  void tempSignalAsyncResponse() {
-    CONDITION_LOCKER(guard, _tempWaitForAsyncResponse);
-    _wasNotified = true;
-    guard.signal();
-  }
-
-  /// TODO This has to stay for a backwards-compatible AQL HTTP API (hasMore).
-  /// So it needs to be renamed.
-  void tempWaitForAsyncResponse() {
-    CONDITION_LOCKER(guard, _tempWaitForAsyncResponse);
-    if (!_wasNotified) {
-      _tempWaitForAsyncResponse.wait();
-    }
-    _wasNotified = false;
-  }
 };
 
 }

arangod/Aql/QueryCursor.cpp

@@ -207,7 +207,8 @@ std::pair<ExecutionState, Result> QueryStreamCursor::dump(VPackBuilder& builder,
                                     << _query->queryString().extract(1024) << "'";
 
   // We will get a different RestHandler on every dump, so we need to update the Callback
-  _query->setContinueHandler(continueHandler);
+  std::shared_ptr<SharedQueryState> ss = _query->sharedState();
+  ss->setContinueHandler(continueHandler);
 
   try {
     ExecutionState state = prepareDump();
@@ -249,9 +250,9 @@ Result QueryStreamCursor::dumpSync(VPackBuilder& builder) {
   LOG_TOPIC(TRACE, Logger::QUERIES) << "executing query " << _id << ": '"
                                     << _query->queryString().extract(1024) << "'";
 
+  std::shared_ptr<SharedQueryState> ss = _query->sharedState();
   // We will get a different RestHandler on every dump, so we need to update the Callback
-  auto continueCallback = [&]() { _query->tempSignalAsyncResponse(); };
+  ss->setContinueCallback();
-  _query->setContinueCallback(continueCallback);
 
   try {
     aql::ExecutionEngine* engine = _query->engine();
@@ -264,7 +265,7 @@ Result QueryStreamCursor::dumpSync(VPackBuilder& builder) {
     while (state == ExecutionState::WAITING) {
       state = prepareDump();
       if (state == ExecutionState::WAITING) {
-        _query->tempWaitForAsyncResponse();
+        ss->waitForAsyncResponse();
       }
     }
 
@@ -350,11 +351,13 @@ Result QueryStreamCursor::writeResult(VPackBuilder &builder) {
     builder.add("cached", VPackValue(false));
 
     if (!hasMore) {
+      std::shared_ptr<SharedQueryState> ss = _query->sharedState();
+      ss->setContinueCallback(); 
+
       QueryResult result;
-      _query->setContinueCallback([&]() { _query->tempSignalAsyncResponse(); });
       ExecutionState state = _query->finalize(result); // will commit transaction
       while (state == ExecutionState::WAITING) {
-        _query->tempWaitForAsyncResponse();
+        ss->waitForAsyncResponse();
         state = _query->finalize(result);
       }
       if (result.extra && result.extra->slice().isObject()) {

arangod/Aql/QueryList.cpp

@@ -183,7 +183,7 @@ int QueryList::kill(TRI_voc_tick_t id) {
   Query* query = (*it).second;
   LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "killing AQL query " << id << " '" << query->queryString() << "'";
 
-  query->killed(true);
+  query->kill();
   return TRI_ERROR_NO_ERROR;
 }
   
@@ -202,7 +202,7 @@ uint64_t QueryList::killAll(bool silent) {
       LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "killing AQL query " << query->id() << " '" << query->queryString() << "'";
     }
     
-    query->killed(true);
+    query->kill();
     ++killed;
   }
 

arangod/Aql/QueryRegistry.cpp

@@ -191,7 +191,7 @@ void QueryRegistry::destroy(std::string const& vocbase, QueryId id,
 
     if (q->second->_isOpen) {
       // query in use by another thread/request
-      q->second->_query->killed(true);
+      q->second->_query->kill();
       return;
     }
 

arangod/Aql/RestAqlHandler.cpp

@@ -676,9 +676,11 @@ bool RestAqlHandler::findQuery(std::string const& idString, Query*& query) {
 RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
                                           VPackSlice const querySlice) {
   auto self = shared_from_this();
-  query->setContinueHandler([this, self]() {
+  std::shared_ptr<SharedQueryState> ss = query->sharedState();
+  ss->setContinueHandler([this, self, ss]() {
     continueHandlerExecution();
   });
+
   bool found;
   std::string const& shardId = _request->header("shard-id", found);
 

arangod/Aql/SharedQueryState.cpp

@@ -0,0 +1,96 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#include "SharedQueryState.h"
+#include "Scheduler/Scheduler.h"
+#include "Scheduler/SchedulerFeature.h"
+
+using namespace arangodb;
+using namespace arangodb::aql;
+
+void SharedQueryState::invalidate() {
+  CONDITION_LOCKER(guard, _condition);
+  _valid = false;
+}
+
+/// this has to stay for a backwards-compatible AQL HTTP API (hasMore).
+void SharedQueryState::waitForAsyncResponse() {
+  CONDITION_LOCKER(guard, _condition);
+  if (!_wasNotified) {
+    _condition.wait();
+  }
+  _wasNotified = false;
+}
+  
+/// @brief continueAfterPause is to be called on the query object to
+/// continue execution in this query part, if the query got paused
+/// because it is waiting for network responses. The idea is that a
+/// RemoteBlock that does an asynchronous cluster-internal request can
+/// register a callback with the asynchronous request and then return
+/// with the result `ExecutionState::WAITING`, which will bubble up
+/// the stack and eventually lead to a suspension of the work on the
+/// RestHandler. In the callback function one can first store the
+/// results in the RemoteBlock object and can then call this method on
+/// the query.
+/// This will lead to the following: The original request that led to
+/// the network communication will be rescheduled on the ioservice and
+/// continues its execution where it left off.
+
+bool SharedQueryState::execute(std::function<bool()> const& cb) {
+  CONDITION_LOCKER(guard, _condition);
+  if (!_valid) {
+    return false;
+  }
+
+  bool res = cb();
+
+  if (_hasHandler) {
+    auto scheduler = SchedulerFeature::SCHEDULER;
+    TRI_ASSERT(scheduler != nullptr);
+    if (scheduler == nullptr) {
+      // We are shutting down
+      return false;
+    }
+    scheduler->post(_continueCallback);
+  } else {
+    _wasNotified = true;
+    guard.signal();
+  }
+
+  return res;
+}
+
+/// @brief setter for the continue callback:
+///        We can either have a handler or a callback
+void SharedQueryState::setContinueCallback() {
+  CONDITION_LOCKER(guard, _condition);
+  _hasHandler = false;
+}
+
+/// @brief setter for the continue handler:
+///        We can either have a handler or a callback
+void SharedQueryState::setContinueHandler(std::function<void()> const& handler) {
+  CONDITION_LOCKER(guard, _condition);
+  _continueCallback = handler;
+  _hasHandler = true;
+}

arangod/Aql/SharedQueryState.h

@@ -0,0 +1,82 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGOD_AQL_SHARED_QUERY_STATE_H
+#define ARANGOD_AQL_SHARED_QUERY_STATE_H 1
+
+#include "Basics/Common.h"
+#include "Basics/ConditionLocker.h"
+#include "Basics/ConditionVariable.h"
+
+namespace arangodb {
+namespace aql {
+
+class SharedQueryState {
+ public:
+  SharedQueryState(SharedQueryState const&) = delete; 
+  SharedQueryState& operator=(SharedQueryState const&) = delete; 
+
+  SharedQueryState() 
+      : _wasNotified(false),
+        _hasHandler(false),
+        _valid(true) {
+  }
+
+  ~SharedQueryState() = default;
+
+  void invalidate();
+
+  bool execute(std::function<bool()> const& cb);
+  
+  /// this has to stay for a backwards-compatible AQL HTTP API (hasMore).
+  void waitForAsyncResponse();
+  
+  /// @brief setter for the continue callback:
+  ///        We can either have a handler or a callback
+  void setContinueCallback();
+
+  /// @brief setter for the continue handler:
+  ///        We can either have a handler or a callback
+  void setContinueHandler(std::function<void()> const& handler);
+
+ private:
+  basics::ConditionVariable _condition;
+  
+  /// @brief a callback function which is used to implement continueAfterPause.
+  /// Typically, the RestHandler using the Query object will put a closure
+  /// in here, which continueAfterPause simply calls.
+  std::function<void()> _continueCallback;
+  
+  bool _wasNotified;
+  
+  /// @brief decide if the _continueCallback needs to be pushed onto the ioservice
+  ///        or if it has to be executed in this thread.
+  bool _hasHandler;
+
+  bool _valid;
+};
+
+}
+}
+
+#endif

arangod/Aql/WakeupQueryCallback.cpp

@@ -23,31 +23,24 @@
 #include "WakeupQueryCallback.h"
 #include "Aql/ExecutionBlock.h"
 #include "Aql/Query.h"
-#include "Scheduler/SchedulerFeature.h"
 
 using namespace arangodb;
 using namespace arangodb::aql;
 
 WakeupQueryCallback::WakeupQueryCallback(ExecutionBlock* initiator,
                                          Query* query)
-    : _initiator(initiator), _query(query) {}
+    : _initiator(initiator), 
+      _query(query),
+      _sharedState(query->sharedState()) {}
+  
+WakeupQueryCallback::~WakeupQueryCallback() {}
 
 bool WakeupQueryCallback::operator()(ClusterCommResult* result) {
+  return _sharedState->execute([&, this]() {
     TRI_ASSERT(_initiator != nullptr);
     TRI_ASSERT(_query != nullptr);
     // TODO Validate that _initiator and _query have not been deleted (ttl)
     // TODO Handle exceptions
-  bool res = _initiator->handleAsyncResult(result);
+    return _initiator->handleAsyncResult(result);
-  if (_query->hasHandler()) {
+  });
-    auto scheduler = SchedulerFeature::SCHEDULER;
-    TRI_ASSERT(scheduler != nullptr);
-    if (scheduler == nullptr) {
-      // We are shutting down
-      return false;
-    }
-    scheduler->post(_query->continueHandler());
-  } else {
-    _query->continueAfterPause();
-  }
-  return res;
 }

arangod/Aql/WakeupQueryCallback.h

@@ -31,16 +31,18 @@ namespace aql {
 
 class ExecutionBlock;
 class Query;
+class SharedQueryState;
 
 struct WakeupQueryCallback : public ClusterCommCallback {
   WakeupQueryCallback(ExecutionBlock* initiator, Query* query);
-  ~WakeupQueryCallback() {};
+  ~WakeupQueryCallback();
 
   bool operator()(ClusterCommResult*) override;
 
  private:
   ExecutionBlock* _initiator;
   Query* _query;
+  std::shared_ptr<SharedQueryState> _sharedState;
 };
 
 } // aql

arangod/CMakeLists.txt

@@ -240,6 +240,7 @@ SET(ARANGOD_SOURCES
   Aql/RegexCache.cpp
   Aql/RestAqlHandler.cpp
   Aql/Scopes.cpp
+  Aql/SharedQueryState.cpp
   Aql/ShortStringStorage.cpp
   Aql/ShortestPathBlock.cpp
   Aql/ShortestPathNode.cpp

arangod/MMFiles/MMFilesPrimaryIndex.cpp

@@ -196,12 +196,11 @@ MMFilesPrimaryIndex::MMFilesPrimaryIndex(
                 {{arangodb::basics::AttributeName(StaticStrings::KeyString,
                                                   false)}}),
             /*unique*/ true , /*sparse*/ false) {
-  size_t indexBuckets = 1;
   auto physical =
     static_cast<arangodb::MMFilesCollection*>(collection.getPhysical());
 
   TRI_ASSERT(physical != nullptr);
-    indexBuckets = static_cast<size_t>(physical->indexBuckets());
+  size_t indexBuckets = static_cast<size_t>(physical->indexBuckets());
 
   if (collection.isAStub()) {
       // in order to reduce memory usage

arangod/Replication/DatabaseInitialSyncer.cpp

@@ -475,6 +475,11 @@ void DatabaseInitialSyncer::orderDumpChunk(std::shared_ptr<Syncer::JobSynchroniz
   
   using ::arangodb::basics::StringUtils::itoa;
       
+  if (isAborted()) {
+    sharedStatus->gotResponse(Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED), nullptr);
+    return;
+  }
+  
   std::string const typeString = (coll->type() == TRI_COL_TYPE_EDGE ? "edge" : "document");
     
   if (!_config.isChild()) {
@@ -715,7 +720,7 @@ Result DatabaseInitialSyncer::fetchCollectionDump(
       }
     }
     
-    if (checkMore) {
+    if (checkMore && !isAborted()) {
       // already fetch next batch in the background, by posting the
       // request to the scheduler, which can run it asynchronously
       
@@ -807,8 +812,11 @@ Result DatabaseInitialSyncer::fetchCollectionDump(
       return Result();
     }
 
-
     batch++;
+    
+    if (isAborted()) {
+      return Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED);
+    }
   }
 
   TRI_ASSERT(false);
@@ -1263,6 +1271,10 @@ Result DatabaseInitialSyncer::handleCollection(VPackSlice const& parameters,
       return res;
     }
     
+    if (isAborted()) {
+      return Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED);
+    }
+
     if (masterName == TRI_COL_NAME_USERS) {
       reloadUsers();
     }

arangod/Replication/Syncer.cpp

@@ -221,7 +221,7 @@ arangodb::Result applyCollectionDumpMarkerInternal(
 
 namespace arangodb {
 
-Syncer::JobSynchronizer::JobSynchronizer(std::shared_ptr<Syncer const> syncer) 
+Syncer::JobSynchronizer::JobSynchronizer(std::shared_ptr<Syncer const> const& syncer) 
     : _syncer(syncer), 
       _gotResponse(false),
       _jobsInFlight(0) {}

arangod/Replication/Syncer.h

@@ -63,7 +63,7 @@ class Syncer : public std::enable_shared_from_this<Syncer> {
     JobSynchronizer(JobSynchronizer const&) = delete;
     JobSynchronizer& operator=(JobSynchronizer const&) = delete;
 
-    explicit JobSynchronizer(std::shared_ptr<Syncer const> syncer); 
+    explicit JobSynchronizer(std::shared_ptr<Syncer const> const& syncer); 
     ~JobSynchronizer() = default;
 
     /// @brief will be called whenever a response for the job comes in

arangod/Replication/TailingSyncer.cpp

@@ -119,18 +119,9 @@ void TailingSyncer::setProgress(std::string const& msg) {
 }
 
 /// @brief abort all ongoing transactions
-void TailingSyncer::abortOngoingTransactions() {
+void TailingSyncer::abortOngoingTransactions() noexcept {
   try {
     // abort all running transactions
-    for (auto& it : _ongoingTransactions) {
-      auto trx = it.second;
-
-      if (trx != nullptr) {
-        trx->abort();
-        delete trx;
-      }
-    }
-
     _ongoingTransactions.clear();
   } catch (...) {
     // ignore errors here
@@ -364,7 +355,7 @@ Result TailingSyncer::processDocument(TRI_replication_operation_e type,
           std::string("unexpected transaction ") + StringUtils::itoa(tid));
     }
 
-    auto trx = (*it).second;
+    std::unique_ptr<ReplicationTransaction>& trx = (*it).second;
 
     if (trx == nullptr) {
       return Result(
@@ -455,14 +446,7 @@ Result TailingSyncer::startTransaction(VPackSlice const& slice) {
 
   if (it != _ongoingTransactions.end()) {
     // found a previous version of the same transaction - should not happen...
-    auto trx = (*it).second;
+    _ongoingTransactions.erase(it);
-
-    _ongoingTransactions.erase(tid);
-
-    if (trx != nullptr) {
-      // abort ongoing trx
-      delete trx;
-    }
   }
 
   TRI_ASSERT(tid > 0);
@@ -474,8 +458,7 @@ Result TailingSyncer::startTransaction(VPackSlice const& slice) {
   Result res = trx->begin();
 
   if (res.ok()) {
-    _ongoingTransactions[tid] = trx.get();
+    _ongoingTransactions[tid] = std::move(trx);
-    trx.release();
   }
 
   return res;
@@ -498,7 +481,7 @@ Result TailingSyncer::abortTransaction(VPackSlice const& slice) {
 
   auto it = _ongoingTransactions.find(tid);
 
-  if (it == _ongoingTransactions.end()) {
+  if (it == _ongoingTransactions.end() || (*it).second == nullptr) {
     // invalid state, no transaction was started.
     return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
   }
@@ -508,17 +491,8 @@ Result TailingSyncer::abortTransaction(VPackSlice const& slice) {
   LOG_TOPIC(TRACE, Logger::REPLICATION)
       << "aborting replication transaction " << tid;
 
-  auto trx = (*it).second;
+  _ongoingTransactions.erase(it);
-  _ongoingTransactions.erase(tid);
+  return Result();
-
-  if (trx != nullptr) {
-    Result res = trx->abort();
-    delete trx;
-
-    return res;
-  }
-
-  return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
 }
 
 /// @brief commits a transaction, based on the VelocyPack provided
@@ -538,7 +512,7 @@ Result TailingSyncer::commitTransaction(VPackSlice const& slice) {
 
   auto it = _ongoingTransactions.find(tid);
 
-  if (it == _ongoingTransactions.end()) {
+  if (it == _ongoingTransactions.end() || (*it).second == nullptr) {
     // invalid state, no transaction was started.
     return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
   }
@@ -548,19 +522,13 @@ Result TailingSyncer::commitTransaction(VPackSlice const& slice) {
   LOG_TOPIC(TRACE, Logger::REPLICATION)
       << "committing replication transaction " << tid;
 
-  auto trx = (*it).second;
+  std::unique_ptr<ReplicationTransaction>& trx = (*it).second;
-  _ongoingTransactions.erase(tid);
-
-  if (trx != nullptr) {
   Result res = trx->commit();
-    delete trx;
   
+  _ongoingTransactions.erase(it);
   return res;
 }
 
-  return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
-}
-
 /// @brief renames a collection, based on the VelocyPack provided
 Result TailingSyncer::renameCollection(VPackSlice const& slice) {
   if (!slice.isObject()) {
@@ -1575,28 +1543,15 @@ Result TailingSyncer::followMasterLog(TRI_voc_tick_t& fetchTick,
 
   setProgress(progress);
 
-  std::string body;
-
-  if (!_ongoingTransactions.empty()) {
   // stringify list of open transactions
-    body.append("[\"");
+  VPackBuilder builder;
-
+  builder.openArray();
-    bool first = true;
-
   for (auto& it : _ongoingTransactions) {
-      if (first) {
+    builder.add(VPackValue(StringUtils::itoa(it.first)));
-        first = false;
-      } else {
-        body.append("\",\"");
   }
+  builder.close();
   
-      body.append(StringUtils::itoa(it.first));
+  std::string body = builder.slice().toJson();
-    }
-
-    body.append("\"]");
-  } else {
-    body.append("[]");
-  }
 
   std::unique_ptr<SimpleHttpResult> response(_state.connection.client->request(
       rest::RequestType::PUT, url, body.c_str(), body.size()));

arangod/Replication/TailingSyncer.h

@@ -70,7 +70,7 @@ class TailingSyncer : public Syncer {
   void setProgress(std::string const&);
   
   /// @brief abort all ongoing transactions
-  void abortOngoingTransactions();
+  void abortOngoingTransactions() noexcept;
 
   /// @brief whether or not a collection should be excluded
   bool skipMarker(TRI_voc_tick_t, arangodb::velocypack::Slice const&);
@@ -176,7 +176,7 @@ class TailingSyncer : public Syncer {
   bool _ignoreDatabaseMarkers;
 
   /// @brief which transactions were open and need to be treated specially
-  std::unordered_map<TRI_voc_tid_t, ReplicationTransaction*>
+  std::unordered_map<TRI_voc_tid_t, std::unique_ptr<ReplicationTransaction>>
       _ongoingTransactions;
 
   /// @brief recycled builder for repeated document creation

arangod/RestHandler/RestCursorHandler.cpp

@@ -48,7 +48,6 @@ RestCursorHandler::RestCursorHandler(
       _query(nullptr),
       _queryResult(),
       _queryRegistry(queryRegistry),
-      _queryLock(),
       _hasStarted(false),
       _queryKilled(false),
       _isValidForFinalize(false) {}
@@ -165,11 +164,12 @@ bool RestCursorHandler::registerQueryOrCursor(VPackSlice const& slice) {
     arangodb::aql::PART_MAIN
   );
 
+  std::shared_ptr<aql::SharedQueryState> ss = query->sharedState();
   auto self = shared_from_this();
-  auto continueHandler = [this, self]() {
+  ss->setContinueHandler([this, self, ss] {
     continueHandlerExecution();
-  };
+  });
-  query->setContinueHandler(continueHandler);
+
   registerQuery(std::move(query));
   return true;
 }
@@ -341,7 +341,7 @@ bool RestCursorHandler::cancelQuery() {
   MUTEX_LOCKER(mutexLocker, _queryLock);
 
   if (_query != nullptr) {
-    _query->killed(true);
+    _query->kill();
     _queryKilled = true;
     _hasStarted = true;
     return true;

arangod/RestServer/AqlFeature.cpp

@@ -60,7 +60,7 @@ AqlFeature* AqlFeature::lease() {
   return aql;
 }
 
-void AqlFeature::unlease() {
+void AqlFeature::unlease() noexcept {
   MUTEX_LOCKER(locker, AqlFeature::_aqlFeatureMutex);
   AqlFeature* aql = AqlFeature::_AQL;
   TRI_ASSERT(aql != nullptr);

arangod/RestServer/AqlFeature.h

@@ -36,7 +36,7 @@ class AqlFeature final : public application_features::ApplicationFeature {
 
  public:
   static AqlFeature* lease();
-  static void unlease();
+  static void unlease() noexcept;
   void start() override final;
   void stop() override final;
  

arangod/RocksDBEngine/RocksDBCollection.cpp

@@ -77,9 +77,9 @@ using namespace arangodb;
 // back on later in its dtor 
 // this is just a performance optimization for small transactions
 struct IndexingDisabler {
-  IndexingDisabler(RocksDBMethods* mthd, bool disableIndexing) 
+  IndexingDisabler(RocksDBMethods* mthd, bool disable) 
-      : mthd(mthd), disableIndexing(disableIndexing) {
+      : mthd(mthd), disableIndexing(disable) {
-    if (disableIndexing) {
+    if (disable) {
       disableIndexing = mthd->DisableIndexing();
     }
   }

arangod/RocksDBEngine/RocksDBIncrementalSync.cpp

@@ -27,7 +27,9 @@
 #include "Replication/DatabaseInitialSyncer.h"
 #include "Replication/utilities.h"
 #include "RocksDBEngine/RocksDBCollection.h"
-#include "RocksDBIterators.h"
+#include "RocksDBEngine/RocksDBIterators.h"
+#include "RocksDBEngine/RocksDBKey.h"
+#include "RocksDBEngine/RocksDBPrimaryIndex.h"
 #include "SimpleHttpClient/SimpleHttpClient.h"
 #include "SimpleHttpClient/SimpleHttpResult.h"
 #include "StorageEngine/PhysicalCollection.h"
@@ -44,6 +46,107 @@
 #include <velocypack/velocypack-aliases.h>
 
 namespace arangodb {
+
+// remove all keys that are below first remote key or beyond last remote key
+Result removeKeysOutsideRange(VPackSlice chunkSlice, 
+                              LogicalCollection* col,
+                              OperationOptions& options,
+                              InitialSyncerIncrementalSyncStats& stats) {
+  size_t const numChunks = chunkSlice.length();
+
+  if (numChunks == 0) {
+    // no need to do anything
+    return Result();
+  }
+  
+  // first chunk
+  SingleCollectionTransaction trx(
+    transaction::StandaloneContext::Create(col->vocbase()),
+    *col,
+    AccessMode::Type::EXCLUSIVE
+  );
+
+  trx.addHint(
+      transaction::Hints::Hint::RECOVERY);  // to turn off waitForSync!
+  trx.addHint(transaction::Hints::Hint::NO_TRACKING);
+  trx.addHint(transaction::Hints::Hint::NO_INDEXING);
+
+  Result res = trx.begin();
+
+  if (!res.ok()) {
+    return Result(
+        res.errorNumber(),
+        std::string("unable to start transaction: ") + res.errorMessage());
+  }
+
+  VPackSlice chunk = chunkSlice.at(0);
+
+  TRI_ASSERT(chunk.isObject());
+  auto lowSlice = chunk.get("low");
+  TRI_ASSERT(lowSlice.isString());
+  StringRef lowRef(lowSlice);
+
+  // last high
+  chunk = chunkSlice.at(numChunks - 1);
+  TRI_ASSERT(chunk.isObject());
+
+  auto highSlice = chunk.get("high");
+  TRI_ASSERT(highSlice.isString());
+  StringRef highRef(highSlice);
+
+  LogicalCollection* coll = trx.documentCollection();
+  auto iterator = createPrimaryIndexIterator(&trx, coll);
+
+  VPackBuilder builder;
+
+  // remove everything from the beginning of the key range until the lowest
+  // remote key
+  iterator.next(
+      [&](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
+        StringRef docKey(RocksDBKey::primaryKey(rocksKey));
+        if (docKey.compare(lowRef) < 0) {
+          builder.clear();
+          builder.add(velocypack::ValuePair(docKey.data(), docKey.size(),
+                                            velocypack::ValueType::String));
+          trx.remove(col->name(), builder.slice(), options);
+          ++stats.numDocsRemoved;
+          // continue iteration
+          return true;
+        }
+
+        // stop iteration
+        return false;
+      },
+      std::numeric_limits<std::uint64_t>::max());
+  
+  // remove everything from the highest remote key until the end of the key range
+  auto index = col->lookupIndex(0); //RocksDBCollection->primaryIndex() is private
+  TRI_ASSERT(index->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX);
+  auto primaryIndex = static_cast<RocksDBPrimaryIndex*>(index.get());
+
+  RocksDBKeyLeaser key(&trx);
+  key->constructPrimaryIndexValue(primaryIndex->objectId(), highRef);
+  iterator.seek(key->string());
+
+  iterator.next(
+      [&](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
+        StringRef docKey(RocksDBKey::primaryKey(rocksKey));
+        if (docKey.compare(highRef) > 0) {
+          builder.clear();
+          builder.add(velocypack::ValuePair(docKey.data(), docKey.size(),
+                                            velocypack::ValueType::String));
+          trx.remove(col->name(), builder.slice(), options);
+          ++stats.numDocsRemoved;
+        }
+
+        // continue iteration until end
+        return true;
+      },
+      std::numeric_limits<std::uint64_t>::max());
+
+  return trx.commit();
+}
+
 Result syncChunkRocksDB(
     DatabaseInitialSyncer& syncer, SingleCollectionTransaction* trx,
     InitialSyncerIncrementalSyncStats& stats,
@@ -505,7 +608,6 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
 
   ManagedDocumentResult mmdr;
   OperationOptions options;
-
   options.silent = true;
   options.ignoreRevs = true;
   options.isRestore = true;
@@ -514,70 +616,17 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
     options.isSynchronousReplicationFrom = syncer._state.leaderId;
   }
 
-  size_t const numChunks = static_cast<size_t>(chunkSlice.length());
+  {
-
     // remove all keys that are below first remote key or beyond last remote key
-  if (numChunks > 0) {
+    Result res = removeKeysOutsideRange(chunkSlice, col, options, stats);
-    // first chunk
-    SingleCollectionTransaction trx(
-      transaction::StandaloneContext::Create(syncer.vocbase()),
-      *col,
-      AccessMode::Type::EXCLUSIVE
-    );
 
-    trx.addHint(
+    if (res.fail()) {
-        transaction::Hints::Hint::RECOVERY);  // to turn off waitForSync!
-    trx.addHint(transaction::Hints::Hint::NO_TRACKING);
-    trx.addHint(transaction::Hints::Hint::NO_INDEXING);
-
-    Result res = trx.begin();
-
-    if (!res.ok()) {
-      return Result(
-          res.errorNumber(),
-          std::string("unable to start transaction: ") + res.errorMessage());
-    }
-
-    VPackSlice chunk = chunkSlice.at(0);
-
-    TRI_ASSERT(chunk.isObject());
-    auto lowSlice = chunk.get("low");
-    TRI_ASSERT(lowSlice.isString());
-
-    // last high
-    chunk = chunkSlice.at(numChunks - 1);
-    TRI_ASSERT(chunk.isObject());
-
-    auto highSlice = chunk.get("high");
-    TRI_ASSERT(highSlice.isString());
-
-    StringRef lowRef(lowSlice);
-    StringRef highRef(highSlice);
-
-    LogicalCollection* coll = trx.documentCollection();
-    auto iterator = createPrimaryIndexIterator(&trx, coll);
-
-    VPackBuilder builder;
-    iterator.next(
-        [&](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
-          StringRef docKey(RocksDBKey::primaryKey(rocksKey));
-          if (docKey.compare(lowRef) < 0 || docKey.compare(highRef) > 0) {
-            builder.clear();
-            builder.add(velocypack::ValuePair(docKey.data(), docKey.size(),
-                                              velocypack::ValueType::String));
-            trx.remove(col->name(), builder.slice(), options);
-            ++stats.numDocsRemoved;
-          }
-        },
-        std::numeric_limits<std::uint64_t>::max());
-
-    res = trx.commit();
-
-    if (!res.ok()) {
       return res;
     }
   }
 
+  size_t const numChunks = static_cast<size_t>(chunkSlice.length());
+
   {
     if (syncer.isAborted()) {
       return Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED);
@@ -749,12 +798,13 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
                 rocksValue);  // we want probably to do this instead
             if (col->readDocument(&trx, documentId, mmdr) == false) {
               TRI_ASSERT(false);
-              return;
+              return true;
             }
             VPackSlice doc(mmdr.vpack());
             docRev = TRI_ExtractRevisionId(doc);
           }
           compareChunk(docKey, docRev);
+          return true;
         },
         std::numeric_limits<std::uint64_t>::max());  // no limit on documents
 

arangod/RocksDBEngine/RocksDBIterators.cpp

@@ -333,9 +333,9 @@ void RocksDBSortedAllIterator::reset() {
   _iterator->Seek(_bounds.start());
 }
 
-RocksDBGenericIterator::RocksDBGenericIterator(rocksdb::ReadOptions& options
+RocksDBGenericIterator::RocksDBGenericIterator(rocksdb::ReadOptions& options,
-                                              ,RocksDBKeyBounds const& bounds
+                                               RocksDBKeyBounds const& bounds,
-                                              ,bool reverse)
+                                               bool reverse)
     : _reverse(reverse)
     , _bounds(bounds)
     , _options(options)
@@ -365,11 +365,15 @@ bool RocksDBGenericIterator::reset() {
 }
 
 bool RocksDBGenericIterator::skip(uint64_t count, uint64_t& skipped) {
-  bool has_more = _iterator->Valid();
+  bool hasMore = _iterator->Valid();
-  while (count > 0 && has_more) {
+  while (count > 0 && hasMore) {
-    has_more = next([&count,&skipped](rocksdb::Slice const&, rocksdb::Slice const&){ --count; ++skipped; }, count /*gets copied*/);
+    hasMore = next([&count, &skipped](rocksdb::Slice const&, rocksdb::Slice const&) { 
+      --count; 
+      ++skipped;
+      return true;
+    }, count /*gets copied*/);
   }
-  return has_more;
+  return hasMore;
 }
 
 bool RocksDBGenericIterator::seek(rocksdb::Slice const& key) {
@@ -397,14 +401,16 @@ bool RocksDBGenericIterator::next(GenericCallback const& cb, size_t limit) {
     TRI_ASSERT(_bounds.objectId() == RocksDBKey::objectId(_iterator->key()));
 #endif
 
-    cb(_iterator->key(),_iterator->value());
+    if (!cb(_iterator->key(),_iterator->value())) {
+      // stop iteration
+      return false;
+    }
     --limit;
     if (_reverse) {
       _iterator->Prev();
     } else {
       _iterator->Next();
     }
-
   }
 
   return hasMore();

arangod/RocksDBEngine/RocksDBIterators.h

@@ -41,7 +41,8 @@ namespace arangodb {
 class RocksDBCollection;
 class RocksDBPrimaryIndex;
 
-typedef std::function<void(rocksdb::Slice const& key, rocksdb::Slice const& value)> GenericCallback;
+/// @brief return false to stop iteration
+typedef std::function<bool(rocksdb::Slice const& key, rocksdb::Slice const& value)> GenericCallback;
 
 /// @brief iterator over all documents in the collection
 /// basically sorted after LocalDocumentId
@@ -134,13 +135,10 @@ class RocksDBGenericIterator {
 
   // the following functions return if the iterator
   // is valid and in bounds on return.
-  bool next(GenericCallback const& cb // void(rocksdb::Slice const& key,rocksd:Slice const& value)
+  bool next(GenericCallback const& cb, size_t count); //number of documents the callback should be applied to
-           , size_t count //number of documents the callback should be applied to
-           );
 
-  bool skip(uint64_t count // documents to skip
+  // documents to skip, skipped documents
-           ,uint64_t& skipped // skipped documents
+  bool skip(uint64_t count, uint64_t& skipped);
-           );
   bool seek(rocksdb::Slice const& key);
   bool reset();
   bool hasMore() const;

arangod/RocksDBEngine/RocksDBReplicationContext.cpp

@@ -295,6 +295,7 @@ RocksDBReplicationResult RocksDBReplicationContext::dumpJson(
     VPackDumper dumper(&adapter, &collectionIter->vpackOptions);
     dumper.dump(velocypack::Slice(rocksValue.data()));
     buff.appendText("}\n");
+    return true;
   };
 
   TRI_ASSERT(collectionIter->iter && !collectionIter->sorted());
@@ -350,12 +351,12 @@ RocksDBReplicationResult RocksDBReplicationContext::dumpVPack(TRI_vocbase_t* voc
 
   VPackBuilder builder(buffer, &collectionIter->vpackOptions);
   auto cb = [&builder](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
-
     builder.openObject();
     builder.add("type", VPackValue(REPLICATION_MARKER_DOCUMENT));
     builder.add(VPackValue("data"));
     builder.add(velocypack::Slice(rocksValue.data()));
     builder.close();
+    return true;
   };
 
   TRI_ASSERT(collectionIter->iter && !collectionIter->sorted());
@@ -413,7 +414,7 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b,
       auto documentId = RocksDBValue::documentId(rocksValue); // we want probably to do this instead
       if (_collection->logical.readDocument(_trx.get(), documentId, _collection->mdr) == false) {
         TRI_ASSERT(false);
-        return;
+        return true;
       }
       VPackSlice doc(_collection->mdr.vpack());
       docRev = TRI_ExtractRevisionId(doc);
@@ -433,6 +434,8 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b,
     builder.clear();
     builder.add(TRI_RidToValuePair(docRev, &ridBuffer[0]));
     hash ^= builder.slice().hashString();
+
+    return true;
   }; //cb
 
   b.openArray(true);
@@ -532,7 +535,7 @@ arangodb::Result RocksDBReplicationContext::dumpKeys(
       auto documentId = RocksDBValue::documentId(rocksValue); // we want probably to do this instead
       if (_collection->logical.readDocument(_trx.get(), documentId, _collection->mdr) == false) {
         TRI_ASSERT(false);
-        return;
+        return true;
       }
       VPackSlice doc(_collection->mdr.vpack());
       docRev = TRI_ExtractRevisionId(doc);
@@ -543,6 +546,8 @@ arangodb::Result RocksDBReplicationContext::dumpKeys(
     b.add(velocypack::ValuePair(docKey.data(), docKey.size(), velocypack::ValueType::String));
     b.add(TRI_RidToValuePair(docRev, &ridBuffer[0]));
     b.close();
+
+    return true;
   };
 
   b.openArray(true);
@@ -619,11 +624,12 @@ arangodb::Result RocksDBReplicationContext::dumpDocuments(
     bool ok = _collection->logical.readDocument(_trx.get(), documentId, _collection->mdr);
     if (!ok) {
       // TODO: do something here?
-      return;
+      return true;
     }
     VPackSlice current(_collection->mdr.vpack());
     TRI_ASSERT(current.isObject());
     b.add(current);
+    return true;
   };
 
   auto buffer = b.buffer();
@@ -657,6 +663,7 @@ arangodb::Result RocksDBReplicationContext::dumpDocuments(
       hasMore = _collection->iter->next(
           [&b](rocksdb::Slice const&, rocksdb::Slice const&) {
             b.add(VPackValue(VPackValueType::Null));
+            return true;
           },
           1);
     } else {

arangod/V8Server/v8-query.cpp

@@ -30,9 +30,9 @@
 #include "Indexes/Index.h"
 #include "StorageEngine/PhysicalCollection.h"
 #include "Transaction/Helpers.h"
+#include "Transaction/V8Context.h"
 #include "Utils/OperationCursor.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Transaction/V8Context.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
@@ -71,8 +71,10 @@ aql::QueryResultV8 AqlQuery(
     arangodb::aql::PART_MAIN
   );
 
+  std::shared_ptr<arangodb::aql::SharedQueryState> ss = query.sharedState();
+  ss->setContinueCallback(); 
+
   aql::QueryResultV8 queryResult;
-  query.setContinueCallback([&query]() { query.tempSignalAsyncResponse(); });
   while (true) {
     auto state = query.executeV8(isolate,
         static_cast<arangodb::aql::QueryRegistry*>(v8g->_queryRegistry),
@@ -80,7 +82,7 @@ aql::QueryResultV8 AqlQuery(
     if (state != aql::ExecutionState::WAITING) {
       break;
     }
-    query.tempWaitForAsyncResponse();
+    ss->waitForAsyncResponse();
   }
 
   if (queryResult.code != TRI_ERROR_NO_ERROR) {

arangod/V8Server/v8-vocbase.cpp

@@ -836,14 +836,17 @@ static void JS_ExecuteAql(v8::FunctionCallbackInfo<v8::Value> const& args) {
     options,
     arangodb::aql::PART_MAIN
   );
+
+  std::shared_ptr<arangodb::aql::SharedQueryState> ss = query.sharedState();
+  ss->setContinueCallback();
+  
   aql::QueryResultV8 queryResult;
-  query.setContinueCallback([&query]() { query.tempSignalAsyncResponse(); });
   while (true) {
     auto state = query.executeV8(isolate, static_cast<arangodb::aql::QueryRegistry*>(v8g->_queryRegistry), queryResult);
     if (state != aql::ExecutionState::WAITING) {
       break;
     }
-    query.tempWaitForAsyncResponse();
+    ss->waitForAsyncResponse();
   }
 
   if (queryResult.code != TRI_ERROR_NO_ERROR) {

js/client/modules/@arangodb/process-utils.js

@@ -831,7 +831,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
       requestOptions.method = 'PUT';
 
       print(coords[0].url + "/_admin/cluster/maintenance");
-      download(coords[0].url + "/_admin/cluster/maintenance", JSON.stringify({ body: "on" }), requestOptions);
+      download(coords[0].url + "/_admin/cluster/maintenance", JSON.stringify("on"), requestOptions);
     }
   } catch (err) {
     print("error while setting cluster maintenance mode:", err);

tests/IResearch/common.cpp

@@ -188,11 +188,13 @@ arangodb::aql::QueryResult executeQuery(
     arangodb::aql::PART_MAIN
   );
 
+  std::shared_ptr<arangodb::aql::SharedQueryState> ss = query.sharedState();
+
   arangodb::aql::QueryResult result;
   while (true) {
     auto state = query.execute(arangodb::QueryRegistryFeature::QUERY_REGISTRY, result);
     if (state == arangodb::aql::ExecutionState::WAITING) {
-      query.tempWaitForAsyncResponse();
+      ss->waitForAsyncResponse();
     } else {
       break;
     }