Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bug fix/3007 (#6019)

This commit is contained in:
Jan 2018-07-30 17:16:47 +02:00 committed by GitHub
parent 74a2ed9b44
commit 2a416f2e33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 515 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
arangod/Aql/EngineInfoContainerCoordinator.cpp
View File

@ -152,12 +152,12 @@ ExecutionEngineResult EngineInfoContainerCoordinator::buildEngines(
TRI_ASSERT(_engineStack.top() == 0);
std::vector<uint64_t> coordinatorQueryIds{};
auto cleanup = [&]() {
// destroy all query snippets in case of error
auto guard = scopeGuard([&dbname, &registry, &coordinatorQueryIds]() {
for (auto const& it : coordinatorQueryIds) {
registry->destroy(dbname, it, TRI_ERROR_INTERNAL);
}
};
TRI_DEFER(cleanup());
});
Query* localQuery = query;
try {
@ -166,9 +166,7 @@ ExecutionEngineResult EngineInfoContainerCoordinator::buildEngines(
if (!first) {
// need a new query instance on the coordinator
localQuery = query->clone(PART_DEPENDENT, false);
if (localQuery == nullptr) {
return ExecutionEngineResult(TRI_ERROR_INTERNAL, "cannot clone query");
}
TRI_ASSERT(localQuery != nullptr);
}
try {
auto res = info.buildEngine(localQuery, registry, dbname,
@ -205,6 +203,6 @@ ExecutionEngineResult EngineInfoContainerCoordinator::buildEngines(
// This deactivates the defered cleanup.
// From here on we rely on the AQL shutdown mechanism.
coordinatorQueryIds.clear();
guard.cancel();
return ExecutionEngineResult(query->engine());
}

6
arangod/Aql/ExecutionEngine.cpp
View File

@ -484,11 +484,13 @@ Result ExecutionEngine::shutdownSync(int errorCode) noexcept {
Result res{TRI_ERROR_INTERNAL};
ExecutionState state = ExecutionState::WAITING;
try {
_query->setContinueCallback([&]() { _query->tempSignalAsyncResponse(); });
std::shared_ptr<SharedQueryState> sharedState = _query->sharedState();
sharedState->setContinueCallback();
while (state == ExecutionState::WAITING) {
std::tie(state, res) = shutdown(errorCode);
if (state == ExecutionState::WAITING) {
_query->tempWaitForAsyncResponse();
sharedState->waitForAsyncResponse();
}
}
} catch (...) {

116
arangod/Aql/Query.cpp
View File

@ -95,14 +95,8 @@ Query::Query(
_killed(false),
_isModificationQuery(false),
_preparedV8Context(false),
_hasHandler(false),
_executionPhase(ExecutionPhase::INITIALIZE) {
AqlFeature* aql = AqlFeature::lease();
if (aql == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
}
_executionPhase(ExecutionPhase::INITIALIZE),
_sharedState(std::make_shared<SharedQueryState>()) {
if (_contextOwnedByExterior) {
// copy transaction options from global state into our local query options
TransactionState* state = transaction::V8Context::getParentState();
@ -153,6 +147,12 @@ Query::Query(
}
_resourceMonitor.setMemoryLimit(_queryOptions.memoryLimit);
AqlFeature* aql = AqlFeature::lease();
if (aql == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
}
}
/// @brief creates a query from VelocyPack
@ -181,14 +181,9 @@ Query::Query(
_killed(false),
_isModificationQuery(false),
_preparedV8Context(false),
_hasHandler(false),
_executionPhase(ExecutionPhase::INITIALIZE) {
AqlFeature* aql = AqlFeature::lease();
if (aql == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
}
_executionPhase(ExecutionPhase::INITIALIZE),
_sharedState(std::make_shared<SharedQueryState>()) {
// populate query options
if (_options != nullptr) {
_queryOptions.fromVelocyPack(_options->slice());
@ -210,6 +205,12 @@ Query::Query(
}
_resourceMonitor.setMemoryLimit(_queryOptions.memoryLimit);
AqlFeature* aql = AqlFeature::lease();
if (aql == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN);
}
}
/// @brief destroys a query
@ -225,10 +226,8 @@ Query::~Query() {
exitContext();
_ast.reset();
_graphs.clear();
for (auto& it : _graphs) {
delete it.second;
}
LOG_TOPIC(DEBUG, Logger::QUERIES)
<< TRI_microtime() - _startTime << " "
<< "Query::~Query this: " << (uintptr_t) this;
@ -284,6 +283,16 @@ Query* Query::clone(QueryPart part, bool withPlan) {
return clone.release();
}
/// @brief whether or not the query is killed
bool Query::killed() const {
return _killed;
}
/// @brief set the query to killed
void Query::kill() {
_killed = true;
}
void Query::setExecutionTime() {
if (_engine != nullptr) {
_engine->_stats.setExecutionTime(TRI_microtime() - _startTime);
@ -745,15 +754,17 @@ ExecutionState Query::execute(QueryRegistry* registry, QueryResult& queryResult)
* @return The result of this query. The result is always complete
*/
QueryResult Query::executeSync(QueryRegistry* registry) {
std::shared_ptr<SharedQueryState> ss = sharedState();
ss->setContinueCallback();
QueryResult queryResult;
setContinueCallback([&]() { tempSignalAsyncResponse(); });
while(true) {
auto state = execute(registry, queryResult);
if (state != aql::ExecutionState::WAITING) {
TRI_ASSERT(state == aql::ExecutionState::DONE);
return queryResult;
}
tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
}
}
@ -764,7 +775,9 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
<< " this: " << (uintptr_t) this;
TRI_ASSERT(registry != nullptr);
setContinueCallback([&]() { tempSignalAsyncResponse(); });
std::shared_ptr<SharedQueryState> ss = sharedState();
ss->setContinueCallback();
try {
bool useQueryCache = canUseQueryCache();
uint64_t queryHash = hash();
@ -781,7 +794,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
ExecContext const* exe = ExecContext::CURRENT;
// got a result from the query cache
if(exe != nullptr) {
if (exe != nullptr) {
for (std::string const& collectionName : cacheEntry->_collections) {
if (!exe->canUseCollection(collectionName, auth::Level::RO)) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_FORBIDDEN);
@ -821,6 +834,8 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
std::unique_ptr<AqlItemBlock> value;
try {
std::shared_ptr<SharedQueryState> ss = sharedState();
if (useQueryCache) {
VPackOptions options = VPackOptions::Defaults;
options.buildUnindexedArrays = true;
@ -837,7 +852,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
state = res.first;
// TODO MAX: We need to let the thread sleep here instead of while loop
while (state == ExecutionState::WAITING) {
tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
res = _engine->getSome(ExecutionBlock::DefaultBatchSize());
state = res.first;
}
@ -850,7 +865,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
size_t const n = value->size();
for (size_t i = 0; i < n; ++i) {
AqlValue const &val = value->getValueReference(i, resultRegister);
AqlValue const& val = value->getValueReference(i, resultRegister);
if (!val.isEmpty()) {
resArray->Set(j++, val.toV8(isolate, _trx));
@ -882,7 +897,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
state = res.first;
// TODO MAX: We need to let the thread sleep here instead of while loop
while (state == ExecutionState::WAITING) {
tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
res = _engine->getSome(ExecutionBlock::DefaultBatchSize());
state = res.first;
}
@ -926,7 +941,7 @@ ExecutionState Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry, Q
// will set warnings, stats, profile and cleanup plan and engine
ExecutionState state = finalize(queryResult);
while (state == ExecutionState::WAITING) {
tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
state = finalize(queryResult);
}
} catch (arangodb::basics::Exception const& ex) {
@ -1147,10 +1162,6 @@ void Query::setEngine(ExecutionEngine* engine) {
_engine.reset(engine);
}
void Query::releaseEngine() {
_engine.release();
}
/// @brief prepare a V8 context for execution for this expression
/// this needs to be called once before executing any V8 function in this
/// expression
@ -1373,10 +1384,12 @@ void Query::enterState(QueryExecutionState::ValueType state) {
void Query::cleanupPlanAndEngineSync(int errorCode, VPackBuilder* statsBuilder) noexcept {
try {
setContinueCallback([&]() { tempSignalAsyncResponse(); });
std::shared_ptr<SharedQueryState> ss = sharedState();
ss->setContinueCallback();
ExecutionState state = cleanupPlanAndEngine(errorCode, statsBuilder);
while (state == ExecutionState::WAITING) {
tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
state = cleanupPlanAndEngine(errorCode, statsBuilder);
}
} catch (...) {
@ -1404,11 +1417,11 @@ ExecutionState Query::cleanupPlanAndEngine(int errorCode, VPackBuilder* statsBui
_engine.reset();
}
if (_trx != nullptr) {
// If the transaction was not committed, it is automatically aborted
delete _trx;
_trx = nullptr;
}
_sharedState->invalidate();
// If the transaction was not committed, it is automatically aborted
delete _trx;
_trx = nullptr;
_plan.reset();
return ExecutionState::DONE;
@ -1432,20 +1445,21 @@ std::shared_ptr<transaction::Context> Query::createTransactionContext() {
Graph const* Query::lookupGraphByName(std::string const& name) {
auto it = _graphs.find(name);
if (it != _graphs.end()) {
return it->second;
if (it == _graphs.end()) {
std::unique_ptr<arangodb::aql::Graph> g(
arangodb::lookupGraphByName(createTransactionContext(), name));
if (g == nullptr) {
return nullptr;
}
auto result = _graphs.emplace(name, std::move(g));
TRI_ASSERT(result.second);
it = result.first;
}
std::unique_ptr<arangodb::aql::Graph> g(
arangodb::lookupGraphByName(createTransactionContext(), name));
if (g == nullptr) {
return nullptr;
}
_graphs.emplace(name, g.get());
return g.release();
TRI_ASSERT((*it).second != nullptr);
return (*it).second.get();
}
/// @brief returns the next query id

99
arangod/Aql/Query.h
View File

@ -36,6 +36,7 @@
#include "Aql/QueryString.h"
#include "Aql/RegexCache.h"
#include "Aql/ResourceUsage.h"
#include "Aql/SharedQueryState.h"
#include "Aql/types.h"
#include "Basics/ConditionLocker.h"
#include "Basics/ConditionVariable.h"
@ -108,6 +109,14 @@ class Query {
TEST_VIRTUAL Query* clone(QueryPart, bool);
public:
/// @brief whether or not the query is killed
bool killed() const;
/// @brief set the query to killed
void kill();
void setExecutionTime();
QueryString const& queryString() const { return _queryString; }
@ -143,12 +152,6 @@ class Query {
/// @brief return the current runtime of the query
double runTime() const { return runTime(TRI_microtime()); }
/// @brief whether or not the query is killed
inline bool killed() const { return _killed; }
/// @brief set the query to killed
inline void killed(bool) { _killed = true; }
/// @brief the part of the query
inline QueryPart part() const { return _part; }
@ -232,8 +235,6 @@ class Query {
/// @brief inject the engine
TEST_VIRTUAL void setEngine(ExecutionEngine* engine);
void releaseEngine();
/// @brief return the transaction, if prepared
TEST_VIRTUAL inline transaction::Methods* trx() { return _trx; }
@ -288,45 +289,9 @@ class Query {
QueryExecutionState::ValueType state() const { return _state; }
/// @brief continueAfterPause is to be called on the query object to
/// continue execution in this query part, if the query got paused
/// because it is waiting for network responses. The idea is that a
/// RemoteBlock that does an asynchronous cluster-internal request can
/// register a callback with the asynchronous request and then return
/// with the result `ExecutionState::WAITING`, which will bubble up
/// the stack and eventually lead to a suspension of the work on the
/// RestHandler. In the callback function one can first store the
/// results in the RemoteBlock object and can then call this method on
/// the query.
/// This will lead to the following: The original request that lead to
/// the network communication will be rescheduled on the ioservice and
/// continues its execution where it left off.
void continueAfterPause() {
TRI_ASSERT(!hasHandler());
_continueCallback();
}
std::function<void()> continueHandler() {
TRI_ASSERT(hasHandler());
return _continueCallback;
}
bool hasHandler() {
return _hasHandler;
}
/// @brief setter for the continue callback:
/// We can either have a handler or a callback
void setContinueCallback(std::function<void()> const& cb) {
_continueCallback = cb;
_hasHandler = false;
}
/// @brief setter for the continue handler:
/// We can either have a handler or a callback
void setContinueHandler(std::function<void()> const& handler) {
_continueCallback = handler;
_hasHandler = true;
/// @brief return the query's shared state
std::shared_ptr<SharedQueryState> sharedState() const {
return _sharedState;
}
private:
@ -339,8 +304,6 @@ class Query {
/// QueryRegistry.
ExecutionPlan* preparePlan();
void setExecutionTime();
/// @brief log a query
void log();
@ -392,7 +355,7 @@ class Query {
V8Context* _context;
/// @brief graphs used in query, identified by name
std::unordered_map<std::string, Graph*> _graphs;
std::unordered_map<std::string, std::unique_ptr<Graph>> _graphs;
/// @brief the actual query string
QueryString _queryString;
@ -460,49 +423,19 @@ class Query {
/// it needs to be run once before any V8-based function is called
bool _preparedV8Context;
/// @brief a callback function which is used to implement continueAfterPause.
/// Typically, the RestHandler using the Query object will put a closure
/// in here, which continueAfterPause simply calls.
std::function<void()> _continueCallback;
/// @brief decide if the _continueCallback needs to be pushed onto the ioservice
/// or if it has to be executed in this thread.
bool _hasHandler;
/// Create the result in this builder. It is also used to determine
/// if we are continuing the query or of we called
std::shared_ptr<arangodb::velocypack::Builder> _resultBuilder;
/// Options for _resultBuilder. Optimally, it's lifetime should be linked to
/// Options for _resultBuilder. Optimally, its lifetime should be linked to
/// it, but this is hard to do.
std::shared_ptr<arangodb::velocypack::Options> _resultBuilderOptions;
/// Track in which phase of execution we are, in order to implement repeatability.
ExecutionPhase _executionPhase;
/// Temporary Section only used during the development of
/// async AQL
/// TODO REMOVE
private:
basics::ConditionVariable _tempWaitForAsyncResponse;
bool _wasNotified = false;
public:
void tempSignalAsyncResponse() {
CONDITION_LOCKER(guard, _tempWaitForAsyncResponse);
_wasNotified = true;
guard.signal();
}
/// TODO This has to stay for a backwards-compatible AQL HTTP API (hasMore).
/// So it needs to be renamed.
void tempWaitForAsyncResponse() {
CONDITION_LOCKER(guard, _tempWaitForAsyncResponse);
if (!_wasNotified) {
_tempWaitForAsyncResponse.wait();
}
_wasNotified = false;
}
/// @brief shared state
std::shared_ptr<SharedQueryState> _sharedState;
};
}

15
arangod/Aql/QueryCursor.cpp
View File

@ -207,7 +207,8 @@ std::pair<ExecutionState, Result> QueryStreamCursor::dump(VPackBuilder& builder,
<< _query->queryString().extract(1024) << "'";
// We will get a different RestHandler on every dump, so we need to update the Callback
_query->setContinueHandler(continueHandler);
std::shared_ptr<SharedQueryState> ss = _query->sharedState();
ss->setContinueHandler(continueHandler);
try {
ExecutionState state = prepareDump();
@ -249,9 +250,9 @@ Result QueryStreamCursor::dumpSync(VPackBuilder& builder) {
LOG_TOPIC(TRACE, Logger::QUERIES) << "executing query " << _id << ": '"
<< _query->queryString().extract(1024) << "'";
std::shared_ptr<SharedQueryState> ss = _query->sharedState();
// We will get a different RestHandler on every dump, so we need to update the Callback
auto continueCallback = [&]() { _query->tempSignalAsyncResponse(); };
_query->setContinueCallback(continueCallback);
ss->setContinueCallback();
try {
aql::ExecutionEngine* engine = _query->engine();
@ -264,7 +265,7 @@ Result QueryStreamCursor::dumpSync(VPackBuilder& builder) {
while (state == ExecutionState::WAITING) {
state = prepareDump();
if (state == ExecutionState::WAITING) {
_query->tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
}
}
@ -350,11 +351,13 @@ Result QueryStreamCursor::writeResult(VPackBuilder &builder) {
builder.add("cached", VPackValue(false));
if (!hasMore) {
std::shared_ptr<SharedQueryState> ss = _query->sharedState();
ss->setContinueCallback();
QueryResult result;
_query->setContinueCallback([&]() { _query->tempSignalAsyncResponse(); });
ExecutionState state = _query->finalize(result); // will commit transaction
while (state == ExecutionState::WAITING) {
_query->tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
state = _query->finalize(result);
}
if (result.extra && result.extra->slice().isObject()) {

4
arangod/Aql/QueryList.cpp
View File

@ -183,7 +183,7 @@ int QueryList::kill(TRI_voc_tick_t id) {
Query* query = (*it).second;
LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "killing AQL query " << id << " '" << query->queryString() << "'";
query->killed(true);
query->kill();
return TRI_ERROR_NO_ERROR;
}
@ -202,7 +202,7 @@ uint64_t QueryList::killAll(bool silent) {
LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "killing AQL query " << query->id() << " '" << query->queryString() << "'";
}
query->killed(true);
query->kill();
++killed;
}

2
arangod/Aql/QueryRegistry.cpp
View File

@ -191,7 +191,7 @@ void QueryRegistry::destroy(std::string const& vocbase, QueryId id,
if (q->second->_isOpen) {
// query in use by another thread/request
q->second->_query->killed(true);
q->second->_query->kill();
return;
}

4
arangod/Aql/RestAqlHandler.cpp
View File

@ -676,9 +676,11 @@ bool RestAqlHandler::findQuery(std::string const& idString, Query*& query) {
RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, Query* query,
VPackSlice const querySlice) {
auto self = shared_from_this();
query->setContinueHandler([this, self]() {
std::shared_ptr<SharedQueryState> ss = query->sharedState();
ss->setContinueHandler([this, self, ss]() {
continueHandlerExecution();
});
bool found;
std::string const& shardId = _request->header("shard-id", found);

96
arangod/Aql/SharedQueryState.cpp Normal file
View File

@ -0,0 +1,96 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "SharedQueryState.h"
#include "Scheduler/Scheduler.h"
#include "Scheduler/SchedulerFeature.h"
using namespace arangodb;
using namespace arangodb::aql;
void SharedQueryState::invalidate() {
CONDITION_LOCKER(guard, _condition);
_valid = false;
}
/// this has to stay for a backwards-compatible AQL HTTP API (hasMore).
void SharedQueryState::waitForAsyncResponse() {
CONDITION_LOCKER(guard, _condition);
if (!_wasNotified) {
_condition.wait();
}
_wasNotified = false;
}
/// @brief continueAfterPause is to be called on the query object to
/// continue execution in this query part, if the query got paused
/// because it is waiting for network responses. The idea is that a
/// RemoteBlock that does an asynchronous cluster-internal request can
/// register a callback with the asynchronous request and then return
/// with the result `ExecutionState::WAITING`, which will bubble up
/// the stack and eventually lead to a suspension of the work on the
/// RestHandler. In the callback function one can first store the
/// results in the RemoteBlock object and can then call this method on
/// the query.
/// This will lead to the following: The original request that led to
/// the network communication will be rescheduled on the ioservice and
/// continues its execution where it left off.
bool SharedQueryState::execute(std::function<bool()> const& cb) {
CONDITION_LOCKER(guard, _condition);
if (!_valid) {
return false;
}
bool res = cb();
if (_hasHandler) {
auto scheduler = SchedulerFeature::SCHEDULER;
TRI_ASSERT(scheduler != nullptr);
if (scheduler == nullptr) {
// We are shutting down
return false;
}
scheduler->post(_continueCallback);
} else {
_wasNotified = true;
guard.signal();
}
return res;
}
/// @brief setter for the continue callback:
/// We can either have a handler or a callback
void SharedQueryState::setContinueCallback() {
CONDITION_LOCKER(guard, _condition);
_hasHandler = false;
}
/// @brief setter for the continue handler:
/// We can either have a handler or a callback
void SharedQueryState::setContinueHandler(std::function<void()> const& handler) {
CONDITION_LOCKER(guard, _condition);
_continueCallback = handler;
_hasHandler = true;
}

82
arangod/Aql/SharedQueryState.h Normal file
View File

@ -0,0 +1,82 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_AQL_SHARED_QUERY_STATE_H
#define ARANGOD_AQL_SHARED_QUERY_STATE_H 1
#include "Basics/Common.h"
#include "Basics/ConditionLocker.h"
#include "Basics/ConditionVariable.h"
namespace arangodb {
namespace aql {
class SharedQueryState {
public:
SharedQueryState(SharedQueryState const&) = delete;
SharedQueryState& operator=(SharedQueryState const&) = delete;
SharedQueryState()
: _wasNotified(false),
_hasHandler(false),
_valid(true) {
}
~SharedQueryState() = default;
void invalidate();
bool execute(std::function<bool()> const& cb);
/// this has to stay for a backwards-compatible AQL HTTP API (hasMore).
void waitForAsyncResponse();
/// @brief setter for the continue callback:
/// We can either have a handler or a callback
void setContinueCallback();
/// @brief setter for the continue handler:
/// We can either have a handler or a callback
void setContinueHandler(std::function<void()> const& handler);
private:
basics::ConditionVariable _condition;
/// @brief a callback function which is used to implement continueAfterPause.
/// Typically, the RestHandler using the Query object will put a closure
/// in here, which continueAfterPause simply calls.
std::function<void()> _continueCallback;
bool _wasNotified;
/// @brief decide if the _continueCallback needs to be pushed onto the ioservice
/// or if it has to be executed in this thread.
bool _hasHandler;
bool _valid;
};
}
}
#endif

31
arangod/Aql/WakeupQueryCallback.cpp
View File

@ -23,31 +23,24 @@
#include "WakeupQueryCallback.h"
#include "Aql/ExecutionBlock.h"
#include "Aql/Query.h"
#include "Scheduler/SchedulerFeature.h"
using namespace arangodb;
using namespace arangodb::aql;
WakeupQueryCallback::WakeupQueryCallback(ExecutionBlock* initiator,
Query* query)
: _initiator(initiator), _query(query) {}
: _initiator(initiator),
_query(query),
_sharedState(query->sharedState()) {}
WakeupQueryCallback::~WakeupQueryCallback() {}
bool WakeupQueryCallback::operator()(ClusterCommResult* result) {
TRI_ASSERT(_initiator != nullptr);
TRI_ASSERT(_query != nullptr);
// TODO Validate that _initiator and _query have not been deleted (ttl)
// TODO Handle exceptions
bool res = _initiator->handleAsyncResult(result);
if (_query->hasHandler()) {
auto scheduler = SchedulerFeature::SCHEDULER;
TRI_ASSERT(scheduler != nullptr);
if (scheduler == nullptr) {
// We are shutting down
return false;
}
scheduler->post(_query->continueHandler());
} else {
_query->continueAfterPause();
}
return res;
return _sharedState->execute([&, this]() {
TRI_ASSERT(_initiator != nullptr);
TRI_ASSERT(_query != nullptr);
// TODO Validate that _initiator and _query have not been deleted (ttl)
// TODO Handle exceptions
return _initiator->handleAsyncResult(result);
});
}

10
arangod/Aql/WakeupQueryCallback.h
View File

@ -31,16 +31,18 @@ namespace aql {
class ExecutionBlock;
class Query;
class SharedQueryState;
struct WakeupQueryCallback : public ClusterCommCallback {
WakeupQueryCallback(ExecutionBlock* initiator, Query* query);
~WakeupQueryCallback() {};
~WakeupQueryCallback();
bool operator()(ClusterCommResult*) override;
private:
ExecutionBlock* _initiator;
Query* _query;
private:
ExecutionBlock* _initiator;
Query* _query;
std::shared_ptr<SharedQueryState> _sharedState;
};
} // aql

1
arangod/CMakeLists.txt
View File

@ -240,6 +240,7 @@ SET(ARANGOD_SOURCES
Aql/RegexCache.cpp
Aql/RestAqlHandler.cpp
Aql/Scopes.cpp
Aql/SharedQueryState.cpp
Aql/ShortStringStorage.cpp
Aql/ShortestPathBlock.cpp
Aql/ShortestPathNode.cpp

5
arangod/MMFiles/MMFilesPrimaryIndex.cpp
View File

@ -196,12 +196,11 @@ MMFilesPrimaryIndex::MMFilesPrimaryIndex(
{{arangodb::basics::AttributeName(StaticStrings::KeyString,
false)}}),
/*unique*/ true , /*sparse*/ false) {
size_t indexBuckets = 1;
auto physical =
static_cast<arangodb::MMFilesCollection*>(collection.getPhysical());
TRI_ASSERT(physical != nullptr);
indexBuckets = static_cast<size_t>(physical->indexBuckets());
TRI_ASSERT(physical != nullptr);
size_t indexBuckets = static_cast<size_t>(physical->indexBuckets());
if (collection.isAStub()) {
// in order to reduce memory usage

16
arangod/Replication/DatabaseInitialSyncer.cpp
View File

@ -474,6 +474,11 @@ void DatabaseInitialSyncer::orderDumpChunk(std::shared_ptr<Syncer::JobSynchroniz
uint64_t chunkSize) {
using ::arangodb::basics::StringUtils::itoa;
if (isAborted()) {
sharedStatus->gotResponse(Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED), nullptr);
return;
}
std::string const typeString = (coll->type() == TRI_COL_TYPE_EDGE ? "edge" : "document");
@ -715,7 +720,7 @@ Result DatabaseInitialSyncer::fetchCollectionDump(
}
}
if (checkMore) {
if (checkMore && !isAborted()) {
// already fetch next batch in the background, by posting the
// request to the scheduler, which can run it asynchronously
@ -807,8 +812,11 @@ Result DatabaseInitialSyncer::fetchCollectionDump(
return Result();
}
batch++;
if (isAborted()) {
return Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED);
}
}
TRI_ASSERT(false);
@ -1262,6 +1270,10 @@ Result DatabaseInitialSyncer::handleCollection(VPackSlice const& parameters,
if (!res.ok()) {
return res;
}
if (isAborted()) {
return Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED);
}
if (masterName == TRI_COL_NAME_USERS) {
reloadUsers();

2
arangod/Replication/Syncer.cpp
View File

@ -221,7 +221,7 @@ arangodb::Result applyCollectionDumpMarkerInternal(
namespace arangodb {
Syncer::JobSynchronizer::JobSynchronizer(std::shared_ptr<Syncer const> syncer)
Syncer::JobSynchronizer::JobSynchronizer(std::shared_ptr<Syncer const> const& syncer)
: _syncer(syncer),
_gotResponse(false),
_jobsInFlight(0) {}

2
arangod/Replication/Syncer.h
View File

@ -63,7 +63,7 @@ class Syncer : public std::enable_shared_from_this<Syncer> {
JobSynchronizer(JobSynchronizer const&) = delete;
JobSynchronizer& operator=(JobSynchronizer const&) = delete;
explicit JobSynchronizer(std::shared_ptr<Syncer const> syncer);
explicit JobSynchronizer(std::shared_ptr<Syncer const> const& syncer);
~JobSynchronizer() = default;
/// @brief will be called whenever a response for the job comes in

87
arangod/Replication/TailingSyncer.cpp
View File

@ -119,18 +119,9 @@ void TailingSyncer::setProgress(std::string const& msg) {
}
/// @brief abort all ongoing transactions
void TailingSyncer::abortOngoingTransactions() {
void TailingSyncer::abortOngoingTransactions() noexcept {
try {
// abort all running transactions
for (auto& it : _ongoingTransactions) {
auto trx = it.second;
if (trx != nullptr) {
trx->abort();
delete trx;
}
}
_ongoingTransactions.clear();
} catch (...) {
// ignore errors here
@ -364,7 +355,7 @@ Result TailingSyncer::processDocument(TRI_replication_operation_e type,
std::string("unexpected transaction ") + StringUtils::itoa(tid));
}
auto trx = (*it).second;
std::unique_ptr<ReplicationTransaction>& trx = (*it).second;
if (trx == nullptr) {
return Result(
@ -455,14 +446,7 @@ Result TailingSyncer::startTransaction(VPackSlice const& slice) {
if (it != _ongoingTransactions.end()) {
// found a previous version of the same transaction - should not happen...
auto trx = (*it).second;
_ongoingTransactions.erase(tid);
if (trx != nullptr) {
// abort ongoing trx
delete trx;
}
_ongoingTransactions.erase(it);
}
TRI_ASSERT(tid > 0);
@ -474,8 +458,7 @@ Result TailingSyncer::startTransaction(VPackSlice const& slice) {
Result res = trx->begin();
if (res.ok()) {
_ongoingTransactions[tid] = trx.get();
trx.release();
_ongoingTransactions[tid] = std::move(trx);
}
return res;
@ -498,7 +481,7 @@ Result TailingSyncer::abortTransaction(VPackSlice const& slice) {
auto it = _ongoingTransactions.find(tid);
if (it == _ongoingTransactions.end()) {
if (it == _ongoingTransactions.end() || (*it).second == nullptr) {
// invalid state, no transaction was started.
return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
}
@ -508,17 +491,8 @@ Result TailingSyncer::abortTransaction(VPackSlice const& slice) {
LOG_TOPIC(TRACE, Logger::REPLICATION)
<< "aborting replication transaction " << tid;
auto trx = (*it).second;
_ongoingTransactions.erase(tid);
if (trx != nullptr) {
Result res = trx->abort();
delete trx;
return res;
}
return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
_ongoingTransactions.erase(it);
return Result();
}
/// @brief commits a transaction, based on the VelocyPack provided
@ -538,7 +512,7 @@ Result TailingSyncer::commitTransaction(VPackSlice const& slice) {
auto it = _ongoingTransactions.find(tid);
if (it == _ongoingTransactions.end()) {
if (it == _ongoingTransactions.end() || (*it).second == nullptr) {
// invalid state, no transaction was started.
return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
}
@ -548,17 +522,11 @@ Result TailingSyncer::commitTransaction(VPackSlice const& slice) {
LOG_TOPIC(TRACE, Logger::REPLICATION)
<< "committing replication transaction " << tid;
auto trx = (*it).second;
_ongoingTransactions.erase(tid);
if (trx != nullptr) {
Result res = trx->commit();
delete trx;
return res;
}
return Result(TRI_ERROR_REPLICATION_UNEXPECTED_TRANSACTION);
std::unique_ptr<ReplicationTransaction>& trx = (*it).second;
Result res = trx->commit();
_ongoingTransactions.erase(it);
return res;
}
/// @brief renames a collection, based on the VelocyPack provided
@ -1575,28 +1543,15 @@ Result TailingSyncer::followMasterLog(TRI_voc_tick_t& fetchTick,
setProgress(progress);
std::string body;
if (!_ongoingTransactions.empty()) {
// stringify list of open transactions
body.append("[\"");
bool first = true;
for (auto& it : _ongoingTransactions) {
if (first) {
first = false;
} else {
body.append("\",\"");
}
body.append(StringUtils::itoa(it.first));
}
body.append("\"]");
} else {
body.append("[]");
// stringify list of open transactions
VPackBuilder builder;
builder.openArray();
for (auto& it : _ongoingTransactions) {
builder.add(VPackValue(StringUtils::itoa(it.first)));
}
builder.close();
std::string body = builder.slice().toJson();
std::unique_ptr<SimpleHttpResult> response(_state.connection.client->request(
rest::RequestType::PUT, url, body.c_str(), body.size()));

4
arangod/Replication/TailingSyncer.h
View File

@ -70,7 +70,7 @@ class TailingSyncer : public Syncer {
void setProgress(std::string const&);
/// @brief abort all ongoing transactions
void abortOngoingTransactions();
void abortOngoingTransactions() noexcept;
/// @brief whether or not a collection should be excluded
bool skipMarker(TRI_voc_tick_t, arangodb::velocypack::Slice const&);
@ -176,7 +176,7 @@ class TailingSyncer : public Syncer {
bool _ignoreDatabaseMarkers;
/// @brief which transactions were open and need to be treated specially
std::unordered_map<TRI_voc_tid_t, ReplicationTransaction*>
std::unordered_map<TRI_voc_tid_t, std::unique_ptr<ReplicationTransaction>>
_ongoingTransactions;
/// @brief recycled builder for repeated document creation

10
arangod/RestHandler/RestCursorHandler.cpp
View File

@ -48,7 +48,6 @@ RestCursorHandler::RestCursorHandler(
_query(nullptr),
_queryResult(),
_queryRegistry(queryRegistry),
_queryLock(),
_hasStarted(false),
_queryKilled(false),
_isValidForFinalize(false) {}
@ -165,11 +164,12 @@ bool RestCursorHandler::registerQueryOrCursor(VPackSlice const& slice) {
arangodb::aql::PART_MAIN
);
std::shared_ptr<aql::SharedQueryState> ss = query->sharedState();
auto self = shared_from_this();
auto continueHandler = [this, self]() {
ss->setContinueHandler([this, self, ss] {
continueHandlerExecution();
};
query->setContinueHandler(continueHandler);
});
registerQuery(std::move(query));
return true;
}
@ -341,7 +341,7 @@ bool RestCursorHandler::cancelQuery() {
MUTEX_LOCKER(mutexLocker, _queryLock);
if (_query != nullptr) {
_query->killed(true);
_query->kill();
_queryKilled = true;
_hasStarted = true;
return true;

2
arangod/RestServer/AqlFeature.cpp
View File

@ -60,7 +60,7 @@ AqlFeature* AqlFeature::lease() {
return aql;
}
void AqlFeature::unlease() {
void AqlFeature::unlease() noexcept {
MUTEX_LOCKER(locker, AqlFeature::_aqlFeatureMutex);
AqlFeature* aql = AqlFeature::_AQL;
TRI_ASSERT(aql != nullptr);

2
arangod/RestServer/AqlFeature.h
View File

@ -36,7 +36,7 @@ class AqlFeature final : public application_features::ApplicationFeature {
public:
static AqlFeature* lease();
static void unlease();
static void unlease() noexcept;
void start() override final;
void stop() override final;

6
arangod/RocksDBEngine/RocksDBCollection.cpp
View File

@ -77,9 +77,9 @@ using namespace arangodb;
// back on later in its dtor
// this is just a performance optimization for small transactions
struct IndexingDisabler {
IndexingDisabler(RocksDBMethods* mthd, bool disableIndexing)
: mthd(mthd), disableIndexing(disableIndexing) {
if (disableIndexing) {
IndexingDisabler(RocksDBMethods* mthd, bool disable)
: mthd(mthd), disableIndexing(disable) {
if (disable) {
disableIndexing = mthd->DisableIndexing();
}
}

174
arangod/RocksDBEngine/RocksDBIncrementalSync.cpp
View File

@ -27,7 +27,9 @@
#include "Replication/DatabaseInitialSyncer.h"
#include "Replication/utilities.h"
#include "RocksDBEngine/RocksDBCollection.h"
#include "RocksDBIterators.h"
#include "RocksDBEngine/RocksDBIterators.h"
#include "RocksDBEngine/RocksDBKey.h"
#include "RocksDBEngine/RocksDBPrimaryIndex.h"
#include "SimpleHttpClient/SimpleHttpClient.h"
#include "SimpleHttpClient/SimpleHttpResult.h"
#include "StorageEngine/PhysicalCollection.h"
@ -44,6 +46,107 @@
#include <velocypack/velocypack-aliases.h>
namespace arangodb {
// remove all keys that are below first remote key or beyond last remote key
Result removeKeysOutsideRange(VPackSlice chunkSlice,
LogicalCollection* col,
OperationOptions& options,
InitialSyncerIncrementalSyncStats& stats) {
size_t const numChunks = chunkSlice.length();
if (numChunks == 0) {
// no need to do anything
return Result();
}
// first chunk
SingleCollectionTransaction trx(
transaction::StandaloneContext::Create(col->vocbase()),
*col,
AccessMode::Type::EXCLUSIVE
);
trx.addHint(
transaction::Hints::Hint::RECOVERY); // to turn off waitForSync!
trx.addHint(transaction::Hints::Hint::NO_TRACKING);
trx.addHint(transaction::Hints::Hint::NO_INDEXING);
Result res = trx.begin();
if (!res.ok()) {
return Result(
res.errorNumber(),
std::string("unable to start transaction: ") + res.errorMessage());
}
VPackSlice chunk = chunkSlice.at(0);
TRI_ASSERT(chunk.isObject());
auto lowSlice = chunk.get("low");
TRI_ASSERT(lowSlice.isString());
StringRef lowRef(lowSlice);
// last high
chunk = chunkSlice.at(numChunks - 1);
TRI_ASSERT(chunk.isObject());
auto highSlice = chunk.get("high");
TRI_ASSERT(highSlice.isString());
StringRef highRef(highSlice);
LogicalCollection* coll = trx.documentCollection();
auto iterator = createPrimaryIndexIterator(&trx, coll);
VPackBuilder builder;
// remove everything from the beginning of the key range until the lowest
// remote key
iterator.next(
[&](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
StringRef docKey(RocksDBKey::primaryKey(rocksKey));
if (docKey.compare(lowRef) < 0) {
builder.clear();
builder.add(velocypack::ValuePair(docKey.data(), docKey.size(),
velocypack::ValueType::String));
trx.remove(col->name(), builder.slice(), options);
++stats.numDocsRemoved;
// continue iteration
return true;
}
// stop iteration
return false;
},
std::numeric_limits<std::uint64_t>::max());
// remove everything from the highest remote key until the end of the key range
auto index = col->lookupIndex(0); //RocksDBCollection->primaryIndex() is private
TRI_ASSERT(index->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX);
auto primaryIndex = static_cast<RocksDBPrimaryIndex*>(index.get());
RocksDBKeyLeaser key(&trx);
key->constructPrimaryIndexValue(primaryIndex->objectId(), highRef);
iterator.seek(key->string());
iterator.next(
[&](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
StringRef docKey(RocksDBKey::primaryKey(rocksKey));
if (docKey.compare(highRef) > 0) {
builder.clear();
builder.add(velocypack::ValuePair(docKey.data(), docKey.size(),
velocypack::ValueType::String));
trx.remove(col->name(), builder.slice(), options);
++stats.numDocsRemoved;
}
// continue iteration until end
return true;
},
std::numeric_limits<std::uint64_t>::max());
return trx.commit();
}
Result syncChunkRocksDB(
DatabaseInitialSyncer& syncer, SingleCollectionTransaction* trx,
InitialSyncerIncrementalSyncStats& stats,
@ -505,7 +608,6 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
ManagedDocumentResult mmdr;
OperationOptions options;
options.silent = true;
options.ignoreRevs = true;
options.isRestore = true;
@ -514,70 +616,17 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
options.isSynchronousReplicationFrom = syncer._state.leaderId;
}
size_t const numChunks = static_cast<size_t>(chunkSlice.length());
{
// remove all keys that are below first remote key or beyond last remote key
Result res = removeKeysOutsideRange(chunkSlice, col, options, stats);
// remove all keys that are below first remote key or beyond last remote key
if (numChunks > 0) {
// first chunk
SingleCollectionTransaction trx(
transaction::StandaloneContext::Create(syncer.vocbase()),
*col,
AccessMode::Type::EXCLUSIVE
);
trx.addHint(
transaction::Hints::Hint::RECOVERY); // to turn off waitForSync!
trx.addHint(transaction::Hints::Hint::NO_TRACKING);
trx.addHint(transaction::Hints::Hint::NO_INDEXING);
Result res = trx.begin();
if (!res.ok()) {
return Result(
res.errorNumber(),
std::string("unable to start transaction: ") + res.errorMessage());
}
VPackSlice chunk = chunkSlice.at(0);
TRI_ASSERT(chunk.isObject());
auto lowSlice = chunk.get("low");
TRI_ASSERT(lowSlice.isString());
// last high
chunk = chunkSlice.at(numChunks - 1);
TRI_ASSERT(chunk.isObject());
auto highSlice = chunk.get("high");
TRI_ASSERT(highSlice.isString());
StringRef lowRef(lowSlice);
StringRef highRef(highSlice);
LogicalCollection* coll = trx.documentCollection();
auto iterator = createPrimaryIndexIterator(&trx, coll);
VPackBuilder builder;
iterator.next(
[&](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
StringRef docKey(RocksDBKey::primaryKey(rocksKey));
if (docKey.compare(lowRef) < 0 || docKey.compare(highRef) > 0) {
builder.clear();
builder.add(velocypack::ValuePair(docKey.data(), docKey.size(),
velocypack::ValueType::String));
trx.remove(col->name(), builder.slice(), options);
++stats.numDocsRemoved;
}
},
std::numeric_limits<std::uint64_t>::max());
res = trx.commit();
if (!res.ok()) {
if (res.fail()) {
return res;
}
}
size_t const numChunks = static_cast<size_t>(chunkSlice.length());
{
if (syncer.isAborted()) {
return Result(TRI_ERROR_REPLICATION_APPLIER_STOPPED);
@ -749,12 +798,13 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
rocksValue); // we want probably to do this instead
if (col->readDocument(&trx, documentId, mmdr) == false) {
TRI_ASSERT(false);
return;
return true;
}
VPackSlice doc(mmdr.vpack());
docRev = TRI_ExtractRevisionId(doc);
}
compareChunk(docKey, docRev);
return true;
},
std::numeric_limits<std::uint64_t>::max()); // no limit on documents

32
arangod/RocksDBEngine/RocksDBIterators.cpp
View File

@ -333,9 +333,9 @@ void RocksDBSortedAllIterator::reset() {
_iterator->Seek(_bounds.start());
}
RocksDBGenericIterator::RocksDBGenericIterator(rocksdb::ReadOptions& options
,RocksDBKeyBounds const& bounds
,bool reverse)
RocksDBGenericIterator::RocksDBGenericIterator(rocksdb::ReadOptions& options,
RocksDBKeyBounds const& bounds,
bool reverse)
: _reverse(reverse)
, _bounds(bounds)
, _options(options)
@ -365,11 +365,15 @@ bool RocksDBGenericIterator::reset() {
}
bool RocksDBGenericIterator::skip(uint64_t count, uint64_t& skipped) {
bool has_more = _iterator->Valid();
while (count > 0 && has_more) {
has_more = next([&count,&skipped](rocksdb::Slice const&, rocksdb::Slice const&){ --count; ++skipped; }, count /*gets copied*/);
bool hasMore = _iterator->Valid();
while (count > 0 && hasMore) {
hasMore = next([&count, &skipped](rocksdb::Slice const&, rocksdb::Slice const&) {
--count;
++skipped;
return true;
}, count /*gets copied*/);
}
return has_more;
return hasMore;
}
bool RocksDBGenericIterator::seek(rocksdb::Slice const& key) {
@ -392,19 +396,21 @@ bool RocksDBGenericIterator::next(GenericCallback const& cb, size_t limit) {
return false;
}
while (limit > 0 && hasMore()){
while (limit > 0 && hasMore()) {
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
TRI_ASSERT(_bounds.objectId() == RocksDBKey::objectId(_iterator->key()));
#endif
cb(_iterator->key(),_iterator->value());
if (!cb(_iterator->key(),_iterator->value())) {
// stop iteration
return false;
}
--limit;
if (_reverse) {
_iterator->Prev();
} else {
_iterator->Next();
}
}
return hasMore();
@ -424,11 +430,11 @@ RocksDBGenericIterator arangodb::createPrimaryIndexIterator(transaction::Methods
options.verify_checksums = false;
auto index = col->lookupIndex(0); //RocksDBCollection->primaryIndex() is private
TRI_ASSERT( index->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX );
TRI_ASSERT(index->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX);
auto primaryIndex = static_cast<RocksDBPrimaryIndex*>(index.get());
auto bounds(RocksDBKeyBounds::PrimaryIndex(primaryIndex->objectId()));
auto iterator = RocksDBGenericIterator(options, bounds);
auto iterator = RocksDBGenericIterator(options, bounds);
TRI_ASSERT(iterator.bounds().objectId() == primaryIndex->objectId());
TRI_ASSERT(iterator.bounds().columnFamily() == RocksDBColumnFamily::primary());
@ -450,7 +456,7 @@ RocksDBGenericIterator arangodb::createDocumentIterator(transaction::Methods* tr
auto rocksColObjectId = static_cast<RocksDBCollection*>(col->getPhysical())->objectId();
auto bounds(RocksDBKeyBounds::CollectionDocuments(rocksColObjectId));
auto iterator = RocksDBGenericIterator(options, bounds);
auto iterator = RocksDBGenericIterator(options, bounds);
TRI_ASSERT(iterator.bounds().objectId() == rocksColObjectId);
TRI_ASSERT(iterator.bounds().columnFamily() == RocksDBColumnFamily::documents());

14
arangod/RocksDBEngine/RocksDBIterators.h
View File

@ -41,7 +41,8 @@ namespace arangodb {
class RocksDBCollection;
class RocksDBPrimaryIndex;
typedef std::function<void(rocksdb::Slice const& key, rocksdb::Slice const& value)> GenericCallback;
/// @brief return false to stop iteration
typedef std::function<bool(rocksdb::Slice const& key, rocksdb::Slice const& value)> GenericCallback;
/// @brief iterator over all documents in the collection
/// basically sorted after LocalDocumentId
@ -134,18 +135,15 @@ class RocksDBGenericIterator {
// the following functions return if the iterator
// is valid and in bounds on return.
bool next(GenericCallback const& cb // void(rocksdb::Slice const& key,rocksd:Slice const& value)
, size_t count //number of documents the callback should be applied to
);
bool next(GenericCallback const& cb, size_t count); //number of documents the callback should be applied to
bool skip(uint64_t count // documents to skip
,uint64_t& skipped // skipped documents
);
// documents to skip, skipped documents
bool skip(uint64_t count, uint64_t& skipped);
bool seek(rocksdb::Slice const& key);
bool reset();
bool hasMore() const;
//return bounds
// return bounds
RocksDBKeyBounds const& bounds() const { return _bounds; }
private:

15
arangod/RocksDBEngine/RocksDBReplicationContext.cpp
View File

@ -295,6 +295,7 @@ RocksDBReplicationResult RocksDBReplicationContext::dumpJson(
VPackDumper dumper(&adapter, &collectionIter->vpackOptions);
dumper.dump(velocypack::Slice(rocksValue.data()));
buff.appendText("}\n");
return true;
};
TRI_ASSERT(collectionIter->iter && !collectionIter->sorted());
@ -350,12 +351,12 @@ RocksDBReplicationResult RocksDBReplicationContext::dumpVPack(TRI_vocbase_t* voc
VPackBuilder builder(buffer, &collectionIter->vpackOptions);
auto cb = [&builder](rocksdb::Slice const& rocksKey, rocksdb::Slice const& rocksValue) {
builder.openObject();
builder.add("type", VPackValue(REPLICATION_MARKER_DOCUMENT));
builder.add(VPackValue("data"));
builder.add(velocypack::Slice(rocksValue.data()));
builder.close();
return true;
};
TRI_ASSERT(collectionIter->iter && !collectionIter->sorted());
@ -413,7 +414,7 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b,
auto documentId = RocksDBValue::documentId(rocksValue); // we want probably to do this instead
if (_collection->logical.readDocument(_trx.get(), documentId, _collection->mdr) == false) {
TRI_ASSERT(false);
return;
return true;
}
VPackSlice doc(_collection->mdr.vpack());
docRev = TRI_ExtractRevisionId(doc);
@ -433,6 +434,8 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b,
builder.clear();
builder.add(TRI_RidToValuePair(docRev, &ridBuffer[0]));
hash ^= builder.slice().hashString();
return true;
}; //cb
b.openArray(true);
@ -532,7 +535,7 @@ arangodb::Result RocksDBReplicationContext::dumpKeys(
auto documentId = RocksDBValue::documentId(rocksValue); // we want probably to do this instead
if (_collection->logical.readDocument(_trx.get(), documentId, _collection->mdr) == false) {
TRI_ASSERT(false);
return;
return true;
}
VPackSlice doc(_collection->mdr.vpack());
docRev = TRI_ExtractRevisionId(doc);
@ -543,6 +546,8 @@ arangodb::Result RocksDBReplicationContext::dumpKeys(
b.add(velocypack::ValuePair(docKey.data(), docKey.size(), velocypack::ValueType::String));
b.add(TRI_RidToValuePair(docRev, &ridBuffer[0]));
b.close();
return true;
};
b.openArray(true);
@ -619,11 +624,12 @@ arangodb::Result RocksDBReplicationContext::dumpDocuments(
bool ok = _collection->logical.readDocument(_trx.get(), documentId, _collection->mdr);
if (!ok) {
// TODO: do something here?
return;
return true;
}
VPackSlice current(_collection->mdr.vpack());
TRI_ASSERT(current.isObject());
b.add(current);
return true;
};
auto buffer = b.buffer();
@ -657,6 +663,7 @@ arangodb::Result RocksDBReplicationContext::dumpDocuments(
hasMore = _collection->iter->next(
[&b](rocksdb::Slice const&, rocksdb::Slice const&) {
b.add(VPackValue(VPackValueType::Null));
return true;
},
1);
} else {

8
arangod/V8Server/v8-query.cpp
View File

@ -30,9 +30,9 @@
#include "Indexes/Index.h"
#include "StorageEngine/PhysicalCollection.h"
#include "Transaction/Helpers.h"
#include "Transaction/V8Context.h"
#include "Utils/OperationCursor.h"
#include "Utils/SingleCollectionTransaction.h"
#include "Transaction/V8Context.h"
#include "V8/v8-conv.h"
#include "V8/v8-utils.h"
#include "V8/v8-vpack.h"
@ -71,8 +71,10 @@ aql::QueryResultV8 AqlQuery(
arangodb::aql::PART_MAIN
);
std::shared_ptr<arangodb::aql::SharedQueryState> ss = query.sharedState();
ss->setContinueCallback();
aql::QueryResultV8 queryResult;
query.setContinueCallback([&query]() { query.tempSignalAsyncResponse(); });
while (true) {
auto state = query.executeV8(isolate,
static_cast<arangodb::aql::QueryRegistry*>(v8g->_queryRegistry),
@ -80,7 +82,7 @@ aql::QueryResultV8 AqlQuery(
if (state != aql::ExecutionState::WAITING) {
break;
}
query.tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
}
if (queryResult.code != TRI_ERROR_NO_ERROR) {

7
arangod/V8Server/v8-vocbase.cpp
View File

@ -836,14 +836,17 @@ static void JS_ExecuteAql(v8::FunctionCallbackInfo<v8::Value> const& args) {
options,
arangodb::aql::PART_MAIN
);
std::shared_ptr<arangodb::aql::SharedQueryState> ss = query.sharedState();
ss->setContinueCallback();
aql::QueryResultV8 queryResult;
query.setContinueCallback([&query]() { query.tempSignalAsyncResponse(); });
while (true) {
auto state = query.executeV8(isolate, static_cast<arangodb::aql::QueryRegistry*>(v8g->_queryRegistry), queryResult);
if (state != aql::ExecutionState::WAITING) {
break;
}
query.tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
}
if (queryResult.code != TRI_ERROR_NO_ERROR) {

2
js/client/modules/@arangodb/process-utils.js
View File

@ -831,7 +831,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
requestOptions.method = 'PUT';
print(coords[0].url + "/_admin/cluster/maintenance");
download(coords[0].url + "/_admin/cluster/maintenance", JSON.stringify({ body: "on" }), requestOptions);
download(coords[0].url + "/_admin/cluster/maintenance", JSON.stringify("on"), requestOptions);
}
} catch (err) {
print("error while setting cluster maintenance mode:", err);

4
tests/IResearch/common.cpp
View File

@ -188,11 +188,13 @@ arangodb::aql::QueryResult executeQuery(
arangodb::aql::PART_MAIN
);
std::shared_ptr<arangodb::aql::SharedQueryState> ss = query.sharedState();
arangodb::aql::QueryResult result;
while (true) {
auto state = query.execute(arangodb::QueryRegistryFeature::QUERY_REGISTRY, result);
if (state == arangodb::aql::ExecutionState::WAITING) {
query.tempWaitForAsyncResponse();
ss->waitForAsyncResponse();
} else {
break;
}