diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index f2e4694973..dddf7a2446 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -587,10 +587,14 @@ std::pair ExecutionEngine::skipSome(size_t atMost) { return _root->skipSome(atMost); } -Result ExecutionEngine::shutdownSync(int errorCode) noexcept { +Result ExecutionEngine::shutdownSync(int errorCode) noexcept try { Result res{TRI_ERROR_INTERNAL}; ExecutionState state = ExecutionState::WAITING; try { + TRI_IF_FAILURE("ExecutionEngine::shutdownSync") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + std::shared_ptr sharedState = _query.sharedState(); if (sharedState != nullptr) { sharedState->setContinueCallback(); @@ -602,10 +606,33 @@ Result ExecutionEngine::shutdownSync(int errorCode) noexcept { } } } + } catch (basics::Exception const& ex) { + res.reset(ex.code(), std::string("unable to shutdown query: ") + ex.what()); + } catch (std::exception const& ex) { + res.reset(TRI_ERROR_INTERNAL, std::string("unable to shutdown query: ") + ex.what()); } catch (...) { res.reset(TRI_ERROR_INTERNAL); } + + if (res.fail() && ServerState::instance()->isCoordinator()) { + // shutdown attempt has failed... + // in a cluster, try to at least abort all other coordinator parts + auto queryRegistry = QueryRegistryFeature::registry(); + if (queryRegistry != nullptr) { + for (auto const& id : _coordinatorQueryIds) { + try { + queryRegistry->destroy(_query.vocbase().name(), id, errorCode, false); + } catch (...) { + // we want to abort all parts, even if aborting other parts fails + } + } + } + } + return res; +} catch (...) { + // nothing we can do here... + return Result(TRI_ERROR_INTERNAL, "unable to shutdown query"); } /// @brief shutdown, will be called exactly once for the whole query diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index b3cacd4977..05e931a488 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -624,6 +624,9 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, Query* q // this is because the request may contain additional data if ((operation == "getSome" || operation == "skipSome") && !query->engine()->initializeCursorCalled()) { + TRI_IF_FAILURE("RestAqlHandler::getSome") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } auto res = query->engine()->initializeCursor(nullptr, 0); if (res.first == ExecutionState::WAITING) { return RestStatus::WAITING; @@ -648,6 +651,9 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, Query* q answerBuilder.add(StaticStrings::Error, VPackValue(res != TRI_ERROR_NO_ERROR)); answerBuilder.add(StaticStrings::Code, VPackValue(res)); } else if (operation == "getSome") { + TRI_IF_FAILURE("RestAqlHandler::getSome") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } auto atMost = VelocyPackHelper::getNumericValue(querySlice, "atMost", ExecutionBlock::DefaultBatchSize()); diff --git a/arangod/RestServer/BootstrapFeature.cpp b/arangod/RestServer/BootstrapFeature.cpp index bd7411a38d..d2d1f171fb 100644 --- a/arangod/RestServer/BootstrapFeature.cpp +++ b/arangod/RestServer/BootstrapFeature.cpp @@ -91,7 +91,7 @@ namespace { /// Initialize certain agency entries, like Plan, system collections /// and various similar things. Only runs through on a SINGLE coordinator. -/// must only return if we are boostrap lead or bootstrap is done +/// must only return if we are bootstrap lead or bootstrap is done void raceForClusterBootstrap(BootstrapFeature& feature) { AgencyComm agency; auto& ci = feature.server().getFeature().clusterInfo(); @@ -109,13 +109,13 @@ void raceForClusterBootstrap(BootstrapFeature& feature) { std::vector({AgencyCommManager::path(), ::bootstrapKey})); if (value.isString()) { // key was found and is a string - std::string boostrapVal = value.copyString(); - if (boostrapVal.find("done") != std::string::npos) { + std::string bootstrapVal = value.copyString(); + if (bootstrapVal.find("done") != std::string::npos) { // all done, let's get out of here: LOG_TOPIC("61e04", TRACE, Logger::STARTUP) << "raceForClusterBootstrap: bootstrap already done"; return; - } else if (boostrapVal == ServerState::instance()->getId()) { + } else if (bootstrapVal == ServerState::instance()->getId()) { agency.removeValues(::bootstrapKey, false); } LOG_TOPIC("49437", DEBUG, Logger::STARTUP) @@ -320,7 +320,7 @@ void BootstrapFeature::start() { ss->setFoxxmaster(myId); // could be empty, but set anyway } - if (v8Enabled) { // runs the single server boostrap JS + if (v8Enabled) { // runs the single server bootstrap JS // will run foxx/manager.js::_startup() and more (start queues, load // routes, etc) LOG_TOPIC("e0c8b", DEBUG, Logger::STARTUP) << "Running server/server.js"; diff --git a/tests/js/server/aql/aql-failures-cluster.js b/tests/js/server/aql/aql-failures-cluster.js new file mode 100644 index 0000000000..f9094dcf4e --- /dev/null +++ b/tests/js/server/aql/aql-failures-cluster.js @@ -0,0 +1,114 @@ +/*jshint globalstrict:false, strict:false, maxlen: 400 */ +/*global fail, assertEqual, AQL_EXECUTE */ + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test failure scenarios +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2010-2012 triagens GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is triAGENS GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +var jsunity = require("jsunity"); +var arangodb = require("@arangodb"); +var db = arangodb.db; +var internal = require("internal"); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite +//////////////////////////////////////////////////////////////////////////////// + +function ahuacatlFailureSuite () { + 'use strict'; + const cn = "UnitTestsAhuacatlFailures"; + const en = "UnitTestsAhuacatlEdgeFailures"; + + let assertFailingQuery = function (query) { + try { + AQL_EXECUTE(query); + fail(); + } catch (err) { + assertEqual(internal.errors.ERROR_DEBUG.code, err.errorNum, query); + } + }; + + return { + + setUpAll: function () { + internal.debugClearFailAt(); + db._drop(cn); + db._create(cn); + db._drop(en); + db._createEdgeCollection(en); + }, + + setUp: function () { + internal.debugClearFailAt(); + }, + + tearDownAll: function () { + internal.debugClearFailAt(); + db._drop(cn); + db._drop(en); + }, + + tearDown: function() { + internal.debugClearFailAt(); + }, + + testShutdownSync : function () { + internal.debugSetFailAt("ExecutionEngine::shutdownSync"); + + let res = AQL_EXECUTE("FOR doc IN " + cn + " RETURN doc").json; + // no real test expectations here, just that the query works and doesn't fail on shutdown + assertEqual(0, res.length); + }, + + testShutdownSyncDiamond : function () { + internal.debugSetFailAt("ExecutionEngine::shutdownSync"); + + let res = AQL_EXECUTE("FOR doc1 IN " + cn + " FOR doc2 IN " + en + " FILTER doc1._key == doc2._key RETURN doc1").json; + // no real test expectations here, just that the query works and doesn't fail on shutdown + assertEqual(0, res.length); + }, + + testShutdownSyncFailInGetSome : function () { + internal.debugSetFailAt("ExecutionEngine::shutdownSync"); + internal.debugSetFailAt("RestAqlHandler::getSome"); + + assertFailingQuery("FOR doc IN " + cn + " RETURN doc"); + }, + + testShutdownSyncDiamondFailInGetSome : function () { + internal.debugSetFailAt("ExecutionEngine::shutdownSync"); + internal.debugSetFailAt("RestAqlHandler::getSome"); + + assertFailingQuery("FOR doc1 IN " + cn + " FOR doc2 IN " + en + " FILTER doc1._key == doc2._key RETURN doc1"); + }, + }; +} + +if (internal.debugCanUseFailAt()) { + jsunity.run(ahuacatlFailureSuite); +} + +return jsunity.done();