try to improve query shutdown in case DB server(s) can't be reached (#10359)

arangod/Aql/ExecutionEngine.cpp

@@ -587,10 +587,14 @@ std::pair<ExecutionState, size_t> ExecutionEngine::skipSome(size_t atMost) {
   return _root->skipSome(atMost);
 }
 
-Result ExecutionEngine::shutdownSync(int errorCode) noexcept {
+Result ExecutionEngine::shutdownSync(int errorCode) noexcept try {
   Result res{TRI_ERROR_INTERNAL};
   ExecutionState state = ExecutionState::WAITING;
   try {
+    TRI_IF_FAILURE("ExecutionEngine::shutdownSync") {
+      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+    }
+
     std::shared_ptr<SharedQueryState> sharedState = _query.sharedState();
     if (sharedState != nullptr) {
       sharedState->setContinueCallback();
@@ -602,10 +606,33 @@ Result ExecutionEngine::shutdownSync(int errorCode) noexcept {
         }
       }
     }
+  } catch (basics::Exception const& ex) {
+    res.reset(ex.code(), std::string("unable to shutdown query: ") + ex.what());
+  } catch (std::exception const& ex) {
+    res.reset(TRI_ERROR_INTERNAL, std::string("unable to shutdown query: ") + ex.what());
   } catch (...) {
     res.reset(TRI_ERROR_INTERNAL);
   }
+
+  if (res.fail() && ServerState::instance()->isCoordinator()) {
+    // shutdown attempt has failed...
+    // in a cluster, try to at least abort all other coordinator parts
+    auto queryRegistry = QueryRegistryFeature::registry();
+    if (queryRegistry != nullptr) {
+      for (auto const& id : _coordinatorQueryIds) {
+        try {
+          queryRegistry->destroy(_query.vocbase().name(), id, errorCode, false);
+        } catch (...) {
+          // we want to abort all parts, even if aborting other parts fails
+        }
+      }
+    }
+  }
+
   return res;
+} catch (...) {
+  // nothing we can do here...
+  return Result(TRI_ERROR_INTERNAL, "unable to shutdown query");
 }
 
 /// @brief shutdown, will be called exactly once for the whole query

arangod/Aql/RestAqlHandler.cpp

@@ -624,6 +624,9 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, Query* q
   // this is because the request may contain additional data
   if ((operation == "getSome" || operation == "skipSome") &&
       !query->engine()->initializeCursorCalled()) {
+    TRI_IF_FAILURE("RestAqlHandler::getSome") {
+      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+    }
     auto res = query->engine()->initializeCursor(nullptr, 0);
     if (res.first == ExecutionState::WAITING) {
       return RestStatus::WAITING;
@@ -648,6 +651,9 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, Query* q
         answerBuilder.add(StaticStrings::Error, VPackValue(res != TRI_ERROR_NO_ERROR));
         answerBuilder.add(StaticStrings::Code, VPackValue(res));
       } else if (operation == "getSome") {
+        TRI_IF_FAILURE("RestAqlHandler::getSome") {
+          THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+        }
         auto atMost =
             VelocyPackHelper::getNumericValue<size_t>(querySlice, "atMost",
                                                       ExecutionBlock::DefaultBatchSize());

arangod/RestServer/BootstrapFeature.cpp

@@ -91,7 +91,7 @@ namespace {
 
 /// Initialize certain agency entries, like Plan, system collections
 /// and various similar things. Only runs through on a SINGLE coordinator.
-/// must only return if we are boostrap lead or bootstrap is done
+/// must only return if we are bootstrap lead or bootstrap is done
 void raceForClusterBootstrap(BootstrapFeature& feature) {
   AgencyComm agency;
   auto& ci = feature.server().getFeature<ClusterFeature>().clusterInfo();
@@ -109,13 +109,13 @@ void raceForClusterBootstrap(BootstrapFeature& feature) {
         std::vector<std::string>({AgencyCommManager::path(), ::bootstrapKey}));
     if (value.isString()) {
       // key was found and is a string
-      std::string boostrapVal = value.copyString();
-      if (boostrapVal.find("done") != std::string::npos) {
+      std::string bootstrapVal = value.copyString();
+      if (bootstrapVal.find("done") != std::string::npos) {
         // all done, let's get out of here:
         LOG_TOPIC("61e04", TRACE, Logger::STARTUP)
             << "raceForClusterBootstrap: bootstrap already done";
         return;
-      } else if (boostrapVal == ServerState::instance()->getId()) {
+      } else if (bootstrapVal == ServerState::instance()->getId()) {
         agency.removeValues(::bootstrapKey, false);
       }
       LOG_TOPIC("49437", DEBUG, Logger::STARTUP)
@@ -320,7 +320,7 @@ void BootstrapFeature::start() {
       ss->setFoxxmaster(myId);  // could be empty, but set anyway
     }
 
-    if (v8Enabled) {  // runs the single server boostrap JS
+    if (v8Enabled) {  // runs the single server bootstrap JS
       // will run foxx/manager.js::_startup() and more (start queues, load
       // routes, etc)
       LOG_TOPIC("e0c8b", DEBUG, Logger::STARTUP) << "Running server/server.js";

tests/js/server/aql/aql-failures-cluster.js

@@ -0,0 +1,114 @@
+/*jshint globalstrict:false, strict:false, maxlen: 400 */
+/*global fail, assertEqual, AQL_EXECUTE */
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test failure scenarios
+///
+/// @file
+///
+/// DISCLAIMER
+///
+/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is triAGENS GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+var jsunity = require("jsunity");
+var arangodb = require("@arangodb");
+var db = arangodb.db;
+var internal = require("internal");
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test suite
+////////////////////////////////////////////////////////////////////////////////
+
+function ahuacatlFailureSuite () {
+  'use strict';
+  const cn = "UnitTestsAhuacatlFailures";
+  const en = "UnitTestsAhuacatlEdgeFailures";
+  
+  let  assertFailingQuery = function (query) {
+    try {
+      AQL_EXECUTE(query);
+      fail();
+    } catch (err) {
+      assertEqual(internal.errors.ERROR_DEBUG.code, err.errorNum, query);
+    }
+  };
+        
+  return {
+
+    setUpAll: function () {
+      internal.debugClearFailAt();
+      db._drop(cn);
+      db._create(cn);
+      db._drop(en);
+      db._createEdgeCollection(en);
+    },
+
+    setUp: function () {
+      internal.debugClearFailAt();
+    },
+
+    tearDownAll: function () {
+      internal.debugClearFailAt();
+      db._drop(cn);
+      db._drop(en);
+    },
+
+    tearDown: function() {
+      internal.debugClearFailAt();
+    },
+
+    testShutdownSync : function () {
+      internal.debugSetFailAt("ExecutionEngine::shutdownSync");
+
+      let res = AQL_EXECUTE("FOR doc IN " + cn + " RETURN doc").json;
+      // no real test expectations here, just that the query works and doesn't fail on shutdown
+      assertEqual(0, res.length);
+    },
+    
+    testShutdownSyncDiamond : function () {
+      internal.debugSetFailAt("ExecutionEngine::shutdownSync");
+
+      let res = AQL_EXECUTE("FOR doc1 IN " + cn + " FOR doc2 IN " + en + " FILTER doc1._key == doc2._key RETURN doc1").json;
+      // no real test expectations here, just that the query works and doesn't fail on shutdown
+      assertEqual(0, res.length);
+    },
+    
+    testShutdownSyncFailInGetSome : function () {
+      internal.debugSetFailAt("ExecutionEngine::shutdownSync");
+      internal.debugSetFailAt("RestAqlHandler::getSome");
+
+      assertFailingQuery("FOR doc IN " + cn + " RETURN doc");
+    },
+
+    testShutdownSyncDiamondFailInGetSome : function () {
+      internal.debugSetFailAt("ExecutionEngine::shutdownSync");
+      internal.debugSetFailAt("RestAqlHandler::getSome");
+
+      assertFailingQuery("FOR doc1 IN " + cn + " FOR doc2 IN " + en + " FILTER doc1._key == doc2._key RETURN doc1");
+    },
+  };
+}
+ 
+if (internal.debugCanUseFailAt()) {
+  jsunity.run(ahuacatlFailureSuite);
+}
+
+return jsunity.done();