From 25ee17c075fe623c111f1151033c3d2959600b66 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 14:50:28 +0200 Subject: [PATCH 01/10] initializeCursor and shutdown methods for ExecutionEngine. --- arangod/Aql/ExecutionEngine.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arangod/Aql/ExecutionEngine.h b/arangod/Aql/ExecutionEngine.h index c61743ca55..79d965e114 100644 --- a/arangod/Aql/ExecutionEngine.h +++ b/arangod/Aql/ExecutionEngine.h @@ -106,6 +106,22 @@ namespace triagens { return _query; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief initializeCursor, could be called multiple times +//////////////////////////////////////////////////////////////////////////////// + + int initializeCursor (AqlItemBlock* items, size_t pos) { + return _root->initializeCursor(items, pos); + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief shutdown, will be called exactly once for the whole query +//////////////////////////////////////////////////////////////////////////////// + + int shutdown () { + return _root->shutdown(); + } + //////////////////////////////////////////////////////////////////////////////// /// @brief getSome //////////////////////////////////////////////////////////////////////////////// From 95264a82abe4a266fc03d2902de5dfd78240515b Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 14:50:52 +0200 Subject: [PATCH 02/10] initializeCursor and shutdown methods for HTTP API for queries. --- arangod/Aql/RestAqlHandler.cpp | 44 ++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index e67eaf0923..7dab728628 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -381,7 +381,8 @@ void RestAqlHandler::deleteQuery (std::string const& idString) { //////////////////////////////////////////////////////////////////////////////// /// @brief PUT method for /_api/aql//, this is using /// the part of the cursor API with side effects. -/// : can be "getSome" or "skip". +/// : can be "getSome" or "skip" or "initializeCursor" or +/// "shutdown". /// The body must be a Json with the following attributes: /// For the "getSome" operation one has to give: /// "atLeast": @@ -402,6 +403,13 @@ void RestAqlHandler::deleteQuery (std::string const& idString) { /// "errorMessage" (if applicable) and "exhausted" (boolean) /// to indicate whether or not the cursor is exhausted. /// If "number" is not given it defaults to 1. +/// For the "initializeCursor" operation, one has to bind the following +/// attributes: +/// "items": This is a serialised AqlItemBlock with usually only one row +/// and the correct number of columns. +/// "pos": The number of the row in "items" to take, usually 0. +/// For the "shutdown" operation no additional arguments are required and +/// an empty JSON object in the body is OK. //////////////////////////////////////////////////////////////////////////////// void RestAqlHandler::useQuery (std::string const& operation, @@ -449,7 +457,7 @@ void RestAqlHandler::useQuery (std::string const& operation, } else if (operation == "skip") { auto number = JsonHelper::getNumericValue(queryJson.json(), - "number", 1); + "number", 1); try { bool exhausted = query->engine()->skip(number); answerBody("exhausted", Json(exhausted)) @@ -462,6 +470,38 @@ void RestAqlHandler::useQuery (std::string const& operation, return; } } + else if (operation == "initializeCursor") { + auto pos = JsonHelper::getNumericValue(queryJson.json(), + "pos", 0); + std::unique_ptr items; + int res; + try { + items.reset(new AqlItemBlock(queryJson.get("items"))); + res = query->engine()->initializeCursor(items.get(), pos); + } + catch (...) { + _queryRegistry->close(vocbase, qId); + generateError(HttpResponse::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, + "initializeCursor lead to an exception"); + return; + } + answerBody("error", res == TRI_ERROR_NO_ERROR ? Json(false) : Json(true)) + ("code", Json(static_cast(res))); + } + else if (operation == "shutdown") { + int res; + try { + res = query->engine()->shutdown(); + } + catch (...) { + _queryRegistry->close(vocbase, qId); + generateError(HttpResponse::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, + "shutdown lead to an exception"); + return; + } + answerBody("error", res == TRI_ERROR_NO_ERROR ? Json(false) : Json(true)) + ("code", Json(static_cast(res))); + } else { _queryRegistry->close(vocbase, qId); generateError(HttpResponse::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND); From 504ae4328d3482774d139a3b0600e07653400bcc Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Fri, 26 Sep 2014 14:56:21 +0200 Subject: [PATCH 03/10] be more verbose about the time & status of sub-processes. --- js/server/modules/org/arangodb/testing.js | 30 +++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/js/server/modules/org/arangodb/testing.js b/js/server/modules/org/arangodb/testing.js index 32ff0ee3db..24c15dad57 100644 --- a/js/server/modules/org/arangodb/testing.js +++ b/js/server/modules/org/arangodb/testing.js @@ -86,6 +86,7 @@ var _ = require("underscore"); var testFuncs = {}; var print = require("internal").print; +var time = require("internal").time; var fs = require("fs"); var download = require("internal").download; var wait = require("internal").wait; @@ -398,17 +399,25 @@ function runThere (options, instanceInfo, file) { function executeAndWait (cmd, args) { var pid = executeExternal(cmd, args); + var startTime = time(); var res = statusExternal(pid, true); + var deltaTime = time() - startTime; + print("Finished: " + res.status + " Signal: " + res.exit + " Time Elapsed: " + deltaTime); + if (res.status === "TERMINATED") { if (res.exit === 0) { - return { status: true, message: "" }; + return { status: true, message: "", duration: deltaTime}; } else { - return { status: false, message: "exit code was " + res.exit}; + return { status: false, message: "exit code was " + res.exit, duration: deltaTime}; } } else { - return { status: false, message: "irregular termination: " + res.status}; + return { + status: false, + message: "irregular termination: " + res.status + "Exit-Signal: " + res.exit, + duration: deltaTime + }; } } @@ -671,19 +680,26 @@ function rubyTests (options, ssl) { "--format", "d", "--require", tmpname, fs.join("UnitTests","HttpInterface",n)]; var pid = executeExternal("rspec", args); + var startTime = time(); var r = statusExternal(pid, true); - + var deltaTime = time() - startTime; + print("Finished: " + r.status + " Signal: " + r.exit + " Time Elapsed: " + deltaTime); if (r.status === "TERMINATED") { if (r.exit === 0) { - result[n] = { status: true, message: "" }; + result[n] = { status: true, message: "", duration: deltaTime }; } else { - result[n] = { status: false, message: "exit code was " + r.exit}; + result[n] = { status: false, message: "exit code was " + r.exit, duration: deltaTime}; } } else { - result[n] = { status: false, message: "irregular termination: " + r.status}; + result[n] = { + status: false, + message: "irregular termination: " + r.status + "Exit-Signal: " + r.exit, + duration: deltaTime + }; } + if (r.status === false && !options.force) { break; } From d003eecac15e5f9d38d61f09f5f3b5bd6241bc39 Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Fri, 26 Sep 2014 15:22:41 +0200 Subject: [PATCH 04/10] We need to pass a reference since we want to return values onto this; and return const references to not duplicate the memory. --- arangod/Aql/ExecutionNode.cpp | 8 ++++---- arangod/Aql/ExecutionNode.h | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 9df96c55db..b6aa670466 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -93,10 +93,10 @@ void ExecutionNode::validateType (int type) { } } -void ExecutionNode::getSortElements(SortElementVector elements, - ExecutionPlan* plan, - triagens::basics::Json const& oneNode, - char const* which) +void ExecutionNode::getSortElements(SortElementVector& elements, + ExecutionPlan* plan, + triagens::basics::Json const& oneNode, + char const* which) { triagens::basics::Json jsonElements = oneNode.get("elements"); if (! jsonElements.isList()){ diff --git a/arangod/Aql/ExecutionNode.h b/arangod/Aql/ExecutionNode.h index 19e31107a3..31e5ac91d8 100644 --- a/arangod/Aql/ExecutionNode.h +++ b/arangod/Aql/ExecutionNode.h @@ -511,7 +511,7 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// /// @brief factory for sort Elements from json. //////////////////////////////////////////////////////////////////////////////// - static void getSortElements(SortElementVector elements, + static void getSortElements(SortElementVector& elements, ExecutionPlan* plan, triagens::basics::Json const& oneNode, char const* which); @@ -1684,7 +1684,7 @@ namespace triagens { /// @brief get Variables Used Here including ASC/DESC //////////////////////////////////////////////////////////////////////////////// - SortElementVector getElements () const { + SortElementVector const & getElements () const { return _elements; } @@ -2799,11 +2799,11 @@ namespace triagens { /// @brief get Variables Used Here including ASC/DESC //////////////////////////////////////////////////////////////////////////////// - SortElementVector getElements () const { + SortElementVector const & getElements () const { return _elements; } - void setElements (SortElementVector const src) { + void setElements (SortElementVector const & src) { _elements = src; } From 93d32839acd27e1e87f20a06eb40d0ef1cc3bc9c Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Fri, 26 Sep 2014 15:53:22 +0200 Subject: [PATCH 05/10] differentiate between ABORT/TERMINATED/other --- js/server/modules/org/arangodb/testing.js | 32 ++++++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/js/server/modules/org/arangodb/testing.js b/js/server/modules/org/arangodb/testing.js index 24c15dad57..06bd5f0f7f 100644 --- a/js/server/modules/org/arangodb/testing.js +++ b/js/server/modules/org/arangodb/testing.js @@ -402,9 +402,9 @@ function executeAndWait (cmd, args) { var startTime = time(); var res = statusExternal(pid, true); var deltaTime = time() - startTime; - print("Finished: " + res.status + " Signal: " + res.exit + " Time Elapsed: " + deltaTime); if (res.status === "TERMINATED") { + print("Finished: " + res.status + " Exitcode: " + res.exit + " Time Elapsed: " + deltaTime); if (res.exit === 0) { return { status: true, message: "", duration: deltaTime}; } @@ -412,10 +412,19 @@ function executeAndWait (cmd, args) { return { status: false, message: "exit code was " + res.exit, duration: deltaTime}; } } - else { + else if (res.status === "ABORTED") { + print("Finished: " + res.status + " Signal: " + res.signal + " Time Elapsed: " + deltaTime); return { status: false, - message: "irregular termination: " + res.status + "Exit-Signal: " + res.exit, + message: "irregular termination: " + res.status + " Exit-Signal: " + res.signal, + duration: deltaTime + }; + } + else { + print("Finished: " + res.status + " Exitcode: " + res.exit + " Time Elapsed: " + deltaTime); + return { + status: false, + message: "irregular termination: " + res.status + " Exit-Code: " + res.exit, duration: deltaTime }; } @@ -683,19 +692,28 @@ function rubyTests (options, ssl) { var startTime = time(); var r = statusExternal(pid, true); var deltaTime = time() - startTime; - print("Finished: " + r.status + " Signal: " + r.exit + " Time Elapsed: " + deltaTime); if (r.status === "TERMINATED") { + print("Finished: " + r.status + " Signal: " + r.exit + " Time Elapsed: " + deltaTime); if (r.exit === 0) { result[n] = { status: true, message: "", duration: deltaTime }; } else { - result[n] = { status: false, message: "exit code was " + r.exit, duration: deltaTime}; + result[n] = { status: false, message: " exit code was " + r.exit, duration: deltaTime}; } } - else { + else if (r.status === "ABORTED") { + print("Finished: " + r.status + " Signal: " + r.exit + " Time Elapsed: " + deltaTime); result[n] = { status: false, - message: "irregular termination: " + r.status + "Exit-Signal: " + r.exit, + message: "irregular termination: " + r.status + " Exit-Signal: " + r.signal, + duration: deltaTime + }; + } + else { + print("Finished: " + r.status + " Exit Status: " + r.exit + " Time Elapsed: " + deltaTime); + result[n] = { + status: false, + message: "irregular termination: " + r.status + " Exit-code: " + r.exit, duration: deltaTime }; } From 865d513dd05e5894688a349f6db125fe8ae25f31 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 16:18:47 +0200 Subject: [PATCH 06/10] Add skipSome to HTTP API and to ExecutionEngine. --- arangod/Aql/ExecutionEngine.h | 8 ++++++++ arangod/Aql/RestAqlHandler.cpp | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/arangod/Aql/ExecutionEngine.h b/arangod/Aql/ExecutionEngine.h index 79d965e114..7e84f24680 100644 --- a/arangod/Aql/ExecutionEngine.h +++ b/arangod/Aql/ExecutionEngine.h @@ -130,6 +130,14 @@ namespace triagens { return _root->getSome(atLeast, atMost); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief skipSome +//////////////////////////////////////////////////////////////////////////////// + + size_t skipSome (size_t atLeast, size_t atMost) { + return _root->skipSome(atLeast, atMost); + } + //////////////////////////////////////////////////////////////////////////////// /// @brief getOne //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 7dab728628..676bd89e82 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -396,6 +396,19 @@ void RestAqlHandler::deleteQuery (std::string const& idString) { /// AqlItemBlock. /// If "atLeast" is not given it defaults to 1, if "atMost" is not /// given it defaults to ExecutionBlock::DefaultBatchSize. +/// For the "skipSome" operation one has to give: +/// "atLeast": +/// "atMost": both must be positive integers, the cursor skips never +/// more than "atMost" items and tries to skip at least +/// "atLeast". Note that it is possible to skip fewer than +/// "atLeast", for example if there are only fewer items +/// left. However, the implementation may skip fewer items +/// than "atLeast" for internal reasons, for example to avoid +/// excessive copying. The result is a JSON object with a +/// single attribute "skipped" containing the number of +/// skipped items. +/// If "atLeast" is not given it defaults to 1, if "atMost" is not +/// given it defaults to ExecutionBlock::DefaultBatchSize. /// For the "skip" operation one should give: /// "number": must be a positive integer, the cursor skips as many items, /// possibly exhausting the cursor. @@ -455,6 +468,23 @@ void RestAqlHandler::useQuery (std::string const& operation, } } } + else if (operation == "getSome") { + auto atLeast = JsonHelper::getNumericValue(queryJson.json(), + "atLeast", 1); + auto atMost = JsonHelper::getNumericValue(queryJson.json(), + "atMost", ExecutionBlock::DefaultBatchSize); + size_t skipped; + try { + skipped = query->engine()->skipSome(atLeast, atMost); + } + catch (...) { + _queryRegistry->close(vocbase, qId); + generateError(HttpResponse::SERVER_ERROR, TRI_ERROR_HTTP_SERVER_ERROR, + "skipSome lead to an exception"); + return; + } + answerBody("skipped", Json(static_cast(skipped))); + } else if (operation == "skip") { auto number = JsonHelper::getNumericValue(queryJson.json(), "number", 1); From 046d4f524dc2ac8c8f7e2f89e673a80adf68cd60 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 16:19:25 +0200 Subject: [PATCH 07/10] Implement getSome for RemoteBlock. --- arangod/Aql/ExecutionBlock.cpp | 65 ++++++++++++++++++- arangod/Aql/ExecutionBlock.h | 25 ++++++- .../RestHandler/RestReplicationHandler.cpp | 2 +- .../aardvark/frontend/js/bootstrap/errors.js | 1 + js/common/bootstrap/errors.js | 1 + lib/Basics/errors.dat | 1 + lib/Basics/voc-errors.cpp | 1 + lib/Basics/voc-errors.h | 14 ++++ 8 files changed, 105 insertions(+), 5 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index 66b4df11cd..d548b558e8 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -28,18 +28,21 @@ #include "Aql/ExecutionBlock.h" #include "Aql/ExecutionEngine.h" #include "Basics/StringUtils.h" +#include "Basics/StringBuffer.h" #include "Basics/json-utilities.h" #include "HashIndex/hash-index.h" #include "Utils/Exception.h" #include "VocBase/edge-collection.h" #include "VocBase/index.h" #include "VocBase/vocbase.h" +#include "Cluster/ClusterComm.h" using namespace triagens::arango; using namespace triagens::aql; using Json = triagens::basics::Json; using JsonHelper = triagens::basics::JsonHelper; +using StringBuffer = triagens::basics::StringBuffer; // ----------------------------------------------------------------------------- // --SECTION-- struct AggregatorGroup @@ -3933,6 +3936,12 @@ size_t ScatterBlock::skipSomeForShard (size_t atLeast, size_t atMost, std::strin // --SECTION-- class RemoteBlock // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief timeout +//////////////////////////////////////////////////////////////////////////////// + +double const RemoteBlock::defaultTimeOut = 3600.0; + //////////////////////////////////////////////////////////////////////////////// /// @brief initialize //////////////////////////////////////////////////////////////////////////////// @@ -3970,7 +3979,61 @@ int RemoteBlock::shutdown () { AqlItemBlock* RemoteBlock::getSome (size_t atLeast, size_t atMost) { - return nullptr; + // For every call we simply forward via HTTP + + ClusterComm* cc = ClusterComm::instance(); + std::unique_ptr res; + + // Later, we probably want to set these sensibly: + ClientTransactionID const clientTransactionId = "AQL"; + CoordTransactionID const coordTransactionId = 1; + + Json body(Json::Array, 2); + body("atLeast", Json(static_cast(atLeast))) + ("atMost", Json(static_cast(atMost))); + std::string bodyString(body.toString()); + std::map headers; + + res.reset(cc->syncRequest(clientTransactionId, + coordTransactionId, + _server, + rest::HttpRequest::HTTP_REQUEST_PUT, + std::string("/_db/") + + _engine->getTransaction()->vocbase()->_name + + "/_api/aql/getSome/" + _queryId, + bodyString, + headers, + 3600.0)); + + if (res->status == CL_COMM_TIMEOUT) { + // No reply, we give up: + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_TIMEOUT, + "timeout in cluster AQL operation"); + } + if (res->status == CL_COMM_ERROR) { + // This could be a broken connection or an Http error: + if (res->result == nullptr || ! res->result->isComplete()) { + // there is no result + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_CONNECTION_LOST, + "lost connection within cluster"); + } + // In this case a proper HTTP error was reported by the DBserver, + THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION); + } + + // If we get here, then res->result is the response which will be + // a serialized AqlItemBlock: + StringBuffer const& responseBodyBuf(res->result->getBody()); + Json responseBodyJson(TRI_UNKNOWN_MEM_ZONE, + TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, + responseBodyBuf.begin())); + if (JsonHelper::getBooleanValue(responseBodyJson.json(), "exhausted", true)) { + return nullptr; + } + else { + auto items = new triagens::aql::AqlItemBlock(responseBodyJson); + return items; + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index 84c4a376c4..600b096eed 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -1638,20 +1638,32 @@ public: class RemoteBlock : public ExecutionBlock { +//////////////////////////////////////////////////////////////////////////////// +/// @brief constructors/destructors +//////////////////////////////////////////////////////////////////////////////// + public: RemoteBlock (ExecutionEngine* engine, RemoteNode const* en, std::string const& server, - std::string const& ownName) + std::string const& ownName, + std::string const& queryId) : ExecutionBlock(engine, en), _server(server), - _ownName(ownName) { + _ownName(ownName), + _queryId(queryId) { } ~RemoteBlock () { } +//////////////////////////////////////////////////////////////////////////////// +/// @brief timeout +//////////////////////////////////////////////////////////////////////////////// + + static double const defaultTimeOut; + //////////////////////////////////////////////////////////////////////////////// /// @brief initialize //////////////////////////////////////////////////////////////////////////////// @@ -1710,11 +1722,18 @@ public: std::string _server; //////////////////////////////////////////////////////////////////////////////// -/// @brief our server, can be like "shard:S1000" or like "server:Claus" +/// @brief our own identity, in case of the coordinator this is empty, +/// in case of the DBservers, this is the shard ID as a string //////////////////////////////////////////////////////////////////////////////// std::string _ownName; +//////////////////////////////////////////////////////////////////////////////// +/// @brief the ID of the query on the server as a string +//////////////////////////////////////////////////////////////////////////////// + + std::string _queryId; + }; } // namespace triagens::aql diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 7f3e6928a3..d83ab979be 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -860,7 +860,7 @@ void RestReplicationHandler::handleTrampolineCoordinator () { } if (res->status == CL_COMM_ERROR) { // This could be a broken connection or an Http error: - if (res->result == 0 || !res->result->isComplete()) { + if (res->result == nullptr || !res->result->isComplete()) { // there is no result delete res; generateError(HttpResponse::BAD, TRI_ERROR_CLUSTER_CONNECTION_LOST, diff --git a/js/apps/system/aardvark/frontend/js/bootstrap/errors.js b/js/apps/system/aardvark/frontend/js/bootstrap/errors.js index b63ede088a..f2e991c0fa 100644 --- a/js/apps/system/aardvark/frontend/js/bootstrap/errors.js +++ b/js/apps/system/aardvark/frontend/js/bootstrap/errors.js @@ -144,6 +144,7 @@ "ERROR_CLUSTER_ONLY_ON_COORDINATOR" : { "code" : 1471, "message" : "this operation is only valid on a coordinator in a cluster" }, "ERROR_CLUSTER_READING_PLAN_AGENCY" : { "code" : 1472, "message" : "error reading Plan in agency" }, "ERROR_CLUSTER_COULD_NOT_TRUNCATE_COLLECTION" : { "code" : 1473, "message" : "could not truncate collection" }, + "ERROR_CLUSTER_AQL_COMMUNICATION" : { "code" : 1474, "message" : "error in cluster internal communication for AQL" }, "ERROR_QUERY_KILLED" : { "code" : 1500, "message" : "query killed" }, "ERROR_QUERY_PARSE" : { "code" : 1501, "message" : "%s" }, "ERROR_QUERY_EMPTY" : { "code" : 1502, "message" : "query is empty" }, diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index b63ede088a..f2e991c0fa 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -144,6 +144,7 @@ "ERROR_CLUSTER_ONLY_ON_COORDINATOR" : { "code" : 1471, "message" : "this operation is only valid on a coordinator in a cluster" }, "ERROR_CLUSTER_READING_PLAN_AGENCY" : { "code" : 1472, "message" : "error reading Plan in agency" }, "ERROR_CLUSTER_COULD_NOT_TRUNCATE_COLLECTION" : { "code" : 1473, "message" : "could not truncate collection" }, + "ERROR_CLUSTER_AQL_COMMUNICATION" : { "code" : 1474, "message" : "error in cluster internal communication for AQL" }, "ERROR_QUERY_KILLED" : { "code" : 1500, "message" : "query killed" }, "ERROR_QUERY_PARSE" : { "code" : 1501, "message" : "%s" }, "ERROR_QUERY_EMPTY" : { "code" : 1502, "message" : "query is empty" }, diff --git a/lib/Basics/errors.dat b/lib/Basics/errors.dat index 7bcc217d8c..69d75dd296 100755 --- a/lib/Basics/errors.dat +++ b/lib/Basics/errors.dat @@ -180,6 +180,7 @@ ERROR_CLUSTER_UNSUPPORTED,1470,"unsupported operation or parameter","Will be rai ERROR_CLUSTER_ONLY_ON_COORDINATOR,1471,"this operation is only valid on a coordinator in a cluster","Will be raised if there is an attempt to run a coordinator-only operation on a different type of node." ERROR_CLUSTER_READING_PLAN_AGENCY,1472,"error reading Plan in agency","Will be raised if a coordinator or DBserver cannot read the Plan in the agency." ERROR_CLUSTER_COULD_NOT_TRUNCATE_COLLECTION,1473,"could not truncate collection","Will be raised if a coordinator cannot truncate all shards of a cluster collection." +ERROR_CLUSTER_AQL_COMMUNICATION,1474,"error in cluster internal communication for AQL","Will be raised if the internal communication of the cluster for AQL produces an error." ################################################################################ ## ArangoDB query errors diff --git a/lib/Basics/voc-errors.cpp b/lib/Basics/voc-errors.cpp index 1e076bc299..4f4f4439d7 100644 --- a/lib/Basics/voc-errors.cpp +++ b/lib/Basics/voc-errors.cpp @@ -140,6 +140,7 @@ void TRI_InitialiseErrorMessages (void) { REG_ERROR(ERROR_CLUSTER_ONLY_ON_COORDINATOR, "this operation is only valid on a coordinator in a cluster"); REG_ERROR(ERROR_CLUSTER_READING_PLAN_AGENCY, "error reading Plan in agency"); REG_ERROR(ERROR_CLUSTER_COULD_NOT_TRUNCATE_COLLECTION, "could not truncate collection"); + REG_ERROR(ERROR_CLUSTER_AQL_COMMUNICATION, "error in cluster internal communication for AQL"); REG_ERROR(ERROR_QUERY_KILLED, "query killed"); REG_ERROR(ERROR_QUERY_PARSE, "%s"); REG_ERROR(ERROR_QUERY_EMPTY, "query is empty"); diff --git a/lib/Basics/voc-errors.h b/lib/Basics/voc-errors.h index 2588dd6992..e8eff0d72d 100644 --- a/lib/Basics/voc-errors.h +++ b/lib/Basics/voc-errors.h @@ -341,6 +341,9 @@ /// - 1473: @LIT{could not truncate collection} /// Will be raised if a coordinator cannot truncate all shards of a cluster /// collection. +/// - 1474: @LIT{error in cluster internal communication for AQL} +/// Will be raised if the internal communication of the cluster for AQL +/// produces an error. /// - 1500: @LIT{query killed} /// Will be raised when a running query is killed by an explicit admin /// command. @@ -1996,6 +1999,17 @@ void TRI_InitialiseErrorMessages (void); #define TRI_ERROR_CLUSTER_COULD_NOT_TRUNCATE_COLLECTION (1473) +//////////////////////////////////////////////////////////////////////////////// +/// @brief 1474: ERROR_CLUSTER_AQL_COMMUNICATION +/// +/// error in cluster internal communication for AQL +/// +/// Will be raised if the internal communication of the cluster for AQL +/// produces an error. +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_ERROR_CLUSTER_AQL_COMMUNICATION (1474) + //////////////////////////////////////////////////////////////////////////////// /// @brief 1500: ERROR_QUERY_KILLED /// From dbed62392f4e81286bdbe4c96e2553403ec7e74b Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 16:30:30 +0200 Subject: [PATCH 08/10] Implement skipSome method for RemoteBlock. --- arangod/Aql/ExecutionBlock.cpp | 55 +++++++++++++++++++++++++++++++++- arangod/Aql/RestAqlHandler.cpp | 3 +- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index d548b558e8..44fff2d704 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -4041,7 +4041,60 @@ AqlItemBlock* RemoteBlock::getSome (size_t atLeast, //////////////////////////////////////////////////////////////////////////////// size_t RemoteBlock::skipSome (size_t atLeast, size_t atMost) { - return 0; + // For every call we simply forward via HTTP + + ClusterComm* cc = ClusterComm::instance(); + std::unique_ptr res; + + // Later, we probably want to set these sensibly: + ClientTransactionID const clientTransactionId = "AQL"; + CoordTransactionID const coordTransactionId = 1; + + Json body(Json::Array, 2); + body("atLeast", Json(static_cast(atLeast))) + ("atMost", Json(static_cast(atMost))); + std::string bodyString(body.toString()); + std::map headers; + + res.reset(cc->syncRequest(clientTransactionId, + coordTransactionId, + _server, + rest::HttpRequest::HTTP_REQUEST_PUT, + std::string("/_db/") + + _engine->getTransaction()->vocbase()->_name + + "/_api/aql/skipSome/" + _queryId, + bodyString, + headers, + 3600.0)); + + if (res->status == CL_COMM_TIMEOUT) { + // No reply, we give up: + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_TIMEOUT, + "timeout in cluster AQL operation"); + } + if (res->status == CL_COMM_ERROR) { + // This could be a broken connection or an Http error: + if (res->result == nullptr || ! res->result->isComplete()) { + // there is no result + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_CONNECTION_LOST, + "lost connection within cluster"); + } + // In this case a proper HTTP error was reported by the DBserver, + THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION); + } + + // If we get here, then res->result is the response which will be + // a serialized AqlItemBlock: + StringBuffer const& responseBodyBuf(res->result->getBody()); + Json responseBodyJson(TRI_UNKNOWN_MEM_ZONE, + TRI_JsonString(TRI_UNKNOWN_MEM_ZONE, + responseBodyBuf.begin())); + if (JsonHelper::getBooleanValue(responseBodyJson.json(), "error", true)) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION); + } + size_t skipped = JsonHelper::getNumericValue(responseBodyJson.json(), + "skipped", 0); + return skipped; } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 676bd89e82..0d56c35f0f 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -483,7 +483,8 @@ void RestAqlHandler::useQuery (std::string const& operation, "skipSome lead to an exception"); return; } - answerBody("skipped", Json(static_cast(skipped))); + answerBody("skipped", Json(static_cast(skipped))) + ("error", Json(false)); } else if (operation == "skip") { auto number = JsonHelper::getNumericValue(queryJson.json(), From 5f5cf846414b85fb71db018f0378c256e2ca7fc0 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 16:49:26 +0200 Subject: [PATCH 09/10] Refactor getSome for RemoteBlock. --- arangod/Aql/ExecutionBlock.cpp | 94 +++++++++++++++++----------------- arangod/Aql/ExecutionBlock.h | 12 ++++- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index 44fff2d704..676270dbac 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -35,7 +35,6 @@ #include "VocBase/edge-collection.h" #include "VocBase/index.h" #include "VocBase/vocbase.h" -#include "Cluster/ClusterComm.h" using namespace triagens::arango; using namespace triagens::aql; @@ -3942,6 +3941,28 @@ size_t ScatterBlock::skipSomeForShard (size_t atLeast, size_t atMost, std::strin double const RemoteBlock::defaultTimeOut = 3600.0; +//////////////////////////////////////////////////////////////////////////////// +/// @brief local helper to throw an exception if a HTTP request went wrong +//////////////////////////////////////////////////////////////////////////////// + +static void throwExceptionAfterBadSyncRequest (ClusterCommResult* res) { + if (res->status == CL_COMM_TIMEOUT) { + // No reply, we give up: + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_TIMEOUT, + "timeout in cluster AQL operation"); + } + if (res->status == CL_COMM_ERROR) { + // This could be a broken connection or an Http error: + if (res->result == nullptr || ! res->result->isComplete()) { + // there is no result + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_CONNECTION_LOST, + "lost connection within cluster"); + } + // In this case a proper HTTP error was reported by the DBserver, + THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION); + } +} + //////////////////////////////////////////////////////////////////////////////// /// @brief initialize //////////////////////////////////////////////////////////////////////////////// @@ -4005,21 +4026,7 @@ AqlItemBlock* RemoteBlock::getSome (size_t atLeast, headers, 3600.0)); - if (res->status == CL_COMM_TIMEOUT) { - // No reply, we give up: - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_TIMEOUT, - "timeout in cluster AQL operation"); - } - if (res->status == CL_COMM_ERROR) { - // This could be a broken connection or an Http error: - if (res->result == nullptr || ! res->result->isComplete()) { - // there is no result - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_CONNECTION_LOST, - "lost connection within cluster"); - } - // In this case a proper HTTP error was reported by the DBserver, - THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION); - } + throwExceptionAfterBadSyncRequest(res.get()); // If we get here, then res->result is the response which will be // a serialized AqlItemBlock: @@ -4040,48 +4047,43 @@ AqlItemBlock* RemoteBlock::getSome (size_t atLeast, /// @brief skipSome //////////////////////////////////////////////////////////////////////////////// -size_t RemoteBlock::skipSome (size_t atLeast, size_t atMost) { - // For every call we simply forward via HTTP +ClusterCommResult* RemoteBlock::sendRequest ( + rest::HttpRequest::HttpRequestType type, + std::string urlPart, + std::string const& body) { ClusterComm* cc = ClusterComm::instance(); - std::unique_ptr res; // Later, we probably want to set these sensibly: ClientTransactionID const clientTransactionId = "AQL"; CoordTransactionID const coordTransactionId = 1; + std::map headers; + + return cc->syncRequest(clientTransactionId, + coordTransactionId, + _server, + type, + std::string("/_db/") + + _engine->getTransaction()->vocbase()->_name + + urlPart + _queryId, + body, + headers, + defaultTimeOut); +} + +size_t RemoteBlock::skipSome (size_t atLeast, size_t atMost) { + // For every call we simply forward via HTTP Json body(Json::Array, 2); body("atLeast", Json(static_cast(atLeast))) ("atMost", Json(static_cast(atMost))); std::string bodyString(body.toString()); - std::map headers; - res.reset(cc->syncRequest(clientTransactionId, - coordTransactionId, - _server, - rest::HttpRequest::HTTP_REQUEST_PUT, - std::string("/_db/") - + _engine->getTransaction()->vocbase()->_name - + "/_api/aql/skipSome/" + _queryId, - bodyString, - headers, - 3600.0)); - - if (res->status == CL_COMM_TIMEOUT) { - // No reply, we give up: - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_TIMEOUT, - "timeout in cluster AQL operation"); - } - if (res->status == CL_COMM_ERROR) { - // This could be a broken connection or an Http error: - if (res->result == nullptr || ! res->result->isComplete()) { - // there is no result - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CLUSTER_CONNECTION_LOST, - "lost connection within cluster"); - } - // In this case a proper HTTP error was reported by the DBserver, - THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_AQL_COMMUNICATION); - } + std::unique_ptr res; + res.reset(sendRequest(rest::HttpRequest::HTTP_REQUEST_PUT, + "/_api/aql/skipSome/", + bodyString)); + throwExceptionAfterBadSyncRequest(res.get()); // If we get here, then res->result is the response which will be // a serialized AqlItemBlock: diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index 600b096eed..1bb8b30763 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -39,6 +39,7 @@ #include "Utils/AqlTransaction.h" #include "Utils/transactions.h" #include "Utils/V8TransactionContext.h" +#include "Cluster/ClusterComm.h" namespace triagens { namespace aql { @@ -1714,11 +1715,20 @@ public: int64_t remaining () final; //////////////////////////////////////////////////////////////////////////////// -/// @brief our server, can be like "shard:S1000" or like "server:Claus" +/// @brief internal method to send a request //////////////////////////////////////////////////////////////////////////////// private: + triagens::arango::ClusterCommResult* sendRequest ( + rest::HttpRequest::HttpRequestType type, + std::string urlPart, + std::string const& body); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief our server, can be like "shard:S1000" or like "server:Claus" +//////////////////////////////////////////////////////////////////////////////// + std::string _server; //////////////////////////////////////////////////////////////////////////////// From 5696c37c1b73b7a3ad0afe2e7b7b3c2fc575a167 Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Fri, 26 Sep 2014 17:16:08 +0200 Subject: [PATCH 10/10] If a whole test suite fails, output the global error message. --- scripts/unittest.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/unittest.js b/scripts/unittest.js index c60623c548..746c99abed 100644 --- a/scripts/unittest.js +++ b/scripts/unittest.js @@ -31,7 +31,9 @@ function resultsToXml(results, baseName) { .text(xmlEscape(String(attrs[a]))).text("\""); } - close && this.text("/"); + if (close) { + this.text("/"); + } this.text(">\n"); return this; @@ -64,6 +66,13 @@ function resultsToXml(results, baseName) { } } + if ((!results[testrun][test].status) && + results[testrun][test].hasOwnProperty('message')) + { + xml.elem("failure"); + xml.text('\n'); + xml.elem("/failure"); + } xml.elem("/testsuite"); var fn = baseName + testrun.replace(/\//g, '_') + '_' + test.replace(/\//g, '_') + ".xml"; //print('Writing: '+ fn);