From 13e24b2db9be2a610deb6cc3467b54ed1e4c7203 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Thu, 10 Oct 2019 08:03:33 -0400 Subject: [PATCH] Convert many uses of ClusterComm to Fuerte (#10154) --- 3rdParty/fuerte/include/fuerte/message.h | 3 +- arangod/Aql/BlocksWithClients.h | 2 - arangod/Aql/ClusterNodes.cpp | 9 +- arangod/Aql/Collection.h | 2 +- ...EngineInfoContainerDBServerServerBased.cpp | 83 +- .../EngineInfoContainerDBServerServerBased.h | 8 +- arangod/Aql/ExecutionBlock.h | 2 - arangod/Aql/ExecutionEngine.cpp | 8 +- arangod/Aql/ExecutionNode.cpp | 1 + arangod/Aql/ShardLocking.cpp | 5 +- .../Aql/SingleRemoteModificationExecutor.cpp | 1 - arangod/Cluster/ClusterMethods.cpp | 4 +- arangod/Cluster/ClusterMethods.h | 3 +- arangod/Cluster/ClusterTraverser.cpp | 48 +- arangod/Cluster/ClusterTrxMethods.cpp | 10 +- arangod/Cluster/ClusterTrxMethods.h | 5 +- arangod/Cluster/SynchronizeShard.cpp | 254 ++-- arangod/Cluster/SynchronizeShard.h | 6 +- arangod/Cluster/TakeoverShardLeadership.cpp | 57 +- arangod/ClusterEngine/MMFilesMethods.cpp | 33 +- arangod/ClusterEngine/RocksDBMethods.cpp | 32 +- arangod/GeneralServer/RestHandler.cpp | 43 +- arangod/GeneralServer/RestHandler.h | 6 +- arangod/Graph/ShortestPathFinder.cpp | 36 +- .../IResearch/IResearchAnalyzerFeature.cpp | 99 +- arangod/Network/Methods.cpp | 61 +- arangod/Network/Methods.h | 39 +- arangod/Network/Utils.cpp | 55 + arangod/Network/Utils.h | 7 + .../RestHandler/RestControlPregelHandler.cpp | 14 +- .../RestHandler/RestControlPregelHandler.h | 2 +- arangod/RestHandler/RestCursorHandler.cpp | 16 +- arangod/RestHandler/RestCursorHandler.h | 2 +- arangod/RestHandler/RestDocumentHandler.cpp | 16 +- arangod/RestHandler/RestDocumentHandler.h | 2 +- arangod/RestHandler/RestJobHandler.cpp | 14 +- arangod/RestHandler/RestJobHandler.h | 2 +- arangod/RestHandler/RestQueryHandler.cpp | 14 +- arangod/RestHandler/RestQueryHandler.h | 2 +- .../RestHandler/RestReplicationHandler.cpp | 140 +-- arangod/RestHandler/RestReplicationHandler.h | 4 +- arangod/RestHandler/RestTasksHandler.cpp | 14 +- arangod/RestHandler/RestTasksHandler.h | 2 +- .../RestHandler/RestTransactionHandler.cpp | 16 +- arangod/RestHandler/RestTransactionHandler.h | 2 +- .../Sharding/ShardDistributionReporter.cpp | 113 +- arangod/Sharding/ShardDistributionReporter.h | 6 +- arangod/Transaction/Manager.cpp | 67 +- arangod/Transaction/Methods.cpp | 45 +- arangod/V8Server/v8-actions.cpp | 63 +- lib/Basics/StaticStrings.cpp | 3 + lib/Basics/StaticStrings.h | 2 + lib/CMakeLists.txt | 1 + lib/Futures/Future.h | 4 + lib/Futures/Utilities.h | 4 + lib/Rest/CommonDefines.cpp | 54 + lib/Rest/CommonDefines.h | 2 + .../ShardDistributionReporterTest.cpp | 1084 ++++------------- 58 files changed, 1142 insertions(+), 1490 deletions(-) create mode 100644 lib/Rest/CommonDefines.cpp diff --git a/3rdParty/fuerte/include/fuerte/message.h b/3rdParty/fuerte/include/fuerte/message.h index a3d2c04ec9..1822a2f3a7 100644 --- a/3rdParty/fuerte/include/fuerte/message.h +++ b/3rdParty/fuerte/include/fuerte/message.h @@ -91,6 +91,7 @@ struct RequestHeader final : public MessageHeader { // accept header accessors ContentType acceptType() const { return _acceptType; } void acceptType(ContentType type) { _acceptType = type; } + void acceptType(std::string const& type) { _acceptType = to_ContentType(type); } // query parameter helpers void addParameter(std::string const& key, std::string const& value); @@ -207,7 +208,7 @@ class Request final : public Message { }; // Response contains the message resulting from a request to a server. -class Response final : public Message { +class Response : public Message { public: Response(ResponseHeader reqHeader = ResponseHeader()) : header(std::move(reqHeader)), _payloadOffset(0) {} diff --git a/arangod/Aql/BlocksWithClients.h b/arangod/Aql/BlocksWithClients.h index 8e5825775e..bdc659a1ae 100644 --- a/arangod/Aql/BlocksWithClients.h +++ b/arangod/Aql/BlocksWithClients.h @@ -47,8 +47,6 @@ namespace transaction { class Methods; } -struct ClusterCommResult; - namespace aql { class AqlItemBlock; struct Collection; diff --git a/arangod/Aql/ClusterNodes.cpp b/arangod/Aql/ClusterNodes.cpp index 75648f3ed1..ca17db8c26 100644 --- a/arangod/Aql/ClusterNodes.cpp +++ b/arangod/Aql/ClusterNodes.cpp @@ -21,6 +21,11 @@ /// @author Max Neunhoeffer //////////////////////////////////////////////////////////////////////////////// +#include + +#include +#include + #include "ClusterNodes.h" #include "Aql/AqlValue.h" @@ -45,11 +50,9 @@ #include "Aql/types.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ServerState.h" - +#include "Logger/LogMacros.h" #include "Transaction/Methods.h" -#include - using namespace arangodb; using namespace arangodb::basics; using namespace arangodb::aql; diff --git a/arangod/Aql/Collection.h b/arangod/Aql/Collection.h index 8d384b6e44..8fb50c24c6 100644 --- a/arangod/Aql/Collection.h +++ b/arangod/Aql/Collection.h @@ -24,7 +24,7 @@ #ifndef ARANGOD_AQL_COLLECTION_H #define ARANGOD_AQL_COLLECTION_H 1 -#include "Cluster/ClusterComm.h" +#include "Cluster/ClusterTypes.h" #include "VocBase/AccessMode.h" #include "VocBase/vocbase.h" diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index fea39403da..50355f61d5 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -29,10 +29,14 @@ #include "Aql/GraphNode.h" #include "Aql/Query.h" #include "Aql/QuerySnippet.h" -#include "Cluster/ClusterComm.h" +#include "Basics/StringUtils.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ClusterTrxMethods.h" #include "Graph/BaseOptions.h" +#include "Logger/LogMacros.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "StorageEngine/TransactionState.h" #include "Utils/CollectionNameResolver.h" @@ -66,10 +70,6 @@ Result ExtractRemoteAndShard(VPackSlice keySlice, size_t& remoteId, std::string& return {TRI_ERROR_NO_ERROR}; } -struct NoopCb final : public arangodb::ClusterCommCallback { - bool operator()(ClusterCommResult*) override { return true; } -}; - } // namespace EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngineShardLists( @@ -278,8 +278,9 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( // Otherwise the locking needs to be empty. TRI_ASSERT(!_closedSnippets.empty() || !_graphNodes.empty()); - auto cc = ClusterComm::instance(); - if (cc == nullptr) { + NetworkFeature const& nf = _query.vocbase().server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool == nullptr) { // nullptr only happens on controlled shutdown return {TRI_ERROR_SHUTTING_DOWN}; } @@ -290,13 +291,15 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( "/_db/" + arangodb::basics::StringUtils::urlEncode(_query.vocbase().name()) + "/_api/aql/setup?ttl=" + std::to_string(ttl)); - auto cleanupGuard = scopeGuard([this, &cc, &queryIds]() { - cleanupEngines(cc, TRI_ERROR_INTERNAL, _query.vocbase().name(), queryIds); + auto cleanupGuard = scopeGuard([this, pool, &queryIds]() { + cleanupEngines(pool, TRI_ERROR_INTERNAL, _query.vocbase().name(), queryIds); }); // Build Lookup Infos VPackBuilder infoBuilder; transaction::Methods* trx = _query.trx(); + network::RequestOptions options; + options.timeout = network::Timeout(SETUP_TIMEOUT); for (auto const& server : dbServers) { std::string const serverDest = "server:" + server; @@ -344,22 +347,29 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( !infoSlice.get("snippets").isEmptyObject()) || infoSlice.hasKey("traverserEngines")); + VPackBuffer buffer(infoSlice.byteSize()); + buffer.append(infoSlice.begin(), infoSlice.byteSize()); + // add the transaction ID header - std::unordered_map headers; + network::Headers headers; ClusterTrxMethods::addAQLTransactionHeader(*trx, server, headers); - CoordTransactionID coordTransactionID = TRI_NewTickServer(); - auto res = cc->syncRequest(coordTransactionID, serverDest, RequestType::POST, - url, infoSlice.toJson(), headers, SETUP_TIMEOUT); + auto res = network::sendRequest(pool, serverDest, fuerte::RestVerb::Post, + url, std::move(buffer), headers, options) + .get(); _query.incHttpRequests(1); - if (res->getErrorCode() != TRI_ERROR_NO_ERROR) { + if (res.fail()) { + int code = network::fuerteToArangoErrorCode(res); + std::string message = network::fuerteToArangoErrorMessage(res); LOG_TOPIC("f9a77", DEBUG, Logger::AQL) - << server << " responded with " << res->getErrorCode() << " -> " - << res->stringifyErrorMessage(); + << server << " responded with " << code << " -> " << message; LOG_TOPIC("41082", TRACE, Logger::AQL) << infoSlice.toJson(); - return {res->getErrorCode(), res->stringifyErrorMessage()}; + return {code, message}; } - std::shared_ptr builder = res->result->getBodyVelocyPack(); - VPackSlice response = builder->slice(); + auto slices = res.response->slices(); + if (slices.empty()) { + return {TRI_ERROR_INTERNAL, "malformed response while building engines"}; + } + VPackSlice response = slices[0]; auto result = parseResponse(response, queryIds, server, serverDest, didCreateEngine); if (!result.ok()) { return result; @@ -447,48 +457,53 @@ Result EngineInfoContainerDBServerServerBased::parseResponse( * they may be leftovers from Coordinator. * Will also clear the list of queryIds after return. * - * @param cc The ClusterComm + * @param pool The ConnectionPool * @param errorCode error Code to be send to DBServers for logging. * @param dbname Name of the database this query is executed in. * @param queryIds A map of QueryIds of the format: (remoteNodeId:shardId) * -> queryid. */ void EngineInfoContainerDBServerServerBased::cleanupEngines( - std::shared_ptr cc, int errorCode, std::string const& dbname, + network::ConnectionPool* pool, int errorCode, std::string const& dbname, MapRemoteToSnippet& queryIds) const { + network::RequestOptions options; + options.timeout = network::Timeout(10.0); // Picked arbitrarily + network::Headers headers; + // Shutdown query snippets std::string url("/_db/" + arangodb::basics::StringUtils::urlEncode(dbname) + "/_api/aql/shutdown/"); - std::vector requests; - auto body = std::make_shared( - "{\"code\":" + std::to_string(errorCode) + "}"); + VPackBuffer body; + VPackBuilder builder(body); + builder.openObject(); + builder.add("code", VPackValue(std::to_string(errorCode))); + builder.close(); for (auto const& it : queryIds) { // it.first == RemoteNodeId, we don't need this // it.second server -> [snippets] for (auto const& serToSnippets : it.second) { auto server = serToSnippets.first; for (auto const& shardId : serToSnippets.second) { - requests.emplace_back(server, rest::RequestType::PUT, url + shardId, body); + // fire and forget + network::sendRequest(pool, server, fuerte::RestVerb::Put, url + shardId, + body, headers, options); } + _query.incHttpRequests(serToSnippets.second.size()); } } // Shutdown traverser engines url = "/_db/" + arangodb::basics::StringUtils::urlEncode(dbname) + "/_internal/traverser/"; - std::unordered_map headers; - std::shared_ptr noBody; + VPackBuffer noBody; - CoordTransactionID coordinatorTransactionID = TRI_NewTickServer(); - auto cb = std::make_shared<::NoopCb>(); - - constexpr double shortTimeout = 10.0; // Picked arbitrarily for (auto const& gn : _graphNodes) { auto allEngines = gn->engines(); for (auto const& engine : *allEngines) { - cc->asyncRequest(coordinatorTransactionID, engine.first, rest::RequestType::DELETE_REQ, - url + basics::StringUtils::itoa(engine.second), noBody, - headers, cb, shortTimeout, false, 2.0); + // fire and forget + network::sendRequestRetry(pool, engine.first, fuerte::RestVerb::Delete, + url + basics::StringUtils::itoa(engine.second), + noBody, headers, options); } _query.incHttpRequests(allEngines->size()); } diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.h b/arangod/Aql/EngineInfoContainerDBServerServerBased.h index e08327d609..1a5666bbaf 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.h +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.h @@ -36,7 +36,9 @@ #include namespace arangodb { -class ClusterComm; +namespace network { +class ConnectionPool; +} namespace velocypack { class Builder; @@ -133,13 +135,13 @@ class EngineInfoContainerDBServerServerBased { * they may be leftovers from Coordinator. * Will also clear the list of queryIds after return. * - * @param cc The ClusterComm + * @param pool The ConnectionPool * @param errorCode error Code to be send to DBServers for logging. * @param dbname Name of the database this query is executed in. * @param queryIds A map of QueryIds of the format: (remoteNodeId:shardId) * -> queryid. */ - void cleanupEngines(std::shared_ptr cc, int errorCode, + void cleanupEngines(network::ConnectionPool* pool, int errorCode, std::string const& dbname, MapRemoteToSnippet& queryIds) const; // Insert a GraphNode that needs to generate TraverserEngines on diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index d55fe49baf..8dbba6258c 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -34,8 +34,6 @@ #include namespace arangodb { -struct ClusterCommResult; - namespace transaction { class Methods; } diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index f94ea26048..54e71a345e 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -39,9 +39,9 @@ #include "Aql/ReturnExecutor.h" #include "Aql/WalkerWorker.h" #include "Basics/ScopeGuard.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ServerState.h" #include "Logger/Logger.h" +#include "Network/NetworkFeature.h" using namespace arangodb; using namespace arangodb::aql; @@ -467,8 +467,10 @@ struct DistributedQueryInstanciator final : public WalkerWorker { // QueryIds are filled by responses of DBServer parts. MapRemoteToSnippet queryIds{}; - auto cleanupGuard = scopeGuard([this, &queryIds]() { - _dbserverParts.cleanupEngines(ClusterComm::instance(), TRI_ERROR_INTERNAL, + NetworkFeature const& nf = _query.vocbase().server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + auto cleanupGuard = scopeGuard([this, pool, &queryIds]() { + _dbserverParts.cleanupEngines(pool, TRI_ERROR_INTERNAL, _query.vocbase().name(), queryIds); }); std::unordered_map nodeAliases; diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 876519d5b6..2b34fb717e 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -61,6 +61,7 @@ #include "Aql/TraversalNode.h" #include "Aql/WalkerWorker.h" #include "Aql/MaterializeExecutor.h" +#include "Basics/VelocyPackHelper.h" #include "Basics/system-compiler.h" #include "Cluster/ServerState.h" #include "Meta/static_assert_size.h" diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index 8d4c92f236..363e667ae7 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -21,6 +21,8 @@ /// @author Michael Hackstein //////////////////////////////////////////////////////////////////////////////// +#include + #include "ShardLocking.h" #include "Aql/Collection.h" @@ -30,8 +32,7 @@ #include "Aql/ModificationNodes.h" #include "Aql/Query.h" #include "Cluster/ClusterFeature.h" - -#include +#include "Logger/LogMacros.h" using namespace arangodb; using namespace arangodb::aql; diff --git a/arangod/Aql/SingleRemoteModificationExecutor.cpp b/arangod/Aql/SingleRemoteModificationExecutor.cpp index 36c824fd77..f373bb4194 100644 --- a/arangod/Aql/SingleRemoteModificationExecutor.cpp +++ b/arangod/Aql/SingleRemoteModificationExecutor.cpp @@ -28,7 +28,6 @@ #include "Aql/SingleRowFetcher.h" #include "Basics/Common.h" #include "Basics/StaticStrings.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "ModificationExecutorTraits.h" diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 910235e240..5baef5dbad 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -791,11 +791,11 @@ int handleGeneralCommErrors(arangodb::ClusterCommResult const* res) { /// @brief creates a copy of all HTTP headers to forward //////////////////////////////////////////////////////////////////////////////// -std::unordered_map getForwardableRequestHeaders(arangodb::GeneralRequest* request) { +network::Headers getForwardableRequestHeaders(arangodb::GeneralRequest* request) { std::unordered_map const& headers = request->headers(); std::unordered_map::const_iterator it = headers.begin(); - std::unordered_map result; + network::Headers result; while (it != headers.end()) { std::string const& key = (*it).first; diff --git a/arangod/Cluster/ClusterMethods.h b/arangod/Cluster/ClusterMethods.h index f2bbd8698a..ba6b23ba0f 100644 --- a/arangod/Cluster/ClusterMethods.h +++ b/arangod/Cluster/ClusterMethods.h @@ -30,6 +30,7 @@ #include "Cluster/ClusterFeature.h" #include "Cluster/TraverserEngineRegistry.h" #include "Futures/Future.h" +#include "Network/types.h" #include "Rest/CommonDefines.h" #include "Rest/GeneralResponse.h" #include "Utils/OperationResult.h" @@ -56,7 +57,7 @@ int handleGeneralCommErrors(arangodb::ClusterCommResult const* res); /// @brief creates a copy of all HTTP headers to forward //////////////////////////////////////////////////////////////////////////////// -std::unordered_map getForwardableRequestHeaders(GeneralRequest*); +network::Headers getForwardableRequestHeaders(GeneralRequest*); //////////////////////////////////////////////////////////////////////////////// /// @brief check if a list of attributes have the same values in two vpack diff --git a/arangod/Cluster/ClusterTraverser.cpp b/arangod/Cluster/ClusterTraverser.cpp index d228b91e0a..448cbf5b97 100644 --- a/arangod/Cluster/ClusterTraverser.cpp +++ b/arangod/Cluster/ClusterTraverser.cpp @@ -22,18 +22,22 @@ //////////////////////////////////////////////////////////////////////////////// #include "ClusterTraverser.h" + +#include +#include + #include "Basics/StaticStrings.h" #include "Basics/VelocyPackHelper.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterMethods.h" #include "Graph/BreadthFirstEnumerator.h" #include "Graph/ClusterTraverserCache.h" #include "Graph/TraverserCache.h" +#include "Logger/LogMacros.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "Transaction/Helpers.h" -#include -#include - using namespace arangodb; using namespace arangodb::graph; @@ -152,9 +156,9 @@ void ClusterTraverser::addVertexToVelocyPack(arangodb::velocypack::StringRef vid void ClusterTraverser::destroyEngines() { // We have to clean up the engines in Coordinator Case. - auto cc = ClusterComm::instance(); - - if (cc != nullptr) { + NetworkFeature const& nf = _trx->vocbase().server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool != nullptr) { // nullptr only happens on controlled server shutdown std::string const url( "/_db/" + arangodb::basics::StringUtils::urlEncode(_trx->vocbase().name()) + @@ -162,24 +166,26 @@ void ClusterTraverser::destroyEngines() { if (_enumerator != nullptr) { _enumerator->incHttpRequests(_engines->size()); - } + } + + VPackBuffer body; + network::Headers headers; + network::RequestOptions options; + options.timeout = network::Timeout(30.0); + for (auto const& it : *_engines) { - arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer(); - std::unordered_map headers; - auto res = cc->syncRequest(coordTransactionID, "server:" + it.first, - RequestType::DELETE_REQ, - url + arangodb::basics::StringUtils::itoa(it.second), - "", headers, 30.0); + auto res = + network::sendRequest(pool, "server:" + it.first, fuerte::RestVerb::Delete, + url + arangodb::basics::StringUtils::itoa(it.second), + body, headers, options); + res.wait(); - if (res->status != CL_COMM_SENT) { - // Note If there was an error on server side we do not have - // CL_COMM_SENT + if (!res.hasValue() || res.get().fail()) { + // Note If there was an error on server side we do not have ok() std::string message("Could not destroy all traversal engines"); - - if (!res->errorMessage.empty()) { - message += std::string(": ") + res->errorMessage; + if (res.hasValue()) { + message += ": " + network::fuerteToArangoErrorMessage(res.get()); } - LOG_TOPIC("8a7a0", ERR, arangodb::Logger::FIXME) << message; } } diff --git a/arangod/Cluster/ClusterTrxMethods.cpp b/arangod/Cluster/ClusterTrxMethods.cpp index 63130b57f7..1a6c6ef2db 100644 --- a/arangod/Cluster/ClusterTrxMethods.cpp +++ b/arangod/Cluster/ClusterTrxMethods.cpp @@ -377,8 +377,9 @@ template void addTransactionHeader> std::unordered_map&); /// @brief add transaction ID header for setting up AQL snippets -void addAQLTransactionHeader(transaction::Methods const& trx, ServerID const& server, - std::unordered_map& headers) { +template +void addAQLTransactionHeader(transaction::Methods const& trx, + ServerID const& server, MapT& headers) { TransactionState& state = *trx.state(); TRI_ASSERT(state.isCoordinator()); if (!ClusterTrxMethods::isElCheapo(trx)) { @@ -402,6 +403,11 @@ void addAQLTransactionHeader(transaction::Methods const& trx, ServerID const& se } headers.emplace(arangodb::StaticStrings::TransactionId, std::move(value)); } +template void addAQLTransactionHeader>( + transaction::Methods const&, ServerID const&, std::map&); +template void addAQLTransactionHeader>( + transaction::Methods const&, ServerID const&, + std::unordered_map&); bool isElCheapo(transaction::Methods const& trx) { return isElCheapo(*trx.state()); diff --git a/arangod/Cluster/ClusterTrxMethods.h b/arangod/Cluster/ClusterTrxMethods.h index 3672522d30..c9d2ea381b 100644 --- a/arangod/Cluster/ClusterTrxMethods.h +++ b/arangod/Cluster/ClusterTrxMethods.h @@ -54,8 +54,9 @@ void addTransactionHeader(transaction::Methods const& trx, ServerID const& server, MapT& headers); /// @brief add transaction ID header for setting up AQL snippets -void addAQLTransactionHeader(transaction::Methods const& trx, ServerID const& server, - std::unordered_map& headers); +template +void addAQLTransactionHeader(transaction::Methods const& trx, + ServerID const& server, MapT& headers); /// @brief check whether this is a kind el cheapo transaction bool isElCheapo(transaction::Methods const& trx); diff --git a/arangod/Cluster/SynchronizeShard.cpp b/arangod/Cluster/SynchronizeShard.cpp index 55ccc4ceb2..66e7226c1c 100644 --- a/arangod/Cluster/SynchronizeShard.cpp +++ b/arangod/Cluster/SynchronizeShard.cpp @@ -27,13 +27,16 @@ #include "Agency/AgencyStrings.h" #include "Agency/TimeString.h" #include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/StringUtils.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ActionDescription.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/FollowerInfo.h" #include "Cluster/MaintenanceFeature.h" #include "Cluster/ServerState.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "Replication/DatabaseInitialSyncer.h" #include "Replication/DatabaseReplicationApplier.h" #include "Replication/DatabaseTailingSyncer.h" @@ -121,14 +124,6 @@ SynchronizeShard::SynchronizeShard(MaintenanceFeature& feature, ActionDescriptio SynchronizeShard::~SynchronizeShard() = default; -class SynchronizeShardCallback : public arangodb::ClusterCommCallback { - public: - explicit SynchronizeShardCallback(SynchronizeShard* callee) {} - virtual bool operator()(arangodb::ClusterCommResult*) override final { - return true; - } -}; - static std::stringstream& AppendShardInformationToMessage( std::string const& database, std::string const& shard, std::string const& planId, std::chrono::system_clock::time_point const& startTime, std::stringstream& msg) { @@ -140,27 +135,26 @@ static std::stringstream& AppendShardInformationToMessage( return msg; } -static arangodb::Result getReadLockId(std::string const& endpoint, +static arangodb::Result getReadLockId(network::ConnectionPool* pool, + std::string const& endpoint, std::string const& database, std::string const& clientId, double timeout, uint64_t& id) { std::string error("startReadLockOnLeader: Failed to get read lock - "); - auto cc = arangodb::ClusterComm::instance(); - if (cc == nullptr) { // nullptr only happens during controlled shutdown + if (pool == nullptr) { // nullptr only happens during controlled shutdown return arangodb::Result(TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); } - auto comres = - cc->syncRequest(TRI_NewTickServer(), endpoint, rest::RequestType::GET, - DB + database + REPL_HOLD_READ_LOCK, std::string(), - std::unordered_map(), timeout); + network::RequestOptions options; + options.timeout = network::Timeout(timeout); + auto res = network::sendRequest(pool, endpoint, fuerte::RestVerb::Get, + DB + database + REPL_HOLD_READ_LOCK, + VPackBuffer(), network::Headers(), options) + .get(); - auto result = comres->result; - - if (result != nullptr && result->getHttpReturnCode() == 200) { - auto const idv = result->getBodyVelocyPack(); - auto const& idSlice = idv->slice(); + if (res.ok() && res.response->statusCode() == fuerte::StatusOK) { + auto const idSlice = res.response->slice(); TRI_ASSERT(idSlice.isObject()); TRI_ASSERT(idSlice.hasKey(ID)); try { @@ -171,11 +165,7 @@ static arangodb::Result getReadLockId(std::string const& endpoint, return arangodb::Result(TRI_ERROR_INTERNAL, error); } } else { - if (result) { - error.append(result->getHttpReturnMessage()); - } else { - error.append(comres->stringifyErrorMessage()); - } + error.append(network::fuerteToArangoErrorMessage(res)); return arangodb::Result(TRI_ERROR_INTERNAL, error); } @@ -211,19 +201,16 @@ static arangodb::Result collectionCount(std::shared_ptrsyncRequest(TRI_NewTickServer(), endpoint, rest::RequestType::PUT, - DB + database + REPL_ADD_FOLLOWER, body.toJson(), - std::unordered_map(), timeout); + network::RequestOptions options; + options.timeout = network::Timeout(timeout); + auto res = network::sendRequest(pool, endpoint, fuerte::RestVerb::Put, + DB + database + REPL_ADD_FOLLOWER, + std::move(*body.steal()), network::Headers(), options) + .get(); - auto result = comres->result; std::string errorMessage( "addShardFollower: could not add us to the leader's follower list. "); - if (result == nullptr || result->getHttpReturnCode() != 200) { + if (res.fail() || res.response->statusCode() != fuerte::StatusOK) { if (lockJobId != 0) { - errorMessage += comres->stringifyErrorMessage(); + errorMessage += network::fuerteToArangoErrorMessage(res); LOG_TOPIC("22e0a", ERR, Logger::MAINTENANCE) << errorMessage; } else { errorMessage += "With shortcut (can happen, no problem)."; @@ -310,12 +300,12 @@ static arangodb::Result addShardFollower(std::string const& endpoint, } } -static arangodb::Result cancelReadLockOnLeader(std::string const& endpoint, +static arangodb::Result cancelReadLockOnLeader(network::ConnectionPool* pool, + std::string const& endpoint, std::string const& database, uint64_t lockJobId, std::string const& clientId, double timeout = 10.0) { - auto cc = arangodb::ClusterComm::instance(); - if (cc == nullptr) { // nullptr only happens during controlled shutdown + if (pool == nullptr) { // nullptr only happens during controlled shutdown return arangodb::Result(TRI_ERROR_SHUTTING_DOWN, "cancelReadLockOnLeader: Shutting down"); } @@ -326,16 +316,15 @@ static arangodb::Result cancelReadLockOnLeader(std::string const& endpoint, body.add(ID, VPackValue(std::to_string(lockJobId))); } - auto comres = - cc->syncRequest(TRI_NewTickServer(), endpoint, rest::RequestType::DELETE_REQ, - DB + database + REPL_HOLD_READ_LOCK, body.toJson(), - std::unordered_map(), timeout); + network::RequestOptions options; + options.timeout = network::Timeout(timeout); + auto res = network::sendRequest(pool, endpoint, fuerte::RestVerb::Delete, + DB + database + REPL_HOLD_READ_LOCK, + std::move(*body.steal()), network::Headers(), options) + .get(); - auto result = comres->result; - - if (result != nullptr && result->getHttpReturnCode() == 404) { - auto const vp = result->getBodyVelocyPack(); - auto const& slice = vp->slice(); + if (res.ok() && res.response && res.response->statusCode() == fuerte::StatusNotFound) { + auto const slice = res.response->slice(); if (slice.isObject()) { VPackSlice s = slice.get(StaticStrings::ErrorNum); if (s.isNumber()) { @@ -348,8 +337,14 @@ static arangodb::Result cancelReadLockOnLeader(std::string const& endpoint, } } - if (result == nullptr || result->getHttpReturnCode() != 200) { - auto errorMessage = comres->stringifyErrorMessage(); + if (res.fail() || res.response->statusCode() != fuerte::StatusOK) { + auto errorMessage = network::fuerteToArangoErrorMessage(res); + // rebuild body since we stole it earlier + VPackBuilder body; + { + VPackObjectBuilder b(&body); + body.add(ID, VPackValue(std::to_string(lockJobId))); + } LOG_TOPIC("52924", ERR, Logger::MAINTENANCE) << "cancelReadLockOnLeader: exception caught for " << body.toJson() << ": " << errorMessage; @@ -360,46 +355,40 @@ static arangodb::Result cancelReadLockOnLeader(std::string const& endpoint, return arangodb::Result(); } -static arangodb::Result cancelBarrier(std::string const& endpoint, +static arangodb::Result cancelBarrier(network::ConnectionPool* pool, + std::string const& endpoint, std::string const& database, int64_t barrierId, std::string const& clientId, double timeout = 120.0) { if (barrierId <= 0) { return Result(); } - auto cc = arangodb::ClusterComm::instance(); - if (cc == nullptr) { // nullptr only happens during controlled shutdown + if (pool == nullptr) { // nullptr only happens during controlled shutdown return arangodb::Result(TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); } - auto comres = - cc->syncRequest(TRI_NewTickServer(), endpoint, rest::RequestType::DELETE_REQ, - DB + database + REPL_BARRIER_API + std::to_string(barrierId), - std::string(), - std::unordered_map(), timeout); + network::RequestOptions options; + options.timeout = network::Timeout(timeout); + auto res = + network::sendRequest(pool, endpoint, fuerte::RestVerb::Delete, + DB + database + REPL_BARRIER_API + std::to_string(barrierId), + VPackBuffer(), network::Headers(), options) + .get(); - // I'm sure that syncRequest cannot return null. But the check doesn't hurt - // and is preferable over a segfault. - TRI_ASSERT(comres != nullptr); - if (comres == nullptr) { - LOG_TOPIC("00aa2", ERR, Logger::MAINTENANCE) - << "CancelBarrier: error: syncRequest returned null"; - return arangodb::Result{TRI_ERROR_INTERNAL}; - } - - if (comres->status == CL_COMM_SENT) { - auto result = comres->result; - if (result == nullptr || - (result->getHttpReturnCode() != 200 && result->getHttpReturnCode() != 204)) { - std::string errorMessage = comres->stringifyErrorMessage(); - LOG_TOPIC("f5733", ERR, Logger::MAINTENANCE) << "CancelBarrier: error" << errorMessage; + if (res.ok()) { + auto* response = res.response.get(); + if (response->statusCode() != fuerte::StatusOK && + response->statusCode() != fuerte::StatusNoContent) { + std::string errorMessage = "got status " + std::to_string(response->statusCode()); + LOG_TOPIC("f5733", ERR, Logger::MAINTENANCE) + << "CancelBarrier: error '" << errorMessage << "'"; return arangodb::Result(TRI_ERROR_INTERNAL, errorMessage); } } else { std::string error( "CancelBarrier: failed to send message to leader : status "); - error += ClusterCommResult::stringifyStatus(comres->status); + error += network::fuerteToArangoErrorMessage(res); LOG_TOPIC("1c48a", ERR, Logger::MAINTENANCE) << error; return arangodb::Result(TRI_ERROR_INTERNAL, error); } @@ -413,32 +402,33 @@ static inline bool isStopping() { return server.isStopping(); } -arangodb::Result SynchronizeShard::getReadLock(std::string const& endpoint, +arangodb::Result SynchronizeShard::getReadLock(network::ConnectionPool* pool, + std::string const& endpoint, std::string const& database, std::string const& collection, std::string const& clientId, uint64_t rlid, bool soft, double timeout) { - auto cc = arangodb::ClusterComm::instance(); - if (cc == nullptr) { // nullptr only happens during controlled shutdown + if (pool == nullptr) { // nullptr only happens during controlled shutdown return arangodb::Result(TRI_ERROR_SHUTTING_DOWN, "startReadLockOnLeader: Shutting down"); } - VPackBuilder body; + VPackBuilder bodyBuilder; { - VPackObjectBuilder o(&body); - body.add(ID, VPackValue(std::to_string(rlid))); - body.add(COLLECTION, VPackValue(collection)); - body.add(TTL, VPackValue(timeout)); - body.add(StaticStrings::ReplicationSoftLockOnly, VPackValue(soft)); + VPackObjectBuilder o(&bodyBuilder); + bodyBuilder.add(ID, VPackValue(std::to_string(rlid))); + bodyBuilder.add(COLLECTION, VPackValue(collection)); + bodyBuilder.add(TTL, VPackValue(timeout)); + bodyBuilder.add(StaticStrings::ReplicationSoftLockOnly, VPackValue(soft)); } + auto body = bodyBuilder.steal(); auto url = DB + database + REPL_HOLD_READ_LOCK; - cc->asyncRequest(TRI_NewTickServer(), endpoint, rest::RequestType::POST, url, - std::make_shared(body.toJson()), - std::unordered_map(), - std::make_shared(this), timeout, true, timeout); + network::RequestOptions options; + options.timeout = network::Timeout(timeout); + auto dummy = network::sendRequest(pool, endpoint, fuerte::RestVerb::Post, url, + *body, network::Headers(), options); // Intentionally do not look at the outcome, even in case of an error // we must make sure that the read lock on the leader is not active! @@ -454,14 +444,12 @@ arangodb::Result SynchronizeShard::getReadLock(std::string const& endpoint, } // Now check that we hold the read lock: - auto putres = cc->syncRequest(TRI_NewTickServer(), endpoint, - rest::RequestType::PUT, url, body.toJson(), - std::unordered_map(), timeout); + auto res = network::sendRequest(pool, endpoint, fuerte::RestVerb::Put, url, + *body, network::Headers(), options) + .get(); - auto result = putres->result; - if (result != nullptr && result->getHttpReturnCode() == 200) { - auto const vp = putres->result->getBodyVelocyPack(); - auto const& slice = vp->slice(); + if (res.ok() && res.response->statusCode() == fuerte::StatusOK) { + auto const slice = res.response->slice(); TRI_ASSERT(slice.isObject()); VPackSlice lockHeld = slice.get("lockHeld"); if (lockHeld.isBoolean() && lockHeld.getBool()) { @@ -470,9 +458,8 @@ arangodb::Result SynchronizeShard::getReadLock(std::string const& endpoint, LOG_TOPIC("b681f", DEBUG, Logger::MAINTENANCE) << "startReadLockOnLeader: Lock not yet acquired..."; } else { - if (result != nullptr && result->getHttpReturnCode() == 404) { - auto const vp = result->getBodyVelocyPack(); - auto const& slice = vp->slice(); + if (res.ok() && res.response->statusCode() == fuerte::StatusNotFound) { + auto const slice = res.response->slice(); if (slice.isObject()) { VPackSlice s = slice.get(StaticStrings::ErrorNum); if (s.isNumber()) { @@ -486,9 +473,9 @@ arangodb::Result SynchronizeShard::getReadLock(std::string const& endpoint, // fall-through to other cases intentional here } + std::string message = network::fuerteToArangoErrorMessage(res); LOG_TOPIC("a82bc", DEBUG, Logger::MAINTENANCE) - << "startReadLockOnLeader: Do not see read lock yet:" - << putres->stringifyErrorMessage(); + << "startReadLockOnLeader: Do not see read lock yet:" << message; } std::this_thread::sleep_for(duration(sleepTime)); @@ -497,13 +484,14 @@ arangodb::Result SynchronizeShard::getReadLock(std::string const& endpoint, LOG_TOPIC("75e2b", ERR, Logger::MAINTENANCE) << "startReadLockOnLeader: giving up"; try { - auto r = cc->syncRequest(TRI_NewTickServer(), endpoint, - rest::RequestType::DELETE_REQ, url, body.toJson(), - std::unordered_map(), timeout); - if (r->result == nullptr || r->result->getHttpReturnCode() != 200) { + auto r = network::sendRequest(pool, endpoint, fuerte::RestVerb::Delete, url, + *body, network::Headers(), options) + .get(); + if (r.fail() || r.response->statusCode() != fuerte::StatusOK) { + std::string addendum = network::fuerteToArangoErrorMessage(r); LOG_TOPIC("4f34d", ERR, Logger::MAINTENANCE) - << "startReadLockOnLeader: cancelation error for shard - " << collection - << " " << r->getErrorCode() << ": " << r->stringifyErrorMessage(); + << "startReadLockOnLeader: cancelation error for shard - " + << collection << ": " << addendum; } } catch (std::exception const& e) { LOG_TOPIC("7fcc9", ERR, Logger::MAINTENANCE) @@ -519,14 +507,17 @@ arangodb::Result SynchronizeShard::startReadLockOnLeader( std::string const& clientId, uint64_t& rlid, bool soft, double timeout) { // Read lock id rlid = 0; - arangodb::Result result = getReadLockId(endpoint, database, clientId, timeout, rlid); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + arangodb::Result result = + getReadLockId(pool, endpoint, database, clientId, timeout, rlid); if (!result.ok()) { LOG_TOPIC("2e5ae", ERR, Logger::MAINTENANCE) << result.errorMessage(); return result; } LOG_TOPIC("c8d18", DEBUG, Logger::MAINTENANCE) << "Got read lock id: " << rlid; - result = getReadLock(endpoint, database, collection, clientId, rlid, soft, timeout); + result = getReadLock(pool, endpoint, database, collection, clientId, rlid, soft, timeout); return result; } @@ -820,8 +811,10 @@ bool SynchronizeShard::first() { << database << "/" << shard << "' for central '" << database << "/" << planId << "'"; try { - auto asResult = - addShardFollower(ep, database, shard, 0, clientId, SyncerId{}, _clientInfoString, 60.0); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + auto asResult = addShardFollower(pool, ep, database, shard, 0, clientId, + SyncerId{}, _clientInfoString, 60.0); if (asResult.ok()) { if (Logger::isEnabled(LogLevel::DEBUG, Logger::MAINTENANCE)) { @@ -918,7 +911,9 @@ bool SynchronizeShard::first() { // From here on, we have to call `cancelBarrier` in case of errors // as well as in the success case! auto barrierId = sy.get(BARRIER_ID).getNumber(); - TRI_DEFER(cancelBarrier(ep, database, barrierId, clientId)); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + TRI_DEFER(cancelBarrier(pool, ep, database, barrierId, clientId)); VPackSlice collections = sy.get(COLLECTIONS); @@ -1018,10 +1013,12 @@ ResultT SynchronizeShard::catchupWithReadLock( return ResultT::error(TRI_ERROR_INTERNAL, errorMessage); } - auto readLockGuard = arangodb::scopeGuard([&]() { + auto readLockGuard = arangodb::scopeGuard([&, this]() { // Always cancel the read lock. // Reported seperately - auto res = cancelReadLockOnLeader(ep, database, lockJobId, clientId, 60.0); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + auto res = cancelReadLockOnLeader(pool, ep, database, lockJobId, clientId, 60.0); if (!res.ok()) { LOG_TOPIC("b15ee", INFO, Logger::MAINTENANCE) << "Could not cancel soft read lock on leader: " << res.errorMessage(); @@ -1061,7 +1058,9 @@ ResultT SynchronizeShard::catchupWithReadLock( } // Stop the read lock again: - res = cancelReadLockOnLeader(ep, database, lockJobId, clientId, 60.0); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + res = cancelReadLockOnLeader(pool, ep, database, lockJobId, clientId, 60.0); // We removed the readlock readLockGuard.cancel(); if (!res.ok()) { @@ -1100,10 +1099,12 @@ Result SynchronizeShard::catchupWithExclusiveLock( "synchronizeOneShard: error in startReadLockOnLeader (hard):" + res.errorMessage(); return {TRI_ERROR_INTERNAL, errorMessage}; } - auto readLockGuard = arangodb::scopeGuard([&]() { + auto readLockGuard = arangodb::scopeGuard([&, this]() { // Always cancel the read lock. // Reported seperately - auto res = cancelReadLockOnLeader(ep, database, lockJobId, clientId, 60.0); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + auto res = cancelReadLockOnLeader(pool, ep, database, lockJobId, clientId, 60.0); if (!res.ok()) { LOG_TOPIC("067a8", INFO, Logger::MAINTENANCE) << "Could not cancel hard read lock on leader: " << res.errorMessage(); @@ -1133,7 +1134,10 @@ Result SynchronizeShard::catchupWithExclusiveLock( return {TRI_ERROR_INTERNAL, errorMessage}; } - res = addShardFollower(ep, database, shard, lockJobId, clientId, syncerId, _clientInfoString, 60.0); + NetworkFeature& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + res = addShardFollower(pool, ep, database, shard, lockJobId, clientId, + syncerId, _clientInfoString, 60.0); if (!res.ok()) { std::string errorMessage( diff --git a/arangod/Cluster/SynchronizeShard.h b/arangod/Cluster/SynchronizeShard.h index 47972831f5..13a8264ed6 100644 --- a/arangod/Cluster/SynchronizeShard.h +++ b/arangod/Cluster/SynchronizeShard.h @@ -33,6 +33,9 @@ #include namespace arangodb { +namespace network { +class ConnectionPool; +} class LogicalCollection; struct SyncerId; @@ -50,7 +53,8 @@ class SynchronizeShard : public ActionBase { void setState(ActionState state) override final; private: - arangodb::Result getReadLock(std::string const& endpoint, std::string const& database, + arangodb::Result getReadLock(network::ConnectionPool* pool, + std::string const& endpoint, std::string const& database, std::string const& collection, std::string const& clientId, uint64_t rlid, bool soft, double timeout = 300.0); diff --git a/arangod/Cluster/TakeoverShardLeadership.cpp b/arangod/Cluster/TakeoverShardLeadership.cpp index b35126963f..032ffaeddc 100644 --- a/arangod/Cluster/TakeoverShardLeadership.cpp +++ b/arangod/Cluster/TakeoverShardLeadership.cpp @@ -27,14 +27,17 @@ #include "MaintenanceFeature.h" #include "ApplicationFeatures/ApplicationServer.h" +#include "Basics/StringUtils.h" #include "Basics/VelocyPackHelper.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "Cluster/FollowerInfo.h" +#include "Futures/Utilities.h" #include "Logger/LogMacros.h" #include "Logger/Logger.h" #include "Logger/LoggerStream.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" #include "Transaction/ClusterUtils.h" #include "Transaction/Methods.h" #include "Transaction/StandaloneContext.h" @@ -54,6 +57,16 @@ using namespace arangodb::application_features; using namespace arangodb::maintenance; using namespace arangodb::methods; +namespace { +static std::string serverPrefix("server:"); + +std::string stripServerPrefix(std::string const& destination) { + TRI_ASSERT(destination.size() >= serverPrefix.size() && + destination.substr(0, serverPrefix.size()) == serverPrefix); + return destination.substr(serverPrefix.size()); +} +} // namespace + TakeoverShardLeadership::TakeoverShardLeadership(MaintenanceFeature& feature, ActionDescription const& desc) : ActionBase(feature, desc) { @@ -97,13 +110,12 @@ TakeoverShardLeadership::TakeoverShardLeadership(MaintenanceFeature& feature, TakeoverShardLeadership::~TakeoverShardLeadership() = default; -static void sendLeaderChangeRequests(std::vector const& currentServers, - std::shared_ptr>& realInsyncFollowers, - std::string const& databaseName, ShardID const& shardID, - std::string const& oldLeader) { - - auto cc = ClusterComm::instance(); - if (cc == nullptr) { +static void sendLeaderChangeRequests(network::ConnectionPool* pool, + std::vector const& currentServers, + std::shared_ptr>& realInsyncFollowers, + std::string const& databaseName, + ShardID const& shardID, std::string const& oldLeader) { + if (pool == nullptr) { // nullptr happens only during controlled shutdown return; } @@ -121,26 +133,31 @@ static void sendLeaderChangeRequests(std::vector const& currentServers std::string const url = "/_db/" + databaseName + "/_api/replication/set-the-leader"; - std::vector requests; - auto body = std::make_shared(bodyBuilder.toJson()); + std::vector futures; + auto body = bodyBuilder.steal(); + network::Headers headers; + network::RequestOptions options; + options.timeout = network::Timeout(3.0); for (auto const& srv : currentServers) { if (srv == sid) { continue; // ignore ourself } LOG_TOPIC("42516", DEBUG, Logger::MAINTENANCE) << "Sending " << bodyBuilder.toJson() << " to " << srv; - requests.emplace_back("server:" + srv, RequestType::PUT, url, body); + auto f = network::sendRequest(pool, "server:" + srv, fuerte::RestVerb::Put, + url, *body, headers, options); + futures.emplace_back(std::move(f)); } - cc->performRequests(requests, 3.0, Logger::COMMUNICATION, false); + auto responses = futures::collectAll(futures).get(); // This code intentionally ignores all errors realInsyncFollowers = std::make_shared>(); - for (auto const& req : requests) { - ClusterCommResult const& result = req.result; - if (result.status == CL_COMM_RECEIVED && result.errorCode == TRI_ERROR_NO_ERROR) { - if (result.result && result.result->getHttpReturnCode() == 200) { - realInsyncFollowers->push_back(result.serverID); + for (auto const& res : responses) { + if (res.hasValue() && res.get().ok()) { + auto& result = res.get(); + if (result.response && result.response->statusCode() == fuerte::StatusOK) { + realInsyncFollowers->push_back(::stripServerPrefix(result.destination)); } } } @@ -178,7 +195,11 @@ static void handleLeadership(LogicalCollection& collection, oldLeader = oldLeader.substr(1); // Update all follower and tell them that we are the leader now - sendLeaderChangeRequests(currentServers, realInsyncFollowers, databaseName, collection.name(), oldLeader); + NetworkFeature& nf = + collection.vocbase().server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + sendLeaderChangeRequests(pool, currentServers, realInsyncFollowers, + databaseName, collection.name(), oldLeader); } } diff --git a/arangod/ClusterEngine/MMFilesMethods.cpp b/arangod/ClusterEngine/MMFilesMethods.cpp index 6355358257..f4d9c8aa89 100644 --- a/arangod/ClusterEngine/MMFilesMethods.cpp +++ b/arangod/ClusterEngine/MMFilesMethods.cpp @@ -22,10 +22,13 @@ #include "MMFilesMethods.h" -#include "Cluster/ClusterComm.h" +#include "Basics/StringUtils.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "ClusterEngine/ClusterEngine.h" +#include "Futures/Utilities.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "VocBase/LogicalCollection.h" @@ -44,8 +47,9 @@ int rotateActiveJournalOnAllDBServers(std::string const& dbname, std::string con auto& server = ce->server(); // Set a few variables needed for our work: - auto cc = ClusterComm::instance(); - if (cc == nullptr) { + NetworkFeature const& nf = server.getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool == nullptr) { // nullptr happens only during controlled shutdown return TRI_ERROR_SHUTTING_DOWN; } @@ -59,25 +63,32 @@ int rotateActiveJournalOnAllDBServers(std::string const& dbname, std::string con std::string const baseUrl = "/_db/" + basics::StringUtils::urlEncode(dbname) + "/_api/collection/"; - std::shared_ptr body; + + VPackBuffer body; + network::Headers headers; + network::RequestOptions options; + options.timeout = network::Timeout(600.0); // now we notify all leader and follower shards std::shared_ptr shardList = collinfo->shardIds(); - std::vector requests; + std::vector futures; for (auto const& shard : *shardList) { for (ServerID const& server : shard.second) { std::string uri = baseUrl + basics::StringUtils::urlEncode(shard.first) + "/rotate"; - requests.emplace_back("server:" + server, arangodb::rest::RequestType::PUT, - std::move(uri), body); + auto f = network::sendRequest(pool, "server:" + server, fuerte::RestVerb::Put, + std::move(uri), body, headers, options); + futures.emplace_back(std::move(f)); } } - size_t nrGood = cc->performRequests(requests, 600.0, Logger::ENGINES, false); - - if (nrGood < requests.size()) { - return TRI_ERROR_FAILED; + auto responses = futures::collectAll(futures).get(); + for (auto const& r : responses) { + if (!r.hasValue() || r.get().fail()) { + return TRI_ERROR_FAILED; + } } + return TRI_ERROR_NO_ERROR; } diff --git a/arangod/ClusterEngine/RocksDBMethods.cpp b/arangod/ClusterEngine/RocksDBMethods.cpp index 2b004947d5..8897011a13 100644 --- a/arangod/ClusterEngine/RocksDBMethods.cpp +++ b/arangod/ClusterEngine/RocksDBMethods.cpp @@ -23,10 +23,12 @@ #include "RocksDBMethods.h" #include "Basics/StringUtils.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "ClusterEngine/ClusterEngine.h" +#include "Futures/Utilities.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "VocBase/LogicalCollection.h" @@ -45,8 +47,9 @@ Result recalculateCountsOnAllDBServers(std::string const& dbname, std::string co auto& server = ce->server(); // Set a few variables needed for our work: - auto cc = ClusterComm::instance(); - if (cc == nullptr) { + NetworkFeature const& nf = server.getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool == nullptr) { // nullptr happens only during controlled shutdown return TRI_ERROR_SHUTTING_DOWN; } @@ -60,25 +63,32 @@ Result recalculateCountsOnAllDBServers(std::string const& dbname, std::string co std::string const baseUrl = "/_db/" + basics::StringUtils::urlEncode(dbname) + "/_api/collection/"; - std::shared_ptr body; + + VPackBuffer body; + network::Headers headers; + network::RequestOptions options; + options.timeout = network::Timeout(600.0); // now we notify all leader and follower shards std::shared_ptr shardList = collinfo->shardIds(); - std::vector requests; + std::vector futures; for (auto const& shard : *shardList) { for (ServerID const& server : shard.second) { std::string uri = baseUrl + basics::StringUtils::urlEncode(shard.first) + "/recalculateCount"; - requests.emplace_back("server:" + server, arangodb::rest::RequestType::PUT, - std::move(uri), body); + auto f = network::sendRequest(pool, "server:" + server, fuerte::RestVerb::Put, + std::move(uri), body, headers, options); + futures.emplace_back(std::move(f)); } } - size_t nrGood = cc->performRequests(requests, 600.0, Logger::ENGINES, false); - - if (nrGood < requests.size()) { - return TRI_ERROR_FAILED; + auto responses = futures::collectAll(futures).get(); + for (auto const& r : responses) { + if (!r.hasValue() || r.get().fail()) { + return TRI_ERROR_FAILED; + } } + return TRI_ERROR_NO_ERROR; } diff --git a/arangod/GeneralServer/RestHandler.cpp b/arangod/GeneralServer/RestHandler.cpp index e4158079bf..8af0258261 100644 --- a/arangod/GeneralServer/RestHandler.cpp +++ b/arangod/GeneralServer/RestHandler.cpp @@ -107,30 +107,20 @@ futures::Future RestHandler::forwardRequest(bool& forwarded) { return futures::makeFuture(Result()); } - // TODO refactor into a more general/customizable method - // - // The below is mostly copied and only lightly modified from - // RestReplicationHandler::handleTrampolineCoordinator; however, that method - // needs some more specific checks regarding headers and param values, so we - // can't just reuse this method there. Maybe we just need to implement some - // virtual methods to handle param/header filtering? - - // TODO verify that vst -> http -> vst conversion works correctly - - uint32_t shortId = forwardingTarget(); - if (shortId == 0) { + std::string serverId = forwardingTarget(); + if (serverId.empty()) { // no need to actually forward return futures::makeFuture(Result()); } - std::string serverId = - server().getFeature().clusterInfo().getCoordinatorByShortID(shortId); - - if (serverId.empty()) { - // no mapping in agency, try to handle the request here - return futures::makeFuture(Result()); + NetworkFeature const& nf = server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool == nullptr) { + // nullptr happens only during controlled shutdown + generateError(rest::ResponseCode::SERVICE_UNAVAILABLE, + TRI_ERROR_SHUTTING_DOWN, "shutting down server"); + return futures::makeFuture(Result(TRI_ERROR_SHUTTING_DOWN)); } - LOG_TOPIC("38d99", DEBUG, Logger::REQUESTS) << "forwarding request " << _request->messageId() << " to " << serverId; @@ -158,14 +148,22 @@ futures::Future RestHandler::forwardRequest(bool& forwarded) { params.append(StringUtils::urlEncode(i.second)); } + network::RequestOptions options; + options.timeout = network::Timeout(300); + options.contentType = rest::contentTypeToString(_request->contentType()); + options.acceptType = rest::contentTypeToString(_request->contentTypeResponse()); + auto requestType = fuerte::from_string(GeneralRequest::translateMethod(_request->requestType())); - auto payload = _request->toVelocyPackBuilderPtr()->steal(); - auto* pool = server().getFeature().pool(); + + VPackStringRef resPayload = _request->rawPayload(); + VPackBuffer payload(resPayload.size()); + payload.append(resPayload.data(), resPayload.size()); + auto future = network::sendRequest(pool, "server:" + serverId, requestType, "/_db/" + StringUtils::urlEncode(dbname) + _request->requestPath() + params, - std::move(*payload), network::Timeout(300), headers); + std::move(payload), std::move(headers), options); auto cb = [this, serverId, useVst, self = shared_from_this()](network::Response&& response) -> Result { int res = network::fuerteToArangoErrorCode(response); @@ -187,6 +185,7 @@ futures::Future RestHandler::forwardRequest(bool& forwarded) { } else { _response->setPayload(std::move(*response.response->stealPayload()), true); } + auto const& resultHeaders = response.response->messageHeader().meta(); for (auto const& it : resultHeaders) { diff --git a/arangod/GeneralServer/RestHandler.h b/arangod/GeneralServer/RestHandler.h index c4a9cc6b51..54d20aaa9a 100644 --- a/arangod/GeneralServer/RestHandler.h +++ b/arangod/GeneralServer/RestHandler.h @@ -139,10 +139,10 @@ class RestHandler : public std::enable_shared_from_this { /// /// This method will be called to determine if the request should be /// forwarded to another server, and if so, which server. If it should be - /// handled by this server, the method should return 0. Otherwise, this - /// method should return a valid (non-zero) short ID (TransactionID) for the + /// handled by this server, the method should return an empty string. + /// Otherwise, this method should return a valid short name for the /// target server. - virtual uint32_t forwardingTarget() { return 0; } + virtual std::string forwardingTarget() { return ""; } void resetResponse(rest::ResponseCode); diff --git a/arangod/Graph/ShortestPathFinder.cpp b/arangod/Graph/ShortestPathFinder.cpp index 5ec092cd09..27ad2d453b 100644 --- a/arangod/Graph/ShortestPathFinder.cpp +++ b/arangod/Graph/ShortestPathFinder.cpp @@ -22,10 +22,16 @@ //////////////////////////////////////////////////////////////////////////////// #include "ShortestPathFinder.h" -#include "Cluster/ClusterComm.h" + +#include "Aql/Query.h" +#include "Basics/StringUtils.h" #include "Cluster/ServerState.h" #include "Graph/ClusterTraverserCache.h" #include "Graph/ShortestPathOptions.h" +#include "Logger/LogMacros.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "Transaction/Methods.h" using namespace arangodb; @@ -37,10 +43,11 @@ ShortestPathFinder::ShortestPathFinder(ShortestPathOptions& options) void ShortestPathFinder::destroyEngines() { if (ServerState::instance()->isCoordinator()) { + NetworkFeature const& nf = + _options.query()->vocbase().server().getFeature(); + network::ConnectionPool* pool = nf.pool(); // We have to clean up the engines in Coordinator Case. - auto cc = ClusterComm::instance(); - - if (cc != nullptr) { + if (pool != nullptr) { auto ch = reinterpret_cast(_options.cache()); // nullptr only happens on controlled server shutdown std::string const url( @@ -50,22 +57,19 @@ void ShortestPathFinder::destroyEngines() { for (auto const& it : *ch->engines()) { incHttpRequests(1); - arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer(); - std::unordered_map headers; - auto res = cc->syncRequest(coordTransactionID, "server:" + it.first, - RequestType::DELETE_REQ, - url + arangodb::basics::StringUtils::itoa(it.second), - "", headers, 30.0); + network::Headers headers; + auto res = + network::sendRequest(pool, "server:" + it.first, fuerte::RestVerb::Delete, + url + arangodb::basics::StringUtils::itoa(it.second), + VPackBuffer(), network::Timeout(30.0), headers) + .get(); - if (res->status != CL_COMM_SENT) { + if (res.error != fuerte::Error::NoError) { // Note If there was an error on server side we do not have // CL_COMM_SENT std::string message("Could not destroy all traversal engines"); - - if (!res->errorMessage.empty()) { - message += std::string(": ") + res->errorMessage; - } - + message += std::string(": ") + + TRI_errno_string(network::fuerteToArangoErrorCode(res)); LOG_TOPIC("d31a4", ERR, arangodb::Logger::FIXME) << message; } } diff --git a/arangod/IResearch/IResearchAnalyzerFeature.cpp b/arangod/IResearch/IResearchAnalyzerFeature.cpp index c41f7ab0ee..47334aa385 100644 --- a/arangod/IResearch/IResearchAnalyzerFeature.cpp +++ b/arangod/IResearch/IResearchAnalyzerFeature.cpp @@ -71,15 +71,18 @@ #include "Basics/error.h" #include "Basics/system-compiler.h" #include "Basics/voc-errors.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "FeaturePhases/V8FeaturePhase.h" +#include "Futures/Utilities.h" #include "IResearchAnalyzerFeature.h" #include "IResearchCommon.h" #include "Logger/LogMacros.h" #include "Logger/LoggerStream.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "Rest/CommonDefines.h" #include "Rest/GeneralRequest.h" #include "RestHandler/RestVocbaseBaseHandler.h" @@ -747,13 +750,17 @@ arangodb::Result visitAnalyzers( // visit analyzers // FIXME TODO find a better way to query a cluster collection // workaround for aql::Query failing to execute on a cluster collection if (arangodb::ServerState::instance()->isDBServer()) { - auto cc = arangodb::ClusterComm::instance(); + arangodb::NetworkFeature const& feature = + vocbase.server().getFeature(); + arangodb::network::ConnectionPool* pool = feature.pool(); - if (!cc) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find 'ClusterComm' instance while visiting Analyzer collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + "'" - ); + if (!pool) { + return arangodb::Result( // result + TRI_ERROR_SHUTTING_DOWN, // code + std::string("failure to find connection pool while visiting Analyzer " + "collection '") + + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + + "', server is likely shutting down"); } auto collection = getAnalyzerCollection(vocbase); @@ -762,8 +769,15 @@ arangodb::Result visitAnalyzers( // visit analyzers return arangodb::Result(); // nothing to load } - static const std::string body("{}"); // RestSimpleQueryHandler::allDocuments() expects opbject (calls get() on slice) - std::vector requests; + // RestSimpleQueryHandler::allDocuments() expects opbject (calls get() on slice) + VPackBuffer buffer; + { + VPackBuilder builder(buffer); + builder.openObject(); + builder.close(); + } + arangodb::network::Headers headers; + std::vector futures; // create a request for every shard //for (auto& entry: collection->errorNum()) { @@ -774,50 +788,53 @@ arangodb::Result visitAnalyzers( // visit analyzers + arangodb::RestVocbaseBaseHandler::SIMPLE_QUERY_ALL_PATH + "?collection=" + shardId; - requests.emplace_back( // add shard request - "shard:" + shardId, // shard - arangodb::rest::RequestType::PUT, // request type as per SimpleQueryHandker - url, // request url - std::shared_ptr(&body, [](std::string const*)->void {}) // body - ); + auto f = arangodb::network::sendRequest(pool, + "shard:" + shardId, // shard + arangodb::fuerte::RestVerb::Put, // request type as per SimpleQueryHandker + url, // request url + buffer, // body + arangodb::network::Timeout(120.0), headers); + futures.emplace_back(std::move(f)); } - // same timeout as in ClusterMethods::getDocumentOnCoordinator() - cc->performRequests( // execute requests - requests, 120.0, arangodb::iresearch::TOPIC, false, false // args - ); + auto results = arangodb::futures::collectAll(futures).get(); + for (auto& r : results) { + auto& res = r.get(); + if (res.error != arangodb::fuerte::Error::NoError) { + return arangodb::Result(arangodb::network::fuerteToArangoErrorCode(res)); + } - for (auto& request: requests) { - if (TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND == request.result.errorCode) { + if (res.response->statusCode() == arangodb::fuerte::StatusNotFound) { continue; // treat missing collection as if there are no analyzers } - if (TRI_ERROR_NO_ERROR != request.result.errorCode) { - return arangodb::Result( // result - request.result.errorCode, request.result.errorMessage // args - ); + std::vector slices = res.response->slices(); + if (slices.empty() || !slices[0].isObject()) { + return arangodb::Result( + TRI_ERROR_INTERNAL, + "got misformed result while visiting Analyzer collection'" + ANALYZER_COLLECTION_NAME + + "' in vocbase '" + vocbase.name() + "'"); } - if (!request.result.answer) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failed to get answer from 'ClusterComm' instance while visiting Analyzer collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + "'" - ); + VPackSlice answer = slices[0]; + arangodb::Result result = + arangodb::network::resultFromBody(answer, TRI_ERROR_NO_ERROR); + if (result.fail()) { + return result; } - auto slice = request.result.answer->payload(); - - if (!slice.hasKey("result")) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failed to parse result from 'ClusterComm' instance while visiting Analyzer collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + "'" - ); + if (!answer.hasKey("result")) { + return arangodb::Result( // result + TRI_ERROR_INTERNAL, // code + std::string( + "failed to parse result while visiting Analyzer collection '") + + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + + "'"); } - auto res = resultVisitor(visitor, vocbase, slice.get("result")); - - if (!res.ok()) { - return res; + result = resultVisitor(visitor, vocbase, answer.get("result")); + if (!result.ok()) { + return result; } } diff --git a/arangod/Network/Methods.cpp b/arangod/Network/Methods.cpp index ada2be4f73..76eaaee0f4 100644 --- a/arangod/Network/Methods.cpp +++ b/arangod/Network/Methods.cpp @@ -65,7 +65,7 @@ std::string Response::serverId() const { template auto prepareRequest(RestVerb type, std::string const& path, T&& payload, - Timeout timeout, Headers headers) { + Headers headers, RequestOptions options) { fuerte::StringMap params; // intentionally empty auto req = fuerte::createRequest(type, path, params, std::forward(payload)); req->header.parseArangoPath(path); // strips /_db// @@ -74,11 +74,18 @@ auto prepareRequest(RestVerb type, std::string const& path, T&& payload, } req->header.setMeta(std::move(headers)); + if (!options.contentType.empty()) { + req->header.contentType(options.contentType); + } + if (!options.acceptType.empty()) { + req->header.acceptType(options.acceptType); + } + TRI_voc_tick_t timeStamp = TRI_HybridLogicalClock(); req->header.addMeta(StaticStrings::HLCHeader, arangodb::basics::HybridLogicalClock::encodeTimeStamp(timeStamp)); - req->timeout(std::chrono::duration_cast(timeout)); + req->timeout(std::chrono::duration_cast(options.timeout)); auto state = ServerState::instance(); if (state->isCoordinator() || state->isDBServer()) { @@ -97,6 +104,16 @@ auto prepareRequest(RestVerb type, std::string const& path, T&& payload, FutureRes sendRequest(ConnectionPool* pool, DestinationId const& destination, RestVerb type, std::string const& path, velocypack::Buffer payload, Timeout timeout, Headers headers) { + RequestOptions options; + options.timeout = timeout; + return sendRequest(pool, std::move(destination), type, std::move(path), + std::move(payload), std::move(headers), options); +} + +/// @brief send a request to a given destination +FutureRes sendRequest(ConnectionPool* pool, DestinationId const& destination, RestVerb type, + std::string const& path, velocypack::Buffer payload, + Headers headers, RequestOptions options) { // FIXME build future.reset(..) if (!pool || !pool->config().clusterInfo) { @@ -113,7 +130,7 @@ FutureRes sendRequest(ConnectionPool* pool, DestinationId const& destination, Re } TRI_ASSERT(!spec.endpoint.empty()); - auto req = prepareRequest(type, path, std::move(payload), timeout, std::move(headers)); + auto req = prepareRequest(type, path, std::move(payload), std::move(headers), options); struct Pack { DestinationId destination; @@ -158,7 +175,7 @@ class RequestsState final : public std::enable_shared_from_this { public: RequestsState(ConnectionPool* pool, DestinationId destination, RestVerb type, std::string path, velocypack::Buffer payload, - Timeout timeout, Headers headers, bool retryNotFound) + Headers headers, RequestOptions options) : _pool(pool), _destination(std::move(destination)), _type(type), @@ -168,9 +185,9 @@ class RequestsState final : public std::enable_shared_from_this { _workItem(nullptr), _promise(), _startTime(std::chrono::steady_clock::now()), - _endTime(_startTime + - std::chrono::duration_cast(timeout)), - _retryOnCollNotFound(retryNotFound) {} + _endTime(_startTime + std::chrono::duration_cast( + options.timeout)), + _options(options) {} ~RequestsState() = default; @@ -187,7 +204,7 @@ class RequestsState final : public std::enable_shared_from_this { std::chrono::steady_clock::time_point const _startTime; std::chrono::steady_clock::time_point const _endTime; - const bool _retryOnCollNotFound; + RequestOptions const _options; public: @@ -215,12 +232,14 @@ class RequestsState final : public std::enable_shared_from_this { callResponse(Error::Canceled, nullptr); return; } - - auto localTO = std::chrono::duration_cast(_endTime - now); - TRI_ASSERT(localTO.count() > 0); + + auto localOptions = _options; + localOptions.timeout = + std::chrono::duration_cast(_endTime - now); + TRI_ASSERT(localOptions.timeout.count() > 0); auto ref = _pool->leaseConnection(spec.endpoint); - auto req = prepareRequest(_type, _path, _payload, localTO, _headers); + auto req = prepareRequest(_type, _path, _payload, _headers, localOptions); auto self = RequestsState::shared_from_this(); auto cb = [self, ref](fuerte::Error err, std::unique_ptr req, @@ -242,7 +261,7 @@ class RequestsState final : public std::enable_shared_from_this { res->statusCode() == fuerte::StatusNoContent) { callResponse(Error::NoError, std::move(res)); break; - } else if (res->statusCode() == fuerte::StatusNotFound && _retryOnCollNotFound && + } else if (res->statusCode() == fuerte::StatusNotFound && _options.retryNotFound && TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND == network::errorCodeFromBody(res->slice())) { LOG_TOPIC("5a8e9", DEBUG, Logger::COMMUNICATION) << "retrying request"; @@ -328,6 +347,18 @@ FutureRes sendRequestRetry(ConnectionPool* pool, DestinationId const& destinatio arangodb::fuerte::RestVerb type, std::string const& path, velocypack::Buffer payload, Timeout timeout, Headers headers, bool retryNotFound) { + RequestOptions options; + options.timeout = timeout; + options.retryNotFound = retryNotFound; + return sendRequestRetry(pool, std::move(destination), type, std::move(path), + std::move(payload), std::move(headers), options); +} + +/// @brief send a request to a given destination, retry until timeout is exceeded +FutureRes sendRequestRetry(ConnectionPool* pool, DestinationId const& destination, + arangodb::fuerte::RestVerb type, std::string const& path, + velocypack::Buffer payload, Headers headers, + RequestOptions options) { if (!pool || !pool->config().clusterInfo) { LOG_TOPIC("59b96", ERR, Logger::COMMUNICATION) << "connection pool unavailable"; @@ -336,8 +367,8 @@ FutureRes sendRequestRetry(ConnectionPool* pool, DestinationId const& destinatio // auto req = prepareRequest(type, path, std::move(payload), timeout, headers); auto rs = std::make_shared(pool, destination, type, path, - std::move(payload), timeout, - std::move(headers), retryNotFound); + std::move(payload), + std::move(headers), options); rs->startRequest(); // will auto reference itself return rs->future(); } diff --git a/arangod/Network/Methods.h b/arangod/Network/Methods.h index 746e90b01f..28277c9f72 100644 --- a/arangod/Network/Methods.h +++ b/arangod/Network/Methods.h @@ -24,9 +24,10 @@ #define ARANGOD_NETWORK_METHODS_H 1 #include "Basics/Result.h" +#include "Basics/StaticStrings.h" #include "Futures/Future.h" -#include "Network/types.h" #include "Network/ConnectionPool.h" +#include "Network/types.h" #include #include @@ -43,7 +44,7 @@ struct Response { DestinationId destination; fuerte::Error error; /// connectivity error std::unique_ptr response; - + bool ok() const { return fuerte::Error::NoError == this->error; } @@ -65,19 +66,49 @@ struct Response { static_assert(std::is_nothrow_move_constructible::value, ""); using FutureRes = arangodb::futures::Future; +// Container for optional (often defaulted) parameters +struct RequestOptions { + Timeout timeout = Timeout(120.0); + std::string contentType = StaticStrings::MimeTypeVPack; + std::string acceptType = StaticStrings::MimeTypeVPack; + bool retryNotFound = false; +}; + /// @brief send a request to a given destination +/// +/// deprecated, use alternative signature FutureRes sendRequest(ConnectionPool* pool, DestinationId const& destination, arangodb::fuerte::RestVerb type, std::string const& path, velocypack::Buffer payload, Timeout timeout, Headers headers = {}); +/// @brief send a request to a given destination +FutureRes sendRequest(ConnectionPool* pool, DestinationId const& destination, + arangodb::fuerte::RestVerb type, std::string const& path, + velocypack::Buffer payload = {}, + Headers headers = {}, RequestOptions options = {}); + +/// @brief send a request to a given destination, retry under certain conditions +/// a retry will be triggered if the connection was lost our could not be established +/// optionally a retry will be performed in the case of until timeout is exceeded +/// +/// deprecated, use alternative signature +FutureRes sendRequestRetry(ConnectionPool* pool, DestinationId const& destination, + arangodb::fuerte::RestVerb type, std::string const& path, + velocypack::Buffer payload, Timeout timeout, + Headers headers = {}, bool retryNotFound = false); + /// @brief send a request to a given destination, retry under certain conditions /// a retry will be triggered if the connection was lost our could not be established /// optionally a retry will be performed in the case of until timeout is exceeded FutureRes sendRequestRetry(ConnectionPool* pool, DestinationId const& destination, arangodb::fuerte::RestVerb type, std::string const& path, - velocypack::Buffer payload, Timeout timeout, - Headers headers = {}, bool retryNotFound = false); + velocypack::Buffer payload = {}, + Headers headers = {}, RequestOptions options = {}); + +using Sender = + std::function, Timeout, Headers)>; } // namespace network } // namespace arangodb diff --git a/arangod/Network/Utils.cpp b/arangod/Network/Utils.cpp index 0801a1610f..8dc5bf6ca4 100644 --- a/arangod/Network/Utils.cpp +++ b/arangod/Network/Utils.cpp @@ -215,6 +215,52 @@ int toArangoErrorCodeInternal(fuerte::Error err) { } } // namespace +fuerte::RestVerb arangoRestVerbToFuerte(rest::RequestType verb) { + switch (verb) { + case rest::RequestType::DELETE_REQ: + return fuerte::RestVerb::Delete; + case rest::RequestType::GET: + return fuerte::RestVerb::Get; + case rest::RequestType::POST: + return fuerte::RestVerb::Post; + case rest::RequestType::PUT: + return fuerte::RestVerb::Put; + case rest::RequestType::HEAD: + return fuerte::RestVerb::Head; + case rest::RequestType::PATCH: + return fuerte::RestVerb::Patch; + case rest::RequestType::OPTIONS: + return fuerte::RestVerb::Options; + case rest::RequestType::ILLEGAL: + return fuerte::RestVerb::Illegal; + } + + return fuerte::RestVerb::Illegal; +} + +rest::RequestType fuerteRestVerbToArango(fuerte::RestVerb verb) { + switch (verb) { + case fuerte::RestVerb::Illegal: + return rest::RequestType::ILLEGAL; + case fuerte::RestVerb::Delete: + return rest::RequestType::DELETE_REQ; + case fuerte::RestVerb::Get: + return rest::RequestType::GET; + case fuerte::RestVerb::Post: + return rest::RequestType::POST; + case fuerte::RestVerb::Put: + return rest::RequestType::PUT; + case fuerte::RestVerb::Head: + return rest::RequestType::HEAD; + case fuerte::RestVerb::Patch: + return rest::RequestType::PATCH; + case fuerte::RestVerb::Options: + return rest::RequestType::OPTIONS; + } + + return rest::RequestType::ILLEGAL; +} + int fuerteToArangoErrorCode(network::Response const& res) { LOG_TOPIC_IF("abcde", ERR, Logger::CLUSTER, res.error != fuerte::Error::NoError) << "cluster error: '" << fuerte::to_string(res.error) @@ -227,5 +273,14 @@ int fuerteToArangoErrorCode(fuerte::Error err) { << "cluster error: '" << fuerte::to_string(err) << "'"; return toArangoErrorCodeInternal(err); } + +std::string fuerteToArangoErrorMessage(network::Response const& res) { + return TRI_errno_string(fuerteToArangoErrorCode(res)); +} + +std::string fuerteToArangoErrorMessage(fuerte::Error err) { + return TRI_errno_string(fuerteToArangoErrorCode(err)); +} + } // namespace network } // namespace arangodb diff --git a/arangod/Network/Utils.h b/arangod/Network/Utils.h index 7594aa579c..92b6d27813 100644 --- a/arangod/Network/Utils.h +++ b/arangod/Network/Utils.h @@ -26,6 +26,7 @@ #include "Basics/Result.h" #include "Network/types.h" +#include "Rest/CommonDefines.h" #include "Utils/OperationResult.h" #include @@ -67,6 +68,12 @@ void errorCodesFromHeaders(network::Headers headers, /// @brief transform response into arango error code int fuerteToArangoErrorCode(network::Response const& res); int fuerteToArangoErrorCode(fuerte::Error err); +std::string fuerteToArangoErrorMessage(network::Response const& res); +std::string fuerteToArangoErrorMessage(fuerte::Error err); + +/// @brief convert between arango and fuerte rest methods +fuerte::RestVerb arangoRestVerbToFuerte(rest::RequestType); +rest::RequestType fuerteRestVerbToArango(fuerte::RestVerb); } // namespace network } // namespace arangodb diff --git a/arangod/RestHandler/RestControlPregelHandler.cpp b/arangod/RestHandler/RestControlPregelHandler.cpp index 76536c5876..8ce1a21b23 100644 --- a/arangod/RestHandler/RestControlPregelHandler.cpp +++ b/arangod/RestHandler/RestControlPregelHandler.cpp @@ -25,6 +25,8 @@ #include "ApplicationFeatures/ApplicationServer.h" #include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "Graph/Graph.h" #include "Graph/GraphManager.h" @@ -73,22 +75,26 @@ RestStatus RestControlPregelHandler::execute() { } /// @brief returns the short id of the server which should handle this request -uint32_t RestControlPregelHandler::forwardingTarget() { +std::string RestControlPregelHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::POST && type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) { - return 0; + return ""; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return 0; + return ""; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } void RestControlPregelHandler::startExecution() { diff --git a/arangod/RestHandler/RestControlPregelHandler.h b/arangod/RestHandler/RestControlPregelHandler.h index 920feb5ada..a0ee5cf45e 100644 --- a/arangod/RestHandler/RestControlPregelHandler.h +++ b/arangod/RestHandler/RestControlPregelHandler.h @@ -38,7 +38,7 @@ class RestControlPregelHandler : public arangodb::RestVocbaseBaseHandler { RestStatus execute() override; protected: - virtual uint32_t forwardingTarget() override; + virtual std::string forwardingTarget() override; private: void startExecution(); diff --git a/arangod/RestHandler/RestCursorHandler.cpp b/arangod/RestHandler/RestCursorHandler.cpp index 56aaef947e..fa76a284fb 100644 --- a/arangod/RestHandler/RestCursorHandler.cpp +++ b/arangod/RestHandler/RestCursorHandler.cpp @@ -26,9 +26,11 @@ #include "Aql/QueryRegistry.h" #include "Basics/Exceptions.h" #include "Basics/MutexLocker.h" +#include "Basics/ScopeGuard.h" #include "Basics/StaticStrings.h" #include "Basics/VelocyPackHelper.h" -#include "Basics/ScopeGuard.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "Transaction/Context.h" #include "Utils/Cursor.h" @@ -368,21 +370,25 @@ RestStatus RestCursorHandler::handleQueryResult() { } /// @brief returns the short id of the server which should handle this request -uint32_t RestCursorHandler::forwardingTarget() { +std::string RestCursorHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { - return 0; + return ""; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return 0; + return ""; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestCursorHandler.h b/arangod/RestHandler/RestCursorHandler.h index ee708292ff..9f3b402b8c 100644 --- a/arangod/RestHandler/RestCursorHandler.h +++ b/arangod/RestHandler/RestCursorHandler.h @@ -88,7 +88,7 @@ class RestCursorHandler : public RestVocbaseBaseHandler { RestStatus processQuery(); /// @brief returns the short id of the server which should handle this request - virtual uint32_t forwardingTarget() override; + virtual std::string forwardingTarget() override; ////////////////////////////////////////////////////////////////////////////// /// @brief unregister the currently running query diff --git a/arangod/RestHandler/RestDocumentHandler.cpp b/arangod/RestHandler/RestDocumentHandler.cpp index 014dc34f7c..4f8835301f 100644 --- a/arangod/RestHandler/RestDocumentHandler.cpp +++ b/arangod/RestHandler/RestDocumentHandler.cpp @@ -25,6 +25,8 @@ #include "Basics/StaticStrings.h" #include "Basics/StringUtils.h" #include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "Transaction/Helpers.h" #include "Transaction/Hints.h" @@ -102,9 +104,9 @@ void RestDocumentHandler::shutdownExecute(bool isFinalized) noexcept { } /// @brief returns the short id of the server which should handle this request -uint32_t RestDocumentHandler::forwardingTarget() { +std::string RestDocumentHandler::forwardingTarget() { if (!ServerState::instance()->isCoordinator()) { - return 0; + return ""; } bool found = false; @@ -113,13 +115,17 @@ uint32_t RestDocumentHandler::forwardingTarget() { uint64_t tid = basics::StringUtils::uint64(value); if (!transaction::isCoordinatorTransactionId(tid)) { TRI_ASSERT(transaction::isLegacyTransactionId(tid)); - return 0; + return ""; } uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } - return 0; + return ""; } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestDocumentHandler.h b/arangod/RestHandler/RestDocumentHandler.h index 92493a7a52..a6d2d4d554 100644 --- a/arangod/RestHandler/RestDocumentHandler.h +++ b/arangod/RestHandler/RestDocumentHandler.h @@ -56,7 +56,7 @@ class RestDocumentHandler : public RestVocbaseBaseHandler { void shutdownExecute(bool isFinalized) noexcept override final; protected: - uint32_t forwardingTarget() override final; + std::string forwardingTarget() override final; private: // inserts a document diff --git a/arangod/RestHandler/RestJobHandler.cpp b/arangod/RestHandler/RestJobHandler.cpp index 0e2c4dea28..e3255b23a7 100644 --- a/arangod/RestHandler/RestJobHandler.cpp +++ b/arangod/RestHandler/RestJobHandler.cpp @@ -28,6 +28,8 @@ #include "Basics/StringUtils.h" #include "Basics/conversions.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "GeneralServer/AsyncJobManager.h" #include "VocBase/ticks.h" @@ -241,20 +243,24 @@ void RestJobHandler::deleteJob() { } /// @brief returns the short id of the server which should handle this request -uint32_t RestJobHandler::forwardingTarget() { +std::string RestJobHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::GET && type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { - return 0; + return ""; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return 0; + return ""; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } diff --git a/arangod/RestHandler/RestJobHandler.h b/arangod/RestHandler/RestJobHandler.h index 790b32d51b..b9b52465f4 100644 --- a/arangod/RestHandler/RestJobHandler.h +++ b/arangod/RestHandler/RestJobHandler.h @@ -85,7 +85,7 @@ class RestJobHandler : public RestBaseHandler { void deleteJob(); protected: - virtual uint32_t forwardingTarget() override; + virtual std::string forwardingTarget() override; private: ////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestQueryHandler.cpp b/arangod/RestHandler/RestQueryHandler.cpp index 98b168a2d1..22a7bde90b 100644 --- a/arangod/RestHandler/RestQueryHandler.cpp +++ b/arangod/RestHandler/RestQueryHandler.cpp @@ -350,9 +350,9 @@ bool RestQueryHandler::parseQuery() { } /// @brief returns the short id of the server which should handle this request -uint32_t RestQueryHandler::forwardingTarget() { +std::string RestQueryHandler::forwardingTarget() { if (!ServerState::instance()->isCoordinator()) { - return 0; + return ""; } bool found = false; @@ -361,11 +361,15 @@ uint32_t RestQueryHandler::forwardingTarget() { uint64_t tid = basics::StringUtils::uint64(value); if (!transaction::isCoordinatorTransactionId(tid)) { TRI_ASSERT(transaction::isLegacyTransactionId(tid)); - return 0; + return ""; } uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } - return 0; + return ""; } diff --git a/arangod/RestHandler/RestQueryHandler.h b/arangod/RestHandler/RestQueryHandler.h index 19a49afd33..a25854c341 100644 --- a/arangod/RestHandler/RestQueryHandler.h +++ b/arangod/RestHandler/RestQueryHandler.h @@ -96,7 +96,7 @@ class RestQueryHandler : public RestVocbaseBaseHandler { bool parseQuery(); - virtual uint32_t forwardingTarget() override; + virtual std::string forwardingTarget() override; }; } // namespace arangodb diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 94b44f0598..59328e0df9 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -31,7 +31,6 @@ #include "Basics/RocksDBUtils.h" #include "Basics/VelocyPackHelper.h" #include "Basics/WriteLocker.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterHelpers.h" #include "Cluster/ClusterMethods.h" @@ -39,6 +38,8 @@ #include "Cluster/ResignShardLeadership.h" #include "GeneralServer/AuthenticationFeature.h" #include "Indexes/Index.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "Replication/DatabaseInitialSyncer.h" #include "Replication/DatabaseReplicationApplier.h" #include "Replication/GlobalInitialSyncer.h" @@ -46,6 +47,7 @@ #include "Replication/ReplicationApplierConfiguration.h" #include "Replication/ReplicationClients.h" #include "Replication/ReplicationFeature.h" +#include "Rest/HttpResponse.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/QueryRegistryFeature.h" #include "RestServer/ServerIdFeature.h" @@ -336,7 +338,7 @@ RestStatus RestReplicationHandler::execute() { // DEL - delete batchid if (ServerState::instance()->isCoordinator()) { - handleTrampolineCoordinator(); + handleUnforwardedTrampolineCoordinator(); } else { handleCommandBatch(); } @@ -366,7 +368,7 @@ RestStatus RestReplicationHandler::execute() { goto BAD_CALL; } if (ServerState::instance()->isCoordinator()) { - handleTrampolineCoordinator(); + handleUnforwardedTrampolineCoordinator(); } else { handleCommandInventory(); } @@ -418,7 +420,7 @@ RestStatus RestReplicationHandler::execute() { } if (ServerState::instance()->isCoordinator()) { - handleTrampolineCoordinator(); + handleUnforwardedTrampolineCoordinator(); } else { handleCommandDump(); } @@ -584,6 +586,29 @@ BAD_CALL: return RestStatus::DONE; } +/// @brief returns the short id of the server which should handle this request +std::string RestReplicationHandler::forwardingTarget() { + if (!ServerState::instance()->isCoordinator()) { + return ""; + } + + auto const& suffixes = _request->suffixes(); + size_t const len = suffixes.size(); + if (len >= 1) { + auto const type = _request->requestType(); + std::string const& command = suffixes[0]; + if ((command == Batch) || (command == Inventory && type == rest::RequestType::GET) || + (command == Dump && type == rest::RequestType::GET)) { + ServerID const& DBserver = _request->value("DBserver"); + if (!DBserver.empty()) { + return DBserver; + } + } + } + + return ""; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief was docuBlock JSF_put_api_replication_makeSlave //////////////////////////////////////////////////////////////////////////////// @@ -640,14 +665,14 @@ void RestReplicationHandler::handleCommandMakeSlave() { } //////////////////////////////////////////////////////////////////////////////// -/// @brief forward a command in the coordinator case +/// @brief handle an unforwarded command in the coordinator case +/// If the request is well-formed and has the DBserver set, then the request +/// should already be forwarded by other means. We should only get here if +/// the request is null or the DBserver parameter is missing. This method +/// now just does a bit of error handling. //////////////////////////////////////////////////////////////////////////////// -void RestReplicationHandler::handleTrampolineCoordinator() { - bool useVst = false; - if (_request->transportType() == Endpoint::TransportType::VST) { - useVst = true; - } +void RestReplicationHandler::handleUnforwardedTrampolineCoordinator() { if (_request == nullptr) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid request"); } @@ -661,100 +686,7 @@ void RestReplicationHandler::handleTrampolineCoordinator() { return; } - std::string const& dbname = _request->databaseName(); - - auto headers = std::make_shared>( - arangodb::getForwardableRequestHeaders(_request.get())); - std::unordered_map values = _request->values(); - std::string params; - - for (auto const& i : values) { - if (i.first != "DBserver") { - if (params.empty()) { - params.push_back('?'); - } else { - params.push_back('&'); - } - params.append(StringUtils::urlEncode(i.first)); - params.push_back('='); - params.append(StringUtils::urlEncode(i.second)); - } - } - - // Set a few variables needed for our work: - auto cc = ClusterComm::instance(); - if (cc == nullptr) { - // nullptr happens only during controlled shutdown - generateError(rest::ResponseCode::SERVICE_UNAVAILABLE, - TRI_ERROR_SHUTTING_DOWN, "shutting down server"); - return; - } - - std::unique_ptr res; - if (!useVst) { - TRI_ASSERT(_request->transportType() == Endpoint::TransportType::HTTP); - - VPackStringRef body = _request->rawPayload(); - // Send a synchronous request to that shard using ClusterComm: - res = cc->syncRequest(TRI_NewTickServer(), "server:" + DBserver, - _request->requestType(), - "/_db/" + StringUtils::urlEncode(dbname) + - _request->requestPath() + params, - body.toString(), *headers, 300.0); - } else { - // do we need to handle multiple payloads here - TODO - // here we switch from vst to http?! - res = cc->syncRequest(TRI_NewTickServer(), "server:" + DBserver, - _request->requestType(), - "/_db/" + StringUtils::urlEncode(dbname) + - _request->requestPath() + params, - _request->payload().toJson(), *headers, 300.0); - } - - if (res->status == CL_COMM_TIMEOUT) { - // No reply, we give up: - generateError(rest::ResponseCode::BAD, TRI_ERROR_CLUSTER_TIMEOUT, - "timeout within cluster"); - return; - } - if (res->status == CL_COMM_BACKEND_UNAVAILABLE) { - // there is no result - generateError(rest::ResponseCode::BAD, TRI_ERROR_CLUSTER_CONNECTION_LOST, - "lost connection within cluster"); - return; - } - if (res->status == CL_COMM_ERROR) { - // This could be a broken connection or an Http error: - TRI_ASSERT(nullptr != res->result && res->result->isComplete()); - // In this case a proper HTTP error was reported by the DBserver, - // we simply forward the result. - // We intentionally fall through here. - } - - bool dummy; - resetResponse(static_cast(res->result->getHttpReturnCode())); - - _response->setContentType( - res->result->getHeaderField(StaticStrings::ContentTypeHeader, dummy)); - - if (!useVst) { - HttpResponse* httpResponse = dynamic_cast(_response.get()); - if (_response == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "invalid response type"); - } - httpResponse->body().swap(&(res->result->getBody())); - } else { - std::shared_ptr builder = res->result->getBodyVelocyPack(); - std::shared_ptr> buf = builder->steal(); - _response->setPayload(std::move(*buf), - true); // do we need to generate the body?! - } - - auto const& resultHeaders = res->result->getHeaderFields(); - for (auto const& it : resultHeaders) { - _response->setHeader(it.first, it.second); - } + TRI_ASSERT(false); // should only get here if request is not well-formed } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestReplicationHandler.h b/arangod/RestHandler/RestReplicationHandler.h index 769a4bcb03..5d13fc6410 100644 --- a/arangod/RestHandler/RestReplicationHandler.h +++ b/arangod/RestHandler/RestReplicationHandler.h @@ -106,6 +106,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler { static std::string const HoldReadLockCollection; protected: + std::string forwardingTarget() override final; + ////////////////////////////////////////////////////////////////////////////// /// @brief creates an error if called on a coordinator server ////////////////////////////////////////////////////////////////////////////// @@ -122,7 +124,7 @@ class RestReplicationHandler : public RestVocbaseBaseHandler { /// @brief forward a command in the coordinator case ////////////////////////////////////////////////////////////////////////////// - void handleTrampolineCoordinator(); + void handleUnforwardedTrampolineCoordinator(); ////////////////////////////////////////////////////////////////////////////// /// @brief returns the cluster inventory, only on coordinator diff --git a/arangod/RestHandler/RestTasksHandler.cpp b/arangod/RestHandler/RestTasksHandler.cpp index ace8277a55..3b179fcce2 100644 --- a/arangod/RestHandler/RestTasksHandler.cpp +++ b/arangod/RestHandler/RestTasksHandler.cpp @@ -25,6 +25,8 @@ #include "ApplicationFeatures/ApplicationServer.h" #include "Basics/VelocyPackHelper.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "V8/JavaScriptSecurityContext.h" #include "V8/v8-globals.h" @@ -72,22 +74,26 @@ RestStatus RestTasksHandler::execute() { } /// @brief returns the short id of the server which should handle this request -uint32_t RestTasksHandler::forwardingTarget() { +std::string RestTasksHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::POST && type != rest::RequestType::PUT && type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) { - return 0; + return ""; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return 0; + return ""; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } void RestTasksHandler::getTasks() { diff --git a/arangod/RestHandler/RestTasksHandler.h b/arangod/RestHandler/RestTasksHandler.h index 673fcf305e..31dc12d938 100644 --- a/arangod/RestHandler/RestTasksHandler.h +++ b/arangod/RestHandler/RestTasksHandler.h @@ -38,7 +38,7 @@ class RestTasksHandler : public arangodb::RestVocbaseBaseHandler { RestStatus execute() override; protected: - virtual uint32_t forwardingTarget() override; + virtual std::string forwardingTarget() override; private: void getTasks(); diff --git a/arangod/RestHandler/RestTransactionHandler.cpp b/arangod/RestHandler/RestTransactionHandler.cpp index 3653d42917..c395ef71c0 100644 --- a/arangod/RestHandler/RestTransactionHandler.cpp +++ b/arangod/RestHandler/RestTransactionHandler.cpp @@ -27,11 +27,13 @@ #include "ApplicationFeatures/ApplicationServer.h" #include "Basics/ReadLocker.h" #include "Basics/WriteLocker.h" +#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "StorageEngine/EngineSelectorFeature.h" +#include "Transaction/Helpers.h" #include "Transaction/Manager.h" #include "Transaction/ManagerFeature.h" -#include "Transaction/Helpers.h" #include "Transaction/Status.h" #include "V8/JavaScriptSecurityContext.h" #include "V8Server/V8Context.h" @@ -325,20 +327,24 @@ bool RestTransactionHandler::cancel() { } /// @brief returns the short id of the server which should handle this request -uint32_t RestTransactionHandler::forwardingTarget() { +std::string RestTransactionHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::GET && type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { - return 0; + return ""; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return 0; + return ""; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); - return (sourceServer == ServerState::instance()->getShortId()) ? 0 : sourceServer; + if (sourceServer == ServerState::instance()->getShortId()) { + return ""; + } + auto& ci = server().getFeature().clusterInfo(); + return ci.getCoordinatorByShortID(sourceServer); } diff --git a/arangod/RestHandler/RestTransactionHandler.h b/arangod/RestHandler/RestTransactionHandler.h index de84690efd..dffb76b768 100644 --- a/arangod/RestHandler/RestTransactionHandler.h +++ b/arangod/RestHandler/RestTransactionHandler.h @@ -48,7 +48,7 @@ class RestTransactionHandler : public arangodb::RestVocbaseBaseHandler { bool cancel() override final; protected: - virtual uint32_t forwardingTarget() override; + virtual std::string forwardingTarget() override; private: void executeGetState(); diff --git a/arangod/Sharding/ShardDistributionReporter.cpp b/arangod/Sharding/ShardDistributionReporter.cpp index 8ed9250ff6..beeab030e8 100644 --- a/arangod/Sharding/ShardDistributionReporter.cpp +++ b/arangod/Sharding/ShardDistributionReporter.cpp @@ -27,9 +27,13 @@ #include "Basics/StringUtils.h" #include "Basics/system-functions.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" +#include "Futures/Utilities.h" +#include "Logger/LogMacros.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "ShardDistributionReporter.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" @@ -260,10 +264,8 @@ static void ReportOffSync(LogicalCollection const* col, ShardMap const* shardIds result.close(); } -ShardDistributionReporter::ShardDistributionReporter(std::shared_ptr cc, - ClusterInfo* ci) - : _cc(cc), _ci(ci) { - TRI_ASSERT(_cc != nullptr); +ShardDistributionReporter::ShardDistributionReporter(ClusterInfo* ci, network::Sender sender) + : _ci(ci), _send(sender) { TRI_ASSERT(_ci != nullptr); } @@ -273,8 +275,15 @@ std::shared_ptr ShardDistributionReporter::instance( application_features::ApplicationServer& server) { if (_theInstance == nullptr) { auto& ci = server.getFeature().clusterInfo(); - _theInstance = - std::make_shared(ClusterComm::instance(), &ci); + auto& nf = server.getFeature(); + auto* pool = nf.pool(); + _theInstance = std::make_shared( + &ci, + [pool](network::DestinationId const& d, arangodb::fuerte::RestVerb v, + std::string const& u, velocypack::Buffer b, + network::Timeout t, network::Headers h) -> network::FutureRes { + return sendRequest(pool, d, v, u, std::move(b), t, std::move(h)); + }); } return _theInstance; } @@ -284,7 +293,6 @@ void ShardDistributionReporter::helperDistributionForDatabase( std::queue>& todoSyncStateCheck, double endtime, std::unordered_map& aliases, bool progress) { if (!todoSyncStateCheck.empty()) { - CoordTransactionID coordId = TRI_NewTickServer(); std::unordered_map counters; std::vector serversToAsk; while (!todoSyncStateCheck.empty()) { @@ -297,9 +305,8 @@ void ShardDistributionReporter::helperDistributionForDatabase( // Send requests for (auto const& s : *(allShards.get())) { double timeleft = endtime - TRI_microtime(); + network::Timeout timeout(timeleft); serversToAsk.clear(); - uint64_t requestsInFlight = 0; - OperationID leaderOpId = 0; auto curServers = cic->servers(s.first); auto& entry = counters[s.first]; // Emplaces a new SyncCountInfo if (curServers.empty() || s.second.empty()) { @@ -314,15 +321,12 @@ void ShardDistributionReporter::helperDistributionForDatabase( "/_api/collection/" + basics::StringUtils::urlEncode(s.first) + "/count"; - auto body = std::make_shared(); + VPackBuffer body; - { - // First Ask the leader - std::unordered_map headers; - leaderOpId = _cc->asyncRequest(coordId, "server:" + s.second.at(0), - rest::RequestType::GET, path, body, - headers, nullptr, timeleft); - } + // First Ask the leader + network::Headers headers; + auto leaderF = _send("server:" + s.second.at(0), fuerte::RestVerb::Get, + path, body, timeout, headers); // Now figure out which servers need to be asked for (auto const& planned : s.second) { @@ -339,71 +343,74 @@ void ShardDistributionReporter::helperDistributionForDatabase( } // Ask them - std::unordered_map headers; + std::vector futures; + futures.reserve(serversToAsk.size()); for (auto const& server : serversToAsk) { - _cc->asyncRequest(coordId, "server:" + server, rest::RequestType::GET, - path, body, headers, nullptr, timeleft); - requestsInFlight++; + network::Headers headers; + auto f = _send("server:" + server, fuerte::RestVerb::Get, path, + body, timeout, headers); + futures.emplace_back(std::move(f)); } // Wait for responses // First wait for Leader { - auto result = _cc->wait(coordId, leaderOpId, ""); - if (result.status != CL_COMM_RECEIVED) { + auto& res = leaderF.get(); + if (fuerteToArangoErrorCode(res) != TRI_ERROR_NO_ERROR || !res.response) { // We did not even get count for leader, use defaults - _cc->drop(coordId, 0, ""); - // Just in case, to get a new state - coordId = TRI_NewTickServer(); continue; } - auto body = result.result->getBodyVelocyPack(); - VPackSlice response = body->slice(); - if (!response.isObject()) { + + std::vector const& slices = res.response->slices(); + if (slices.empty() || !slices[0].isObject()) { LOG_TOPIC("c02b2", WARN, arangodb::Logger::CLUSTER) << "Received invalid response for count. Shard " - "distribution " - "inaccurate"; + << "distribution inaccurate"; continue; } - response = response.get("count"); + + VPackSlice response = slices[0].get("count"); if (!response.isNumber()) { LOG_TOPIC("fe868", WARN, arangodb::Logger::CLUSTER) << "Received invalid response for count. Shard " - "distribution " - "inaccurate"; + << "distribution inaccurate"; continue; } + entry.total = response.getNumber(); entry.current = entry.total; // << We use this to flip around min/max test } - // Now wait for others - while (requestsInFlight > 0) { - auto result = _cc->wait(coordId, 0, ""); - requestsInFlight--; - if (result.status != CL_COMM_RECEIVED) { - // We do not care for errors of any kind. - // We can continue here because all other requests will be - // handled by the accumulated timeout - continue; - } else { - auto body = result.result->getBodyVelocyPack(); - VPackSlice response = body->slice(); - if (!response.isObject()) { + { + auto responses = futures::collectAll(futures).get(); + for (futures::Try const& response : responses) { + if (!response.hasValue() || + fuerteToArangoErrorCode(response.get()) != TRI_ERROR_NO_ERROR || + !response.get().response) { + // We do not care for errors of any kind. + // We can continue here because all other requests will be + // handled by the accumulated timeout + continue; + } + + auto& res = response.get(); + std::vector const& slices = res.response->slices(); + if (slices.empty() || !slices[0].isObject()) { LOG_TOPIC("fcbb3", WARN, arangodb::Logger::CLUSTER) << "Received invalid response for count. Shard " - "distribution inaccurate"; + << "distribution inaccurate"; continue; } - response = response.get("count"); - if (!response.isNumber()) { + + VPackSlice answer = slices[0].get("count"); + if (!answer.isNumber()) { LOG_TOPIC("8d7b0", WARN, arangodb::Logger::CLUSTER) << "Received invalid response for count. Shard " - "distribution inaccurate"; + << "distribution inaccurate"; continue; } - uint64_t other = response.getNumber(); + + uint64_t other = answer.getNumber(); if (other < entry.total) { // If we have more in total we need the minimum of other // counts diff --git a/arangod/Sharding/ShardDistributionReporter.h b/arangod/Sharding/ShardDistributionReporter.h index f139c8411b..4d9b7cb2d9 100644 --- a/arangod/Sharding/ShardDistributionReporter.h +++ b/arangod/Sharding/ShardDistributionReporter.h @@ -24,12 +24,12 @@ #define ARANGOD_CLUSTER_SHARD_DISTRIBUTED_REPORTER_H 1 #include "Basics/Common.h" +#include "Network/Methods.h" #include namespace arangodb { -class ClusterComm; class ClusterInfo; namespace velocypack { @@ -54,7 +54,7 @@ class ShardDistributionReporter { static std::shared_ptr _theInstance; public: - ShardDistributionReporter(std::shared_ptr cc, ClusterInfo* ci); + ShardDistributionReporter(ClusterInfo* ci, network::Sender sender); ~ShardDistributionReporter(); @@ -75,8 +75,8 @@ class ShardDistributionReporter { std::unordered_map& aliases, bool progress); private: - std::shared_ptr _cc; ClusterInfo* _ci; + network::Sender _send; }; } // namespace cluster } // namespace arangodb diff --git a/arangod/Transaction/Manager.cpp b/arangod/Transaction/Manager.cpp index 206b18b99a..84856689da 100644 --- a/arangod/Transaction/Manager.cpp +++ b/arangod/Transaction/Manager.cpp @@ -26,14 +26,16 @@ #include "Basics/ReadLocker.h" #include "Basics/WriteLocker.h" #include "Basics/system-functions.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" +#include "Futures/Utilities.h" #include "GeneralServer/AuthenticationFeature.h" #include "Logger/LogMacros.h" #include "Logger/Logger.h" #include "Logger/LoggerStream.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" #include "StorageEngine/TransactionState.h" @@ -872,24 +874,28 @@ void Manager::toVelocyPack(VPackBuilder& builder, std::string const& database, TRI_ASSERT(ServerState::instance()->isCoordinator()); auto& ci = _feature.server().getFeature().clusterInfo(); - std::shared_ptr cc = ClusterComm::instance(); - if (cc == nullptr) { + NetworkFeature const& nf = _feature.server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool == nullptr) { THROW_ARANGO_EXCEPTION(TRI_ERROR_SHUTTING_DOWN); } - std::vector requests; + std::vector futures; auto auth = AuthenticationFeature::instance(); + network::RequestOptions options; + options.timeout = network::Timeout(30.0); + + VPackBuffer body; + for (auto const& coordinator : ci.getCurrentCoordinators()) { if (coordinator == ServerState::instance()->getId()) { // ourselves! continue; } - auto headers = std::make_unique>(); + network::Headers headers; if (auth != nullptr && auth->isActive()) { - // when in superuser mode, username is empty - // in this case ClusterComm will add the default superuser token if (!username.empty()) { VPackBuilder builder; { @@ -897,31 +903,38 @@ void Manager::toVelocyPack(VPackBuilder& builder, std::string const& database, payload->add("preferred_username", VPackValue(username)); } VPackSlice slice = builder.slice(); - headers->emplace(StaticStrings::Authorization, - "bearer " + auth->tokenCache().generateJwt(slice)); + headers.emplace(StaticStrings::Authorization, + "bearer " + auth->tokenCache().generateJwt(slice)); + } else { + headers.emplace(StaticStrings::Authorization, + "bearer " + auth->tokenCache().jwtToken()); } } - requests.emplace_back("server:" + coordinator, rest::RequestType::GET, - "/_db/" + database + "/_api/transaction?local=true", - std::make_shared(), std::move(headers)); + auto f = network::sendRequest(pool, "server:" + coordinator, fuerte::RestVerb::Get, + "/_db/" + database + + "/_api/transaction?local=true", + body, std::move(headers), options); + futures.emplace_back(std::move(f)); } - if (!requests.empty()) { - size_t nrGood = cc->performRequests(requests, 30.0, Logger::COMMUNICATION, false); - - if (nrGood != requests.size()) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_BACKEND_UNAVAILABLE); - } - for (auto const& it : requests) { - if (it.result.result && it.result.result->getHttpReturnCode() == 200) { - auto const body = it.result.result->getBodyVelocyPack(); - VPackSlice slice = body->slice(); - if (slice.isObject()) { - slice = slice.get("transactions"); - if (slice.isArray()) { - for (auto const& it : VPackArrayIterator(slice)) { - builder.add(it); + if (!futures.empty()) { + auto responses = futures::collectAll(futures).get(); + for (auto const& it : responses) { + if (!it.hasValue()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_BACKEND_UNAVAILABLE); + } + auto& res = it.get(); + if (res.response && res.response->statusCode() == fuerte::StatusOK) { + auto slices = res.response->slices(); + if (!slices.empty()) { + VPackSlice slice = slices[0]; + if (slice.isObject()) { + slice = slice.get("transactions"); + if (slice.isArray()) { + for (auto const& it : VPackArrayIterator(slice)) { + builder.add(it); + } } } } diff --git a/arangod/Transaction/Methods.cpp b/arangod/Transaction/Methods.cpp index a2fd4023be..d75fb34c7f 100644 --- a/arangod/Transaction/Methods.cpp +++ b/arangod/Transaction/Methods.cpp @@ -37,7 +37,6 @@ #include "Basics/VelocyPackHelper.h" #include "Basics/encoding.h" #include "Basics/system-compiler.h" -#include "Cluster/ClusterComm.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterMethods.h" #include "Cluster/ClusterTrxMethods.h" @@ -709,10 +708,10 @@ std::pair transaction::Methods::findIndexHandleForAndNode( } /// @brief Find out if any of the given requests has ended in a refusal -static bool findRefusal(std::vector const& requests) { - for (auto const& it : requests) { - if (it.done && it.result.status == CL_COMM_RECEIVED && - it.result.answer_code == rest::ResponseCode::NOT_ACCEPTABLE) { +static bool findRefusal(std::vector> const& responses) { + for (auto const& it : responses) { + if (it.hasValue() && it.get().ok() && + it.get().response->statusCode() == fuerte::StatusNotAcceptable) { return true; } } @@ -2457,42 +2456,43 @@ Future transaction::Methods::truncateLocal(std::string const& c TRI_ASSERT(!_state->hasHint(Hints::Hint::FROM_TOPLEVEL_AQL)); // Now replicate the good operations on all followers: - auto cc = arangodb::ClusterComm::instance(); - - if (cc != nullptr) { + NetworkFeature const& nf = vocbase().server().getFeature(); + network::ConnectionPool* pool = nf.pool(); + if (pool != nullptr) { // nullptr only happens on controlled shutdown std::string path = "/_db/" + arangodb::basics::StringUtils::urlEncode(vocbase().name()) + "/_api/collection/" + arangodb::basics::StringUtils::urlEncode(collectionName) + "/truncate?isSynchronousReplication=" + ServerState::instance()->getId(); - auto body = std::make_shared(); + VPackBuffer body; // Now prepare the requests: - std::vector requests; - requests.reserve(followers->size()); + std::vector futures; + futures.reserve(followers->size()); for (auto const& f : *followers) { - auto headers = std::make_unique>(); - ClusterTrxMethods::addTransactionHeader(*this, f, *headers); - requests.emplace_back("server:" + f, arangodb::rest::RequestType::PUT, - path, body, std::move(headers)); + network::Headers headers; + ClusterTrxMethods::addTransactionHeader(*this, f, headers); + auto future = network::sendRequest(pool, "server:" + f, fuerte::RestVerb::Put, + path, body, std::move(headers)); + futures.emplace_back(std::move(future)); } - cc->performRequests(requests, 120.0, Logger::REPLICATION, false); + auto responses = futures::collectAll(futures).get(); // If any would-be-follower refused to follow there must be a // new leader in the meantime, in this case we must not allow // this operation to succeed, we simply return with a refusal // error (note that we use the follower version, since we have // lost leadership): - if (findRefusal(requests)) { + if (findRefusal(responses)) { return futures::makeFuture(OperationResult(TRI_ERROR_CLUSTER_SHARD_LEADER_RESIGNED)); } // we drop all followers that were not successful: for (size_t i = 0; i < followers->size(); ++i) { bool replicationWorked = - requests[i].done && requests[i].result.status == CL_COMM_RECEIVED && - (requests[i].result.answer_code == rest::ResponseCode::ACCEPTED || - requests[i].result.answer_code == rest::ResponseCode::OK); + responses[i].hasValue() && responses[i].get().ok() && + (responses[i].get().response->statusCode() == fuerte::StatusAccepted || + responses[i].get().response->statusCode() == fuerte::StatusOK); if (!replicationWorked) { auto const& followerInfo = collection->followers(); Result res = followerInfo->remove((*followers)[i]); @@ -2542,11 +2542,6 @@ futures::Future transaction::Methods::countAsync(std::string co futures::Future transaction::Methods::countCoordinator( std::string const& collectionName, transaction::CountType type) { auto& feature = vocbase().server().getFeature(); - auto cc = ClusterComm::instance(); - if (cc == nullptr) { - // nullptr happens only during controlled shutdown - return futures::makeFuture(OperationResult(TRI_ERROR_SHUTTING_DOWN)); - } ClusterInfo& ci = feature.clusterInfo(); // First determine the collection ID from the name: diff --git a/arangod/V8Server/v8-actions.cpp b/arangod/V8Server/v8-actions.cpp index 3c23ac8b9b..b94297ccc3 100644 --- a/arangod/V8Server/v8-actions.cpp +++ b/arangod/V8Server/v8-actions.cpp @@ -33,12 +33,16 @@ #include "Basics/conversions.h" #include "Basics/files.h" #include "Basics/tri-strings.h" -#include "Cluster/ClusterComm.h" -#include "Cluster/ClusterFeature.h" +#include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" +#include "Futures/Utilities.h" #include "GeneralServer/GeneralServer.h" #include "GeneralServer/ServerSecurityFeature.h" +#include "Logger/LogMacros.h" #include "Logger/Logger.h" +#include "Network/Methods.h" +#include "Network/NetworkFeature.h" +#include "Network/Utils.h" #include "Rest/GeneralRequest.h" #include "Rest/HttpRequest.h" #include "Rest/HttpResponse.h" @@ -1415,41 +1419,42 @@ static int clusterSendToAllServers(std::string const& dbname, arangodb::rest::RequestType const& method, std::string const& body) { auto& server = application_features::ApplicationServer::server(); - ClusterInfo& ci = server.getFeature().clusterInfo(); - auto cc = ClusterComm::instance(); - if (cc == nullptr) { + network::ConnectionPool* pool = server.getFeature().pool(); + if (!pool || !pool->config().clusterInfo) { + LOG_TOPIC("98fc7", ERR, Logger::COMMUNICATION) << "Network pool unavailable."; return TRI_ERROR_SHUTTING_DOWN; } + ClusterInfo& ci = *pool->config().clusterInfo; + + network::Headers headers; + fuerte::RestVerb verb = network::arangoRestVerbToFuerte(method); std::string url = "/_db/" + StringUtils::urlEncode(dbname) + "/" + path; + auto timeout = std::chrono::seconds(3600); + + std::vector> futures; // Have to propagate to DB Servers - std::vector DBServers; - CoordTransactionID coordTransactionID = TRI_NewTickServer(); - auto reqBodyString = std::make_shared(body); - - DBServers = ci.getCurrentDBServers(); - std::unordered_map headers; + std::vector DBServers = ci.getCurrentDBServers(); for (auto const& sid : DBServers) { - cc->asyncRequest(coordTransactionID, "server:" + sid, method, url, - reqBodyString, headers, nullptr, 3600.0); + VPackBuffer buffer(body.size()); + buffer.append(body); + auto f = network::sendRequest(pool, "server:" + sid, verb, url, + std::move(buffer), timeout, headers); + futures.emplace_back(std::move(f)); } - // Now listen to the results: - size_t count = DBServers.size(); - - for (; count > 0; count--) { - auto res = cc->wait(coordTransactionID, 0, "", 0.0); - if (res.status == CL_COMM_TIMEOUT) { - cc->drop(coordTransactionID, 0, ""); - return TRI_ERROR_CLUSTER_TIMEOUT; - } - if (res.status == CL_COMM_ERROR || res.status == CL_COMM_DROPPED || - res.status == CL_COMM_BACKEND_UNAVAILABLE) { - cc->drop(coordTransactionID, 0, ""); - return TRI_ERROR_INTERNAL; - } - } - return TRI_ERROR_NO_ERROR; + return futures::collectAll(futures) + .thenValue([](std::vector>&& responses) -> int { + for (futures::Try const& tryRes : responses) { + network::Response const& res = tryRes.get(); // throws exceptions upwards + int commError = network::fuerteToArangoErrorCode(res); + if (commError != TRI_ERROR_NO_ERROR) { + return commError; + } + } + return TRI_ERROR_NO_ERROR; + }) + .get(); } #endif diff --git a/lib/Basics/StaticStrings.cpp b/lib/Basics/StaticStrings.cpp index c84b505000..307a37cf32 100644 --- a/lib/Basics/StaticStrings.cpp +++ b/lib/Basics/StaticStrings.cpp @@ -210,6 +210,9 @@ std::string const StaticStrings::XContentTypeOptions("x-content-type-options"); std::string const StaticStrings::XArangoFrontend("x-arango-frontend"); // mime types +std::string const StaticStrings::MimeTypeDump( + "application/x-arango-dump; charset=utf-8"); +std::string const StaticStrings::MimeTypeHtml("text/html; charset=utf-8"); std::string const StaticStrings::MimeTypeJson( "application/json; charset=utf-8"); std::string const StaticStrings::MimeTypeText("text/plain; charset=utf-8"); diff --git a/lib/Basics/StaticStrings.h b/lib/Basics/StaticStrings.h index 7242c3ae2c..dbf1f40dc7 100644 --- a/lib/Basics/StaticStrings.h +++ b/lib/Basics/StaticStrings.h @@ -196,6 +196,8 @@ class StaticStrings { static std::string const XArangoFrontend; // mime types + static std::string const MimeTypeDump; + static std::string const MimeTypeHtml; static std::string const MimeTypeJson; static std::string const MimeTypeText; static std::string const MimeTypeVPack; diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 62b21643c4..74d500508d 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -223,6 +223,7 @@ add_library(arango STATIC Random/RandomFeature.cpp Random/RandomGenerator.cpp Random/UniformCharacter.cpp + Rest/CommonDefines.cpp Rest/GeneralRequest.cpp Rest/GeneralResponse.cpp Rest/VstRequest.cpp diff --git a/lib/Futures/Future.h b/lib/Futures/Future.h index 0fcfdf3367..1073674225 100644 --- a/lib/Futures/Future.h +++ b/lib/Futures/Future.h @@ -25,6 +25,7 @@ #include #include +#include #include #include "Basics/debugging.h" @@ -213,6 +214,9 @@ class Future { // True when the result (or exception) is ready bool isReady() const { return getState().hasResult(); } + /// True if the future already has a callback set + bool hasCallback() const { return getState().hasCallback(); } + /// True if the result is a value (not an exception) bool hasValue() const { TRI_ASSERT(isReady()); diff --git a/lib/Futures/Utilities.h b/lib/Futures/Utilities.h index 9dcf1b50ad..0c01145ef6 100644 --- a/lib/Futures/Utilities.h +++ b/lib/Futures/Utilities.h @@ -23,6 +23,10 @@ #ifndef ARANGOD_FUTURES_UTILITIES_H #define ARANGOD_FUTURES_UTILITIES_H 1 +#include +#include +#include +#include #include #include "Futures/Future.h" diff --git a/lib/Rest/CommonDefines.cpp b/lib/Rest/CommonDefines.cpp new file mode 100644 index 0000000000..758768c993 --- /dev/null +++ b/lib/Rest/CommonDefines.cpp @@ -0,0 +1,54 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Dan Larkin-York +//////////////////////////////////////////////////////////////////////////////// + +#include +#include + +#include "CommonDefines.h" + +#include "Basics/StaticStrings.h" + +namespace arangodb { +namespace rest { + +std::string contentTypeToString(ContentType type) { + switch (type) { + case ContentType::VPACK: + return StaticStrings::MimeTypeVPack; + case ContentType::TEXT: + return StaticStrings::MimeTypeText; + case ContentType::HTML: + return StaticStrings::MimeTypeHtml; + case ContentType::DUMP: + return StaticStrings::MimeTypeDump; + case ContentType::CUSTOM: + return ""; // use value from headers + case ContentType::UNSET: + case ContentType::JSON: + default: + return StaticStrings::MimeTypeJson; + } +} + +} // namespace rest +} // namespace arangodb diff --git a/lib/Rest/CommonDefines.h b/lib/Rest/CommonDefines.h index 0ff3ba689c..2b22c0a8b8 100644 --- a/lib/Rest/CommonDefines.h +++ b/lib/Rest/CommonDefines.h @@ -78,6 +78,8 @@ enum class ContentType { UNSET }; +std::string contentTypeToString(ContentType type); + enum class EncodingType { DEFLATE, UNSET diff --git a/tests/Sharding/ShardDistributionReporterTest.cpp b/tests/Sharding/ShardDistributionReporterTest.cpp index 8d7e03e999..f2019be8d2 100644 --- a/tests/Sharding/ShardDistributionReporterTest.cpp +++ b/tests/Sharding/ShardDistributionReporterTest.cpp @@ -25,13 +25,24 @@ /// @author Copyright 2017, ArangoDB GmbH, Cologne, Germany //////////////////////////////////////////////////////////////////////////////// +#include +#include + #include "gtest/gtest.h" #include "fakeit.hpp" -#include "../IResearch/common.h" +#include +#include +#include + +#include "IResearch/common.h" +#include "Mocks/LogLevels.h" +#include "Mocks/StorageEngineMock.h" + #include "ApplicationFeatures/ApplicationServer.h" #include "Cluster/ClusterComm.h" +#include "Futures/Utilities.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/QueryRegistryFeature.h" #include "Sharding/ShardDistributionReporter.h" @@ -40,15 +51,6 @@ #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" -#include "Mocks/StorageEngineMock.h" - -#include -#include -#include - -#include -#include - using namespace arangodb; using namespace arangodb::cluster; using namespace arangodb::httpclient; @@ -88,30 +90,38 @@ static void VerifyNumbers(VPackSlice result, std::string const& colName, VPackSlice total = progress.get("total"); ASSERT_TRUE(total.isNumber()); - ASSERT_TRUE(total.getNumber() == testTotal); + ASSERT_EQ(total.getNumber(), testTotal); VPackSlice current = progress.get("current"); ASSERT_TRUE(current.isNumber()); ASSERT_EQ(current.getNumber(), testCurrent); } -static std::shared_ptr buildCountBody(uint64_t count) { - auto res = std::make_shared(); - res->openObject(); - res->add("count", VPackValue(count)); - res->close(); - return res; +static std::unique_ptr generateCountResponse(uint64_t count) { + VPackBuffer responseBuffer; + VPackBuilder builder(responseBuffer); + builder.openObject(); + builder.add("count", VPackValue(count)); + builder.close(); + + fuerte::ResponseHeader header; + header.contentType(fuerte::ContentType::VPack); + std::unique_ptr response(new fuerte::Response(std::move(header))); + response->setPayload(std::move(responseBuffer), /*offset*/ 0); + + TRI_ASSERT(!response->slices().empty()); + + return response; } -class ShardDistributionReporterTest : public ::testing::Test { +class ShardDistributionReporterTest + : public ::testing::Test, + public arangodb::tests::LogSuppressor { protected: arangodb::application_features::ApplicationServer server; StorageEngineMock engine; std::vector> features; - fakeit::Mock commMock; - ClusterComm& cc; - fakeit::Mock infoMock; ClusterInfo& ci; @@ -150,12 +160,9 @@ class ShardDistributionReporterTest : public ::testing::Test { // Fake the collections std::vector> allCollections; - ShardDistributionReporter testee; - ShardDistributionReporterTest() : server(nullptr, nullptr), engine(server), - cc(commMock.get()), ci(infoMock.get()), cicInst(infoCurrentMock.get()), cic(&cicInst, [](CollectionInfoCurrent*) {}), @@ -173,8 +180,7 @@ class ShardDistributionReporterTest : public ::testing::Test { dbserver3short("DBServer3"), json(arangodb::velocypack::Parser::fromJson( "{ \"cid\" : \"1337\", \"name\": \"UnitTestCollection\" }")), - shards(std::make_shared()), - testee(std::shared_ptr(&cc, [](ClusterComm*) {}), &ci) { + shards(std::make_shared()) { aliases[dbserver1] = dbserver1short; aliases[dbserver2] = dbserver2short; aliases[dbserver3] = dbserver3short; @@ -250,34 +256,6 @@ TEST_F(ShardDistributionReporterTest, uint64_t shard3LeaderCount = 4651; uint64_t shard3FollowerCount = 912; - // Moking HttpResults - fakeit::Mock db1s2CountMock; - SimpleHttpResult& httpdb1s2Count = db1s2CountMock.get(); - - fakeit::Mock db1s3CountMock; - SimpleHttpResult& httpdb1s3Count = db1s3CountMock.get(); - - fakeit::Mock db2s2CountMock; - SimpleHttpResult& httpdb2s2Count = db2s2CountMock.get(); - - fakeit::Mock db3s2CountMock; - SimpleHttpResult& httpdb3s2Count = db3s2CountMock.get(); - - fakeit::Mock db3s3CountMock; - SimpleHttpResult& httpdb3s3Count = db3s3CountMock.get(); - - bool gotFirstRequest = false; - CoordTransactionID cordTrxId = 0; - - std::queue responses; - ClusterCommResult leaderS2Response; - OperationID leaderS2Id; - bool leaderS2Delivered = true; - - ClusterCommResult leaderS3Response; - OperationID leaderS3Id; - bool leaderS3Delivered = true; - shards->emplace(s1, std::vector{dbserver1, dbserver2, dbserver3}); shards->emplace(s2, std::vector{dbserver2, dbserver1, dbserver3}); shards->emplace(s3, std::vector{dbserver3, dbserver1, dbserver2}); @@ -296,144 +274,54 @@ TEST_F(ShardDistributionReporterTest, return currentShards[sid]; }); - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(db1s2CountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(shard2LowFollowerCount); }); + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(reqType == fuerte::RestVerb::Get); + EXPECT_TRUE(headerFields.empty()); + // This feature has at most 2s to do its job + // otherwise default values will be returned + EXPECT_TRUE(timeout.count() <= 2.0); - fakeit::When(ConstOverloadedMethod(db1s3CountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(shard3FollowerCount); }); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - fakeit::When(ConstOverloadedMethod(db2s2CountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(shard2LeaderCount); }); + // '/_db/UnitTestDB/_api/collection/' + shard.shard + '/count' + if (destination == "server:" + dbserver1) { + // off-sync follows s2,s3 + if (path == "/_db/UnitTestDB/_api/collection/" + s2 + "/count") { + response.response = generateCountResponse(shard2LowFollowerCount); + } else { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s3 + "/count"); + response.response = generateCountResponse(shard3FollowerCount); + } + } else if (destination == "server:" + dbserver2) { + // Leads s2 + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s2 + "/count"); + response.response = generateCountResponse(shard2LeaderCount); + } else if (destination == "server:" + dbserver3) { + // Leads s3 + // off-sync follows s2 + if (path == "/_db/UnitTestDB/_api/collection/" + s2 + "/count") { + response.response = generateCountResponse(shard2HighFollowerCount); + } else { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s3 + "/count"); + response.response = generateCountResponse(shard3LeaderCount); + } + } else { + // Unknown Server!! + EXPECT_TRUE(false); + } - fakeit::When(ConstOverloadedMethod(db3s2CountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(shard2HighFollowerCount); }); - - fakeit::When(ConstOverloadedMethod(db3s3CountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(shard3LeaderCount); }); - - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(initTimeout == -1.0); // Default - EXPECT_TRUE(!singleRequest); // we want to use keep-alive - EXPECT_TRUE(callback == nullptr); // We actively wait - EXPECT_TRUE(reqtype == rest::RequestType::GET); // count is only get! - EXPECT_TRUE(headerFields.empty()); // Nono headers - - // This feature has at most 2s to do its job - // otherwise default values will be returned - EXPECT_TRUE(timeout <= 2.0); - - if (!gotFirstRequest) { - gotFirstRequest = true; - cordTrxId = coordTransactionID; - } else { - // We always use the same id - EXPECT_EQ(cordTrxId, coordTransactionID); - } - - OperationID opId = TRI_NewTickServer(); - - ClusterCommResult response; - response.coordTransactionID = cordTrxId; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - // '/_db/UnitTestDB/_api/collection/' + shard.shard + '/count' - if (destination == "server:" + dbserver1) { - // off-sync follows s2,s3 - if (path == "/_db/UnitTestDB/_api/collection/" + s2 + "/count") { - response.result = - std::shared_ptr(&httpdb1s2Count, - [](SimpleHttpResult*) {}); - } else { - EXPECT_TRUE(path == - "/_db/UnitTestDB/_api/collection/" + s3 + "/count"); - response.result = - std::shared_ptr(&httpdb1s3Count, - [](SimpleHttpResult*) {}); - } - } else if (destination == "server:" + dbserver2) { - // Leads s2 - EXPECT_TRUE(path == - "/_db/UnitTestDB/_api/collection/" + s2 + "/count"); - response.result = std::shared_ptr(&httpdb2s2Count, - [](SimpleHttpResult*) {}); - leaderS2Response = response; - leaderS2Id = opId; - leaderS2Delivered = false; - return opId; - } else if (destination == "server:" + dbserver3) { - // Leads s3 - // off-sync follows s2 - if (path == "/_db/UnitTestDB/_api/collection/" + s2 + "/count") { - response.result = - std::shared_ptr(&httpdb3s2Count, - [](SimpleHttpResult*) {}); - } else { - EXPECT_TRUE(path == - "/_db/UnitTestDB/_api/collection/" + s3 + "/count"); - response.result = - std::shared_ptr(&httpdb3s3Count, - [](SimpleHttpResult*) {}); - leaderS3Response = response; - leaderS3Id = opId; - leaderS3Delivered = false; - return opId; - } - } else { - // Unknown Server!! - EXPECT_TRUE(false); - } - - responses.push(response); - return opId; - }); - - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - EXPECT_EQ(coordTransactionID, cordTrxId); - EXPECT_TRUE(shardID == ""); // Superfluous - EXPECT_TRUE(timeout == 0.0); // Default, the requests has timeout already - - if (operationID == leaderS2Id && !leaderS2Delivered) { - EXPECT_TRUE(leaderS2Id != 0); - EXPECT_TRUE(!leaderS2Delivered); - leaderS2Delivered = true; - return leaderS2Response; - } - - if (operationID == leaderS3Id && !leaderS3Delivered) { - EXPECT_TRUE(leaderS3Id != 0); - EXPECT_TRUE(!leaderS3Delivered); - leaderS3Delivered = true; - return leaderS3Response; - } - - EXPECT_TRUE(operationID == 0); // We do not wait for a specific one - - EXPECT_TRUE(!responses.empty()); - ClusterCommResult last = responses.front(); - responses.pop(); - return last; - }); + return futures::makeFuture(std::move(response)); + }; { // testing distribution for database - gotFirstRequest = false; VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getDistributionForDatabase(dbname, resultBuilder); VPackSlice result = resultBuilder.slice(); @@ -656,8 +544,8 @@ TEST_F(ShardDistributionReporterTest, } { // testing distribution for collection database - gotFirstRequest = false; VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VPackSlice result = resultBuilder.slice(); @@ -755,11 +643,11 @@ TEST_F(ShardDistributionReporterTest, VPackSlice total = progress.get("total"); ASSERT_TRUE(total.isNumber()); - ASSERT_TRUE(total.getNumber() == shard2LeaderCount); + ASSERT_EQ(total.getNumber(), shard2LeaderCount); VPackSlice current = progress.get("current"); ASSERT_TRUE(current.isNumber()); - ASSERT_TRUE(current.getNumber() == shard2LowFollowerCount); + ASSERT_EQ(current.getNumber(), shard2LowFollowerCount); } } @@ -895,107 +783,6 @@ TEST_F(ShardDistributionReporterTest, } } } - -TEST_F(ShardDistributionReporterTest, - testing_distribution_for_database_a_single_collection_of_three_shards_and_three_replicas) { - shards->emplace(s1, std::vector{dbserver1, dbserver2, dbserver3}); - - col->setShardMap(shards); - - currentShards.emplace(s1, std::vector{dbserver1}); - - allCollections.emplace_back( - std::shared_ptr(col.get(), [](LogicalCollection*) {})); - - fakeit::When(Method(infoCurrentMock, servers)).AlwaysDo([&](ShardID const& sid) { - EXPECT_TRUE(sid == s1); - return currentShards[sid]; - }); - - // Moking HttpResults - fakeit::Mock leaderCountMock; - SimpleHttpResult& lCount = leaderCountMock.get(); - - fakeit::Mock followerOneCountMock; - SimpleHttpResult& f1Count = followerOneCountMock.get(); - - fakeit::Mock followerTwoCountMock; - SimpleHttpResult& f2Count = followerTwoCountMock.get(); - - uint64_t leaderCount = 1337; - uint64_t smallerFollowerCount = 456; - uint64_t largerFollowerCount = 1111; - - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(leaderCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(leaderCount); }); - - fakeit::When(ConstOverloadedMethod(followerOneCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(largerFollowerCount); }); - - fakeit::When(ConstOverloadedMethod(followerTwoCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(smallerFollowerCount); }); - - ClusterCommResult leaderRes; - ClusterCommResult follower1Res; - ClusterCommResult follower2Res; - - bool returnedFirstFollower = false; - - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - - OperationID opId = TRI_NewTickServer(); - - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.result = - std::shared_ptr(&lCount, [](SimpleHttpResult*) {}); - leaderRes = response; - } else if (destination == "server:" + dbserver2) { - response.result = - std::shared_ptr(&f1Count, [](SimpleHttpResult*) {}); - follower1Res = response; - } else if (destination == "server:" + dbserver3) { - response.result = - std::shared_ptr(&f2Count, [](SimpleHttpResult*) {}); - follower2Res = response; - } else { - EXPECT_TRUE(false); - } - return opId; - }); - - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - if (returnedFirstFollower) { - return follower2Res; - } else { - returnedFirstFollower = true; - return follower1Res; - } - }); -} - TEST_F(ShardDistributionReporterTest, testing_collection_distribution_for_database_a_single_collection_of_three_shards_and_three_replicas) { shards->emplace(s1, std::vector{dbserver1, dbserver2, dbserver3}); @@ -1012,88 +799,33 @@ TEST_F(ShardDistributionReporterTest, return currentShards[sid]; }); - // Moking HttpResults - fakeit::Mock leaderCountMock; - SimpleHttpResult& lCount = leaderCountMock.get(); - - fakeit::Mock followerOneCountMock; - SimpleHttpResult& f1Count = followerOneCountMock.get(); - - fakeit::Mock followerTwoCountMock; - SimpleHttpResult& f2Count = followerTwoCountMock.get(); - uint64_t leaderCount = 1337; uint64_t smallerFollowerCount = 456; uint64_t largerFollowerCount = 1111; - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(leaderCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(leaderCount); }); + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - fakeit::When(ConstOverloadedMethod(followerOneCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(largerFollowerCount); }); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - fakeit::When(ConstOverloadedMethod(followerTwoCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(smallerFollowerCount); }); + if (destination == "server:" + dbserver1) { + response.response = generateCountResponse(leaderCount); + } else if (destination == "server:" + dbserver2) { + response.response = generateCountResponse(largerFollowerCount); + } else if (destination == "server:" + dbserver3) { + response.response = generateCountResponse(smallerFollowerCount); + } else { + EXPECT_TRUE(false); + } - ClusterCommResult leaderRes; - ClusterCommResult follower1Res; - ClusterCommResult follower2Res; - - bool returnedFirstFollower = false; - - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - - OperationID opId = TRI_NewTickServer(); - - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.result = - std::shared_ptr(&lCount, [](SimpleHttpResult*) {}); - leaderRes = response; - } else if (destination == "server:" + dbserver2) { - response.result = - std::shared_ptr(&f1Count, [](SimpleHttpResult*) {}); - follower1Res = response; - } else if (destination == "server:" + dbserver3) { - response.result = - std::shared_ptr(&f2Count, [](SimpleHttpResult*) {}); - follower2Res = response; - } else { - EXPECT_TRUE(false); - } - return opId; - }); - - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - if (returnedFirstFollower) { - return follower2Res; - } else { - returnedFirstFollower = true; - return follower1Res; - } - }); + return futures::makeFuture(std::move(response)); + }; { // testing collection distribution for database { // Both followers have a smaller amount of documents @@ -1102,6 +834,7 @@ TEST_F(ShardDistributionReporterTest, { // The minimum should be reported VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VerifyNumbers(resultBuilder.slice(), colName, s1, leaderCount, smallerFollowerCount); @@ -1109,12 +842,12 @@ TEST_F(ShardDistributionReporterTest, } { // Both followers have a larger amount of documents - returnedFirstFollower = false; smallerFollowerCount = 1987; largerFollowerCount = 2345; { // The maximum should be reported VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VerifyNumbers(resultBuilder.slice(), colName, s1, leaderCount, largerFollowerCount); @@ -1127,6 +860,7 @@ TEST_F(ShardDistributionReporterTest, { // The lesser should be reported VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VerifyNumbers(resultBuilder.slice(), colName, s1, leaderCount, smallerFollowerCount); @@ -1151,68 +885,34 @@ TEST_F(ShardDistributionReporterTest, return currentShards[sid]; }); - ClusterCommResult leaderRes; + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - CoordTransactionID coordId = 0; - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - OperationID opId = TRI_NewTickServer(); - coordId = coordTransactionID; + if (destination == "server:" + dbserver1) { + response.error = fuerte::Error::Timeout; + } else if (destination == "server:" + dbserver2) { + } else if (destination == "server:" + dbserver3) { + } else { + EXPECT_TRUE(false); + } - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.status = CL_COMM_TIMEOUT; - } else if (destination == "server:" + dbserver2) { - } else if (destination == "server:" + dbserver3) { - } else { - EXPECT_TRUE(false); - } - return opId; - }); - - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - // If we get here we tried to wait for followers and we cannot use the answer... - EXPECT_TRUE(false); - return leaderRes; - }); - - fakeit::When(Method(commMock, drop)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - OperationID const operationID, ShardID const& shardID) { - // We need to abort this trx - EXPECT_TRUE(coordTransactionID == coordId); - // For all operations and shards - EXPECT_TRUE(operationID == 0); - EXPECT_TRUE(shardID == ""); - }); + return futures::makeFuture(std::move(response)); + }; { // It should use the defaults total: 1 current: 0 VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getDistributionForDatabase(dbname, resultBuilder); VerifyAttributes(resultBuilder.slice(), colName, s1); } - - { // It needs to call drop - fakeit::Verify(Method(commMock, drop)).Exactly(1); - } } TEST_F(ShardDistributionReporterTest, @@ -1232,112 +932,38 @@ TEST_F(ShardDistributionReporterTest, }); uint64_t leaderCount = 1337; - uint64_t smallerFollowerCount = 456; uint64_t largerFollowerCount = 1111; - // Moking HttpResults - fakeit::Mock leaderCountMock; - SimpleHttpResult& lCount = leaderCountMock.get(); + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - fakeit::Mock followerOneCountMock; - SimpleHttpResult& f1Count = followerOneCountMock.get(); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - fakeit::Mock followerTwoCountMock; - SimpleHttpResult& f2Count = followerTwoCountMock.get(); + if (destination == "server:" + dbserver1) { + response.response = generateCountResponse(leaderCount); + } else if (destination == "server:" + dbserver2) { + response.response = generateCountResponse(largerFollowerCount); + } else if (destination == "server:" + dbserver3) { + response.error = fuerte::Error::Timeout; + } else { + EXPECT_TRUE(false); + } - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(leaderCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(leaderCount); }); - - fakeit::When(ConstOverloadedMethod(followerOneCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(largerFollowerCount); }); - - fakeit::When(ConstOverloadedMethod(followerTwoCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { - EXPECT_TRUE(false); - return buildCountBody(smallerFollowerCount); - }); - - ClusterCommResult leaderRes; - ClusterCommResult follower1Res; - ClusterCommResult follower2Res; - bool returnedFirstFollower = false; - - CoordTransactionID coordId = 0; - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - - OperationID opId = TRI_NewTickServer(); - coordId = coordTransactionID; - - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.result = - std::shared_ptr(&lCount, [](SimpleHttpResult*) {}); - leaderRes = response; - } else if (destination == "server:" + dbserver2) { - response.result = - std::shared_ptr(&f1Count, [](SimpleHttpResult*) {}); - follower1Res = response; - } else if (destination == "server:" + dbserver3) { - response.status = CL_COMM_TIMEOUT; - response.result = - std::shared_ptr(&f2Count, [](SimpleHttpResult*) {}); - follower2Res = response; - } else { - EXPECT_TRUE(false); - } - - return opId; - }); - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - if (returnedFirstFollower) { - return follower2Res; - } else { - returnedFirstFollower = true; - return follower1Res; - } - }); - - fakeit::When(Method(commMock, drop)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - OperationID const operationID, ShardID const& shardID) { - // We need to abort this trx - EXPECT_TRUE(coordTransactionID == coordId); - // For all operations and shards - EXPECT_TRUE(operationID == 0); - EXPECT_TRUE(shardID == ""); - }); + return futures::makeFuture(std::move(response)); + }; { // It should use the leader and the other one VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getDistributionForDatabase(dbname, resultBuilder); VerifyAttributes(resultBuilder.slice(), colName, s1); } - - { // It should not call drop - fakeit::Verify(Method(commMock, drop)).Exactly(0); - } } TEST_F(ShardDistributionReporterTest, @@ -1357,116 +983,37 @@ TEST_F(ShardDistributionReporterTest, }); uint64_t leaderCount = 1337; - uint64_t smallerFollowerCount = 456; - uint64_t largerFollowerCount = 1111; - // Moking HttpResults - fakeit::Mock leaderCountMock; - SimpleHttpResult& lCount = leaderCountMock.get(); + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - fakeit::Mock followerOneCountMock; - SimpleHttpResult& f1Count = followerOneCountMock.get(); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - fakeit::Mock followerTwoCountMock; - SimpleHttpResult& f2Count = followerTwoCountMock.get(); + if (destination == "server:" + dbserver1) { + response.response = generateCountResponse(leaderCount); + } else if (destination == "server:" + dbserver2) { + response.error = fuerte::Error::Timeout; + } else if (destination == "server:" + dbserver3) { + response.error = fuerte::Error::Timeout; + } else { + EXPECT_TRUE(false); + } - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(leaderCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(leaderCount); }); - - fakeit::When(ConstOverloadedMethod(followerOneCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { - EXPECT_TRUE(false); - return buildCountBody(largerFollowerCount); - }); - - fakeit::When(ConstOverloadedMethod(followerTwoCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { - EXPECT_TRUE(false); - return buildCountBody(smallerFollowerCount); - }); - - ClusterCommResult leaderRes; - ClusterCommResult follower1Res; - ClusterCommResult follower2Res; - bool returnedFirstFollower = false; - - CoordTransactionID coordId = 0; - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - - OperationID opId = TRI_NewTickServer(); - coordId = coordTransactionID; - - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.result = - std::shared_ptr(&lCount, [](SimpleHttpResult*) {}); - leaderRes = response; - } else if (destination == "server:" + dbserver2) { - response.status = CL_COMM_TIMEOUT; - response.result = - std::shared_ptr(&f1Count, [](SimpleHttpResult*) {}); - follower1Res = response; - } else if (destination == "server:" + dbserver3) { - response.status = CL_COMM_TIMEOUT; - response.result = - std::shared_ptr(&f2Count, [](SimpleHttpResult*) {}); - follower2Res = response; - } else { - EXPECT_TRUE(false); - } - - return opId; - }); - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - if (returnedFirstFollower) { - return follower2Res; - } else { - returnedFirstFollower = true; - return follower1Res; - } - }); - - fakeit::When(Method(commMock, drop)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - OperationID const operationID, ShardID const& shardID) { - // We need to abort this trx - EXPECT_TRUE(coordTransactionID == coordId); - // For all operations and shards - EXPECT_TRUE(operationID == 0); - EXPECT_TRUE(shardID == ""); - }); + return futures::makeFuture(std::move(response)); + }; { // It should use the leader VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getDistributionForDatabase(dbname, resultBuilder); VerifyAttributes(resultBuilder.slice(), colName, s1); } - - { // It should not call drop - fakeit::Verify(Method(commMock, drop)).Exactly(0); - } } TEST_F(ShardDistributionReporterTest, @@ -1485,68 +1032,34 @@ TEST_F(ShardDistributionReporterTest, return currentShards[sid]; }); - ClusterCommResult leaderRes; + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - CoordTransactionID coordId = 0; - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - OperationID opId = TRI_NewTickServer(); - coordId = coordTransactionID; + if (destination == "server:" + dbserver1) { + response.error = fuerte::Error::Timeout; + } else if (destination == "server:" + dbserver2) { + } else if (destination == "server:" + dbserver3) { + } else { + EXPECT_TRUE(false); + } - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.status = CL_COMM_TIMEOUT; - } else if (destination == "server:" + dbserver2) { - } else if (destination == "server:" + dbserver3) { - } else { - EXPECT_TRUE(false); - } - return opId; - }); - - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - // If we get here we tried to wait for followers and we cannot use the answer... - EXPECT_TRUE(false); - return leaderRes; - }); - - fakeit::When(Method(commMock, drop)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - OperationID const operationID, ShardID const& shardID) { - // We need to abort this trx - EXPECT_TRUE(coordTransactionID == coordId); - // For all operations and shards - EXPECT_TRUE(operationID == 0); - EXPECT_TRUE(shardID == ""); - }); + return futures::makeFuture(std::move(response)); + }; { // It should use the defaults total: 1 current: 0 VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VerifyNumbers(resultBuilder.slice(), colName, s1, 1, 0); } - - { // It needs to call drop - fakeit::Verify(Method(commMock, drop)).Exactly(1); - } } TEST_F(ShardDistributionReporterTest, @@ -1566,112 +1079,38 @@ TEST_F(ShardDistributionReporterTest, }); uint64_t leaderCount = 1337; - uint64_t smallerFollowerCount = 456; uint64_t largerFollowerCount = 1111; - // Moking HttpResults - fakeit::Mock leaderCountMock; - SimpleHttpResult& lCount = leaderCountMock.get(); + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - fakeit::Mock followerOneCountMock; - SimpleHttpResult& f1Count = followerOneCountMock.get(); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - fakeit::Mock followerTwoCountMock; - SimpleHttpResult& f2Count = followerTwoCountMock.get(); + if (destination == "server:" + dbserver1) { + response.response = generateCountResponse(leaderCount); + } else if (destination == "server:" + dbserver2) { + response.response = generateCountResponse(largerFollowerCount); + } else if (destination == "server:" + dbserver3) { + response.error = fuerte::Error::Timeout; + } else { + EXPECT_TRUE(false); + } - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(leaderCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(leaderCount); }); - - fakeit::When(ConstOverloadedMethod(followerOneCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(largerFollowerCount); }); - - fakeit::When(ConstOverloadedMethod(followerTwoCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { - EXPECT_TRUE(false); - return buildCountBody(smallerFollowerCount); - }); - - ClusterCommResult leaderRes; - ClusterCommResult follower1Res; - ClusterCommResult follower2Res; - bool returnedFirstFollower = false; - - CoordTransactionID coordId = 0; - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - - OperationID opId = TRI_NewTickServer(); - coordId = coordTransactionID; - - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.result = - std::shared_ptr(&lCount, [](SimpleHttpResult*) {}); - leaderRes = response; - } else if (destination == "server:" + dbserver2) { - response.result = - std::shared_ptr(&f1Count, [](SimpleHttpResult*) {}); - follower1Res = response; - } else if (destination == "server:" + dbserver3) { - response.status = CL_COMM_TIMEOUT; - response.result = - std::shared_ptr(&f2Count, [](SimpleHttpResult*) {}); - follower2Res = response; - } else { - EXPECT_TRUE(false); - } - - return opId; - }); - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - if (returnedFirstFollower) { - return follower2Res; - } else { - returnedFirstFollower = true; - return follower1Res; - } - }); - - fakeit::When(Method(commMock, drop)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - OperationID const operationID, ShardID const& shardID) { - // We need to abort this trx - EXPECT_TRUE(coordTransactionID == coordId); - // For all operations and shards - EXPECT_TRUE(operationID == 0); - EXPECT_TRUE(shardID == ""); - }); + return futures::makeFuture(std::move(response)); + }; { // It should use the leader and the other one VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VerifyNumbers(resultBuilder.slice(), colName, s1, leaderCount, largerFollowerCount); } - - { // It should not call drop - fakeit::Verify(Method(commMock, drop)).Exactly(0); - } } TEST_F(ShardDistributionReporterTest, @@ -1691,114 +1130,35 @@ TEST_F(ShardDistributionReporterTest, }); uint64_t leaderCount = 1337; - uint64_t smallerFollowerCount = 456; - uint64_t largerFollowerCount = 1111; - // Moking HttpResults - fakeit::Mock leaderCountMock; - SimpleHttpResult& lCount = leaderCountMock.get(); + // Mocking sendRequest for count calls + auto sender = [&, this](network::DestinationId const& destination, + arangodb::fuerte::RestVerb reqType, std::string const& path, + velocypack::Buffer body, network::Timeout timeout, + network::Headers headerFields) -> network::FutureRes { + EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - fakeit::Mock followerOneCountMock; - SimpleHttpResult& f1Count = followerOneCountMock.get(); + network::Response response; + response.destination = destination; + response.error = fuerte::Error::NoError; - fakeit::Mock followerTwoCountMock; - SimpleHttpResult& f2Count = followerTwoCountMock.get(); + if (destination == "server:" + dbserver1) { + response.response = generateCountResponse(leaderCount); + } else if (destination == "server:" + dbserver2) { + response.error = fuerte::Error::Timeout; + } else if (destination == "server:" + dbserver3) { + response.error = fuerte::Error::Timeout; + } else { + EXPECT_TRUE(false); + } - // Mocking HTTP Response - fakeit::When(ConstOverloadedMethod(leaderCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { return buildCountBody(leaderCount); }); - - fakeit::When(ConstOverloadedMethod(followerOneCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { - EXPECT_TRUE(false); - return buildCountBody(largerFollowerCount); - }); - - fakeit::When(ConstOverloadedMethod(followerTwoCountMock, getBodyVelocyPack, - std::shared_ptr())) - .AlwaysDo([&]() { - EXPECT_TRUE(false); - return buildCountBody(smallerFollowerCount); - }); - - ClusterCommResult leaderRes; - ClusterCommResult follower1Res; - ClusterCommResult follower2Res; - bool returnedFirstFollower = false; - - CoordTransactionID coordId = 0; - // Mocking the ClusterComm for count calls - fakeit::When(Method(commMock, asyncRequest)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - std::string const& destination, rest::RequestType reqtype, - std::string const& path, std::shared_ptr body, - std::unordered_map const& headerFields, - std::shared_ptr callback, ClusterCommTimeout timeout, - bool singleRequest, ClusterCommTimeout initTimeout) -> OperationID { - EXPECT_TRUE(path == "/_db/UnitTestDB/_api/collection/" + s1 + "/count"); - - OperationID opId = TRI_NewTickServer(); - coordId = coordTransactionID; - - ClusterCommResult response; - response.coordTransactionID = coordTransactionID; - response.operationID = opId; - response.answer_code = rest::ResponseCode::OK; - response.status = CL_COMM_RECEIVED; - - if (destination == "server:" + dbserver1) { - response.result = - std::shared_ptr(&lCount, [](SimpleHttpResult*) {}); - leaderRes = response; - } else if (destination == "server:" + dbserver2) { - response.status = CL_COMM_TIMEOUT; - response.result = - std::shared_ptr(&f1Count, [](SimpleHttpResult*) {}); - follower1Res = response; - } else if (destination == "server:" + dbserver3) { - response.status = CL_COMM_TIMEOUT; - response.result = - std::shared_ptr(&f2Count, [](SimpleHttpResult*) {}); - follower2Res = response; - } else { - EXPECT_TRUE(false); - } - - return opId; - }); - fakeit::When(Method(commMock, wait)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, OperationID const operationID, - ShardID const& shardID, ClusterCommTimeout timeout) { - if (operationID != 0) { - return leaderRes; - } - if (returnedFirstFollower) { - return follower2Res; - } else { - returnedFirstFollower = true; - return follower1Res; - } - }); - - fakeit::When(Method(commMock, drop)) - .AlwaysDo([&](CoordTransactionID const coordTransactionID, - OperationID const operationID, ShardID const& shardID) { - // We need to abort this trx - EXPECT_TRUE(coordTransactionID == coordId); - // For all operations and shards - EXPECT_TRUE(operationID == 0); - EXPECT_TRUE(shardID == ""); - }); + return futures::makeFuture(std::move(response)); + }; { // It should use the leader and the default current of 0 VPackBuilder resultBuilder; + ShardDistributionReporter testee(&ci, sender); testee.getCollectionDistributionForDatabase(dbname, colName, resultBuilder); VerifyNumbers(resultBuilder.slice(), colName, s1, leaderCount, 0); } - - { // It should not call drop - fakeit::Verify(Method(commMock, drop)).Exactly(0); - } }