Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Actually use the given format to execute a query (#10484) 

* Actually use the given format to execute a query

* Send the serialization format of AQL from Coordinators.
This commit is contained in:
Michael Hackstein 2019-11-21 09:10:32 +01:00 committed by GitHub
parent 688a680023
commit 3090e49258
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 68 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
arangod/Aql/EngineInfoContainerDBServerServerBased.cpp
View File

@ -23,6 +23,7 @@
#include "EngineInfoContainerDBServerServerBased.h" #include "EngineInfoContainerDBServerServerBased.h"
#include "Aql/AqlItemBlockSerializationFormat.h"
#include "Aql/Ast.h" #include "Aql/Ast.h"
#include "Aql/Collection.h" #include "Aql/Collection.h"
#include "Aql/ExecutionNode.h" #include "Aql/ExecutionNode.h"
@ -78,8 +79,7 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi
: _node(node), _hasShard(false) { : _node(node), _hasShard(false) {
auto const& edges = _node->edgeColls(); auto const& edges = _node->edgeColls();
TRI_ASSERT(!edges.empty()); TRI_ASSERT(!edges.empty());
std::unordered_set<std::string> const& restrictToShards = std::unordered_set<std::string> const& restrictToShards = query.queryOptions().shardIds;
query.queryOptions().shardIds;
// Extract the local shards for edge collections. // Extract the local shards for edge collections.
for (auto const& col : edges) { for (auto const& col : edges) {
_edgeCollections.emplace_back( _edgeCollections.emplace_back(
@ -197,7 +197,7 @@ void EngineInfoContainerDBServerServerBased::injectVertexColletions(GraphNode* g
auto const& vCols = graphNode->vertexColls(); auto const& vCols = graphNode->vertexColls();
if (vCols.empty()) { if (vCols.empty()) {
std::map<std::string, Collection*> const* allCollections = std::map<std::string, Collection*> const* allCollections =
_query.collections()->collections(); _query.collections()->collections();
auto& resolver = _query.resolver(); auto& resolver = _query.resolver();
for (auto const& it : *allCollections) { for (auto const& it : *allCollections) {
// If resolver cannot resolve this collection // If resolver cannot resolve this collection
@ -296,7 +296,7 @@ Result EngineInfoContainerDBServerServerBased::buildEngines(
network::RequestOptions options; network::RequestOptions options;
options.database = _query.vocbase().name(); options.database = _query.vocbase().name();
options.timeout = network::Timeout(SETUP_TIMEOUT); options.timeout = network::Timeout(SETUP_TIMEOUT);
options.skipScheduler = true; // hack to speed up future.get() options.skipScheduler = true; // hack to speed up future.get()
options.param("ttl", std::to_string(_query.queryOptions().ttl)); options.param("ttl", std::to_string(_query.queryOptions().ttl));
for (auto const& server : dbServers) { for (auto const& server : dbServers) {
@ -323,6 +323,8 @@ Result EngineInfoContainerDBServerServerBased::buildEngines(
TRI_ASSERT(didCreateEngine.size() == _graphNodes.size()); TRI_ASSERT(didCreateEngine.size() == _graphNodes.size());
TRI_ASSERT(infoBuilder.isOpenObject()); TRI_ASSERT(infoBuilder.isOpenObject());
infoBuilder.add(StaticStrings::SerializationFormat,
VPackValue(static_cast<int>(aql::SerializationFormat::SHADOWROWS)));
infoBuilder.close(); // Base object infoBuilder.close(); // Base object
TRI_ASSERT(infoBuilder.isClosed()); TRI_ASSERT(infoBuilder.isClosed());
@ -468,7 +470,7 @@ void EngineInfoContainerDBServerServerBased::cleanupEngines(
network::RequestOptions options; network::RequestOptions options;
options.database = dbname; options.database = dbname;
options.timeout = network::Timeout(10.0); // Picked arbitrarily options.timeout = network::Timeout(10.0); // Picked arbitrarily
options.skipScheduler = true; // hack to speed up future.get() options.skipScheduler = true; // hack to speed up future.get()
// Shutdown query snippets // Shutdown query snippets
std::string url("/_api/aql/shutdown/"); std::string url("/_api/aql/shutdown/");
@ -485,7 +487,7 @@ void EngineInfoContainerDBServerServerBased::cleanupEngines(
for (auto const& shardId : serToSnippets.second) { for (auto const& shardId : serToSnippets.second) {
// fire and forget // fire and forget
network::sendRequest(pool, server, fuerte::RestVerb::Put, url + shardId, network::sendRequest(pool, server, fuerte::RestVerb::Put, url + shardId,
/*copy*/body, options); /*copy*/ body, options);
} }
_query.incHttpRequests(serToSnippets.second.size()); _query.incHttpRequests(serToSnippets.second.size());
} }
@ -500,8 +502,7 @@ void EngineInfoContainerDBServerServerBased::cleanupEngines(
for (auto const& engine : *allEngines) { for (auto const& engine : *allEngines) {
// fire and forget // fire and forget
network::sendRequest(pool, engine.first, fuerte::RestVerb::Delete, network::sendRequest(pool, engine.first, fuerte::RestVerb::Delete,
url + basics::StringUtils::itoa(engine.second), url + basics::StringUtils::itoa(engine.second), noBody, options);
noBody, options);
} }
_query.incHttpRequests(allEngines->size()); _query.incHttpRequests(allEngines->size());
} }

32
arangod/Aql/ExecutionEngine.cpp
View File

@ -41,8 +41,8 @@
#include "Basics/ScopeGuard.h" #include "Basics/ScopeGuard.h"
#include "Cluster/ServerState.h" #include "Cluster/ServerState.h"
#include "Futures/Utilities.h" #include "Futures/Utilities.h"
#include "Logger/Logger.h"
#include "Logger/LogMacros.h" #include "Logger/LogMacros.h"
#include "Logger/Logger.h"
#include "Network/Methods.h" #include "Network/Methods.h"
#include "Network/NetworkFeature.h" #include "Network/NetworkFeature.h"
#include "Network/Utils.h" #include "Network/Utils.h"
@ -489,8 +489,8 @@ struct DistributedQueryInstanciator final : public WalkerWorker<ExecutionNode> {
// however every engine gets injected the list of locked shards. // however every engine gets injected the list of locked shards.
std::vector<uint64_t> coordinatorQueryIds{}; std::vector<uint64_t> coordinatorQueryIds{};
res = _coordinatorParts.buildEngines(_query, registry, _query.vocbase().name(), res = _coordinatorParts.buildEngines(_query, registry, _query.vocbase().name(),
_query.queryOptions().shardIds, queryIds, _query.queryOptions().shardIds,
coordinatorQueryIds); queryIds, coordinatorQueryIds);
if (res.ok()) { if (res.ok()) {
TRI_ASSERT(_query.engine() != nullptr); TRI_ASSERT(_query.engine() != nullptr);
@ -506,14 +506,16 @@ struct DistributedQueryInstanciator final : public WalkerWorker<ExecutionNode> {
void ExecutionEngine::kill() { void ExecutionEngine::kill() {
// kill coordinator parts // kill coordinator parts
// TODO: this doesn't seem to be necessary and sometimes even show adverse effects // TODO: this doesn't seem to be necessary and sometimes even show adverse
// so leaving this deactivated for now // effects so leaving this deactivated for now
// auto queryRegistry = QueryRegistryFeature::registry(); /*
// if (queryRegistry != nullptr) { auto queryRegistry = QueryRegistryFeature::registry();
// for (auto const& id : _coordinatorQueryIds) { if (queryRegistry != nullptr) {
// queryRegistry->kill(&(_query.vocbase()), id); for (auto const& id : _coordinatorQueryIds) {
// } queryRegistry->kill(&(_query.vocbase()), id);
// } }
}
*/
// kill DB server parts // kill DB server parts
// RemoteNodeId -> DBServerId -> [snippetId] // RemoteNodeId -> DBServerId -> [snippetId]
@ -654,7 +656,8 @@ std::pair<ExecutionState, Result> ExecutionEngine::shutdown(int errorCode) {
/// @brief create an execution engine from a plan /// @brief create an execution engine from a plan
ExecutionEngine* ExecutionEngine::instantiateFromPlan(QueryRegistry& queryRegistry, ExecutionEngine* ExecutionEngine::instantiateFromPlan(QueryRegistry& queryRegistry,
Query& query, ExecutionPlan& plan, Query& query, ExecutionPlan& plan,
bool planRegisters) { bool planRegisters,
SerializationFormat format) {
auto role = arangodb::ServerState::instance()->getRole(); auto role = arangodb::ServerState::instance()->getRole();
plan.findVarUsage(); plan.findVarUsage();
@ -688,7 +691,7 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan(QueryRegistry& queryRegist
TRI_ASSERT(root != nullptr); TRI_ASSERT(root != nullptr);
} else { } else {
// instantiate the engine on a local server // instantiate the engine on a local server
engine.reset(new ExecutionEngine(query, SerializationFormat::SHADOWROWS)); engine.reset(new ExecutionEngine(query, format));
SingleServerQueryInstanciator inst(*engine); SingleServerQueryInstanciator inst(*engine);
plan.root()->walk(inst); plan.root()->walk(inst);
@ -709,7 +712,8 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan(QueryRegistry& queryRegist
bool const returnInheritedResults = !arangodb::ServerState::isDBServer(role); bool const returnInheritedResults = !arangodb::ServerState::isDBServer(role);
if (returnInheritedResults) { if (returnInheritedResults) {
auto returnNode = dynamic_cast<ExecutionBlockImpl<IdExecutor<BlockPassthrough::Enable, void>>*>(root); auto returnNode =
dynamic_cast<ExecutionBlockImpl<IdExecutor<BlockPassthrough::Enable, void>>*>(root);
TRI_ASSERT(returnNode != nullptr); TRI_ASSERT(returnNode != nullptr);
engine->resultRegister(returnNode->getOutputRegisterId()); engine->resultRegister(returnNode->getOutputRegisterId());
} else { } else {

4
arangod/Aql/ExecutionEngine.h
View File

@ -58,7 +58,9 @@ class ExecutionEngine {
public: public:
// @brief create an execution engine from a plan // @brief create an execution engine from a plan
static ExecutionEngine* instantiateFromPlan(QueryRegistry&, Query&, ExecutionPlan&, bool); static ExecutionEngine* instantiateFromPlan(QueryRegistry& queryRegistry, Query& query,
ExecutionPlan& plan, bool planRegisters,
SerializationFormat format);
TEST_VIRTUAL Result createBlocks(std::vector<ExecutionNode*> const& nodes, TEST_VIRTUAL Result createBlocks(std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const& restrictToShards, std::unordered_set<std::string> const& restrictToShards,

10
arangod/Aql/Query.cpp
View File

@ -276,8 +276,8 @@ Query* Query::clone(QueryPart part, bool withPlan) {
} }
bool Query::killed() const { bool Query::killed() const {
if(_queryOptions.timeout > std::numeric_limits<double>::epsilon()) { if (_queryOptions.timeout > std::numeric_limits<double>::epsilon()) {
if(TRI_microtime() > (_startTime + _queryOptions.timeout)) { if (TRI_microtime() > (_startTime + _queryOptions.timeout)) {
return true; return true;
} }
} }
@ -451,7 +451,7 @@ void Query::prepare(QueryRegistry* registry, SerializationFormat format) {
// this is confusing and should be fixed! // this is confusing and should be fixed!
std::unique_ptr<ExecutionEngine> engine( std::unique_ptr<ExecutionEngine> engine(
ExecutionEngine::instantiateFromPlan(*registry, *this, *plan, ExecutionEngine::instantiateFromPlan(*registry, *this, *plan,
!_queryString.empty())); !_queryString.empty(), format));
if (_engine == nullptr) { if (_engine == nullptr) {
_engine = std::move(engine); _engine = std::move(engine);
@ -655,7 +655,7 @@ ExecutionState Query::execute(QueryRegistry* registry, QueryResult& queryResult)
_resultBuilder->openArray(); _resultBuilder->openArray();
_executionPhase = ExecutionPhase::EXECUTE; _executionPhase = ExecutionPhase::EXECUTE;
} }
[[fallthrough]]; [[fallthrough]];
case ExecutionPhase::EXECUTE: { case ExecutionPhase::EXECUTE: {
TRI_ASSERT(_resultBuilder != nullptr); TRI_ASSERT(_resultBuilder != nullptr);
TRI_ASSERT(_resultBuilder->isOpenArray()); TRI_ASSERT(_resultBuilder->isOpenArray());
@ -734,7 +734,7 @@ ExecutionState Query::execute(QueryRegistry* registry, QueryResult& queryResult)
_executionPhase = ExecutionPhase::FINALIZE; _executionPhase = ExecutionPhase::FINALIZE;
} }
[[fallthrough]]; [[fallthrough]];
case ExecutionPhase::FINALIZE: { case ExecutionPhase::FINALIZE: {
// will set warnings, stats, profile and cleanup plan and engine // will set warnings, stats, profile and cleanup plan and engine
return finalize(queryResult); return finalize(queryResult);

23
arangod/Aql/RestAqlHandler.cpp
View File

@ -189,9 +189,8 @@ void RestAqlHandler::setupClusterQuery() {
// If we have a new format then it has to be included here. // If we have a new format then it has to be included here.
// If not default to classic (old coordinator will not send it) // If not default to classic (old coordinator will not send it)
SerializationFormat format = static_cast<SerializationFormat>( SerializationFormat format = static_cast<SerializationFormat>(
VelocyPackHelper::getNumericValue<int>(querySlice, "serializationFormat", VelocyPackHelper::getNumericValue<int>(querySlice, StaticStrings::SerializationFormat,
static_cast<int>(SerializationFormat::CLASSIC))); static_cast<int>(SerializationFormat::CLASSIC)));
// Now we need to create shared_ptr<VPackBuilder> // Now we need to create shared_ptr<VPackBuilder>
// That contains the old-style cluster snippet in order // That contains the old-style cluster snippet in order
// to prepare create a Query object. // to prepare create a Query object.
@ -315,7 +314,6 @@ bool RestAqlHandler::registerSnippets(
} }
try { try {
if (needToLock) { if (needToLock) {
// Directly try to lock only the first snippet is required to be locked. // Directly try to lock only the first snippet is required to be locked.
// For all others locking is pointless // For all others locking is pointless
@ -340,7 +338,7 @@ bool RestAqlHandler::registerSnippets(
// No need to cleanup... // No need to cleanup...
} }
QueryId qId = query->id(); // not true in general QueryId qId = query->id(); // not true in general
TRI_ASSERT(qId > 0); TRI_ASSERT(qId > 0);
_queryRegistry->insert(qId, query.get(), ttl, true, false); _queryRegistry->insert(qId, query.get(), ttl, true, false);
query.release(); query.release();
@ -443,7 +441,7 @@ RestStatus RestAqlHandler::useQuery(std::string const& operation, std::string co
return RestStatus::DONE; return RestStatus::DONE;
} }
if (!_query) { // the PUT verb if (!_query) { // the PUT verb
TRI_ASSERT(this->state() == RestHandler::HandlerState::EXECUTE); TRI_ASSERT(this->state() == RestHandler::HandlerState::EXECUTE);
_query = findQuery(idString); _query = findQuery(idString);
@ -451,9 +449,8 @@ RestStatus RestAqlHandler::useQuery(std::string const& operation, std::string co
return RestStatus::DONE; return RestStatus::DONE;
} }
std::shared_ptr<SharedQueryState> ss = _query->sharedState(); std::shared_ptr<SharedQueryState> ss = _query->sharedState();
ss->setWakeupHandler([self = shared_from_this()] { ss->setWakeupHandler(
return self->wakeupHandler(); [self = shared_from_this()] { return self->wakeupHandler(); });
});
} }
TRI_ASSERT(_qId > 0); TRI_ASSERT(_qId > 0);
@ -640,7 +637,6 @@ Query* RestAqlHandler::findQuery(std::string const& idString) {
// handle for useQuery // handle for useQuery
RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, RestStatus RestAqlHandler::handleUseQuery(std::string const& operation,
VPackSlice const querySlice) { VPackSlice const querySlice) {
std::string const& shardId = _request->header("shard-id"); std::string const& shardId = _request->header("shard-id");
// upon first usage, the "initializeCursor" method must be called // upon first usage, the "initializeCursor" method must be called
@ -668,7 +664,7 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation,
VPackBuffer<uint8_t> answerBuffer; VPackBuffer<uint8_t> answerBuffer;
VPackBuilder answerBuilder(answerBuffer); VPackBuilder answerBuilder(answerBuffer);
answerBuilder.openObject(/*unindexed*/true); answerBuilder.openObject(/*unindexed*/ true);
if (operation == "getSome") { if (operation == "getSome") {
TRI_IF_FAILURE("RestAqlHandler::getSome") { TRI_IF_FAILURE("RestAqlHandler::getSome") {
@ -756,8 +752,8 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation,
answerBuilder.add(StaticStrings::Error, VPackValue(res.fail())); answerBuilder.add(StaticStrings::Error, VPackValue(res.fail()));
answerBuilder.add(StaticStrings::Code, VPackValue(res.errorNumber())); answerBuilder.add(StaticStrings::Code, VPackValue(res.errorNumber()));
} else if (operation == "shutdown") { } else if (operation == "shutdown") {
int errorCode = int errorCode = VelocyPackHelper::getNumericValue<int>(querySlice, StaticStrings::Code,
VelocyPackHelper::getNumericValue<int>(querySlice, StaticStrings::Code, TRI_ERROR_INTERNAL); TRI_ERROR_INTERNAL);
ExecutionState state; ExecutionState state;
Result res; Result res;
@ -789,8 +785,7 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation,
} }
answerBuilder.close(); answerBuilder.close();
generateResult(rest::ResponseCode::OK, std::move(answerBuffer), generateResult(rest::ResponseCode::OK, std::move(answerBuffer), transactionContext);
transactionContext);
return RestStatus::DONE; return RestStatus::DONE;
} }

7
lib/Basics/StaticStrings.cpp
View File

@ -217,8 +217,7 @@ std::string const StaticStrings::MimeTypeDump(
std::string const StaticStrings::MimeTypeHtml("text/html; charset=utf-8"); std::string const StaticStrings::MimeTypeHtml("text/html; charset=utf-8");
std::string const StaticStrings::MimeTypeJson( std::string const StaticStrings::MimeTypeJson(
"application/json; charset=utf-8"); "application/json; charset=utf-8");
std::string const StaticStrings::MimeTypeJsonNoEncoding( std::string const StaticStrings::MimeTypeJsonNoEncoding("application/json");
"application/json");
std::string const StaticStrings::MimeTypeText("text/plain; charset=utf-8"); std::string const StaticStrings::MimeTypeText("text/plain; charset=utf-8");
std::string const StaticStrings::MimeTypeVPack("application/x-velocypack"); std::string const StaticStrings::MimeTypeVPack("application/x-velocypack");
std::string const StaticStrings::MultiPartContentType("multipart/form-data"); std::string const StaticStrings::MultiPartContentType("multipart/form-data");
@ -278,4 +277,6 @@ std::string const StaticStrings::Old("old");
std::string const StaticStrings::UpgradeEnvName( std::string const StaticStrings::UpgradeEnvName(
"ARANGODB_UPGRADE_DURING_RESTORE"); "ARANGODB_UPGRADE_DURING_RESTORE");
std::string const StaticStrings::BackupToDeleteName("DIRECTORY_TO_DELETE"); std::string const StaticStrings::BackupToDeleteName("DIRECTORY_TO_DELETE");
std::string const StaticStrings::BackupSearchToDeleteName("DIRECTORY_TO_DELETE_SEARCH"); std::string const StaticStrings::BackupSearchToDeleteName(
"DIRECTORY_TO_DELETE_SEARCH");
std::string const StaticStrings::SerializationFormat("serializationFormat");

1
lib/Basics/StaticStrings.h
View File

@ -254,6 +254,7 @@ class StaticStrings {
static std::string const UpgradeEnvName; static std::string const UpgradeEnvName;
static std::string const BackupToDeleteName; static std::string const BackupToDeleteName;
static std::string const BackupSearchToDeleteName; static std::string const BackupSearchToDeleteName;
static std::string const SerializationFormat;
}; };
} // namespace arangodb } // namespace arangodb