Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
arangodb/arangod/Aql/ExecutionEngine.cpp

1417 lines
53 KiB
C++
Raw Blame History

////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "ExecutionEngine.h"
#include "Aql/BasicBlocks.h"
#include "Aql/CalculationBlock.h"
#include "Aql/ClusterBlocks.h"
#include "Aql/CollectBlock.h"
#include "Aql/CollectNode.h"
#include "Aql/CollectOptions.h"
#include "Aql/EnumerateCollectionBlock.h"
#include "Aql/EnumerateListBlock.h"
#include "Aql/ExecutionNode.h"
#include "Aql/IndexBlock.h"
#include "Aql/ModificationBlocks.h"
#include "Aql/SortBlock.h"
#include "Aql/SubqueryBlock.h"
#include "Aql/TraversalBlock.h"
#include "Aql/ShortestPathBlock.h"
#include "Aql/ShortestPathNode.h"
#include "Aql/WalkerWorker.h"
#include "Basics/Exceptions.h"
#include "Basics/VelocyPackHelper.h"
#include "Cluster/ClusterComm.h"
#include "Cluster/ClusterInfo.h"
#include "Cluster/TraverserEngineRegistry.h"
#include "Logger/Logger.h"
#include "Transaction/Methods.h"
#include "VocBase/ticks.h"
using namespace arangodb;
using namespace arangodb::aql;
/// @brief helper function to create a block
static ExecutionBlock* CreateBlock(
ExecutionEngine* engine, ExecutionNode const* en,
std::unordered_map<ExecutionNode*, ExecutionBlock*> const& cache) {
switch (en->getType()) {
case ExecutionNode::SINGLETON: {
return new SingletonBlock(engine, static_cast<SingletonNode const*>(en));
}
case ExecutionNode::INDEX: {
return new IndexBlock(engine, static_cast<IndexNode const*>(en));
}
case ExecutionNode::ENUMERATE_COLLECTION: {
return new EnumerateCollectionBlock(
engine, static_cast<EnumerateCollectionNode const*>(en));
}
case ExecutionNode::ENUMERATE_LIST: {
return new EnumerateListBlock(engine,
static_cast<EnumerateListNode const*>(en));
}
case ExecutionNode::TRAVERSAL: {
return new TraversalBlock(engine, static_cast<TraversalNode const*>(en));
}
case ExecutionNode::SHORTEST_PATH: {
return new ShortestPathBlock(engine, static_cast<ShortestPathNode const*>(en));
}
case ExecutionNode::CALCULATION: {
return new CalculationBlock(engine,
static_cast<CalculationNode const*>(en));
}
case ExecutionNode::FILTER: {
return new FilterBlock(engine, static_cast<FilterNode const*>(en));
}
case ExecutionNode::LIMIT: {
return new LimitBlock(engine, static_cast<LimitNode const*>(en));
}
case ExecutionNode::SORT: {
return new SortBlock(engine, static_cast<SortNode const*>(en));
}
case ExecutionNode::COLLECT: {
auto aggregationMethod =
static_cast<CollectNode const*>(en)->aggregationMethod();
if (aggregationMethod ==
CollectOptions::CollectMethod::COLLECT_METHOD_HASH) {
return new HashedCollectBlock(engine,
static_cast<CollectNode const*>(en));
} else if (aggregationMethod ==
CollectOptions::CollectMethod::COLLECT_METHOD_SORTED) {
return new SortedCollectBlock(engine,
static_cast<CollectNode const*>(en));
}
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"cannot instantiate CollectBlock with "
"undetermined aggregation method");
}
case ExecutionNode::SUBQUERY: {
auto es = static_cast<SubqueryNode const*>(en);
auto it = cache.find(es->getSubquery());
TRI_ASSERT(it != cache.end());
return new SubqueryBlock(engine, static_cast<SubqueryNode const*>(en),
it->second);
}
case ExecutionNode::RETURN: {
return new ReturnBlock(engine, static_cast<ReturnNode const*>(en));
}
case ExecutionNode::REMOVE: {
return new RemoveBlock(engine, static_cast<RemoveNode const*>(en));
}
case ExecutionNode::INSERT: {
return new InsertBlock(engine, static_cast<InsertNode const*>(en));
}
case ExecutionNode::UPDATE: {
return new UpdateBlock(engine, static_cast<UpdateNode const*>(en));
}
case ExecutionNode::REPLACE: {
return new ReplaceBlock(engine, static_cast<ReplaceNode const*>(en));
}
case ExecutionNode::UPSERT: {
return new UpsertBlock(engine, static_cast<UpsertNode const*>(en));
}
case ExecutionNode::NORESULTS: {
return new NoResultsBlock(engine, static_cast<NoResultsNode const*>(en));
}
case ExecutionNode::SCATTER: {
auto shardIds =
static_cast<ScatterNode const*>(en)->collection()->shardIds();
return new ScatterBlock(engine, static_cast<ScatterNode const*>(en),
*shardIds);
}
case ExecutionNode::DISTRIBUTE: {
auto shardIds =
static_cast<DistributeNode const*>(en)->collection()->shardIds();
return new DistributeBlock(
engine, static_cast<DistributeNode const*>(en), *shardIds,
static_cast<DistributeNode const*>(en)->collection());
}
case ExecutionNode::GATHER: {
return new GatherBlock(engine, static_cast<GatherNode const*>(en));
}
case ExecutionNode::REMOTE: {
auto remote = static_cast<RemoteNode const*>(en);
return new RemoteBlock(engine, remote, remote->server(),
remote->ownName(), remote->queryId());
}
}
return nullptr;
}
/// @brief create the engine
ExecutionEngine::ExecutionEngine(Query* query)
: _stats(),
_itemBlockManager(query->resourceMonitor()),
_blocks(),
_root(nullptr),
_query(query),
_resultRegister(0),
_wasShutdown(false),
_previouslyLockedShards(nullptr),
_lockedShards(nullptr) {
_blocks.reserve(8);
}
/// @brief destroy the engine, frees all assigned blocks
ExecutionEngine::~ExecutionEngine() {
try {
shutdown(TRI_ERROR_INTERNAL);
} catch (...) {
// shutdown can throw - ignore it in the destructor
}
for (auto& it : _blocks) {
delete it;
}
}
struct Instanciator final : public WalkerWorker<ExecutionNode> {
ExecutionEngine* engine;
ExecutionBlock* root;
std::unordered_map<ExecutionNode*, ExecutionBlock*> cache;
explicit Instanciator(ExecutionEngine* engine)
: engine(engine), root(nullptr) {}
~Instanciator() {}
virtual void after(ExecutionNode* en) override final {
ExecutionBlock* block = nullptr;
{
if (en->getType() == ExecutionNode::TRAVERSAL) {
// We have to prepare the options before we build the block
static_cast<TraversalNode*>(en)->prepareOptions();
}
std::unique_ptr<ExecutionBlock> eb(CreateBlock(engine, en, cache));
if (eb == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "illegal node type");
}
// do we need to adjust the root node?
auto const nodeType = en->getType();
if (nodeType == ExecutionNode::DISTRIBUTE ||
nodeType == ExecutionNode::SCATTER ||
nodeType == ExecutionNode::GATHER) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_INTERNAL, "logic error, got cluster node in local query");
}
engine->addBlock(eb.get());
if (!en->hasParent()) {
// yes. found a new root!
root = eb.get();
}
block = eb.release();
}
TRI_ASSERT(block != nullptr);
// Now add dependencies:
for (auto const& it : en->getDependencies()) {
auto it2 = cache.find(it);
TRI_ASSERT(it2 != cache.end());
TRI_ASSERT(it2->second != nullptr);
block->addDependency(it2->second);
}
cache.emplace(en, block);
}
};
// Here is a description of how the instantiation of an execution plan
// works in the cluster. See below for a complete example
//
// The instantiation of this works as follows:
// (0) Variable usage and register planning is done in the global plan
// (1) A walk with subqueries is done on the whole plan
// The purpose is to plan how many ExecutionEngines we need, where they
// have to be instantiated and which plan nodes belong to each of them.
// Such a walk is depth first and visits subqueries after it has visited
// the dependencies of the subquery node recursively. Whenever the
// walk passes by a RemoteNode it switches location between coordinator
// and DBserver and starts a new engine. The nodes of an engine are
// collected in the after method.
// This walk results in a list of engines and a list of nodes for
// each engine. It follows that the order in these lists is as follows:
// The first engine is the main one on the coordinator, it has id 0.
// The order of the engines is exactly as they are discovered in the
// walk. That is, engines closer to the root are earlier and engines
// in subqueries are later. The nodes in each engine are always
// done in a way such that a dependency D of a node N is earlier in the
// list as N, and a subquery node is later in the list than the nodes
// of the subquery.
// (2) buildEngines is called with that data. It proceeds engine by engine,
// starting from the back of the list. This means that an engine that
// is referred to in a RemoteNode (because its nodes are dependencies
// of that node) are always already instantiated before the RemoteNode
// is instantiated. The corresponding query ids are collected in a
// global hash table, for which the key consists of the id of the
// RemoteNode using the query and the actual query id. For each engine,
// the nodes are instantiated along the list of nodes for that engine.
// This means that all dependencies of a node N are already instantiated
// when N is instantiated. We distinguish the coordinator and the
// DBserver case. In the former one we have to clone a part of the
// plan and in the latter we have to send a part to a DBserver via HTTP.
//
// Here is a fully worked out example:
//
// FOR i IN [1,2]
// FOR d IN coll
// FILTER d.pass == i
// LET s = (FOR e IN coll2 FILTER e.name == d.name RETURN e)
// RETURN {d:d, s:s}
//
// this is optimized to, variable and register planning is done in this plan:
//
// Singleton
// ^
// EnumList [1,2] Singleton
// ^ ^
// Scatter (2) Enum coll2
// ^ ^
// Remote Calc e.name==d.name
// ^ ^
// Enum coll Filter (3)
// ^ ^
// Calc d.pass==i Remote
// ^ ^
// Filter (1) Gather
// ^ ^
// Remote Return
// ^ ^
// Gather |
// ^ |
// Subquery -------------------/
// ^
// Calc {d:d, s:s}
// ^
// Return (0)
//
// There are 4 engines here, their corresponding root nodes are labelled
// in the above picture in round brackets with the ids of the engine.
// Engines 1 and 3 have to be replicated for each shard of coll or coll2
// respectively, and sent to the right DBserver via HTTP. Engine 0 is the
// main one on the coordinator and engine 2 is a non-main part on the
// coordinator. Recall that the walk goes first to the dependencies before
// it visits the nodes of the subquery. Thus, the walk builds up the lists
// in this order:
// engine 0: [Remote, Gather, Remote, Gather, Return, Subquery, Calc, Return]
// engine 1: [Remote, Enum coll, Calc d.pass==i, Filter]
// engine 2: [Singleton, EnumList [1,2], Scatter]
// engine 3: [Singleton, Enum coll2, Calc e.name==d.name, Filter]
// buildEngines will then do engines in the order 3, 2, 1, 0 and for each
// of them the nodes from left to right in these lists. In the end, we have
// a proper instantiation of the whole thing.
struct CoordinatorInstanciator : public WalkerWorker<ExecutionNode> {
enum EngineLocation { COORDINATOR, DBSERVER };
struct EngineInfo {
EngineInfo(EngineLocation location, size_t id, arangodb::aql::QueryPart p,
size_t idOfRemoteNode)
: location(location),
id(id),
nodes(),
part(p),
idOfRemoteNode(idOfRemoteNode),
collection(nullptr),
auxiliaryCollections(),
populated(false) {
}
void populate() {
// mop: compiler should inline that I suppose :S
auto collectionFn = [&](Collection* col) -> void {
if (col->isSatellite()) {
auxiliaryCollections.emplace(col);
} else {
collection = col;
}
};
Collection* localCollection = nullptr;
for (auto en = nodes.rbegin(); en != nodes.rend(); ++en) {
// find the collection to be used
if ((*en)->getType() == ExecutionNode::ENUMERATE_COLLECTION) {
localCollection = const_cast<Collection*>(
static_cast<EnumerateCollectionNode*>((*en))->collection());
collectionFn(localCollection);
} else if ((*en)->getType() == ExecutionNode::INDEX) {
localCollection = const_cast<Collection*>(
static_cast<IndexNode*>((*en))->collection());
collectionFn(localCollection);
} else if ((*en)->getType() == ExecutionNode::INSERT ||
(*en)->getType() == ExecutionNode::UPDATE ||
(*en)->getType() == ExecutionNode::REPLACE ||
(*en)->getType() == ExecutionNode::REMOVE ||
(*en)->getType() == ExecutionNode::UPSERT) {
localCollection = const_cast<Collection*>(
static_cast<ModificationNode*>((*en))->collection());
collectionFn(localCollection);
}
}
// mop: no non satellite collection found
if (collection == nullptr) {
// mop: just take the last satellite then
collection = localCollection;
}
// mop: ok we are actually only working with a satellite...
// so remove its shardId from the auxiliaryShards again
if (collection != nullptr && collection->isSatellite()) {
auxiliaryCollections.erase(collection);
}
populated = true;
}
Collection* getCollection() {
if (!populated) {
populate();
}
TRI_ASSERT(collection != nullptr);
return collection;
}
std::unordered_set<Collection*> getAuxiliaryCollections() {
if (!populated) {
populate();
}
return auxiliaryCollections;
}
EngineLocation const location;
size_t const id;
std::vector<ExecutionNode*> nodes;
arangodb::aql::QueryPart part; // only relevant for DBserver parts
size_t idOfRemoteNode; // id of the remote node
Collection* collection;
std::unordered_set<Collection*> auxiliaryCollections;
bool populated;
// in the original plan that needs this engine
};
Query* query;
QueryRegistry* queryRegistry;
ExecutionBlock* root;
EngineLocation currentLocation;
size_t currentEngineId;
std::vector<EngineInfo> engines;
std::vector<size_t> engineStack; // stack of engine ids, used for
// RemoteNodes
std::unordered_set<std::string> collNamesSeenOnDBServer;
// names of sharded collections that we have already seen on a DBserver
// this is relevant to decide whether or not the engine there is a main
// query or a dependent one.
std::unordered_map<std::string, std::string> queryIds;
std::unordered_set<Collection*> auxiliaryCollections;
// this map allows to find the queries which are the parts of the big
// query. There are two cases, the first is for the remote queries on
// the DBservers, for these, the key is:
// itoa(ID of RemoteNode in original plan) + ":" + shardId
// and the value is the
// queryId on DBserver
// with a * appended, if it is a PART_MAIN query.
// The second case is a query, which lives on the coordinator but is not
// the main query. For these, we store
// itoa(ID of RemoteNode in original plan) + "/" + <name of vocbase>
// and the value is the
// queryId used in the QueryRegistry
// this is built up when we instantiate the various engines on the
// DBservers and used when we instantiate the ones on the
// coordinator. Note that the main query and engine is not put into
// this map at all.
std::unordered_map<traverser::TraverserEngineID, std::unordered_set<std::string>> traverserEngines;
// This map allows to find all traverser engine parts of the query.
// The first value is the engine id. The second value is a list of
// shards this engine is responsible for.
// All shards that are not yet in queryIds have to be locked by
// one of the traverserEngines.
// TraverserEngines will always give the PART_MAIN to other parts
// of the queries if they desire them.
CoordinatorInstanciator(Query* query, QueryRegistry* queryRegistry)
: query(query),
queryRegistry(queryRegistry),
root(nullptr),
currentLocation(COORDINATOR),
currentEngineId(0),
engines() {
TRI_ASSERT(query != nullptr);
TRI_ASSERT(queryRegistry != nullptr);
engines.emplace_back(COORDINATOR, 0, PART_MAIN, 0);
}
~CoordinatorInstanciator() {}
/// @brief generatePlanForOneShard
void generatePlanForOneShard(VPackBuilder& builder, size_t nr,
EngineInfo* info, QueryId& connectedId,
std::string const& shardId, bool verbose) {
// copy the relevant fragment of the plan for each shard
// Note that in these parts of the query there are no SubqueryNodes,
// since they are all on the coordinator!
ExecutionPlan plan(query->ast());
ExecutionNode* previous = nullptr;
for (ExecutionNode const* current : info->nodes) {
auto clone = current->clone(&plan, false, false);
// UNNECESSARY, because clone does it: plan.registerNode(clone);
if (current->getType() == ExecutionNode::REMOTE) {
// update the remote node with the information about the query
static_cast<RemoteNode*>(clone)
->server("server:" + arangodb::ServerState::instance()->getId());
static_cast<RemoteNode*>(clone)->ownName(shardId);
static_cast<RemoteNode*>(clone)->queryId(connectedId);
// only one of the remote blocks is responsible for forwarding the
// initializeCursor and shutDown requests
// for simplicity, we always use the first remote block if we have more
// than one
static_cast<RemoteNode*>(clone)->isResponsibleForInitializeCursor(nr == 0);
}
if (previous != nullptr) {
clone->addDependency(previous);
}
previous = clone;
}
plan.root(previous);
plan.setVarUsageComputed();
return plan.root()->toVelocyPack(builder, verbose);
}
/// @brief distributePlanToShard, send a single plan to one shard
void distributePlanToShard(arangodb::CoordTransactionID& coordTransactionID,
EngineInfo* info,
QueryId& connectedId, std::string const& shardId,
VPackSlice const& planSlice) {
Collection* collection = info->getCollection();
// create a JSON representation of the plan
VPackBuilder result;
result.openObject();
result.add("plan", VPackValue(VPackValueType::Object));
VPackBuilder tmp;
query->ast()->variables()->toVelocyPack(tmp);
result.add("initialize", VPackValue(false));
result.add("variables", tmp.slice());
result.add("collections", VPackValue(VPackValueType::Array));
result.openObject();
result.add("name", VPackValue(shardId));
result.add("type", VPackValue(AccessMode::typeString(collection->accessType)));
result.close();
// mop: this is currently only working for satellites and hardcoded to their structure
for (auto auxiliaryCollection: info->getAuxiliaryCollections()) {
TRI_ASSERT(auxiliaryCollection->isSatellite());
// add the collection
result.openObject();
auto auxiliaryShards = auxiliaryCollection->shardIds();
result.add("name", VPackValue((*auxiliaryShards)[0]));
result.add("type", VPackValue(AccessMode::typeString(collection->accessType)));
result.close();
}
result.close(); // collections
result.add(VPackObjectIterator(planSlice));
result.close(); // plan
if (info->part == arangodb::aql::PART_MAIN) {
result.add("part", VPackValue("main"));
} else {
result.add("part", VPackValue("dependent"));
}
result.add("options", VPackValue(VPackValueType::Object));
result.add("optimizer", VPackValue(VPackValueType::Object));
result.add("rules", VPackValue(VPackValueType::Array));
result.add(VPackValue("-all"));
result.close(); // options.optimizer.rules
result.close(); // options.optimizer
double tracing = query->getNumericOption<double>("tracing", 0.0);
result.add("tracing", VPackValue(tracing));
double satelliteSyncWait = query->getNumericOption<double>("satelliteSyncWait", 60.0);
result.add("satelliteSyncWait", VPackValue(satelliteSyncWait));
result.close(); // options
result.close();
TRI_ASSERT(result.isClosed());
auto body = std::make_shared<std::string const>(result.slice().toJson());
//LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "GENERATED A PLAN FOR THE REMOTE SERVERS: " << *(body.get());
// << "\n";
auto cc = arangodb::ClusterComm::instance();
if (cc != nullptr) {
// nullptr only happens on controlled shutdown
std::string const url("/_db/"
+ arangodb::basics::StringUtils::urlEncode(collection->vocbase->name()) +
"/_api/aql/instantiate");
auto headers = std::make_unique<std::unordered_map<std::string, std::string>>();
(*headers)["X-Arango-Nolock"] = shardId; // Prevent locking
cc->asyncRequest("", coordTransactionID, "shard:" + shardId,
arangodb::rest::RequestType::POST,
url, body, headers, nullptr, 30.0);
}
}
/// @brief aggregateQueryIds, get answers for all shards in a Scatter/Gather
void aggregateQueryIds(EngineInfo* info,
std::shared_ptr<arangodb::ClusterComm>& cc,
arangodb::CoordTransactionID& coordTransactionID,
Collection* collection) {
// pick up the remote query ids
auto shardIds = collection->shardIds();
std::string error;
int count = 0;
int nrok = 0;
int errorCode = TRI_ERROR_NO_ERROR;
for (count = (int)shardIds->size(); count > 0; count--) {
auto res = cc->wait("", coordTransactionID, 0, "", 30.0);
if (res.status == arangodb::CL_COMM_RECEIVED) {
if (res.answer_code == arangodb::rest::ResponseCode::OK ||
res.answer_code == arangodb::rest::ResponseCode::CREATED ||
res.answer_code == arangodb::rest::ResponseCode::ACCEPTED) {
// query instantiated without problems
nrok++;
VPackSlice tmp = res.answer->payload().get("queryId");
std::string queryId;
if (tmp.isString()) {
queryId = tmp.copyString();
}
// std::cout << "DB SERVER ANSWERED WITHOUT ERROR: " <<
// res.answer->body() << ", REMOTENODEID: " << info.idOfRemoteNode <<
// " SHARDID:" << res.shardID << ", QUERYID: " << queryId << "\n";
std::string theID =
arangodb::basics::StringUtils::itoa(info->idOfRemoteNode) + ":" +
res.shardID;
if (info->part == arangodb::aql::PART_MAIN) {
queryId += "*";
}
queryIds.emplace(theID, queryId);
} else {
error += "DB SERVER ANSWERED WITH ERROR: ";
error += res.answer->payload().toJson();
error += "\n";
}
} else {
error += res.stringifyErrorMessage();
if (errorCode == TRI_ERROR_NO_ERROR) {
errorCode = res.getErrorCode();
}
}
}
//LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "GOT ALL RESPONSES FROM DB SERVERS: " << nrok << "\n";
if (nrok != (int)shardIds->size()) {
if (errorCode == TRI_ERROR_NO_ERROR) {
errorCode = TRI_ERROR_INTERNAL; // must have an error
}
THROW_ARANGO_EXCEPTION_MESSAGE(errorCode, error);
}
}
/// @brief distributePlansToShards, for a single Scatter/Gather block
void distributePlansToShards(EngineInfo* info, QueryId connectedId) {
//LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "distributePlansToShards: " << info.id;
Collection* collection = info->getCollection();
auto auxiliaryCollections = info->getAuxiliaryCollections();
for (auto const& auxiliaryCollection: auxiliaryCollections) {
TRI_ASSERT(auxiliaryCollection->shardIds()->size() == 1);
auxiliaryCollection->setCurrentShard((*auxiliaryCollection->shardIds())[0]);
}
// now send the plan to the remote servers
arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer();
auto cc = arangodb::ClusterComm::instance();
if (cc != nullptr) {
// nullptr only happens on controlled shutdown
// iterate over all shards of the collection
size_t nr = 0;
auto shardIds = collection->shardIds();
for (auto const& shardId : *shardIds) {
// inject the current shard id into the collection
VPackBuilder b;
collection->setCurrentShard(shardId);
generatePlanForOneShard(b, nr++, info, connectedId, shardId, true);
distributePlanToShard(coordTransactionID, info,
connectedId, shardId,
b.slice());
}
collection->resetCurrentShard();
for (auto const& auxiliaryCollection: auxiliaryCollections) {
TRI_ASSERT(auxiliaryCollection->shardIds()->size() == 1);
auxiliaryCollection->resetCurrentShard();
}
aggregateQueryIds(info, cc, coordTransactionID, collection);
}
}
/// @brief buildEngineCoordinator, for a single piece
ExecutionEngine* buildEngineCoordinator(EngineInfo* info) {
Query* localQuery = query;
bool needToClone = info->id > 0; // use the original for the main part
if (needToClone) {
// need a new query instance on the coordinator
localQuery = query->clone(PART_DEPENDENT, false);
if (localQuery == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"cannot clone query");
}
}
try {
auto engine = std::make_unique<ExecutionEngine>(localQuery);
localQuery->engine(engine.get());
std::unordered_map<ExecutionNode*, ExecutionBlock*> cache;
RemoteNode* remoteNode = nullptr;
for (auto en = info->nodes.begin(); en != info->nodes.end(); ++en) {
auto const nodeType = (*en)->getType();
if (nodeType == ExecutionNode::REMOTE) {
remoteNode = static_cast<RemoteNode*>((*en));
continue;
}
// for all node types but REMOTEs, we create blocks
ExecutionBlock* eb = CreateBlock(engine.get(), (*en), cache);
if (eb == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"illegal node type");
}
try {
engine.get()->addBlock(eb);
} catch (...) {
delete eb;
throw;
}
for (auto const& dep : (*en)->getDependencies()) {
auto d = cache.find(dep);
if (d != cache.end()) {
// add regular dependencies
TRI_ASSERT((*d).second != nullptr);
eb->addDependency((*d).second);
}
}
if (nodeType == ExecutionNode::GATHER) {
// we found a gather node
if (remoteNode == nullptr) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"expecting a remoteNode");
}
// now we'll create a remote node for each shard and add it to the
// gather node
auto gatherNode = static_cast<GatherNode const*>(*en);
Collection const* collection = gatherNode->collection();
auto shardIds = collection->shardIds();
for (auto const& shardId : *shardIds) {
std::string theId =
arangodb::basics::StringUtils::itoa(remoteNode->id()) + ":" +
shardId;
auto it = queryIds.find(theId);
if (it == queryIds.end()) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL,
"could not find query id in list");
}
std::string idThere = it->second;
if (idThere.back() == '*') {
idThere.pop_back();
}
ExecutionBlock* r = new RemoteBlock(engine.get(), remoteNode,
"shard:" + shardId, // server
"", // ownName
idThere); // queryId
try {
engine.get()->addBlock(r);
} catch (...) {
delete r;
throw;
}
TRI_ASSERT(r != nullptr);
eb->addDependency(r);
}
}
// the last block is always the root
engine->root(eb);
// put it into our cache:
cache.emplace(*en, eb);
}
TRI_ASSERT(engine->root() != nullptr);
// localQuery is stored in the engine
return engine.release();
} catch (...) {
localQuery->engine(nullptr); // engine is already destroyed by unique_ptr
if (needToClone) {
delete localQuery;
}
throw;
}
}
/// @brief Build traverser engines on DBServers. Coordinator still uses
/// traversal block.
void buildTraverserEnginesForNode(TraversalNode* en) {
// We have to initialize all options. After this point the node
// is not cloneable any more.
en->prepareOptions();
VPackBuilder optsBuilder;
auto opts = en->options();
opts->buildEngineInfo(optsBuilder);
// All info in opts is identical for each traverser engine.
// Only the shards are different.
std::vector<std::unique_ptr<arangodb::aql::Collection>> const& edges = en->edgeColls();
// Here we create a mapping
// ServerID => ResponsibleShards
// Where Responsible shards is divided in edgeCollections and vertexCollections
// For edgeCollections the Ordering is important for the index access.
// Also the same edgeCollection can be included twice (iff direction is ANY)
auto clusterInfo = arangodb::ClusterInfo::instance();
std::unordered_map<
ServerID,
std::pair<std::vector<std::vector<ShardID>>,
std::unordered_map<std::string, std::vector<ShardID>>>>
mappingServerToCollections;
auto servers = clusterInfo->getCurrentDBServers();
size_t length = edges.size();
// Initialize on engine for every server known to this cluster
// Thanks to locking mechanism we cannot leave any out, even it
// is not responsible for anything...
for (auto s : servers) {
// We insert at lease an empty vector for every edge collection
// Used in the traverser.
auto& info = mappingServerToCollections[s];
// We need to exactly maintain the ordering.
// A server my be responsible for a shard in edge collection 1 but not 0 or 2.
info.first.resize(length);
}
for (size_t i = 0; i < length; ++i) {
auto shardIds = edges[i]->shardIds();
for (auto const& shard : *shardIds) {
auto serverList = clusterInfo->getResponsibleServer(shard);
TRI_ASSERT(!serverList->empty());
auto& pair = mappingServerToCollections[(*serverList)[0]];
pair.first[i].emplace_back(shard);
}
}
std::vector<std::unique_ptr<arangodb::aql::Collection>> const& vertices =
en->vertexColls();
if (vertices.empty()) {
std::unordered_set<std::string> knownEdges;
for (auto const& it : edges) {
knownEdges.emplace(it->getName());
}
// This case indicates we do not have a named graph. We simply use
// ALL collections known to this query.
auto cs = query->collections()->collections();
for (auto const& collection : (*cs)) {
for (auto& entry : mappingServerToCollections) {
entry.second.second.emplace(collection.second->getName(),
std::vector<ShardID>());
}
if (knownEdges.find(collection.second->getName()) == knownEdges.end()) {
// This collection is not one of the edge collections used in this
// graph.
auto shardIds = collection.second->shardIds();
for (auto const& shard : *shardIds) {
auto serverList = clusterInfo->getResponsibleServer(shard);
TRI_ASSERT(!serverList->empty());
auto& pair = mappingServerToCollections[(*serverList)[0]];
pair.second[collection.second->getName()].emplace_back(shard);
}
}
}
} else {
// This Traversal is started with a GRAPH. It knows all relevant collections.
for (auto const& it : vertices) {
for (auto& entry : mappingServerToCollections) {
entry.second.second.emplace(it->getName(),
std::vector<ShardID>());
}
auto shardIds = it->shardIds();
for (auto const& shard : *shardIds) {
auto serverList = clusterInfo->getResponsibleServer(shard);
TRI_ASSERT(!serverList->empty());
auto& pair = mappingServerToCollections[(*serverList)[0]];
pair.second[it->getName()].emplace_back(shard);
}
}
}
// Now we create a VPack Object containing the relevant information
// for the Traverser Engines.
// First the options (which are identical for all engines.
// Second the list of shards this engine is responsible for.
// Shards are not overlapping between Engines as there is exactly
// one engine per server.
//
// The resulting JSON is a s follows:
//
// {
// "options": <options.toVelocyPack>,
// "variables": [<vars used in conditions>], // optional
// "shards": {
// "edges" : [
// [ <shards of edge collection 1> ],
// [ <shards of edge collection 2> ]
// ],
// "vertices" : {
// "v1": [<shards of v1>], // may be empty
// "v2": [<shards of v2>] // may be empty
// }
// }
// }
std::string const url("/_db/" + arangodb::basics::StringUtils::urlEncode(
query->vocbase()->name()) +
"/_internal/traverser");
auto cc = arangodb::ClusterComm::instance();
if (cc == nullptr) {
// nullptr only happens on controlled shutdown
return;
}
bool hasVars = false;
VPackBuilder varInfo;
std::vector<aql::Variable const*> vars;
en->getConditionVariables(vars);
if (!vars.empty()) {
hasVars = true;
varInfo.openArray();
for (auto v : vars) {
v->toVelocyPack(varInfo);
}
varInfo.close();
}
VPackBuilder engineInfo;
for (auto const& list : mappingServerToCollections) {
std::unordered_set<std::string> shardSet;
engineInfo.clear();
engineInfo.openObject();
engineInfo.add(VPackValue("options"));
engineInfo.add(optsBuilder.slice());
if (hasVars) {
engineInfo.add(VPackValue("variables"));
engineInfo.add(varInfo.slice());
}
engineInfo.add(VPackValue("shards"));
engineInfo.openObject();
engineInfo.add(VPackValue("vertices"));
engineInfo.openObject();
for (auto const& col : list.second.second) {
engineInfo.add(VPackValue(col.first));
engineInfo.openArray();
for (auto const& v : col.second) {
shardSet.emplace(v);
engineInfo.add(VPackValue(v));
}
engineInfo.close(); // this collection
}
engineInfo.close(); // vertices
engineInfo.add(VPackValue("edges"));
engineInfo.openArray();
for (auto const& edgeShards : list.second.first) {
engineInfo.openArray();
for (auto const& e : edgeShards) {
shardSet.emplace(e);
engineInfo.add(VPackValue(e));
}
engineInfo.close();
}
engineInfo.close(); // edges
engineInfo.close(); // shards
en->enhanceEngineInfo(engineInfo);
engineInfo.close(); // base
if (!shardSet.empty()) {
arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer();
std::unordered_map<std::string, std::string> headers;
auto res = cc->syncRequest("", coordTransactionID, "server:" + list.first,
RequestType::POST, url, engineInfo.toJson(),
headers, 30.0);
if (res->status != CL_COMM_SENT) {
// Note If there was an error on server side we do not have CL_COMM_SENT
std::string message("could not start all traversal engines");
if (res->errorMessage.length() > 0) {
message += std::string(" : ") + res->errorMessage;
}
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, message);
} else {
// Only if the result was successful we will get here
arangodb::basics::StringBuffer& body = res->result->getBody();
std::shared_ptr<VPackBuilder> builder =
VPackParser::fromJson(body.c_str(), body.length());
VPackSlice resultSlice = builder->slice();
if (!resultSlice.isNumber()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_INTERNAL, "got unexpected response from engine lock request");
}
auto engineId = resultSlice.getNumericValue<traverser::TraverserEngineID>();
TRI_ASSERT(engineId != 0);
traverserEngines.emplace(engineId, shardSet);
en->addEngine(engineId, list.first);
}
}
}
}
/// @brief buildEngines, build engines on DBservers and coordinator
ExecutionEngine* buildEngines() {
ExecutionEngine* engine = nullptr;
QueryId id = 0;
for (auto it = engines.rbegin(); it != engines.rend(); ++it) {
EngineInfo* info = &(*it);
//LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "Doing engine: " << it->id << " location:"
// << it->location;
if (info->location == COORDINATOR) {
// create a coordinator-based engine
engine = buildEngineCoordinator(info);
TRI_ASSERT(engine != nullptr);
if ((*it).id > 0) {
// create a remote id for the engine that we can pass to
// the plans to be created for the DBServers
id = TRI_NewTickServer();
try {
queryRegistry->insert(id, engine->getQuery(), 600.0);
} catch (...) {
delete engine->getQuery();
// This deletes the new query as well as the engine
throw;
}
try {
std::string queryId = arangodb::basics::StringUtils::itoa(id);
std::string theID =
arangodb::basics::StringUtils::itoa(it->idOfRemoteNode) + "/" +
engine->getQuery()->vocbase()->name();
queryIds.emplace(theID, queryId);
} catch (...) {
queryRegistry->destroy(engine->getQuery()->vocbase(), id,
TRI_ERROR_INTERNAL);
// This deletes query, engine and entry in QueryRegistry
throw;
}
}
} else {
// create an engine on a remote DB server
// hand in the previous engine's id
distributePlansToShards(info, id);
}
}
TRI_ASSERT(engine != nullptr);
// return the last created coordinator-based engine
// this is the local engine that we'll use to run the query
return engine;
}
/// @brief before method for collection of pieces phase
virtual bool before(ExecutionNode* en) override final {
auto const nodeType = en->getType();
if (nodeType == ExecutionNode::REMOTE) {
// got a remote node
// this indicates the end of an execution section
engineStack.push_back(currentEngineId);
// begin a new engine
// flip current location
currentLocation =
(currentLocation == COORDINATOR ? DBSERVER : COORDINATOR);
currentEngineId = engines.size();
QueryPart part = PART_DEPENDENT;
if (currentLocation == DBSERVER) {
auto rn = static_cast<RemoteNode*>(en);
Collection const* coll = rn->collection();
if (collNamesSeenOnDBServer.find(coll->name) ==
collNamesSeenOnDBServer.end()) {
part = PART_MAIN;
collNamesSeenOnDBServer.insert(coll->name);
}
}
// For the coordinator we do not care about main or part:
engines.emplace_back(currentLocation, currentEngineId, part, en->id());
}
if (nodeType == ExecutionNode::TRAVERSAL) {
buildTraverserEnginesForNode(static_cast<TraversalNode*>(en));
}
return false;
}
/// @brief after method for collection of pieces phase
virtual void after(ExecutionNode* en) override final {
auto const nodeType = en->getType();
if (nodeType == ExecutionNode::REMOTE) {
currentEngineId = engineStack.back();
engineStack.pop_back();
currentLocation = engines[currentEngineId].location;
}
// assign the current node to the current engine
engines[currentEngineId].nodes.emplace_back(en);
}
};
/// @brief shutdown, will be called exactly once for the whole query
int ExecutionEngine::shutdown(int errorCode) {
if (_root != nullptr && !_wasShutdown) {
// Take care of locking prevention measures in the cluster:
if (_lockedShards != nullptr) {
if (transaction::Methods::_makeNolockHeaders == _lockedShards) {
transaction::Methods::_makeNolockHeaders = _previouslyLockedShards;
}
delete _lockedShards;
_lockedShards = nullptr;
_previouslyLockedShards = nullptr;
}
// prevent a duplicate shutdown
int res = _root->shutdown(errorCode);
_wasShutdown = true;
return res;
}
return TRI_ERROR_NO_ERROR;
}
/// @brief create an execution engine from a plan
ExecutionEngine* ExecutionEngine::instantiateFromPlan(
QueryRegistry* queryRegistry, Query* query, ExecutionPlan* plan,
bool planRegisters) {
auto role = arangodb::ServerState::instance()->getRole();
bool const isCoordinator =
arangodb::ServerState::instance()->isCoordinator(role);
bool const isDBServer = arangodb::ServerState::instance()->isDBServer(role);
TRI_ASSERT(queryRegistry != nullptr);
ExecutionEngine* engine = nullptr;
try {
if (!plan->varUsageComputed()) {
plan->findVarUsage();
}
if (planRegisters) {
plan->planRegisters();
}
ExecutionBlock* root = nullptr;
if (isCoordinator) {
// instantiate the engine on the coordinator
auto inst =
std::make_unique<CoordinatorInstanciator>(query, queryRegistry);
plan->root()->walk(inst.get());
try {
engine = inst.get()->buildEngines();
root = engine->root();
// Now find all shards that take part:
if (transaction::Methods::_makeNolockHeaders != nullptr) {
engine->_lockedShards = new std::unordered_set<std::string>(
*transaction::Methods::_makeNolockHeaders);
engine->_previouslyLockedShards = transaction::Methods::_makeNolockHeaders;
} else {
engine->_lockedShards = new std::unordered_set<std::string>();
engine->_previouslyLockedShards = nullptr;
}
std::map<std::string, std::pair<std::string, bool>> forLocking;
for (auto& q : inst.get()->queryIds) {
std::string theId = q.first;
std::string queryId = q.second;
// std::cout << "queryIds: " << theId << " : " << queryId <<
// std::endl;
auto pos = theId.find(':');
if (pos != std::string::npos) {
// So this is a remote one on a DBserver:
if (queryId.back() == '*') { // only the PART_MAIN one!
queryId.pop_back();
std::string shardId = theId.substr(pos + 1);
engine->_lockedShards->insert(shardId);
forLocking.emplace(shardId, std::make_pair(queryId, false));
}
}
}
// TODO is this enough? Do we have to somehow inform the other engines?
for (auto& te : inst.get()->traverserEngines) {
std::string traverserId = arangodb::basics::StringUtils::itoa(te.first);
for (auto const& shardId : te.second) {
if (forLocking.find(shardId) == forLocking.end()) {
// No other node stated that it is responsible for locking this shard.
// So the traverser engine has to step in.
forLocking.emplace(shardId, std::make_pair(traverserId, true));
}
}
}
// Second round, this time we deal with the coordinator pieces
// and tell them the lockedShards as well, we need to copy, since
// they want to delete independently:
for (auto& q : inst.get()->queryIds) {
std::string theId = q.first;
std::string queryId = q.second;
// std::cout << "queryIds: " << theId << " : " << queryId <<
// std::endl;
auto pos = theId.find('/');
if (pos != std::string::npos) {
// std::cout << "Setting lockedShards for query ID "
// << queryId << std::endl;
QueryId qId = arangodb::basics::StringUtils::uint64(queryId);
TRI_vocbase_t* vocbase = query->vocbase();
Query* q = queryRegistry->open(vocbase, qId);
q->engine()->setLockedShards(
new std::unordered_set<std::string>(*engine->_lockedShards));
queryRegistry->close(vocbase, qId);
// std::cout << "Setting lockedShards done." << std::endl;
}
}
// Now lock them all in the right order:
for (auto& p : forLocking) {
std::string const& shardId = p.first;
std::string const& queryId = p.second.first;
bool isTraverserEngine = p.second.second;
// Lock shard on DBserver:
arangodb::CoordTransactionID coordTransactionID = TRI_NewTickServer();
auto cc = arangodb::ClusterComm::instance();
if (cc == nullptr) {
// nullptr only happens on controlled shutdown
THROW_ARANGO_EXCEPTION( TRI_ERROR_SHUTTING_DOWN);
}
TRI_vocbase_t* vocbase = query->vocbase();
std::unique_ptr<ClusterCommResult> res;
std::unordered_map<std::string, std::string> headers;
if (isTraverserEngine) {
std::string const url(
"/_db/" +
arangodb::basics::StringUtils::urlEncode(vocbase->name()) +
"/_internal/traverser/lock/" + queryId + "/" + shardId);
res = cc->syncRequest("", coordTransactionID, "shard:" + shardId,
RequestType::PUT, url, "", headers, 30.0);
} else {
std::string const url(
"/_db/" +
arangodb::basics::StringUtils::urlEncode(vocbase->name()) +
"/_api/aql/lock/" + queryId);
res = cc->syncRequest("", coordTransactionID, "shard:" + shardId,
RequestType::PUT, url, "{}", headers, 30.0);
}
if (res->status != CL_COMM_SENT) {
std::string message("could not lock all shards");
if (res->errorMessage.length() > 0) {
message += std::string(" : ") + res->errorMessage;
}
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, message);
}
}
transaction::Methods::_makeNolockHeaders = engine->_lockedShards;
} catch (...) {
// We need to destroy all queries that we have built and stuffed
// into the QueryRegistry as well as those that we have pushed to
// the DBservers via HTTP:
TRI_vocbase_t* vocbase = query->vocbase();
auto cc = arangodb::ClusterComm::instance();
if (cc != nullptr) {
// nullptr only happens during controlled shutdown
for (auto& q : inst.get()->queryIds) {
std::string theId = q.first;
std::string queryId = q.second;
auto pos = theId.find(':');
if (pos != std::string::npos) {
// So this is a remote one on a DBserver:
std::string shardId = theId.substr(pos + 1);
// Remove query from DBserver:
arangodb::CoordTransactionID coordTransactionID =
TRI_NewTickServer();
if (queryId.back() == '*') {
queryId.pop_back();
}
std::string const url(
"/_db/" +
arangodb::basics::StringUtils::urlEncode(vocbase->name()) +
"/_api/aql/shutdown/" + queryId);
std::unordered_map<std::string, std::string> headers;
auto res =
cc->syncRequest("", coordTransactionID, "shard:" + shardId,
arangodb::rest::RequestType::PUT,
url, "{\"code\": 0}", headers, 120.0);
// Ignore result, we need to try to remove all.
// However, log the incident if we have an errorMessage.
if (!res->errorMessage.empty()) {
std::string msg("while trying to unregister query ");
msg += queryId + ": " + res->stringifyErrorMessage();
LOG_TOPIC(WARN, arangodb::Logger::FIXME) << msg;
}
} else {
// Remove query from registry:
try {
queryRegistry->destroy(
vocbase, arangodb::basics::StringUtils::uint64(queryId),
TRI_ERROR_INTERNAL);
} catch (...) {
// Ignore problems
}
}
}
// Also we need to destroy all traverser engines that have been pushed to DBServers
{
std::string const url(
"/_db/" +
arangodb::basics::StringUtils::urlEncode(vocbase->name()) +
"/_internal/traverser/");
for (auto& te : inst.get()->traverserEngines) {
std::string traverserId = arangodb::basics::StringUtils::itoa(te.first);
arangodb::CoordTransactionID coordTransactionID =
TRI_NewTickServer();
std::unordered_map<std::string, std::string> headers;
// NOTE: te.second is the list of shards. So we just send delete
// to the first of those shards
auto res = cc->syncRequest(
"", coordTransactionID, "shard:" + *(te.second.begin()),
RequestType::DELETE_REQ, url + traverserId, "", headers, 30.0);
// Ignore result, we need to try to remove all.
// However, log the incident if we have an errorMessage.
if (!res->errorMessage.empty()) {
std::string msg("while trying to unregister traverser engine ");
msg += traverserId + ": " + res->stringifyErrorMessage();
LOG_TOPIC(WARN, arangodb::Logger::FIXME) << msg;
}
}
}
}
throw;
}
} else {
// instantiate the engine on a local server
engine = new ExecutionEngine(query);
auto inst = std::make_unique<Instanciator>(engine);
plan->root()->walk(inst.get());
root = inst.get()->root;
TRI_ASSERT(root != nullptr);
}
TRI_ASSERT(root != nullptr);
// inspect the root block of the query
if (!isDBServer &&
root->getPlanNode()->getType() == ExecutionNode::RETURN) {
// it's a return node. now tell it to not copy its results from above,
// but directly return it. we also need to note the RegisterId the
// caller needs to look into when fetching the results
// in short: this avoids copying the return values
engine->resultRegister(
static_cast<ReturnBlock*>(root)->returnInheritedResults());
}
engine->_root = root;
if (plan->isResponsibleForInitialize()) {
root->initialize();
root->initializeCursor(nullptr, 0);
}
return engine;
} catch (...) {
if (!isCoordinator) {
delete engine;
}
throw;
}
}
/// @brief add a block to the engine
void ExecutionEngine::addBlock(ExecutionBlock* block) {
TRI_ASSERT(block != nullptr);
_blocks.emplace_back(block);
}
Reference in New Issue View Git Blame Copy Permalink