Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add TTL to query options to standardize usage across components. (#8203)

This commit is contained in:
Dan Larkin-York 2019-02-21 14:29:37 -05:00 committed by Jan
parent fcc1700d33
commit cd54271af0
13 changed files with 298 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG
View File

@ -1,6 +1,9 @@
devel
-----
* `--query.registry-ttl` is now honored in single-server mode, and cursor TTLs
are now honored on DBServers in cluster mode
* add "TTL" index type, for optional auto-expiration of documents
* disable selection of index types "persistent" and "skiplist" in the web
@ -24,7 +27,6 @@ devel
* fixed JS AQL query objects with empty query strings not being recognized as AQL queries
>>>>>>> 492d05c1f1bd5cff4e067ae7d6593b9f03cebe37
* report run-time openssl version (for dynamically linked executables)

4
Documentation/Examples/arangod.json
View File

@ -2163,9 +2163,9 @@
},
"query.registry-ttl" : {
"category" : "option",
"default" : 600,
"default" : 0,
"deprecatedIn" : null,
"description" : "default time-to-live of query snippets (in seconds)",
"description" : "default time-to-live of cursors and query snippets (in seconds); if <= 0, value will default to 30 for single-server instances or 600 for cluster instances",
"dynamic" : false,
"enterpriseOnly" : false,
"hidden" : true,

4
arangod/Aql/EngineInfoContainerCoordinator.cpp
View File

@ -21,7 +21,6 @@
/// @author Michael Hackstein
////////////////////////////////////////////////////////////////////////////////
#include "EngineInfoContainerCoordinator.h"
#include "Aql/AqlItemBlock.h"
#include "Aql/AqlResult.h"
#include "Aql/ClusterBlocks.h"
@ -31,6 +30,7 @@
#include "Aql/ExecutionNode.h"
#include "Aql/Query.h"
#include "Aql/QueryRegistry.h"
#include "EngineInfoContainerCoordinator.h"
#include "VocBase/ticks.h"
using namespace arangodb;
@ -86,7 +86,7 @@ Result EngineInfoContainerCoordinator::EngineInfo::buildEngine(
// For _id == 0 this thread will always maintain the handle to
// the engine and will clean up. We do not keep track of it seperately
if (_id != 0) {
double ttl = queryRegistry->defaultTTL();
double ttl = query->queryOptions().ttl;
TRI_ASSERT(ttl > 0);
try {
queryRegistry->insert(_id, query, ttl, true, false);

21
arangod/Aql/EngineInfoContainerDBServer.cpp
View File

@ -21,7 +21,6 @@
/// @author Michael Hackstein
////////////////////////////////////////////////////////////////////////////////
#include "EngineInfoContainerDBServer.h"
#include "Aql/AqlItemBlock.h"
#include "Aql/ClusterNodes.h"
#include "Aql/Collection.h"
@ -36,6 +35,7 @@
#include "Cluster/ClusterComm.h"
#include "Cluster/ServerState.h"
#include "Cluster/TraverserEngineRegistry.h"
#include "EngineInfoContainerDBServer.h"
#include "Graph/BaseOptions.h"
#include "RestServer/QueryRegistryFeature.h"
#include "StorageEngine/TransactionState.h"
@ -98,7 +98,6 @@ GatherNode* findFirstGather(ExecutionNode const& root) {
return nullptr;
}
ScatterNode* findFirstScatter(ExecutionNode const& root) {
ExecutionNode* node = root.getFirstDependency();
@ -153,9 +152,7 @@ EngineInfoContainerDBServer::EngineInfo::EngineInfo(EngineInfo&& other) noexcept
TRI_ASSERT(source.collection);
}
void operator()(ViewSource const& source) {
TRI_ASSERT(source.view);
}
void operator()(ViewSource const& source) { TRI_ASSERT(source.view); }
} visitor;
boost::apply_visitor(visitor, _source);
@ -197,11 +194,8 @@ void EngineInfoContainerDBServer::EngineInfo::addNode(ExecutionNode* node) {
// can't do it on DB servers since only parts of the plan will be sent
viewNode.volatility(true);
_source = ViewSource(
*viewNode.view().get(),
findFirstGather(viewNode),
findFirstScatter(viewNode)
);
_source = ViewSource(*viewNode.view().get(), findFirstGather(viewNode),
findFirstScatter(viewNode));
break;
}
#endif
@ -979,12 +973,7 @@ Result EngineInfoContainerDBServer::buildEngines(MapRemoteToSnippet& queryIds,
return {TRI_ERROR_SHUTTING_DOWN};
}
double ttl = QueryRegistryFeature::DefaultQueryTTL;
auto* registry = QueryRegistryFeature::registry();
if (registry != nullptr) {
ttl = registry->defaultTTL();
}
TRI_ASSERT(ttl > 0);
double ttl = _query->queryOptions().ttl;
std::string const url(
"/_db/" + arangodb::basics::StringUtils::urlEncode(_query->vocbase().name()) +

13
arangod/Aql/QueryOptions.cpp
View File

@ -21,9 +21,10 @@
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "QueryOptions.h"
#include "ApplicationFeatures/ApplicationServer.h"
#include "Aql/QueryCache.h"
#include "Aql/QueryRegistry.h"
#include "QueryOptions.h"
#include "RestServer/QueryRegistryFeature.h"
#include <velocypack/Builder.h>
@ -39,6 +40,7 @@ QueryOptions::QueryOptions()
maxWarningCount(10),
literalSizeThreshold(-1),
satelliteSyncWait(60.0),
ttl(0),
profile(PROFILE_LEVEL_NONE),
allPlans(false),
verbosePlans(false),
@ -62,6 +64,9 @@ QueryOptions::QueryOptions()
memoryLimit = globalLimit;
}
// get global default ttl
ttl = q->registry()->defaultTTL();
// use global "failOnWarning" value
failOnWarning = q->failOnWarning();
@ -111,6 +116,11 @@ void QueryOptions::fromVelocyPack(VPackSlice const& slice) {
satelliteSyncWait = value.getNumber<double>();
}
value = slice.get("ttl");
if (value.isNumber()) {
ttl = value.getNumber<double>();
}
// boolean options
value = slice.get("profile");
if (value.isBool()) {
@ -214,6 +224,7 @@ void QueryOptions::toVelocyPack(VPackBuilder& builder, bool disableOptimizerRule
builder.add("maxWarningCount", VPackValue(maxWarningCount));
builder.add("literalSizeThreshold", VPackValue(literalSizeThreshold));
builder.add("satelliteSyncWait", VPackValue(satelliteSyncWait));
builder.add("ttl", VPackValue(ttl));
builder.add("profile", VPackValue(static_cast<uint32_t>(profile)));
builder.add("allPlans", VPackValue(allPlans));
builder.add("verbosePlans", VPackValue(verbosePlans));

3
arangod/Aql/QueryOptions.h
View File

@ -54,13 +54,14 @@ struct QueryOptions {
void fromVelocyPack(arangodb::velocypack::Slice const&);
void toVelocyPack(arangodb::velocypack::Builder&, bool disableOptimizerRules) const;
TEST_VIRTUAL ProfileLevel getProfileLevel () {return profile; };
TEST_VIRTUAL ProfileLevel getProfileLevel() { return profile; };
size_t memoryLimit;
size_t maxNumberOfPlans;
size_t maxWarningCount;
int64_t literalSizeThreshold;
double satelliteSyncWait;
double ttl;
/// Level 0 nothing, Level 1 profile, Level 2,3 log tracing info
ProfileLevel profile;
bool allPlans;

41
arangod/Aql/RestAqlHandler.cpp
View File

@ -93,6 +93,29 @@ RestAqlHandler::RestAqlHandler(GeneralRequest* request, GeneralResponse* respons
// variables: [ <variables> ]
// }
std::pair<double, std::shared_ptr<VPackBuilder>> RestAqlHandler::getPatchedOptionsWithTTL(
VPackSlice const& optionsSlice) const {
auto options = std::make_shared<VPackBuilder>();
double ttl = _queryRegistry->defaultTTL();
{
VPackObjectBuilder guard(options.get());
TRI_ASSERT(optionsSlice.isObject());
for (auto const& pair : VPackObjectIterator(optionsSlice)) {
if (pair.key.isEqualString("ttl")) {
ttl = VelocyPackHelper::getNumericValue<double>(optionsSlice, "ttl", ttl);
ttl = _request->parsedValue<double>("ttl", ttl);
if (ttl <= 0) {
ttl = _queryRegistry->defaultTTL();
}
options->add("ttl", VPackValue(ttl));
} else {
options->add(pair.key.stringRef(), pair.value);
}
}
}
return std::make_pair(ttl, options);
}
void RestAqlHandler::setupClusterQuery() {
// We should not intentionally call this method
// on the wrong server. So fail during maintanence.
@ -175,7 +198,9 @@ void RestAqlHandler::setupClusterQuery() {
// variables: <variables slice>
// }
auto options = std::make_shared<VPackBuilder>(VPackBuilder::clone(optionsSlice));
std::shared_ptr<VPackBuilder> options;
double ttl;
std::tie(ttl, options) = getPatchedOptionsWithTTL(optionsSlice);
// Build the collection information
VPackBuilder collectionBuilder;
@ -210,12 +235,6 @@ void RestAqlHandler::setupClusterQuery() {
}
collectionBuilder.close();
// Now the query is ready to go, store it in the registry and return:
double ttl = _request->parsedValue<double>("ttl", _queryRegistry->defaultTTL());
if (ttl <= 0) {
ttl = _queryRegistry->defaultTTL();
}
// creates a StandaloneContext or a leasing context
auto ctx = transaction::SmartContext::Create(_vocbase);
@ -396,8 +415,9 @@ void RestAqlHandler::createQueryFromVelocyPack() {
return;
}
auto options = std::make_shared<VPackBuilder>(
VPackBuilder::clone(querySlice.get("options")));
std::shared_ptr<VPackBuilder> options;
double ttl;
std::tie(ttl, options) = getPatchedOptionsWithTTL(querySlice.get("options"));
std::string const part =
VelocyPackHelper::getStringValue(querySlice, "part", "");
@ -420,9 +440,6 @@ void RestAqlHandler::createQueryFromVelocyPack() {
return;
}
// Now the query is ready to go, store it in the registry and return:
double ttl = _request->parsedValue<double>("ttl", _queryRegistry->defaultTTL());
_qId = TRI_NewTickServer();
try {
_queryRegistry->insert(_qId, query.get(), ttl, true, false);

3
arangod/Aql/RestAqlHandler.h
View File

@ -135,6 +135,9 @@ class RestAqlHandler : public RestVocbaseBaseHandler {
// dig out vocbase from context and query from ID, handle errors
bool findQuery(std::string const& idString, Query*& query);
// generate patched options with TTL extracted from request
std::pair<double, std::shared_ptr<VPackBuilder>> getPatchedOptionsWithTTL(VPackSlice const& optionsSlice) const;
// our query registry
QueryRegistry* _queryRegistry;

9
arangod/RestHandler/RestCursorHandler.cpp
View File

@ -21,7 +21,6 @@
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "RestCursorHandler.h"
#include "Aql/Query.h"
#include "Aql/QueryRegistry.h"
#include "Basics/Exceptions.h"
@ -29,6 +28,7 @@
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h"
#include "Cluster/ServerState.h"
#include "RestCursorHandler.h"
#include "Transaction/Context.h"
#include "Utils/Cursor.h"
#include "Utils/CursorRepository.h"
@ -151,7 +151,8 @@ RestStatus RestCursorHandler::registerQueryOrCursor(VPackSlice const& slice) {
bool stream = VelocyPackHelper::getBooleanValue(opts, "stream", false);
size_t batchSize = VelocyPackHelper::getNumericValue<size_t>(opts, "batchSize", 1000);
double ttl = VelocyPackHelper::getNumericValue<double>(opts, "ttl", 30);
double ttl = VelocyPackHelper::getNumericValue<double>(opts, "ttl",
_queryRegistry->defaultTTL());
bool count = VelocyPackHelper::getBooleanValue(opts, "count", false);
if (stream) {
@ -444,7 +445,9 @@ void RestCursorHandler::buildOptions(VPackSlice const& slice) {
}
VPackSlice ttl = slice.get("ttl");
_options->add("ttl", VPackValue(ttl.isNumber() ? ttl.getNumber<double>() : 30));
_options->add("ttl", VPackValue(ttl.isNumber() && ttl.getNumber<double>() > 0
? ttl.getNumber<double>()
: _queryRegistry->defaultTTL()));
}
//////////////////////////////////////////////////////////////////////////////

9
arangod/RestServer/QueryRegistryFeature.cpp
View File

@ -51,7 +51,7 @@ QueryRegistryFeature::QueryRegistryFeature(application_features::ApplicationServ
_queryCacheMaxResultsSize(0),
_queryCacheMaxEntrySize(0),
_queryCacheIncludeSystem(false),
_queryRegistryTTL(DefaultQueryTTL) {
_queryRegistryTTL(0) {
setOptional(false);
startsAfter("V8Phase");
@ -123,7 +123,9 @@ void QueryRegistryFeature::collectOptions(std::shared_ptr<ProgramOptions> option
new UInt64Parameter(&_maxQueryPlans));
options->addOption("--query.registry-ttl",
"default time-to-live of query snippets (in seconds)",
"default time-to-live of cursors and query snippets (in "
"seconds); if <= 0, value will default to 30 for "
"single-server instances or 600 for cluster instances",
new DoubleParameter(&_queryRegistryTTL),
arangodb::options::makeFlags(arangodb::options::Flags::Hidden));
}
@ -157,7 +159,8 @@ void QueryRegistryFeature::prepare() {
arangodb::aql::QueryCache::instance()->properties(properties);
if (_queryRegistryTTL <= 0) {
_queryRegistryTTL = DefaultQueryTTL;
// set to default value based on instance type
_queryRegistryTTL = ServerState::instance()->isSingleServer() ? 30 : 600;
}
// create the query registery

2
arangod/RestServer/QueryRegistryFeature.h
View File

@ -38,8 +38,6 @@ class QueryRegistryFeature final : public application_features::ApplicationFeatu
return QUERY_REGISTRY.load(std::memory_order_acquire);
}
static constexpr double DefaultQueryTTL = 600.0;
explicit QueryRegistryFeature(application_features::ApplicationServer& server);
void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;

404
tests/Aql/EngineInfoContainerCoordinatorTest.cpp
View File

@ -45,7 +45,6 @@ namespace tests {
namespace engine_info_container_coordinator_test {
TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
SECTION("it should always start with an open snippet, with queryID 0") {
EngineInfoContainerCoordinator testee;
QueryId res = testee.closeSnippet();
@ -88,7 +87,6 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
// 3. query->engine();
SECTION("it should create an ExecutionEngine for the first snippet") {
std::unordered_set<std::string> const restrictToShards;
MapRemoteToSnippet queryIds;
std::unordered_set<ShardID> lockedShards;
@ -121,7 +119,6 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
// Section: Mock Functions
// ------------------------------
fakeit::When(Method(mockQuery, setEngine)).Do([&](ExecutionEngine* eng) -> void {
// We expect that the snippet injects a new engine into our
// query.
@ -133,12 +130,12 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::When(Method(mockQuery, trx)).Return(&trx);
fakeit::When(Method(mockQuery, engine)).Return(&myEngine).Return(&myEngine);
fakeit::When(Method(mockTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockEngine, createBlocks)).Return(Result{TRI_ERROR_NO_ERROR});
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock* ()))
.AlwaysReturn(&rootBlock);
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock * ())).AlwaysReturn(&rootBlock);
// ------------------------------
// Section: Run the test
@ -147,9 +144,8 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
EngineInfoContainerCoordinator testee;
testee.addNode(&sNode);
ExecutionEngineResult result = testee.buildEngines(
&query, &registry, dbname, restrictToShards, queryIds, lockedShards
);
ExecutionEngineResult result =
testee.buildEngines(&query, &registry, dbname, restrictToShards, queryIds, lockedShards);
REQUIRE(result.ok());
ExecutionEngine* engine = result.engine();
@ -199,30 +195,39 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::Mock<ExecutionEngine> mockSecondEngine;
ExecutionEngine& mySecondEngine = mockSecondEngine.get();
fakeit::Mock<Query> mockQuery;
Query& query = mockQuery.get();
fakeit::Mock<Query> mockQueryClone;
Query& queryClone = mockQueryClone.get();
fakeit::Mock<QueryRegistry> mockRegistry;
fakeit::When(Method(mockRegistry, defaultTTL)).AlwaysReturn(600.0);
QueryRegistry& registry = mockRegistry.get();
fakeit::Mock<QueryOptions> mockQueryOptions;
QueryOptions& lqueryOptions = mockQueryOptions.get();
lqueryOptions.ttl = 600;
fakeit::Mock<Query> mockQuery;
Query& query = mockQuery.get();
fakeit::When(ConstOverloadedMethod(mockQuery, queryOptions, QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQuery, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
fakeit::Mock<Query> mockQueryClone;
Query& queryClone = mockQueryClone.get();
fakeit::When(ConstOverloadedMethod(mockQueryClone, queryOptions, QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQueryClone, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
fakeit::Mock<transaction::Methods> mockTrx;
transaction::Methods& trx = mockTrx.get();
fakeit::Mock<transaction::Methods> mockSecondTrx;
transaction::Methods& secondTrx = mockSecondTrx.get();
// ------------------------------
// Section: Mock Functions
// ------------------------------
fakeit::When(Method(mockQuery, setEngine)).Do([&](ExecutionEngine* eng) -> void {
// We expect that the snippet injects a new engine into our
// query.
// However we have to return a mocked engine later
@ -233,19 +238,18 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::When(Method(mockQuery, trx)).Return(&trx);
fakeit::When(Method(mockQuery, engine)).Return(&myEngine).Return(&myEngine);
fakeit::When(Method(mockTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockEngine, createBlocks)).Do([&](
std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&,
MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &fNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock* ()))
.AlwaysReturn(&block);
fakeit::When(Method(mockTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockEngine, createBlocks))
.Do([&](std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&, MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &fNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock * ())).AlwaysReturn(&block);
// Mock query clone
fakeit::When(Method(mockQuery, clone)).Do([&](QueryPart part, bool withPlan) -> Query* {
@ -265,32 +269,32 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::When(Method(mockQueryClone, trx)).Return(&secondTrx);
fakeit::When(Method(mockQueryClone, engine)).Return(&mySecondEngine);
fakeit::When(Method(mockSecondTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockSecondTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockSecondEngine, createBlocks)).Do([&](
std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&,
MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &sNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockSecondEngine, root, ExecutionBlock* ()))
fakeit::When(Method(mockSecondEngine, createBlocks))
.Do([&](std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&, MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &sNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockSecondEngine, root, ExecutionBlock * ()))
.AlwaysReturn(&block);
// Mock the Registry
fakeit::When(Method(mockRegistry, insert)).Do([&] (QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(isPrepared == true);
REQUIRE(keepLease == false);
REQUIRE(timeout == 600.0);
REQUIRE(query == &queryClone);
secondId = id;
});
fakeit::When(Method(mockRegistry, insert))
.Do([&](QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(isPrepared == true);
REQUIRE(keepLease == false);
REQUIRE(timeout == 600.0);
REQUIRE(query == &queryClone);
secondId = id;
});
// ------------------------------
// Section: Run the test
@ -306,9 +310,8 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
// Close the second snippet
testee.closeSnippet();
ExecutionEngineResult result = testee.buildEngines(
&query, &registry, dbname, restrictToShards, queryIds, lockedShards
);
ExecutionEngineResult result =
testee.buildEngines(&query, &registry, dbname, restrictToShards, queryIds, lockedShards);
REQUIRE(result.ok());
ExecutionEngine* engine = result.engine();
@ -345,7 +348,7 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
QueryId thirdId = 0;
std::string dbname = "TestDB";
auto setEngineCallback = [] (ExecutionEngine* eng) -> void {
auto setEngineCallback = [](ExecutionEngine* eng) -> void {
// We expect that the snippet injects a new engine into our
// query.
// However we have to return a mocked engine later
@ -406,15 +409,32 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::Mock<ExecutionEngine> mockThirdEngine;
ExecutionEngine& myThirdEngine = mockThirdEngine.get();
fakeit::Mock<QueryOptions> mockQueryOptions;
QueryOptions& lqueryOptions = mockQueryOptions.get();
lqueryOptions.ttl = 600;
fakeit::Mock<Query> mockQuery;
Query& query = mockQuery.get();
fakeit::When(ConstOverloadedMethod(mockQuery, queryOptions, QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQuery, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
// We need two query clones
fakeit::Mock<Query> mockQueryClone;
Query& queryClone = mockQueryClone.get();
fakeit::When(ConstOverloadedMethod(mockQueryClone, queryOptions, QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQueryClone, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
fakeit::Mock<Query> mockQuerySecondClone;
Query& querySecondClone = mockQuerySecondClone.get();
fakeit::When(ConstOverloadedMethod(mockQuerySecondClone, queryOptions,
QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQuerySecondClone, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
fakeit::Mock<QueryRegistry> mockRegistry;
fakeit::When(Method(mockRegistry, defaultTTL)).AlwaysReturn(600.0);
@ -436,91 +456,91 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::When(Method(mockQuery, setEngine)).Do(setEngineCallback);
fakeit::When(Method(mockQuery, trx)).Return(&trx);
fakeit::When(Method(mockQuery, engine)).Return(&myEngine).Return(&myEngine);
fakeit::When(Method(mockTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockEngine, createBlocks)).Do([&](
std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&,
MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 3);
REQUIRE(nodes[0] == &fbNode);
REQUIRE(nodes[1] == &sbNode);
REQUIRE(nodes[2] == &tbNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock* ()))
.AlwaysReturn(&block);
fakeit::When(Method(mockQuery, clone)).Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &queryClone;
}).Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &querySecondClone;
});
fakeit::When(Method(mockTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockEngine, createBlocks))
.Do([&](std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&, MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 3);
REQUIRE(nodes[0] == &fbNode);
REQUIRE(nodes[1] == &sbNode);
REQUIRE(nodes[2] == &tbNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock * ())).AlwaysReturn(&block);
fakeit::When(Method(mockQuery, clone))
.Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &queryClone;
})
.Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &querySecondClone;
});
// Mock first clone
fakeit::When(Method(mockQueryClone, setEngine)).Do(setEngineCallback);
fakeit::When(Method(mockQueryClone, engine)).Return(&mySecondEngine);
fakeit::When(Method(mockQueryClone, trx)).Return(&secondTrx);
fakeit::When(Method(mockSecondTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockSecondEngine, createBlocks)).Do([&](
std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&,
MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &aNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockSecondEngine, root, ExecutionBlock* ()))
fakeit::When(Method(mockSecondTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockSecondEngine, createBlocks))
.Do([&](std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&, MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &aNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockSecondEngine, root, ExecutionBlock * ()))
.AlwaysReturn(&block);
// Mock second clone
fakeit::When(Method(mockQuerySecondClone, setEngine)).Do(setEngineCallback);
fakeit::When(Method(mockQuerySecondClone, engine)).Return(&myThirdEngine);
fakeit::When(Method(mockQuerySecondClone, trx)).Return(&thirdTrx);
fakeit::When(Method(mockThirdTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockThirdEngine, createBlocks)).Do([&](
std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&,
MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &bNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockThirdEngine, root, ExecutionBlock* ()))
fakeit::When(Method(mockThirdTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockThirdEngine, createBlocks))
.Do([&](std::vector<ExecutionNode*> const& nodes,
std::unordered_set<std::string> const&, MapRemoteToSnippet const&) {
REQUIRE(nodes.size() == 1);
REQUIRE(nodes[0] == &bNode);
return Result{TRI_ERROR_NO_ERROR};
});
fakeit::When(ConstOverloadedMethod(mockThirdEngine, root, ExecutionBlock * ()))
.AlwaysReturn(&block);
// Mock the Registry
// NOTE: This expects an ordering of the engines first of the stack will be handled
// first. With same fakeit magic we could make this ordering independent which is
// is fine as well for the production code.
fakeit::When(Method(mockRegistry, insert)).Do([&] (QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(isPrepared == true);
REQUIRE(keepLease == false);
REQUIRE(timeout == 600.0);
REQUIRE(query == &queryClone);
secondId = id;
}).Do([&] (QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(timeout == 600.0);
REQUIRE(keepLease == false);
REQUIRE(query == &querySecondClone);
thirdId = id;
});
// NOTE: This expects an ordering of the engines first of the stack will be
// handled first. With same fakeit magic we could make this ordering
// independent which is is fine as well for the production code.
fakeit::When(Method(mockRegistry, insert))
.Do([&](QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(isPrepared == true);
REQUIRE(keepLease == false);
REQUIRE(timeout == 600.0);
REQUIRE(query == &queryClone);
secondId = id;
})
.Do([&](QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(timeout == 600.0);
REQUIRE(keepLease == false);
REQUIRE(query == &querySecondClone);
thirdId = id;
});
// ------------------------------
// Section: Run the test
@ -541,9 +561,8 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
testee.addNode(&tbNode);
ExecutionEngineResult result = testee.buildEngines(
&query, &registry, dbname, restrictToShards, queryIds, lockedShards
);
ExecutionEngineResult result =
testee.buildEngines(&query, &registry, dbname, restrictToShards, queryIds, lockedShards);
REQUIRE(result.ok());
ExecutionEngine* engine = result.engine();
@ -602,11 +621,23 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::Mock<ExecutionEngine> mockSecondEngine;
ExecutionEngine& mySecondEngine = mockSecondEngine.get();
fakeit::Mock<QueryOptions> mockQueryOptions;
QueryOptions& lqueryOptions = mockQueryOptions.get();
lqueryOptions.ttl = 600;
fakeit::Mock<Query> mockQuery;
Query& query = mockQuery.get();
fakeit::When(ConstOverloadedMethod(mockQuery, queryOptions, QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQuery, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
fakeit::Mock<Query> mockQueryClone;
Query& queryClone = mockQueryClone.get();
fakeit::When(ConstOverloadedMethod(mockQueryClone, queryOptions, QueryOptions const&()))
.AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; });
fakeit::When(OverloadedMethod(mockQueryClone, queryOptions, QueryOptions & ()))
.AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; });
fakeit::Mock<QueryRegistry> mockRegistry;
fakeit::When(Method(mockRegistry, defaultTTL)).AlwaysReturn(600.0);
@ -622,9 +653,7 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
// Section: Mock Functions
// ------------------------------
fakeit::When(Method(mockQuery, setEngine)).Do([&](ExecutionEngine* eng) -> void {
// We expect that the snippet injects a new engine into our
// query.
// However we have to return a mocked engine later
@ -634,12 +663,12 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
});
fakeit::When(Method(mockQuery, engine)).Return(&myEngine).Return(&myEngine);
fakeit::When(Method(mockQuery, trx)).Return(&trx);
fakeit::When(Method(mockTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockEngine, createBlocks)).AlwaysReturn(Result{TRI_ERROR_NO_ERROR});
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock* ()))
.AlwaysReturn(&block);
fakeit::When(ConstOverloadedMethod(mockEngine, root, ExecutionBlock * ())).AlwaysReturn(&block);
fakeit::When(Method(mockQueryClone, setEngine)).Do([&](ExecutionEngine* eng) -> void {
// We expect that the snippet injects a new engine into our
@ -652,19 +681,21 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::When(Method(mockQueryClone, engine)).Return(&mySecondEngine);
fakeit::When(Method(mockQueryClone, trx)).Return(&secondTrx);
fakeit::When(Method(mockSecondTrx, setLockedShards)).AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockSecondTrx, setLockedShards))
.AlwaysDo([&](std::unordered_set<std::string> const& lockedShards) {
return;
});
fakeit::When(Method(mockSecondEngine, createBlocks)).AlwaysReturn(Result{TRI_ERROR_NO_ERROR});
fakeit::When(ConstOverloadedMethod(mockSecondEngine, root, ExecutionBlock* ()))
fakeit::When(ConstOverloadedMethod(mockSecondEngine, root, ExecutionBlock * ()))
.AlwaysReturn(&block);
fakeit::When(OverloadedMethod(mockRegistry, destroy, void(std::string const&, QueryId, int))).Do([&]
(std::string const& vocbase, QueryId id, int errorCode) {
REQUIRE(vocbase == dbname);
REQUIRE(id == secondId);
REQUIRE(errorCode == TRI_ERROR_INTERNAL);
});
fakeit::When(OverloadedMethod(mockRegistry, destroy,
void(std::string const&, QueryId, int)))
.Do([&](std::string const& vocbase, QueryId id, int errorCode) {
REQUIRE(vocbase == dbname);
REQUIRE(id == secondId);
REQUIRE(errorCode == TRI_ERROR_INTERNAL);
});
// ------------------------------
// Section: Run the test
@ -690,27 +721,30 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
SECTION("cloning of a query fails") {
// Mock the Registry
fakeit::When(Method(mockRegistry, insert)).Do([&] (QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(timeout == 600.0);
REQUIRE(isPrepared == true);
REQUIRE(keepLease == false);
REQUIRE(query == &queryClone);
secondId = id;
});
fakeit::When(Method(mockRegistry, insert))
.Do([&](QueryId id, Query* query, double timeout, bool isPrepared, bool keepLease) {
REQUIRE(id != 0);
REQUIRE(query != nullptr);
REQUIRE(timeout == 600.0);
REQUIRE(isPrepared == true);
REQUIRE(keepLease == false);
REQUIRE(query == &queryClone);
secondId = id;
});
SECTION("it throws an error") {
// Mock query clone
fakeit::When(Method(mockQuery, clone)).Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &queryClone;
}).Throw(arangodb::basics::Exception(TRI_ERROR_DEBUG, __FILE__, __LINE__));
fakeit::When(Method(mockQuery, clone))
.Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &queryClone;
})
.Throw(arangodb::basics::Exception(TRI_ERROR_DEBUG, __FILE__, __LINE__));
ExecutionEngineResult result = testee.buildEngines(
&query, &registry, dbname, restrictToShards, queryIds, lockedShards
);
ExecutionEngineResult result =
testee.buildEngines(&query, &registry, dbname, restrictToShards,
queryIds, lockedShards);
REQUIRE(!result.ok());
// Make sure we check the right thing here
REQUIRE(result.errorNumber() == TRI_ERROR_DEBUG);
@ -718,20 +752,21 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
SECTION("it returns nullptr") {
// Mock query clone
fakeit::When(Method(mockQuery, clone)).Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &queryClone;
}).Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return nullptr;
});
fakeit::When(Method(mockQuery, clone))
.Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return &queryClone;
})
.Do([&](QueryPart part, bool withPlan) -> Query* {
REQUIRE(part == PART_DEPENDENT);
REQUIRE(withPlan == false);
return nullptr;
});
ExecutionEngineResult result = testee.buildEngines(
&query, &registry, dbname, restrictToShards, queryIds, lockedShards
);
ExecutionEngineResult result =
testee.buildEngines(&query, &registry, dbname, restrictToShards,
queryIds, lockedShards);
REQUIRE(!result.ok());
// Make sure we check the right thing here
REQUIRE(result.errorNumber() == TRI_ERROR_INTERNAL);
@ -753,7 +788,9 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
fakeit::Verify(Method(mockRegistry, insert)).Exactly(1);
// Assert unregister of second engine.
fakeit::Verify(OverloadedMethod(mockRegistry, destroy, void(std::string const&, QueryId, int))).Exactly(1);
fakeit::Verify(OverloadedMethod(mockRegistry, destroy,
void(std::string const&, QueryId, int)))
.Exactly(1);
}
/*
@ -805,8 +842,7 @@ TEST_CASE("EngineInfoContainerCoordinator", "[aql][cluster][coordinator]") {
}
*/
}
}
} // test
} // aql
} // arangodb
} // namespace engine_info_container_coordinator_test
} // namespace tests
} // namespace arangodb

16
tests/rb/HttpInterface/api-cursor-spec.rb
View File

@ -499,7 +499,7 @@ describe ArangoDB do
it "creates a cursor that will expire" do
cmd = api
body = "{ \"query\" : \"FOR u IN #{@cn} LIMIT 5 RETURN u.n\", \"count\" : true, \"batchSize\" : 1, \"ttl\" : 10 }"
body = "{ \"query\" : \"FOR u IN #{@cn} LIMIT 5 RETURN u.n\", \"count\" : true, \"batchSize\" : 1, \"ttl\" : 2 }"
doc = ArangoDB.log_post("#{prefix}-create-ttl", cmd, :body => body)
doc.code.should eq(201)
@ -550,13 +550,13 @@ describe ArangoDB do
# when it really vanishes, as this depends on thread scheduling, state
# of the cleanup thread etc.
# sleep 10 # this should delete the cursor on the server
# doc = ArangoDB.log_put("#{prefix}-create-ttl", cmd)
# doc.code.should eq(404)
# doc.headers['content-type'].should eq("application/json; charset=utf-8")
# doc.parsed_response['error'].should eq(true)
# doc.parsed_response['errorNum'].should eq(1600)
# doc.parsed_response['code'].should eq(404)
sleep 10 # this should delete the cursor on the server
doc = ArangoDB.log_put("#{prefix}-create-ttl", cmd)
doc.code.should eq(404)
doc.headers['content-type'].should eq("application/json; charset=utf-8")
doc.parsed_response['error'].should eq(true)
doc.parsed_response['errorNum'].should eq(1600)
doc.parsed_response['code'].should eq(404)
end
it "creates a cursor that will not expire" do