Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bug fix/adjust 3.5 rc3 (#9146) 

* fix for windows

* Fix some uses of atomics (#9141)

* Remove std::move as it prevents copy elision (#9144)

* bug-fix/internal-issue-#549 (#9086)

* do not persist legacy analyzers into _analyzers table

* fix arangosearch upgrade in cluster

* get rid of Vasiliy's shit

* address review comments

* ensure link is synchronized after creation in upgrade

* fix compilation error

* minor cleanup

* fix tests

* distribute '_analyzers' collection as '_graphs'

* comment out Vasiliy's shit part 2

* Remove ArangoSearch views upgrade limit
This commit is contained in:
KVS85 2019-05-30 23:20:33 +03:00 committed by GitHub
parent d2034289a6
commit fab06069db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 1447 additions and 1879 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
arangod/Cluster/ClusterFeature.h
View File

@ -50,6 +50,9 @@ class ClusterFeature : public application_features::ApplicationFeature {
std::string agencyPrefix() const { return _agencyPrefix; }
/// @return role argument as it was supplied by a user
std::string const& myRole() const noexcept { return _myRole; }
void syncDBServerStatusQuo();
protected:

1
arangod/Cluster/ServerState.h
View File

@ -153,7 +153,6 @@ class ServerState {
/// @brief whether or not the role is a cluster-related role
static bool isClusterRole(ServerState::RoleEnum role) {
TRI_ASSERT(role != ServerState::ROLE_UNDEFINED);
return (role == ServerState::ROLE_DBSERVER || role == ServerState::ROLE_COORDINATOR);
}

4
arangod/ClusterEngine/ClusterSelectivityEstimates.cpp
View File

@ -43,7 +43,7 @@ void ClusterSelectivityEstimates::flush() {
_updating.store(false, std::memory_order_release);
});
std::atomic_store<InternalData>(&_data, std::shared_ptr<InternalData>());
std::atomic_store(&_data, std::shared_ptr<InternalData>());
}
IndexEstMap ClusterSelectivityEstimates::get(bool allowUpdating, TRI_voc_tid_t tid) {
@ -116,5 +116,5 @@ void ClusterSelectivityEstimates::set(IndexEstMap const& estimates) {
}
// finally update the cache
std::atomic_store<ClusterSelectivityEstimates::InternalData>(&_data, std::make_shared<ClusterSelectivityEstimates::InternalData>(estimates, ttl));
std::atomic_store(&_data, std::make_shared<ClusterSelectivityEstimates::InternalData>(estimates, ttl));
}

200
arangod/IResearch/IResearchAnalyzerFeature.cpp
View File

@ -325,143 +325,97 @@ std::string normalizedAnalyzerName(
return database.append(2, ANALYZER_PREFIX_DELIM).append(analyzer);
}
bool iresearchAnalyzerLegacyAnalyzers( // upgrade task
TRI_vocbase_t& vocbase, // upgraded vocbase
arangodb::velocypack::Slice const& upgradeParams // upgrade params
) {
auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::iresearch::IResearchAnalyzerFeature // feature type
////////////////////////////////////////////////////////////////////////////////
/// @brief creates '_analyzers' collection
////////////////////////////////////////////////////////////////////////////////
bool setupAnalyzersCollection(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& /*upgradeParams*/) {
return arangodb::methods::Collections::createSystem(vocbase, ANALYZER_COLLECTION_NAME).ok();
}
////////////////////////////////////////////////////////////////////////////////
/// @brief drops '_iresearch_analyzers' collection
////////////////////////////////////////////////////////////////////////////////
bool dropLegacyAnalyzersCollection(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& /*upgradeParams*/) {
// drop legacy collection if upgrading the system vocbase and collection found
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::SystemDatabaseFeature // feature type
>();
if (!analyzers) {
LOG_TOPIC("6b6b5", WARN, arangodb::iresearch::TOPIC)
<< "failure to find '" << arangodb::iresearch::IResearchAnalyzerFeature::name() << "' feature while registering legacy static analyzers with vocbase '" << vocbase.name() << "'";
if (!sysDatabase) {
LOG_TOPIC("8783e", WARN, arangodb::iresearch::TOPIC)
<< "failure to find '" << arangodb::SystemDatabaseFeature::name() << "' feature while registering legacy static analyzers with vocbase '" << vocbase.name() << "'";
TRI_set_errno(TRI_ERROR_INTERNAL);
return false; // internal error
}
// drop legacy collection if upgrading the system vocbase and collection found
{
auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::SystemDatabaseFeature // feature type
>();
auto sysVocbase = sysDatabase->use();
if (!sysDatabase) {
LOG_TOPIC("8783e", WARN, arangodb::iresearch::TOPIC)
<< "failure to find '" << arangodb::SystemDatabaseFeature::name() << "' feature while registering legacy static analyzers with vocbase '" << vocbase.name() << "'";
TRI_set_errno(TRI_ERROR_INTERNAL);
TRI_ASSERT(sysVocbase.get() == &vocbase || sysVocbase->name() == vocbase.name());
#endif
return false; // internal error
}
static std::string const LEGACY_ANALYZER_COLLECTION_NAME("_iresearch_analyzers");
auto sysVocbase = sysDatabase->use();
if (sysVocbase && sysVocbase->name() == vocbase.name()) { // upgrading system vocbase
static std::string const LEGACY_ANALYZER_COLLECTION_NAME("_iresearch_analyzers");
arangodb::methods::Collections::lookup( // find legacy analyzer collection
*sysVocbase, // vocbase to search
LEGACY_ANALYZER_COLLECTION_NAME, // collection name to search
[](std::shared_ptr<arangodb::LogicalCollection> const& col)->void { // callback if found
if (col) {
arangodb::methods::Collections::drop(*col, true, -1.0); // -1.0 same as in RestCollectionHandler
}
}
);
}
}
// register the text analyzers with the current vocbase
{
// NOTE: ArangoDB strings coming from JavaScript user input are UTF-8 encoded
static const std::vector<std::pair<irs::string_ref, irs::string_ref>> legacyAnalzyers = {
{ "text_de", "{ \"locale\": \"de.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_en", "{ \"locale\": \"en.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_es", "{ \"locale\": \"es.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_fi", "{ \"locale\": \"fi.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_fr", "{ \"locale\": \"fr.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_it", "{ \"locale\": \"it.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_nl", "{ \"locale\": \"nl.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_no", "{ \"locale\": \"no.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_pt", "{ \"locale\": \"pt.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_ru", "{ \"locale\": \"ru.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_sv", "{ \"locale\": \"sv.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
{ "text_zh", "{ \"locale\": \"zh.UTF-8\", \"ignored_words\": [ ] }" }, // empty stop word list
};
static const irs::flags legacyAnalyzerFeatures = { // add norms + frequency/position for by_phrase
irs::frequency::type(), // frequency feature
irs::norm::type(), // norm feature
irs::position::type(), // position feature
};
static const irs::string_ref legacyAnalyzerType("text");
bool success = true;
// register each legacy static analyzer with the current vocbase
for (auto& entry: legacyAnalzyers) {
auto name = normalizedAnalyzerName(vocbase.name(), entry.first);
auto& type = legacyAnalyzerType;
auto& properties = entry.second;
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
auto res = analyzers->emplace( // add analyzer
result, name, type, properties, legacyAnalyzerFeatures // args
);
if (!res.ok()) {
LOG_TOPIC("ec566", WARN, arangodb::iresearch::TOPIC)
<< "failure while registering a legacy static analyzer '" << name << "' with vocbase '" << vocbase.name() << "': " << res.errorNumber() << " " << res.errorMessage();
success = false;
} else if (!result.first) {
LOG_TOPIC("1dc1d", WARN, arangodb::iresearch::TOPIC)
<< "failure while registering a legacy static analyzer '" << name << "' with vocbase '" << vocbase.name() << "'";
success = false;
// find legacy analyzer collection
arangodb::Result dropRes;
auto const lookupRes = arangodb::methods::Collections::lookup(
vocbase,
LEGACY_ANALYZER_COLLECTION_NAME,
[&dropRes](std::shared_ptr<arangodb::LogicalCollection> const& col)->void { // callback if found
if (col) {
dropRes = arangodb::methods::Collections::drop(*col, true, -1.0); // -1.0 same as in RestCollectionHandler
}
}
);
return success;
if (lookupRes.ok()) {
return dropRes.ok();
}
return lookupRes.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND);
}
void registerUpgradeTasks() {
auto* upgrade = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::UpgradeFeature // feature type
>("Upgrade");
using namespace arangodb;
using namespace arangodb::application_features;
using namespace arangodb::methods;
auto* upgrade = ApplicationServer::lookupFeature<UpgradeFeature>("Upgrade");
if (!upgrade) {
return; // nothing to register with (OK if no tasks actually need to be applied)
}
// register legacy static analyzers with each vocbase found in DatabaseFeature
// required for backward compatibility for e.g. TOKENS(...) function
// NOTE: db-servers do not have a dedicated collection for storing analyzers,
// instead they get their cache populated from coordinators
{
arangodb::methods::Upgrade::Task task;
task.name = "IResearhAnalyzer legacy analyzers";
task.description = // description
"register legacy static analyzers with each vocbase found in DatabaseFeature";
task.systemFlag = arangodb::methods::Upgrade::Flags::DATABASE_ALL;
task.clusterFlags = // flags
arangodb::methods::Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL // any 1 single coordinator
| arangodb::methods::Upgrade::Flags::CLUSTER_NONE // local server
;
task.databaseFlags = arangodb::methods::Upgrade::Flags::DATABASE_UPGRADE;
task.action = &iresearchAnalyzerLegacyAnalyzers;
upgrade->addTask(std::move(task));
// FIXME TODO find out why CLUSTER_COORDINATOR_GLOBAL will only work with DATABASE_INIT (hardcoded in Upgrade::clusterBootstrap(...))
task.name = "IResearhAnalyzer legacy analyzers";
task.description =
"register legacy static analyzers with each vocbase found in DatabaseFeature";
task.systemFlag = arangodb::methods::Upgrade::Flags::DATABASE_ALL;
task.clusterFlags = // flags
arangodb::methods::Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL; // any 1 single coordinator
task.databaseFlags = arangodb::methods::Upgrade::Flags::DATABASE_INIT;
task.action = &iresearchAnalyzerLegacyAnalyzers;
upgrade->addTask(std::move(task));
}
upgrade->addTask({
"setupAnalyzers", // name
"setup _analyzers collection", // description
Upgrade::Flags::DATABASE_ALL, // system flags
Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL // cluster flags
| Upgrade::Flags::CLUSTER_NONE,
Upgrade::Flags::DATABASE_INIT // database flags
| Upgrade::Flags::DATABASE_UPGRADE
| Upgrade::Flags::DATABASE_EXISTING,
&setupAnalyzersCollection // action
});
upgrade->addTask({
"dropLegacyAnalyzersCollection", // name
"drop _iresearch_analyzers collection", // description
Upgrade::Flags::DATABASE_SYSTEM, // system flags
Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL // cluster flags
| Upgrade::Flags::CLUSTER_NONE,
Upgrade::Flags::DATABASE_INIT // database flags
| Upgrade::Flags::DATABASE_UPGRADE,
&dropLegacyAnalyzersCollection // action
});
}
////////////////////////////////////////////////////////////////////////////////
@ -477,16 +431,13 @@ std::pair<irs::string_ref, irs::string_ref> splitAnalyzerName( // split name
// search for vocbase prefix ending with '::'
for (size_t i = 1, count = analyzer.size(); i < count; ++i) {
if (analyzer[i] == ANALYZER_PREFIX_DELIM // current is delim
&& analyzer[i - 1] == ANALYZER_PREFIX_DELIM // previous is also delim
) {
&& analyzer[i - 1] == ANALYZER_PREFIX_DELIM) { // previous is also delim
auto vocbase = i > 1 // non-empty prefix, +1 for first delimiter char
? irs::string_ref(analyzer.c_str(), i - 1) // -1 for the first ':' delimiter
: irs::string_ref::EMPTY
;
: irs::string_ref::EMPTY;
auto name = i < count - 1 // have suffix
? irs::string_ref(analyzer.c_str() + i + 1, count - i - 1) // +-1 for the suffix after '::'
: irs::string_ref::EMPTY // do not point after end of buffer
;
: irs::string_ref::EMPTY; // do not point after end of buffer
return std::make_pair(vocbase, name); // prefixed analyzer name
}
@ -1258,19 +1209,6 @@ IResearchAnalyzerFeature::AnalyzerPool::ptr IResearchAnalyzerFeature::get( // fi
analyzers.emplace(irs::make_hashed_ref(name, std::hash<irs::string_ref>()), pool);
}
auto* databaseFeature =
application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
"Database");
// check if DB is currently being upgraded (load all legacy built-in analyzers)
bool const inUpgrade = databaseFeature
? (databaseFeature->upgrade() || databaseFeature->checkVersion())
: false;
if (!inUpgrade) {
return;
}
// register the text analyzers
{
// Note: ArangoDB strings coming from JavaScript user input are UTF-8 encoded

133
arangod/IResearch/IResearchFeature.cpp
View File

@ -38,6 +38,7 @@
#include "Basics/SmallVector.h"
#include "Cluster/ClusterInfo.h"
#include "Cluster/ServerState.h"
#include "Cluster/ClusterFeature.h"
#include "Containers.h"
#include "IResearchCommon.h"
#include "IResearchFeature.h"
@ -166,39 +167,53 @@ size_t computeThreadPoolSize(size_t threads, size_t threadsLimit) {
size_t(std::thread::hardware_concurrency()) / 4));
}
bool iresearchViewUpgradeVersion0_1(TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& upgradeParams) {
std::vector<std::shared_ptr<arangodb::LogicalView>> views;
bool upgradeSingleServerArangoSearchView0_1(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& /*upgradeParams*/) {
using arangodb::application_features::ApplicationServer;
if (arangodb::ServerState::instance()->isCoordinator()) {
auto* ci = arangodb::ClusterInfo::instance();
// NOTE: during the upgrade 'ClusterFeature' is disabled which means 'ClusterFeature::validateOptions(...)'
// hasn't been called and server role in 'ServerState' is not set properly.
// In order to upgrade ArangoSearch views from version 0 to version 1 we need to
// differentiate between single server and cluster, therefore we temporary set role in 'ServerState',
// actually supplied by a user, only for the duration of task to avoid other upgrade tasks, that
// potentially rely on the original behaviour, to be affected.
struct ServerRoleGuard {
ServerRoleGuard() {
auto const* clusterFeature = ApplicationServer::lookupFeature<arangodb::ClusterFeature>("Cluster");
auto* state = arangodb::ServerState::instance();
if (!ci) {
LOG_TOPIC("1804b", WARN, arangodb::iresearch::TOPIC)
<< "failure to find 'ClusterInfo' instance while upgrading "
"IResearchView from version 0 to version 1";
if (state && clusterFeature && !clusterFeature->isEnabled()) {
auto const role = arangodb::ServerState::stringToRole(clusterFeature->myRole());
return false; // internal error
// only for cluster
if (arangodb::ServerState::isClusterRole(role)) {
_originalRole = state->getRole();
state->setRole(role);
_state = state;
}
}
}
views = ci->getViews(vocbase.name());
} else if (arangodb::ServerState::instance()->isSingleServer() ||
arangodb::ServerState::instance()->isDBServer()) {
views = vocbase.views();
} else {
~ServerRoleGuard() {
if (_state) {
// restore the original server role
_state->setRole(_originalRole);
}
}
arangodb::ServerState* _state{};
arangodb::ServerState::RoleEnum _originalRole{arangodb::ServerState::ROLE_UNDEFINED};
} guard;
if (!arangodb::ServerState::instance()->isSingleServer() &&
!arangodb::ServerState::instance()->isDBServer()) {
return true; // not applicable for other ServerState roles
}
for (auto& view : views) {
if (arangodb::ServerState::instance()->isCoordinator()) {
if (!arangodb::LogicalView::cast<arangodb::iresearch::IResearchViewCoordinator>(
view.get())) {
continue; // not an IResearchViewCoordinator
}
} else { // single-server and db-server
if (!arangodb::LogicalView::cast<arangodb::iresearch::IResearchView>(view.get())) {
continue; // not an IResearchView
}
for (auto& view : vocbase.views()) {
if (!arangodb::LogicalView::cast<arangodb::iresearch::IResearchView>(view.get())) {
continue; // not an IResearchView
}
arangodb::velocypack::Builder builder;
@ -227,7 +242,7 @@ bool iresearchViewUpgradeVersion0_1(TRI_vocbase_t& vocbase,
return false; // required field is missing
}
auto version = versionSlice.getNumber<uint32_t>();
auto const version = versionSlice.getNumber<uint32_t>();
if (0 != version) {
continue; // no upgrade required
@ -248,33 +263,28 @@ bool iresearchViewUpgradeVersion0_1(TRI_vocbase_t& vocbase,
irs::utf8_path dataPath;
if (arangodb::ServerState::instance()->isSingleServer() ||
arangodb::ServerState::instance()->isDBServer()) {
auto* dbPathFeature =
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>(
"DatabasePath");
auto* dbPathFeature = ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
if (!dbPathFeature) {
LOG_TOPIC("67c7e", WARN, arangodb::iresearch::TOPIC)
<< "failure to find feature 'DatabasePath' while upgrading "
"IResearchView from version 0 to version 1";
if (!dbPathFeature) {
LOG_TOPIC("67c7e", WARN, arangodb::iresearch::TOPIC)
<< "failure to find feature 'DatabasePath' while upgrading "
"IResearchView from version 0 to version 1";
return false; // required feature is missing
}
// original algorithm for computing data-store path
static const std::string subPath("databases");
static const std::string dbPath("database-");
dataPath = irs::utf8_path(dbPathFeature->directory());
dataPath /= subPath;
dataPath /= dbPath;
dataPath += std::to_string(vocbase.id());
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
dataPath += "-";
dataPath += std::to_string(view->id());
return false; // required feature is missing
}
// original algorithm for computing data-store path
static const std::string subPath("databases");
static const std::string dbPath("database-");
dataPath = irs::utf8_path(dbPathFeature->directory());
dataPath /= subPath;
dataPath /= dbPath;
dataPath += std::to_string(vocbase.id());
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
dataPath += "-";
dataPath += std::to_string(view->id());
res = view->drop(); // drop view (including all links)
if (!res.ok()) {
@ -541,27 +551,14 @@ void registerUpgradeTasks() {
{
arangodb::methods::Upgrade::Task task;
task.name = "IResearhView version 0->1";
task.description =
"move IResearch data-store from IResearchView to IResearchLink";
task.name = "upgradeArangoSearch0_1";
task.description = "store ArangoSearch index on per linked collection basis";
task.systemFlag = arangodb::methods::Upgrade::Flags::DATABASE_ALL;
task.clusterFlags = arangodb::methods::Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL // any 1 single coordinator
| arangodb::methods::Upgrade::Flags::CLUSTER_DB_SERVER_LOCAL // db-server
| arangodb::methods::Upgrade::Flags::CLUSTER_LOCAL // any cluster node (filtred out in task) FIXME TODO without this db-server never ges invoked
| arangodb::methods::Upgrade::Flags::CLUSTER_NONE // local server
;
task.clusterFlags = arangodb::methods::Upgrade::Flags::CLUSTER_DB_SERVER_LOCAL // db-server
| arangodb::methods::Upgrade::Flags::CLUSTER_NONE // local server
| arangodb::methods::Upgrade::Flags::CLUSTER_LOCAL;
task.databaseFlags = arangodb::methods::Upgrade::Flags::DATABASE_UPGRADE;
task.action = &iresearchViewUpgradeVersion0_1;
upgrade->addTask(std::move(task));
// FIXME TODO find out why CLUSTER_COORDINATOR_GLOBAL will only work with DATABASE_INIT (hardcoded in Upgrade::clusterBootstrap(...))
task.name = "IResearhView version 0->1";
task.description =
"move IResearch data-store from IResearchView to IResearchLink";
task.systemFlag = arangodb::methods::Upgrade::Flags::DATABASE_ALL;
task.clusterFlags = arangodb::methods::Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL; // any 1 single coordinator
task.databaseFlags = arangodb::methods::Upgrade::Flags::DATABASE_INIT;
task.action = &iresearchViewUpgradeVersion0_1;
task.action = &upgradeSingleServerArangoSearchView0_1;
upgrade->addTask(std::move(task));
}
}

12
arangod/IResearch/IResearchLink.cpp
View File

@ -731,18 +731,6 @@ bool IResearchLink::hasSelectivityEstimate() const {
arangodb::Result IResearchLink::init(
arangodb::velocypack::Slice const& definition,
InitCallback const& initCallback /* = { }*/ ) {
auto* databaseFeature =
arangodb::application_features::ApplicationServer::getFeature<arangodb::DatabaseFeature>(
"Database");
bool checkVersion = databaseFeature->checkVersion();
if (checkVersion) {
LOG_TOPIC("3541f", FATAL, arangodb::iresearch::TOPIC)
<< "Upgrading views is not supported in 3.5RC1, please drop all the existing views and manually recreate them after the upgrade is complete";
FATAL_ERROR_EXIT();
}
// disassociate from view if it has not been done yet
if (!unload().ok()) {
return arangodb::Result(TRI_ERROR_INTERNAL, "failed to unload link");

20
arangod/IResearch/IResearchLinkHelper.cpp
View File

@ -36,6 +36,7 @@
#include "Logger/Logger.h"
#include "Logger/LogMacros.h"
#include "RestServer/SystemDatabaseFeature.h"
#include "RestServer/DatabaseFeature.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "StorageEngine/StorageEngine.h"
#include "Transaction/Methods.h"
@ -118,6 +119,23 @@ arangodb::Result createLink( // create link
std::string("failed to create link between arangosearch view '") + view.name() + "' and collection '" + collection.name() + "'"
);
}
// ensure link is synchronized after upgrade in single-server
if (arangodb::ServerState::instance()->isSingleServer()) {
auto* db = arangodb::DatabaseFeature::DATABASE;
if (db && (db->checkVersion() || db->upgrade())) {
// FIXME find a better way to retrieve an IResearch Link
// cannot use static_cast/reinterpret_cast since Index is not related to
// IResearchLink
auto impl = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(link);
if (impl) {
return impl->commit();
}
}
}
} catch (arangodb::basics::Exception const& e) {
return arangodb::Result(e.code(), e.what());
}
@ -853,4 +871,4 @@ namespace iresearch {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

16
arangod/IResearch/IResearchView.cpp
View File

@ -241,27 +241,11 @@ struct IResearchView::ViewFactory : public arangodb::ViewFactory {
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& definition,
uint64_t planVersion) const override {
auto* databaseFeature =
application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
"Database");
// check if DB is currently being upgraded (skip validation checks)
bool const inUpgrade = databaseFeature
? (databaseFeature->upgrade() || databaseFeature->checkVersion())
: false;
if (inUpgrade) {
LOG_TOPIC("3541e", FATAL, arangodb::iresearch::TOPIC)
<< "Upgrading views is not supported in 3.5RC2, please drop all the existing views and manually recreate them after the upgrade is complete";
FATAL_ERROR_EXIT();
}
std::string error;
IResearchViewMeta meta;
IResearchViewMetaState metaState;
if (!meta.init(definition, error) // parse definition
|| (meta._version == 0 && !inUpgrade) // version 0 must be upgraded to split data-store on a per-link basis
|| meta._version > LATEST_VERSION // ensure version is valid
|| (ServerState::instance()->isSingleServer() // init metaState for SingleServer
&& !metaState.init(definition, error))) {

43
arangod/VocBase/Methods/Collections.cpp
View File

@ -377,6 +377,49 @@ void Collections::enumerate(TRI_vocbase_t* vocbase,
return TRI_ERROR_NO_ERROR;
}
/*static*/ Result Collections::createSystem(
TRI_vocbase_t& vocbase,
std::string const& name) {
FuncCallback const noop = [](std::shared_ptr<LogicalCollection> const&)->void{};
auto res = methods::Collections::lookup(vocbase, name, noop);
if (res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) {
uint32_t defaultReplFactor = 1;
auto* cl = application_features::ApplicationServer::lookupFeature<ClusterFeature>("Cluster");
if (cl != nullptr) {
defaultReplFactor = cl->systemReplicationFactor();
}
VPackBuilder bb;
{
VPackObjectBuilder scope(&bb);
bb.add("isSystem", VPackSlice::trueSlice());
bb.add("waitForSync", VPackSlice::falseSlice());
bb.add("journalSize", VPackValue(1024 * 1024));
bb.add("replicationFactor", VPackValue(defaultReplFactor));
if (name != "_graphs") {
// that forces all collections to be on the same physical DBserver
bb.add("distributeShardsLike", VPackValue("_graphs"));
}
}
res = Collections::create(
vocbase, // vocbase to create in
name, // collection name top create
TRI_COL_TYPE_DOCUMENT, // collection type to create
bb.slice(), // collection definition to create
true, // waitsForSyncReplication
true, // enforceReplicationFactor
noop); // callback
}
return res;
}
Result Collections::load(TRI_vocbase_t& vocbase, LogicalCollection* coll) {
TRI_ASSERT(coll != nullptr);

2
arangod/VocBase/Methods/Collections.h
View File

@ -88,6 +88,8 @@ struct Collections {
FuncCallback callback // invoke on collection creation
);
static Result createSystem(TRI_vocbase_t& vocbase, std::string const& name);
static Result load(TRI_vocbase_t& vocbase, LogicalCollection* coll);
static Result unload(TRI_vocbase_t* vocbase, LogicalCollection* coll);

62
arangod/VocBase/Methods/UpgradeTasks.cpp
View File

@ -62,49 +62,12 @@ using basics::VelocyPackHelper;
namespace {
/// create a collection if it does not exists.
bool createSystemCollection(TRI_vocbase_t& vocbase, std::string const& name) {
auto res = methods::Collections::lookup(
vocbase, // vocbase to search
name, // collection to find
[](std::shared_ptr<LogicalCollection> const&) -> void {});
if (res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) {
uint32_t defaultReplFactor = 1;
ClusterFeature* cl =
ApplicationServer::getFeature<ClusterFeature>("Cluster");
if (cl != nullptr) {
defaultReplFactor = cl->systemReplicationFactor();
}
VPackBuilder bb;
bb.openObject();
bb.add("isSystem", VPackSlice::trueSlice());
bb.add("waitForSync", VPackSlice::falseSlice());
bb.add("journalSize", VPackValue(1024 * 1024));
bb.add("replicationFactor", VPackValue(defaultReplFactor));
if (name != "_graphs") {
bb.add("distributeShardsLike", VPackValue("_graphs"));
}
bb.close();
res = Collections::create(
vocbase, // vocbase to create in
name, // collection name top create
TRI_COL_TYPE_DOCUMENT, // collection type to create
bb.slice(), // collection definition to create
/*waitsForSyncReplication*/ true,
/*enforceReplicationFactor*/ true,
[](std::shared_ptr<LogicalCollection> const&) -> void {});
}
void createSystemCollection(TRI_vocbase_t& vocbase, std::string const& name) {
auto const res = methods::Collections::createSystem(vocbase, name);
if (res.fail()) {
THROW_ARANGO_EXCEPTION(res);
}
return true;
}
/// create an index if it does not exist
@ -203,12 +166,14 @@ bool UpgradeTasks::upgradeGeoIndexes(TRI_vocbase_t& vocbase,
bool UpgradeTasks::setupGraphs(TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_graphs");
::createSystemCollection(vocbase, "_graphs"); // throws on error
return true;
}
bool UpgradeTasks::setupUsers(TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_users");
::createSystemCollection(vocbase, "_users"); // throws on error
return true;
}
bool UpgradeTasks::createUsersIndex(TRI_vocbase_t& vocbase,
@ -278,16 +243,19 @@ bool UpgradeTasks::addDefaultUserOther(TRI_vocbase_t& vocbase,
bool UpgradeTasks::setupAqlFunctions(TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_aqlfunctions");
::createSystemCollection(vocbase, "_aqlfunctions"); // throws on error
return true;
}
bool UpgradeTasks::setupQueues(TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_queues");
::createSystemCollection(vocbase, "_queues"); // throws on error
return true;
}
bool UpgradeTasks::setupJobs(TRI_vocbase_t& vocbase, arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_jobs");
::createSystemCollection(vocbase, "_jobs"); // throws on error
return true;
}
bool UpgradeTasks::createJobsIndex(TRI_vocbase_t& vocbase,
@ -312,7 +280,8 @@ bool UpgradeTasks::createJobsIndex(TRI_vocbase_t& vocbase,
}
bool UpgradeTasks::setupApps(TRI_vocbase_t& vocbase, arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_apps");
::createSystemCollection(vocbase, "_apps"); // throws on error
return true;
}
bool UpgradeTasks::createAppsIndex(TRI_vocbase_t& vocbase,
@ -328,7 +297,8 @@ bool UpgradeTasks::createAppsIndex(TRI_vocbase_t& vocbase,
bool UpgradeTasks::setupAppBundles(TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& slice) {
return ::createSystemCollection(vocbase, "_appbundles");
::createSystemCollection(vocbase, "_appbundles"); // throws on error
return true;
}
bool UpgradeTasks::persistLocalDocumentIds(TRI_vocbase_t& vocbase,

18
arangosh/Shell/V8ClientConnection.cpp
View File

@ -98,7 +98,7 @@ void V8ClientConnection::createConnection() {
}
if (_lastHttpReturnCode == 200) {
std::atomic_store<fuerte::Connection>(&_connection, newConnection);
std::atomic_store(&_connection, newConnection);
std::shared_ptr<VPackBuilder> parsedBody;
VPackSlice body;
@ -158,7 +158,7 @@ void V8ClientConnection::createConnection() {
}
void V8ClientConnection::setInterrupted(bool interrupted) {
auto connection = std::atomic_load<fuerte::Connection>(&_connection);
auto connection = std::atomic_load(&_connection);
if (interrupted && connection != nullptr) {
shutdownConnection();
} else if (!interrupted && connection == nullptr) {
@ -167,7 +167,7 @@ void V8ClientConnection::setInterrupted(bool interrupted) {
}
bool V8ClientConnection::isConnected() const {
auto connection = std::atomic_load<fuerte::Connection>(&_connection);
auto connection = std::atomic_load(&_connection);
if (connection) {
return connection->state() == fuerte::Connection::State::Connected;
}
@ -175,7 +175,7 @@ bool V8ClientConnection::isConnected() const {
}
std::string V8ClientConnection::endpointSpecification() const {
auto connection = std::atomic_load<fuerte::Connection>(&_connection);
auto connection = std::atomic_load(&_connection);
if (connection) {
return connection->endpoint();
}
@ -225,7 +225,7 @@ void V8ClientConnection::reconnect(ClientFeature* client) {
_builder.authenticationType(fuerte::AuthenticationType::Basic);
}
auto oldConnection = std::atomic_exchange<fuerte::Connection>(&_connection, std::shared_ptr<fuerte::Connection>());
auto oldConnection = std::atomic_exchange(&_connection, std::shared_ptr<fuerte::Connection>());
if (oldConnection) {
oldConnection->cancel();
}
@ -1481,7 +1481,7 @@ v8::Local<v8::Value> V8ClientConnection::requestData(
}
req->timeout(std::chrono::duration_cast<std::chrono::milliseconds>(_requestTimeout));
auto connection = std::atomic_load<fuerte::Connection>(&_connection);
auto connection = std::atomic_load(&_connection);
if (!connection) {
TRI_V8_SET_EXCEPTION_MESSAGE(TRI_SIMPLE_CLIENT_COULD_NOT_CONNECT,
"not connected");
@ -1540,7 +1540,7 @@ v8::Local<v8::Value> V8ClientConnection::requestDataRaw(
}
req->timeout(std::chrono::duration_cast<std::chrono::milliseconds>(_requestTimeout));
auto connection = std::atomic_load<fuerte::Connection>(&_connection);
auto connection = std::atomic_load(&_connection);
if (!connection) {
TRI_V8_SET_EXCEPTION_MESSAGE(TRI_SIMPLE_CLIENT_COULD_NOT_CONNECT,
"not connected");
@ -1823,9 +1823,9 @@ void V8ClientConnection::initServer(v8::Isolate* isolate, v8::Local<v8::Context>
}
void V8ClientConnection::shutdownConnection() {
auto connection = std::atomic_load<fuerte::Connection>(&_connection);
auto connection = std::atomic_load(&_connection);
if (connection) {
connection->cancel();
std::atomic_store<fuerte::Connection>(&_connection, std::shared_ptr<fuerte::Connection>());
std::atomic_store(&_connection, std::shared_ptr<fuerte::Connection>());
}
}

12
lib/Futures/Future.h
View File

@ -326,7 +326,7 @@ class Future {
}));
}
});
return std::move(future);
return future;
}
/// Variant: callable accepts T&&, returns future
@ -358,7 +358,7 @@ class Future {
}
}
});
return std::move(future);
return future;
}
/// Variant: callable accepts Try<T&&>, returns value
@ -380,7 +380,7 @@ class Future {
return futures::invoke(std::forward<DF>(fn), std::move(t));
}));
});
return std::move(future);
return future;
}
/// Variant: callable accepts Try<T&&>, returns future
@ -404,7 +404,7 @@ class Future {
pr.setException(std::current_exception());
}
});
return std::move(future);
return future;
}
/// Variant: function returns void and accepts Try<T>&&
@ -442,7 +442,7 @@ class Future {
pr.setTry(std::move(t));
}
});
return std::move(future);
return future;
}
/// Set an error continuation for this Future where the continuation can
@ -476,7 +476,7 @@ class Future {
pr.setTry(std::move(t));
}
});
return std::move(future);
return future;
}
private:

1749
tests/IResearch/IResearchAnalyzerFeature-test.cpp
View File

File diff suppressed because it is too large Load Diff

957
tests/IResearch/IResearchFeature-test.cpp
View File

@ -177,484 +177,485 @@ TEST_F(IResearchFeatureTest, test_start) {
};
}
//FIXME uncomment
//TEST_F(IResearchFeatureTest, test_upgrade0_1) {
// // version 0 data-source path
// auto getPersistedPath0 = [](arangodb::LogicalView const& view)->irs::utf8_path {
// auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
// EXPECT_TRUE(dbPathFeature);
// irs::utf8_path dataPath(dbPathFeature->directory());
// dataPath /= "databases";
// dataPath /= "database-";
// dataPath += std::to_string(view.vocbase().id());
// dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
// dataPath += "-";
// dataPath += std::to_string(view.id());
// return dataPath;
// };
//
// // version 1 data-source path
// auto getPersistedPath1 = [](arangodb::iresearch::IResearchLink const& link)->irs::utf8_path {
// auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
// EXPECT_TRUE(dbPathFeature);
// irs::utf8_path dataPath(dbPathFeature->directory());
// dataPath /= "databases";
// dataPath /= "database-";
// dataPath += std::to_string(link.collection().vocbase().id());
// dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
// dataPath += "-";
// dataPath += std::to_string(link.collection().id());
// dataPath += "_";
// dataPath += std::to_string(link.id());
// return dataPath;
// };
//
// // test single-server (no directory)
// {
// auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
// auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
// auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
// auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
//
// // create a new instance of an ApplicationServer and fill it with the required features
// // cannot use the existing server since its features already have some state
// std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
// arangodb::application_features::ApplicationServer::server,
// [](arangodb::application_features::ApplicationServer* ptr)->void {
// arangodb::application_features::ApplicationServer::server = ptr;
// }
// );
// arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
// arangodb::application_features::ApplicationServer server(nullptr, nullptr);
// arangodb::iresearch::IResearchFeature feature(server);
// arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
// arangodb::DatabasePathFeature* dbPathFeature;
// server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
// server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
// server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
// server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
// TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
// server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchAnalyzerFeature::start()
// server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
// server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
// analyzerFeature->prepare(); // add static analyzers
// feature.prepare(); // register iresearch view type
// feature.start(); // register upgrade tasks
// server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
//
// arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
// auto versionFilename = StorageEngineMock::versionFilenameResult;
// auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
// StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
// ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
// ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
//
// TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
// auto logicalCollection = vocbase.createCollection(collectionJson->slice());
// ASSERT_TRUE((false == !logicalCollection));
// auto logicalView0 = vocbase.createView(viewJson->slice());
// ASSERT_TRUE((false == !logicalView0));
// bool created;
// auto index = logicalCollection->createIndex(linkJson->slice(), created);
// ASSERT_TRUE((created));
// ASSERT_TRUE((false == !index));
// auto link0 = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
// ASSERT_TRUE((false == !link0));
//
// index->unload(); // release file handles
// bool result;
// auto linkDataPath = getPersistedPath1(*link0);
// EXPECT_TRUE((linkDataPath.remove())); // remove link directory
// auto viewDataPath = getPersistedPath0(*logicalView0);
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure no view directory
// arangodb::velocypack::Builder builder;
// builder.openObject();
// EXPECT_TRUE((logicalView0->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
//
// EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
// auto logicalView1 = vocbase.lookupView(logicalView0->name());
// EXPECT_TRUE((false == !logicalView1)); // ensure view present after upgrade
// EXPECT_TRUE((logicalView0->id() == logicalView1->id())); // ensure same id for view
// auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
// EXPECT_TRUE((false == !link1)); // ensure link present after upgrade
// EXPECT_TRUE((link0->id() != link1->id())); // ensure new link
// linkDataPath = getPersistedPath1(*link1);
// EXPECT_TRUE((linkDataPath.exists(result) && result)); // ensure link directory created after upgrade
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not present
// viewDataPath = getPersistedPath0(*logicalView1);
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not created
// builder.clear();
// builder.openObject();
// EXPECT_TRUE((logicalView1->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
// }
//
// // test single-server (with directory)
// {
// auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
// auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
// auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
// auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
//
// // create a new instance of an ApplicationServer and fill it with the required features
// // cannot use the existing server since its features already have some state
// std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
// arangodb::application_features::ApplicationServer::server,
// [](arangodb::application_features::ApplicationServer* ptr)->void {
// arangodb::application_features::ApplicationServer::server = ptr;
// }
// );
// arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
// arangodb::application_features::ApplicationServer server(nullptr, nullptr);
// arangodb::iresearch::IResearchFeature feature(server);
// arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
// arangodb::DatabasePathFeature* dbPathFeature;
// server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
// server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
// server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
// server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
// TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
// server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchLinkHelper::normalize(...)
// server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
// server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
// analyzerFeature->prepare(); // add static analyzers
// feature.prepare(); // register iresearch view type
// feature.start(); // register upgrade tasks
// server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
//
// arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
// auto versionFilename = StorageEngineMock::versionFilenameResult;
// auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
// StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
// ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
// ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
//
// TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
// auto logicalCollection = vocbase.createCollection(collectionJson->slice());
// ASSERT_TRUE((false == !logicalCollection));
// auto logicalView0 = vocbase.createView(viewJson->slice());
// ASSERT_TRUE((false == !logicalView0));
// bool created;
// auto index = logicalCollection->createIndex(linkJson->slice(), created);
// ASSERT_TRUE((created));
// ASSERT_TRUE((false == !index));
// auto link0 = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
// ASSERT_TRUE((false == !link0));
//
// index->unload(); // release file handles
// bool result;
// auto linkDataPath = getPersistedPath1(*link0);
// EXPECT_TRUE((linkDataPath.remove())); // remove link directory
// auto viewDataPath = getPersistedPath0(*logicalView0);
// EXPECT_TRUE((viewDataPath.exists(result) && !result));
// EXPECT_TRUE((viewDataPath.mkdir())); // create view directory
// EXPECT_TRUE((viewDataPath.exists(result) && result));
// arangodb::velocypack::Builder builder;
// builder.openObject();
// EXPECT_TRUE((logicalView0->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
//
// EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
// auto logicalView1 = vocbase.lookupView(logicalView0->name());
// EXPECT_TRUE((false == !logicalView1)); // ensure view present after upgrade
// EXPECT_TRUE((logicalView0->id() == logicalView1->id())); // ensure same id for view
// auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
// EXPECT_TRUE((false == !link1)); // ensure link present after upgrade
// EXPECT_TRUE((link0->id() != link1->id())); // ensure new link
// linkDataPath = getPersistedPath1(*link1);
// EXPECT_TRUE((linkDataPath.exists(result) && result)); // ensure link directory created after upgrade
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory removed after upgrade
// viewDataPath = getPersistedPath0(*logicalView1);
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not created
// builder.clear();
// builder.openObject();
// EXPECT_TRUE((logicalView1->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
//
// server.getFeature<arangodb::DatabaseFeature>("Database")->unprepare();
// }
//
// // test coordinator
// {
// auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"id\": \"1\", \"name\": \"testCollection\", \"shards\":{} }");
// auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
// auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"id\": 42, \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
// auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
// auto collectionId = std::to_string(1);
// auto viewId = std::to_string(42);
//
// auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
// arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
// auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
//
// // create a new instance of an ApplicationServer and fill it with the required features
// // cannot use the existing server since its features already have some state
// std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
// arangodb::application_features::ApplicationServer::server,
// [](arangodb::application_features::ApplicationServer* ptr)->void {
// arangodb::application_features::ApplicationServer::server = ptr;
// }
// );
// arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
// arangodb::application_features::ApplicationServer server(nullptr, nullptr);
// arangodb::DatabaseFeature* database;
// arangodb::iresearch::IResearchFeature feature(server);
// arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
// server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
// TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
// server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterComm::instance()
// server.addFeature(new arangodb::ClusterFeature(server)); // required to create ClusterInfo instance
// server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
// server.addFeature(database = new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
// server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
// server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
// server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchLinkHelper::normalize(...)
// server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
// server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
//
// #if USE_ENTERPRISE
// server.addFeature(new arangodb::LdapFeature(server)); // required for AuthenticationFeature with USE_ENTERPRISE
// #endif
//
// analyzerFeature->prepare(); // add static analyzers
// feature.prepare(); // register iresearch view type
// feature.start(); // register upgrade tasks
// server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
// server.getFeature<arangodb::ClusterFeature>("Cluster")->prepare(); // create ClusterInfo instance
// server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
// server.getFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
// arangodb::AgencyCommManager::MANAGER->start(); // initialize agency
// arangodb::DatabaseFeature::DATABASE = database; // required for ClusterInfo::createCollectionCoordinator(...)
// const_cast<arangodb::IndexFactory&>(engine.indexFactory()).emplace( // required for Indexes::ensureIndex(...)
// arangodb::iresearch::DATA_SOURCE_TYPE.name(),
// arangodb::iresearch::IResearchLinkCoordinator::factory()
// );
// auto* ci = arangodb::ClusterInfo::instance();
// ASSERT_TRUE((nullptr != ci));
// TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature
//
// ASSERT_TRUE((TRI_ERROR_NO_ERROR == database->createDatabase(1, "testDatabase", vocbase)));
// ASSERT_TRUE((ci->createDatabaseCoordinator(vocbase->name(), arangodb::velocypack::Slice::emptyObjectSlice(), 0.0).ok()));
// ASSERT_TRUE((ci->createCollectionCoordinator(vocbase->name(), collectionId, 0, 1, false, collectionJson->slice(), 0.0).ok()));
// auto logicalCollection = ci->getCollection(vocbase->name(), collectionId);
// ASSERT_TRUE((false == !logicalCollection));
// EXPECT_TRUE((ci->createViewCoordinator(vocbase->name(), viewId, viewJson->slice()).ok()));
// auto logicalView0 = ci->getView(vocbase->name(), viewId);
// ASSERT_TRUE((false == !logicalView0));
//
// // simulate heartbeat thread (create index in current)
// {
// auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id());
// auto const value = arangodb::velocypack::Parser::fromJson("{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"1\" } ] } }");
// EXPECT_TRUE(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful());
// }
// arangodb::velocypack::Builder tmp;
// ASSERT_TRUE((arangodb::methods::Indexes::ensureIndex(logicalCollection.get(), linkJson->slice(), true, tmp).ok()));
// logicalCollection = ci->getCollection(vocbase->name(), collectionId);
// ASSERT_TRUE((false == !logicalCollection));
// auto link0 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView0);
// ASSERT_TRUE((false == !link0));
//
// arangodb::velocypack::Builder builder;
// builder.openObject();
// EXPECT_TRUE((logicalView0->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
//
// // simulate heartbeat thread (create index in current)
// {
// auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id());
// auto const value = arangodb::velocypack::Parser::fromJson("{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"2\" } ] } }");
// EXPECT_TRUE(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful());
// }
// EXPECT_TRUE((arangodb::methods::Upgrade::clusterBootstrap(*vocbase).ok())); // run upgrade
// auto logicalCollection2 = ci->getCollection(vocbase->name(), collectionId);
// ASSERT_TRUE((false == !logicalCollection2));
// auto logicalView1 = ci->getView(vocbase->name(), viewId);
// EXPECT_TRUE((false == !logicalView1)); // ensure view present after upgrade
// EXPECT_TRUE((logicalView0->id() == logicalView1->id())); // ensure same id for view
// auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection2, *logicalView1);
// EXPECT_TRUE((false == !link1)); // ensure link present after upgrade
// EXPECT_TRUE((link0->id() != link1->id())); // ensure new link
// builder.clear();
// builder.openObject();
// EXPECT_TRUE((logicalView1->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
//
// server.getFeature<arangodb::DatabaseFeature>("Database")->unprepare();
// }
//
// // test db-server (no directory)
// {
// auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
// auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
// auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
// auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
//
// auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
// arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
// auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
//
// // create a new instance of an ApplicationServer and fill it with the required features
// // cannot use the existing server since its features already have some state
// std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
// arangodb::application_features::ApplicationServer::server,
// [](arangodb::application_features::ApplicationServer* ptr)->void {
// arangodb::application_features::ApplicationServer::server = ptr;
// }
// );
// arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
// arangodb::application_features::ApplicationServer server(nullptr, nullptr);
// arangodb::iresearch::IResearchFeature feature(server);
// arangodb::DatabasePathFeature* dbPathFeature;
// arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
// server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
// server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
// server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
// server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
// server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
// server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
// server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
// server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
// server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
// analyzerFeature->prepare(); // add static analyzers
// feature.prepare(); // register iresearch view type
// feature.start(); // register upgrade tasks
// server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
// server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
// server.getFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
//
// arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
// auto versionFilename = StorageEngineMock::versionFilenameResult;
// auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
// StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
// ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
// ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
//
// TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
// auto logicalCollection = vocbase.createCollection(collectionJson->slice());
// ASSERT_TRUE((false == !logicalCollection));
// auto logicalView = vocbase.createView(viewJson->slice());
// ASSERT_TRUE((false == !logicalView));
// auto* view = dynamic_cast<arangodb::iresearch::IResearchView*>(logicalView.get());
// ASSERT_TRUE((false == !view));
// bool created;
// auto index = logicalCollection->createIndex(linkJson->slice(), created);
// ASSERT_TRUE((created));
// ASSERT_TRUE((false == !index));
// auto link = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
// ASSERT_TRUE((false == !link));
// ASSERT_TRUE((view->link(link->self()).ok())); // link will not notify view in 'vocbase', hence notify manually
//
// index->unload(); // release file handles
// bool result;
// auto linkDataPath = getPersistedPath1(*link);
// EXPECT_TRUE((linkDataPath.remove())); // remove link directory
// auto viewDataPath = getPersistedPath0(*logicalView);
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure no view directory
// arangodb::velocypack::Builder builder;
// builder.openObject();
// EXPECT_TRUE((logicalView->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
//
// EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
// logicalView = vocbase.lookupView(logicalView->name());
// EXPECT_TRUE((true == !logicalView)); // ensure view removed after upgrade
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not present
//
// server.getFeature<arangodb::DatabaseFeature>("Database")->unprepare();
// }
//
// // test db-server (with directory)
// {
// auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
// auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
// auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
// auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
//
// auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
// arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
// auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
//
// // create a new instance of an ApplicationServer and fill it with the required features
// // cannot use the existing server since its features already have some state
// std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
// arangodb::application_features::ApplicationServer::server,
// [](arangodb::application_features::ApplicationServer* ptr)->void {
// arangodb::application_features::ApplicationServer::server = ptr;
// }
// );
// arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
// arangodb::application_features::ApplicationServer server(nullptr, nullptr);
// arangodb::iresearch::IResearchFeature feature(server);
// arangodb::DatabasePathFeature* dbPathFeature;
// arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
// server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
// server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
// server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
// server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
// server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
// server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
// server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
// server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
// server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
// analyzerFeature->prepare(); // add static analyzers
// feature.prepare(); // register iresearch view type
// feature.start(); // register upgrade tasks
// server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
// server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
// server.getFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
//
// arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
// auto versionFilename = StorageEngineMock::versionFilenameResult;
// auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
// StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
// ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
// ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
//
// engine.views.clear();
// TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
// auto logicalCollection = vocbase.createCollection(collectionJson->slice());
// ASSERT_TRUE((false == !logicalCollection));
// auto logicalView = vocbase.createView(viewJson->slice());
// ASSERT_TRUE((false == !logicalView));
// auto* view = dynamic_cast<arangodb::iresearch::IResearchView*>(logicalView.get());
// ASSERT_TRUE((false == !view));
// bool created;
// auto index = logicalCollection->createIndex(linkJson->slice(), created);
// ASSERT_TRUE((created));
// ASSERT_TRUE((false == !index));
// auto link = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
// ASSERT_TRUE((false == !link));
// ASSERT_TRUE((view->link(link->self()).ok())); // link will not notify view in 'vocbase', hence notify manually
//
// index->unload(); // release file handles
// bool result;
// auto linkDataPath = getPersistedPath1(*link);
// EXPECT_TRUE((linkDataPath.remove())); // remove link directory
// auto viewDataPath = getPersistedPath0(*logicalView);
// EXPECT_TRUE((viewDataPath.exists(result) && !result));
// EXPECT_TRUE((viewDataPath.mkdir())); // create view directory
// EXPECT_TRUE((viewDataPath.exists(result) && result));
// arangodb::velocypack::Builder builder;
// builder.openObject();
// EXPECT_TRUE((logicalView->properties(builder, true, true).ok()));
// builder.close();
// EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
//
// EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
// logicalView = vocbase.lookupView(logicalView->name());
// EXPECT_TRUE((true == !logicalView)); // ensure view removed after upgrade
// EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory removed after upgrade
// }
//}
TEST_F(IResearchFeatureTest, test_upgrade0_1) {
// version 0 data-source path
auto getPersistedPath0 = [](arangodb::LogicalView const& view)->irs::utf8_path {
auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
EXPECT_TRUE(dbPathFeature);
irs::utf8_path dataPath(dbPathFeature->directory());
dataPath /= "databases";
dataPath /= "database-";
dataPath += std::to_string(view.vocbase().id());
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
dataPath += "-";
dataPath += std::to_string(view.id());
return dataPath;
};
// version 1 data-source path
auto getPersistedPath1 = [](arangodb::iresearch::IResearchLink const& link)->irs::utf8_path {
auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
EXPECT_TRUE(dbPathFeature);
irs::utf8_path dataPath(dbPathFeature->directory());
dataPath /= "databases";
dataPath /= "database-";
dataPath += std::to_string(link.collection().vocbase().id());
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
dataPath += "-";
dataPath += std::to_string(link.collection().id());
dataPath += "_";
dataPath += std::to_string(link.id());
return dataPath;
};
// test single-server (no directory)
{
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
// create a new instance of an ApplicationServer and fill it with the required features
// cannot use the existing server since its features already have some state
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
arangodb::application_features::ApplicationServer::server,
[](arangodb::application_features::ApplicationServer* ptr)->void {
arangodb::application_features::ApplicationServer::server = ptr;
}
);
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
arangodb::DatabasePathFeature* dbPathFeature;
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchAnalyzerFeature::start()
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
auto versionFilename = StorageEngineMock::versionFilenameResult;
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
ASSERT_TRUE((false == !logicalCollection));
auto logicalView0 = vocbase.createView(viewJson->slice());
ASSERT_TRUE((false == !logicalView0));
bool created;
auto index = logicalCollection->createIndex(linkJson->slice(), created);
ASSERT_TRUE((created));
ASSERT_TRUE((false == !index));
auto link0 = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
ASSERT_TRUE((false == !link0));
index->unload(); // release file handles
bool result;
auto linkDataPath = getPersistedPath1(*link0);
EXPECT_TRUE((linkDataPath.remove())); // remove link directory
auto viewDataPath = getPersistedPath0(*logicalView0);
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure no view directory
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE((logicalView0->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
auto logicalView1 = vocbase.lookupView(logicalView0->name());
EXPECT_TRUE((false == !logicalView1)); // ensure view present after upgrade
EXPECT_TRUE((logicalView0->id() == logicalView1->id())); // ensure same id for view
auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
EXPECT_TRUE((false == !link1)); // ensure link present after upgrade
EXPECT_TRUE((link0->id() != link1->id())); // ensure new link
linkDataPath = getPersistedPath1(*link1);
EXPECT_TRUE((linkDataPath.exists(result) && result)); // ensure link directory created after upgrade
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not present
viewDataPath = getPersistedPath0(*logicalView1);
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not created
builder.clear();
builder.openObject();
EXPECT_TRUE((logicalView1->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
}
// test single-server (with directory)
{
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
// create a new instance of an ApplicationServer and fill it with the required features
// cannot use the existing server since its features already have some state
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
arangodb::application_features::ApplicationServer::server,
[](arangodb::application_features::ApplicationServer* ptr)->void {
arangodb::application_features::ApplicationServer::server = ptr;
}
);
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
arangodb::DatabasePathFeature* dbPathFeature;
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchLinkHelper::normalize(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
auto versionFilename = StorageEngineMock::versionFilenameResult;
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
ASSERT_TRUE((false == !logicalCollection));
auto logicalView0 = vocbase.createView(viewJson->slice());
ASSERT_TRUE((false == !logicalView0));
bool created;
auto index = logicalCollection->createIndex(linkJson->slice(), created);
ASSERT_TRUE((created));
ASSERT_TRUE((false == !index));
auto link0 = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
ASSERT_TRUE((false == !link0));
index->unload(); // release file handles
bool result;
auto linkDataPath = getPersistedPath1(*link0);
EXPECT_TRUE((linkDataPath.remove())); // remove link directory
auto viewDataPath = getPersistedPath0(*logicalView0);
EXPECT_TRUE((viewDataPath.exists(result) && !result));
EXPECT_TRUE((viewDataPath.mkdir())); // create view directory
EXPECT_TRUE((viewDataPath.exists(result) && result));
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE((logicalView0->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
auto logicalView1 = vocbase.lookupView(logicalView0->name());
EXPECT_TRUE((false == !logicalView1)); // ensure view present after upgrade
EXPECT_TRUE((logicalView0->id() == logicalView1->id())); // ensure same id for view
auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
EXPECT_TRUE((false == !link1)); // ensure link present after upgrade
EXPECT_TRUE((link0->id() != link1->id())); // ensure new link
linkDataPath = getPersistedPath1(*link1);
EXPECT_TRUE((linkDataPath.exists(result) && result)); // ensure link directory created after upgrade
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory removed after upgrade
viewDataPath = getPersistedPath0(*logicalView1);
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not created
builder.clear();
builder.openObject();
EXPECT_TRUE((logicalView1->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
server.getFeature<arangodb::DatabaseFeature>("Database")->unprepare();
}
// test coordinator
{
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"id\": \"1\", \"name\": \"testCollection\", \"shards\":{} }");
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"id\": 42, \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
auto collectionId = std::to_string(1);
auto viewId = std::to_string(42);
auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
// create a new instance of an ApplicationServer and fill it with the required features
// cannot use the existing server since its features already have some state
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
arangodb::application_features::ApplicationServer::server,
[](arangodb::application_features::ApplicationServer* ptr)->void {
arangodb::application_features::ApplicationServer::server = ptr;
}
);
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::DatabaseFeature* database;
arangodb::iresearch::IResearchFeature feature(server);
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterComm::instance()
server.addFeature(new arangodb::ClusterFeature(server)); // required to create ClusterInfo instance
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
server.addFeature(database = new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchLinkHelper::normalize(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
#if USE_ENTERPRISE
server.addFeature(new arangodb::LdapFeature(server)); // required for AuthenticationFeature with USE_ENTERPRISE
#endif
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
server.getFeature<arangodb::ClusterFeature>("Cluster")->prepare(); // create ClusterInfo instance
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
server.getFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
arangodb::AgencyCommManager::MANAGER->start(); // initialize agency
arangodb::DatabaseFeature::DATABASE = database; // required for ClusterInfo::createCollectionCoordinator(...)
const_cast<arangodb::IndexFactory&>(engine.indexFactory()).emplace( // required for Indexes::ensureIndex(...)
arangodb::iresearch::DATA_SOURCE_TYPE.name(),
arangodb::iresearch::IResearchLinkCoordinator::factory()
);
auto* ci = arangodb::ClusterInfo::instance();
ASSERT_TRUE((nullptr != ci));
TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature
ASSERT_TRUE((TRI_ERROR_NO_ERROR == database->createDatabase(1, "testDatabase", vocbase)));
ASSERT_TRUE((ci->createDatabaseCoordinator(vocbase->name(), arangodb::velocypack::Slice::emptyObjectSlice(), 0.0).ok()));
ASSERT_TRUE((ci->createCollectionCoordinator(vocbase->name(), collectionId, 0, 1, false, collectionJson->slice(), 0.0).ok()));
auto logicalCollection = ci->getCollection(vocbase->name(), collectionId);
ASSERT_TRUE((false == !logicalCollection));
EXPECT_TRUE((ci->createViewCoordinator(vocbase->name(), viewId, viewJson->slice()).ok()));
auto logicalView0 = ci->getView(vocbase->name(), viewId);
ASSERT_TRUE((false == !logicalView0));
// simulate heartbeat thread (create index in current)
{
auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id());
auto const value = arangodb::velocypack::Parser::fromJson("{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"1\" } ] } }");
EXPECT_TRUE(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful());
}
arangodb::velocypack::Builder tmp;
ASSERT_TRUE((arangodb::methods::Indexes::ensureIndex(logicalCollection.get(), linkJson->slice(), true, tmp).ok()));
logicalCollection = ci->getCollection(vocbase->name(), collectionId);
ASSERT_TRUE((false == !logicalCollection));
auto link0 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView0);
ASSERT_TRUE((false == !link0));
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE((logicalView0->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
// ensure no upgrade on coordinator
// simulate heartbeat thread (create index in current)
{
auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id());
auto const value = arangodb::velocypack::Parser::fromJson("{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"2\" } ] } }");
EXPECT_TRUE(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful());
}
EXPECT_TRUE((arangodb::methods::Upgrade::clusterBootstrap(*vocbase).ok())); // run upgrade
auto logicalCollection2 = ci->getCollection(vocbase->name(), collectionId);
ASSERT_TRUE((false == !logicalCollection2));
auto logicalView1 = ci->getView(vocbase->name(), viewId);
EXPECT_TRUE((false == !logicalView1)); // ensure view present after upgrade
EXPECT_TRUE((logicalView0->id() == logicalView1->id())); // ensure same id for view
auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection2, *logicalView1);
EXPECT_TRUE((false == !link1)); // ensure link present after upgrade
EXPECT_TRUE((link0->id() == link1->id())); // ensure new link
builder.clear();
builder.openObject();
EXPECT_TRUE((logicalView1->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 after upgrade
server.getFeature<arangodb::DatabaseFeature>("Database")->unprepare();
}
// test db-server (no directory)
{
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
// create a new instance of an ApplicationServer and fill it with the required features
// cannot use the existing server since its features already have some state
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
arangodb::application_features::ApplicationServer::server,
[](arangodb::application_features::ApplicationServer* ptr)->void {
arangodb::application_features::ApplicationServer::server = ptr;
}
);
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::DatabasePathFeature* dbPathFeature;
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
server.getFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
auto versionFilename = StorageEngineMock::versionFilenameResult;
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
ASSERT_TRUE((false == !logicalCollection));
auto logicalView = vocbase.createView(viewJson->slice());
ASSERT_TRUE((false == !logicalView));
auto* view = dynamic_cast<arangodb::iresearch::IResearchView*>(logicalView.get());
ASSERT_TRUE((false == !view));
bool created;
auto index = logicalCollection->createIndex(linkJson->slice(), created);
ASSERT_TRUE((created));
ASSERT_TRUE((false == !index));
auto link = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
ASSERT_TRUE((false == !link));
ASSERT_TRUE((view->link(link->self()).ok())); // link will not notify view in 'vocbase', hence notify manually
index->unload(); // release file handles
bool result;
auto linkDataPath = getPersistedPath1(*link);
EXPECT_TRUE((linkDataPath.remove())); // remove link directory
auto viewDataPath = getPersistedPath0(*logicalView);
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure no view directory
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE((logicalView->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
logicalView = vocbase.lookupView(logicalView->name());
EXPECT_TRUE((true == !logicalView)); // ensure view removed after upgrade
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory not present
server.getFeature<arangodb::DatabaseFeature>("Database")->unprepare();
}
// test db-server (with directory)
{
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
// create a new instance of an ApplicationServer and fill it with the required features
// cannot use the existing server since its features already have some state
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
arangodb::application_features::ApplicationServer::server,
[](arangodb::application_features::ApplicationServer* ptr)->void {
arangodb::application_features::ApplicationServer::server = ptr;
}
);
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::DatabasePathFeature* dbPathFeature;
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
server.getFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
auto versionFilename = StorageEngineMock::versionFilenameResult;
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
ASSERT_TRUE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
ASSERT_TRUE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
engine.views.clear();
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
ASSERT_TRUE((false == !logicalCollection));
auto logicalView = vocbase.createView(viewJson->slice());
ASSERT_TRUE((false == !logicalView));
auto* view = dynamic_cast<arangodb::iresearch::IResearchView*>(logicalView.get());
ASSERT_TRUE((false == !view));
bool created;
auto index = logicalCollection->createIndex(linkJson->slice(), created);
ASSERT_TRUE((created));
ASSERT_TRUE((false == !index));
auto link = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
ASSERT_TRUE((false == !link));
ASSERT_TRUE((view->link(link->self()).ok())); // link will not notify view in 'vocbase', hence notify manually
index->unload(); // release file handles
bool result;
auto linkDataPath = getPersistedPath1(*link);
EXPECT_TRUE((linkDataPath.remove())); // remove link directory
auto viewDataPath = getPersistedPath0(*logicalView);
EXPECT_TRUE((viewDataPath.exists(result) && !result));
EXPECT_TRUE((viewDataPath.mkdir())); // create view directory
EXPECT_TRUE((viewDataPath.exists(result) && result));
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE((logicalView->properties(builder, true, true).ok()));
builder.close();
EXPECT_TRUE((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
EXPECT_TRUE((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
// EXPECT_TRUE(arangodb::methods::Upgrade::clusterBootstrap(vocbase).ok()); // run upgrade
logicalView = vocbase.lookupView(logicalView->name());
EXPECT_TRUE((true == !logicalView)); // ensure view removed after upgrade
EXPECT_TRUE((viewDataPath.exists(result) && !result)); // ensure view directory removed after upgrade
}
}
TEST_F(IResearchFeatureTest, IResearch_version_test) {
EXPECT_TRUE(IResearch_version == arangodb::rest::Version::getIResearchVersion());

2
tests/IResearch/IResearchOrder-test.cpp
View File

@ -182,7 +182,7 @@ void assertOrderExecutionFail(std::string const& queryString,
return assertOrder(true, false, queryString, expected, exprCtx, bindVars, refName);
}
void assertOrderParseFail(std::string const& queryString, size_t parseCode) {
void assertOrderParseFail(std::string const& queryString, int parseCode) {
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1,
"testVocbase");

7
tests/IResearch/IResearchView-test.cpp
View File

@ -1097,14 +1097,13 @@ TEST_F(IResearchViewTest, test_instantiate) {
EXPECT_TRUE((false == !view));
}
// no-longer supported version version
/*
// intantiate view from old version
{
auto json = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
arangodb::LogicalView::ptr view;
EXPECT_TRUE((!arangodb::iresearch::IResearchView::factory().instantiate(view, vocbase, json->slice(), 0).ok()));
EXPECT_TRUE((true == !view));
EXPECT_TRUE(arangodb::iresearch::IResearchView::factory().instantiate(view, vocbase, json->slice(), 0).ok());
EXPECT_TRUE(nullptr != view);
}
*/

4
tests/IResearch/IResearchViewCoordinator-test.cpp
View File

@ -109,10 +109,10 @@ class IResearchViewCoordinatorTest : public ::testing::Test {
std::string testFilesystemPath;
IResearchViewCoordinatorTest() : engine(server), server(nullptr, nullptr) {
// need 2 connections or Agency callbacks will fail
auto* agencyCommManager = new AgencyCommManagerMock("arango");
agency = agencyCommManager->addConnection<GeneralClientConnectionAgencyMock>(_agencyStore);
agency = agencyCommManager->addConnection<GeneralClientConnectionAgencyMock>(
_agencyStore); // need 2 connections or Agency callbacks will fail
agency = agencyCommManager->addConnection<GeneralClientConnectionAgencyMock>(_agencyStore);
arangodb::AgencyCommManager::MANAGER.reset(agencyCommManager);
arangodb::EngineSelectorFeature::ENGINE = &engine;

46
tests/RestHandler/RestAnalyzerHandler-test.cpp
View File

@ -910,8 +910,10 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
};
auto status = handler.execute();
EXPECT_TRUE((arangodb::RestStatus::DONE == status));
@ -934,7 +936,7 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
EXPECT_TRUE((subSlice.hasKey("name") && subSlice.get("name").isString()));
EXPECT_TRUE((subSlice.hasKey("type") && subSlice.get("type").isString()));
EXPECT_TRUE((subSlice.hasKey("properties") &&
(subSlice.get("properties").isString() ||
(subSlice.get("properties").isObject() ||
subSlice.get("properties").isNull())));
EXPECT_TRUE((subSlice.hasKey("features") && subSlice.get("features").isArray()));
EXPECT_TRUE((1 == expected.erase(subSlice.get("name").copyString())));
@ -964,7 +966,9 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
};
auto status = handler.execute();
EXPECT_TRUE((arangodb::RestStatus::DONE == status));
@ -987,7 +991,7 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
EXPECT_TRUE((subSlice.hasKey("name") && subSlice.get("name").isString()));
EXPECT_TRUE((subSlice.hasKey("type") && subSlice.get("type").isString()));
EXPECT_TRUE((subSlice.hasKey("properties") &&
(subSlice.get("properties").isString() ||
(subSlice.get("properties").isObject() ||
subSlice.get("properties").isNull())));
EXPECT_TRUE((subSlice.hasKey("features") && subSlice.get("features").isArray()));
EXPECT_TRUE((1 == expected.erase(subSlice.get("name").copyString())));
@ -1018,9 +1022,11 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"testVocbase::testAnalyzer2",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"testVocbase::testAnalyzer2",
};
auto status = handler.execute();
EXPECT_TRUE((arangodb::RestStatus::DONE == status));
@ -1043,7 +1049,7 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
EXPECT_TRUE((subSlice.hasKey("name") && subSlice.get("name").isString()));
EXPECT_TRUE((subSlice.hasKey("type") && subSlice.get("type").isString()));
EXPECT_TRUE((subSlice.hasKey("properties") &&
(subSlice.get("properties").isString() ||
(subSlice.get("properties").isObject() ||
subSlice.get("properties").isNull())));
EXPECT_TRUE((subSlice.hasKey("features") && subSlice.get("features").isArray()));
EXPECT_TRUE((1 == expected.erase(subSlice.get("name").copyString())));
@ -1074,8 +1080,10 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
};
auto status = handler.execute();
EXPECT_TRUE((arangodb::RestStatus::DONE == status));
@ -1098,7 +1106,7 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
EXPECT_TRUE((subSlice.hasKey("name") && subSlice.get("name").isString()));
EXPECT_TRUE((subSlice.hasKey("type") && subSlice.get("type").isString()));
EXPECT_TRUE((subSlice.hasKey("properties") &&
(subSlice.get("properties").isString() ||
(subSlice.get("properties").isObject() ||
subSlice.get("properties").isNull())));
EXPECT_TRUE((subSlice.hasKey("features") && subSlice.get("features").isArray()));
EXPECT_TRUE((1 == expected.erase(subSlice.get("name").copyString())));
@ -1129,8 +1137,10 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
"testVocbase::testAnalyzer2",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
"testVocbase::testAnalyzer2",
};
auto status = handler.execute();
EXPECT_TRUE((arangodb::RestStatus::DONE == status));
@ -1153,7 +1163,7 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
EXPECT_TRUE((subSlice.hasKey("name") && subSlice.get("name").isString()));
EXPECT_TRUE((subSlice.hasKey("type") && subSlice.get("type").isString()));
EXPECT_TRUE((subSlice.hasKey("properties") &&
(subSlice.get("properties").isString() ||
(subSlice.get("properties").isObject() ||
subSlice.get("properties").isNull())));
EXPECT_TRUE((subSlice.hasKey("features") && subSlice.get("features").isArray()));
EXPECT_TRUE((1 == expected.erase(subSlice.get("name").copyString())));
@ -1184,7 +1194,9 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
};
auto status = handler.execute();
EXPECT_TRUE((arangodb::RestStatus::DONE == status));
@ -1207,7 +1219,7 @@ TEST_F(RestAnalyzerHandlerTest, test_list) {
EXPECT_TRUE((subSlice.hasKey("name") && subSlice.get("name").isString()));
EXPECT_TRUE((subSlice.hasKey("type") && subSlice.get("type").isString()));
EXPECT_TRUE((subSlice.hasKey("properties") &&
(subSlice.get("properties").isString() ||
(subSlice.get("properties").isObject() ||
subSlice.get("properties").isNull())));
EXPECT_TRUE((subSlice.hasKey("features") && subSlice.get("features").isArray()));
EXPECT_TRUE((1 == expected.erase(subSlice.get("name").copyString())));

35
tests/V8Server/v8-analyzers-test.cpp
View File

@ -1695,8 +1695,10 @@ TEST_F(V8AnalyzersTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
};
auto result =
v8::Function::Cast(*fn_list)->CallAsFunction(context, fn_list,
@ -1759,8 +1761,11 @@ TEST_F(V8AnalyzersTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
};
auto result =
v8::Function::Cast(*fn_list)->CallAsFunction(context, fn_list,
static_cast<int>(args.size()),
@ -1823,9 +1828,11 @@ TEST_F(V8AnalyzersTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"testVocbase::testAnalyzer2",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"testVocbase::testAnalyzer2",
};
auto result =
v8::Function::Cast(*fn_list)->CallAsFunction(context, fn_list,
@ -1889,8 +1896,10 @@ TEST_F(V8AnalyzersTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
};
auto result =
v8::Function::Cast(*fn_list)->CallAsFunction(context, fn_list,
@ -1954,8 +1963,10 @@ TEST_F(V8AnalyzersTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
"testVocbase::testAnalyzer2",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
"testVocbase::testAnalyzer2",
};
auto result =
v8::Function::Cast(*fn_list)->CallAsFunction(context, fn_list,
@ -2019,7 +2030,9 @@ TEST_F(V8AnalyzersTest, test_list) {
userManager->setAuthInfo(userMap); // set user map to avoid loading configuration from system database
std::set<std::string> expected = {
"identity",
"identity", "text_de", "text_en", "text_es", "text_fi",
"text_fr", "text_it", "text_nl", "text_no", "text_pt",
"text_ru", "text_sv", "text_zh",
};
auto result =
v8::Function::Cast(*fn_list)->CallAsFunction(context, fn_list,