Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

bug-fix/arangosearch-upgrade (#9010) 

* fix upgrade

* fix catch tests

* Update arangod/IResearch/IResearchAnalyzerFeature.cpp

Co-Authored-By: Jan <jsteemann@users.noreply.github.com>

* Update arangod/IResearch/IResearchAnalyzerFeature.cpp

Co-Authored-By: Jan <jsteemann@users.noreply.github.com>
This commit is contained in:
Andrey Abramov 2019-05-16 10:38:39 +03:00 committed by GitHub
parent 0ea4d35a17
commit b7cfa24518
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 133 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
arangod/IResearch/IResearchAnalyzerFeature.cpp
View File

@ -804,14 +804,14 @@ irs::analysis::analyzer::ptr IResearchAnalyzerFeature::AnalyzerPool::get() const
}
IResearchAnalyzerFeature::IResearchAnalyzerFeature(arangodb::application_features::ApplicationServer& server)
: ApplicationFeature(server, IResearchAnalyzerFeature::name()),
_analyzers(getStaticAnalyzers()) { // load static analyzers
: ApplicationFeature(server, IResearchAnalyzerFeature::name()) {
setOptional(true);
startsAfter("V8Phase");
startsAfter("AQLFunctions"); // used for registering IResearch analyzer functions
startsAfter("SystemDatabase"); // used for getting the system database
// containing the persisted configuration
// used for registering IResearch analyzer functions
startsAfter("AQLFunctions");
// used for getting the system database
// containing the persisted configuration
startsAfter("SystemDatabase");
}
/*static*/ bool IResearchAnalyzerFeature::canUse( // check permissions
@ -1168,8 +1168,8 @@ IResearchAnalyzerFeature::AnalyzerPool::ptr IResearchAnalyzerFeature::get( // fi
Instance() {
// register the indentity analyzer
{
static const irs::flags extraFeatures = {irs::frequency::type(), irs::norm::type()};
static const irs::string_ref name("identity");
irs::flags const extraFeatures = {irs::frequency::type(), irs::norm::type()};
irs::string_ref const name("identity");
PTR_NAMED(AnalyzerPool, pool, name);
if (!pool || !pool->init(IdentityAnalyzer::type().name(),
@ -1178,12 +1178,68 @@ IResearchAnalyzerFeature::AnalyzerPool::ptr IResearchAnalyzerFeature::get( // fi
<< "failure creating an arangosearch static analyzer instance "
"for name '"
<< name << "'";
throw irs::illegal_state(); // this should never happen, treat as an
// assertion failure
// this should never happen, treat as an assertion failure
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "failed to create arangosearch static analyzer");
}
analyzers.emplace(irs::make_hashed_ref(name, std::hash<irs::string_ref>()), pool);
}
auto* databaseFeature =
application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
"Database");
// check if DB is currently being upgraded (load all legacy built-in analyzers)
bool const inUpgrade = databaseFeature
? (databaseFeature->upgrade() || databaseFeature->checkVersion())
: false;
if (!inUpgrade) {
return;
}
// register the text analyzers
{
// Note: ArangoDB strings coming from JavaScript user input are UTF-8 encoded
std::vector<std::pair<irs::string_ref, irs::string_ref>> const textAnalzyers = {
{"text_de", "{ \"locale\": \"de.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_en", "{ \"locale\": \"en.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_es", "{ \"locale\": \"es.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_fi", "{ \"locale\": \"fi.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_fr", "{ \"locale\": \"fr.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_it", "{ \"locale\": \"it.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_nl", "{ \"locale\": \"nl.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_no", "{ \"locale\": \"no.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_pt", "{ \"locale\": \"pt.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_ru", "{ \"locale\": \"ru.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_sv", "{ \"locale\": \"sv.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
{"text_zh", "{ \"locale\": \"zh.UTF-8\", \"ignored_words\": [ ] " "}"}, // empty stop word list
};
irs::flags const extraFeatures = {
irs::frequency::type(), irs::norm::type(), irs::position::type()
}; // add norms + frequency/position for by_phrase
irs::string_ref const type("text");
for (auto& entry : textAnalzyers) {
auto& name = entry.first;
auto& args = entry.second;
PTR_NAMED(AnalyzerPool, pool, name);
if (!pool || !pool->init(type, args, extraFeatures)) {
LOG_TOPIC("e25f5", WARN, arangodb::iresearch::TOPIC)
<< "failure creating an arangosearch static analyzer instance "
"for name '"
<< name << "'";
// this should never happen, treat as an assertion failure
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "failed to create arangosearch static analyzer instance");
}
analyzers.emplace(irs::make_hashed_ref(name, std::hash<irs::string_ref>()), pool);
}
}
}
};
static const Instance instance;
@ -1609,6 +1665,9 @@ void IResearchAnalyzerFeature::prepare() {
// load all known analyzers
::iresearch::analysis::analyzers::init();
// load all static analyzers
_analyzers = getStaticAnalyzers();
}
arangodb::Result IResearchAnalyzerFeature::remove( // remove analyzer

9
arangod/IResearch/IResearchView.cpp
View File

@ -242,10 +242,15 @@ struct IResearchView::ViewFactory : public arangodb::ViewFactory {
arangodb::velocypack::Slice const& definition,
uint64_t planVersion) const override {
auto* databaseFeature =
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
"Database");
// check if DB is currently being upgraded (skip validation checks)
bool const inUpgrade = databaseFeature
? (databaseFeature->upgrade() || databaseFeature->checkVersion())
: false;
std::string error;
bool inUpgrade = databaseFeature ? databaseFeature->upgrade() : false; // check if DB is currently being upgraded (skip validation checks)
IResearchViewMeta meta;
IResearchViewMetaState metaState;

37
tests/IResearch/IResearchAnalyzerFeature-test.cpp
View File

@ -595,6 +595,7 @@ SECTION("test_emplace") {
{
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
CHECK((true == feature.emplace(result, "identity", "identity", irs::string_ref::NIL, irs::flags{ irs::frequency::type(), irs::norm::type() }).ok()));
CHECK((false == !result.first));
auto pool = feature.get("identity");
@ -616,6 +617,7 @@ SECTION("test_get") {
{
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
REQUIRE((feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer", "TestAnalyzer", "abc").ok()));
@ -884,6 +886,7 @@ SECTION("test_identity") {
// test registered 'identity'
{
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
CHECK((false == !feature.get("identity")));
auto pool = feature.get("identity");
REQUIRE((false == !pool));
@ -1054,6 +1057,7 @@ SECTION("test_static_analyzer_features") {
// test registered 'identity'
{
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
for (auto& analyzerEntry : staticAnalyzers()) {
CHECK((false == !feature.get(analyzerEntry.first)));
auto pool = feature.get(analyzerEntry.first);
@ -1266,6 +1270,7 @@ SECTION("test_persistence") {
};
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // load static analyzers
feature.start(); // load persisted analyzers
feature.visit([&expected](
@ -1506,6 +1511,7 @@ SECTION("test_remove") {
// remove existing
{
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
// add analyzer
{
@ -1921,12 +1927,37 @@ SECTION("test_remove") {
// remove static analyzer
{
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
CHECK((false == !feature.get("identity")));
CHECK((false == feature.remove("identity").ok()));
CHECK((false == !feature.get("identity")));
}
}
SECTION("test_prepare") {
auto before = StorageEngineMock::inRecoveryResult;
StorageEngineMock::inRecoveryResult = true;
auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
CHECK(feature.visit([](auto) { return false; })); // ensure feature is empty after creation
feature.prepare(); // add static analyzers
// check static analyzers
auto expected = staticAnalyzers();
feature.visit([&expected, &feature](
arangodb::iresearch::IResearchAnalyzerFeature::AnalyzerPool::ptr const& analyzer
)->bool {
auto itr = expected.find(analyzer->name());
CHECK((itr != expected.end()));
CHECK((itr->second.type == analyzer->type()));
CHECK((itr->second.properties == analyzer->properties()));
CHECK((itr->second.features.is_subset_of(feature.get(analyzer->name())->features())));
expected.erase(itr);
return true;
});
CHECK((expected.empty()));
}
SECTION("test_start") {
auto* database = arangodb::application_features::ApplicationServer::lookupFeature<
arangodb::SystemDatabaseFeature
@ -1951,6 +1982,7 @@ SECTION("test_start") {
StorageEngineMock::inRecoveryResult = true;
auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
feature.start(); // load persisted analyzers
CHECK((nullptr == vocbase->lookupCollection(ANALYZER_COLLECTION_NAME)));
@ -1995,6 +2027,7 @@ SECTION("test_start") {
StorageEngineMock::inRecoveryResult = true;
auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
feature.start(); // load persisted analyzers
CHECK((nullptr != vocbase->lookupCollection(ANALYZER_COLLECTION_NAME)));
@ -2031,6 +2064,7 @@ SECTION("test_start") {
}
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
feature.start(); // load persisted analyzers
CHECK((nullptr == vocbase->lookupCollection(ANALYZER_COLLECTION_NAME)));
@ -2072,6 +2106,7 @@ SECTION("test_start") {
}
arangodb::iresearch::IResearchAnalyzerFeature feature(s.server);
feature.prepare(); // add static analyzers
feature.start(); // load persisted analyzers
CHECK((nullptr != vocbase->lookupCollection(ANALYZER_COLLECTION_NAME)));
@ -3757,4 +3792,4 @@ SECTION("test_visit") {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

22
tests/IResearch/IResearchFeature-test.cpp
View File

@ -232,15 +232,17 @@ SECTION("test_upgrade0_1") {
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
arangodb::DatabasePathFeature* dbPathFeature;
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchAnalyzerFeature::start()
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
@ -313,15 +315,17 @@ SECTION("test_upgrade0_1") {
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
arangodb::DatabasePathFeature* dbPathFeature;
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchLinkHelper::normalize(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
@ -405,13 +409,14 @@ SECTION("test_upgrade0_1") {
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::DatabaseFeature* database;
arangodb::iresearch::IResearchFeature feature(server);
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
TRI_vocbase_t system(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 0, TRI_VOC_SYSTEM_DATABASE);
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterComm::instance()
server.addFeature(new arangodb::ClusterFeature(server)); // required to create ClusterInfo instance
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
server.addFeature(database = new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
server.addFeature(new arangodb::SystemDatabaseFeature(server, &system)); // required for IResearchLinkHelper::normalize(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
@ -421,6 +426,7 @@ SECTION("test_upgrade0_1") {
server.addFeature(new arangodb::LdapFeature(server)); // required for AuthenticationFeature with USE_ENTERPRISE
#endif
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
@ -512,15 +518,17 @@ SECTION("test_upgrade0_1") {
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::DatabasePathFeature* dbPathFeature;
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
@ -592,15 +600,17 @@ SECTION("test_upgrade0_1") {
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchFeature feature(server);
arangodb::DatabasePathFeature* dbPathFeature;
arangodb::iresearch::IResearchAnalyzerFeature* analyzerFeature{};
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(analyzerFeature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
analyzerFeature->prepare(); // add static analyzers
feature.prepare(); // register iresearch view type
feature.start(); // register upgrade tasks
server.getFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
@ -1097,4 +1107,4 @@ SECTION("test_async") {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

1
tests/IResearch/IResearchLinkMeta-test.cpp
View File

@ -598,6 +598,7 @@ SECTION("test_writeDefaults") {
SECTION("test_writeCustomizedValues") {
arangodb::iresearch::IResearchAnalyzerFeature analyzers(s.server);
analyzers.prepare(); // add static analyzers
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult emplaceResult;
arangodb::iresearch::IResearchLinkMeta meta;

3
tests/RestHandler/RestAnalyzerHandler-test.cpp
View File

@ -383,6 +383,7 @@ SECTION("test_get") {
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
analyzers->prepare(); // add static analyzers
// create system vocbase
{
@ -1130,4 +1131,4 @@ SECTION("test_remove") {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

3
tests/V8Server/v8-analyzers-test.cpp
View File

@ -909,6 +909,7 @@ SECTION("test_get") {
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
analyzers->prepare(); // add static analyzers
// create system vocbase
{
@ -2000,4 +2001,4 @@ SECTION("test_remove") {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------