diff --git a/CHANGELOG b/CHANGELOG index 815cb63f9d..e3c66ccaaf 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,34 @@ v3.5.3 (XXXX-XX-XX) ------------------- +* Fixed UPSERT matching. + + Empty objects in the `UPSERT { ... }` expression will not be omitted anymore: + + db._collection("col").insert({ "find" : "me" }); + db._query(` UPSERT { "find" : "me", "foo" : {} } + UPDATE { "foo" : "not gonna happen" } + INSERT { "find" : "me", "foo" : {} } + INTO col + `) + + This will now correctly insert a document instead of updating the existing, + that only partially machtes the upsert-expression. + +* Fixed undefined behaviour with creation of ArangoSearch links with custom + analyzers in cluster environment. + +* Fixed internal issue #651: analyzer duplication in _analyzers collection. + +* Fixed internal issue #4597: rebalanceShards API cannot work on any database + other than the _system database. + +* Stop putting system services in _apps on single server, this has never + worked on cluster and was not needed. + +* Fixed issue #10371: K_SHORTEST_PATHS LIMIT 1 can not return the shortest path. + Now the shortest path is returned as the first one in such queries. + * Improve killability of some types of cluster AQL queries. Previously, several cluster queries, especially those containing a `DistributeNode` in their execution plans, did not respond to a kill instruction. diff --git a/arangod/Aql/Ast.cpp b/arangod/Aql/Ast.cpp index 57a643ddc4..ec7726d02d 100644 --- a/arangod/Aql/Ast.cpp +++ b/arangod/Aql/Ast.cpp @@ -2665,7 +2665,6 @@ AstNode* Ast::makeConditionFromExample(AstNode const* node) { TRI_ASSERT(object->type == NODE_TYPE_OBJECT); auto const n = object->numMembers(); - for (size_t i = 0; i < n; ++i) { auto member = object->getMember(i); @@ -2680,8 +2679,8 @@ AstNode* Ast::makeConditionFromExample(AstNode const* node) { auto value = member->getMember(0); - if (value->type == NODE_TYPE_OBJECT) { - createCondition(value); + if (value->type == NODE_TYPE_OBJECT && value->numMembers() != 0) { + createCondition(value); } else { auto access = variable; for (auto const& it : attributeParts) { diff --git a/arangod/Aql/QueryString.h b/arangod/Aql/QueryString.h index 90128da26a..2bcae6d669 100644 --- a/arangod/Aql/QueryString.h +++ b/arangod/Aql/QueryString.h @@ -55,6 +55,7 @@ class QueryString { ~QueryString() = default; public: + std::string const& string() const noexcept { return _queryString; } char const* data() const { return _queryString.data(); } size_t size() const { return _queryString.size(); } size_t length() const { return _queryString.size(); } diff --git a/arangod/Graph/KShortestPathsFinder.cpp b/arangod/Graph/KShortestPathsFinder.cpp index c02b4d58fc..04e11e33f8 100644 --- a/arangod/Graph/KShortestPathsFinder.cpp +++ b/arangod/Graph/KShortestPathsFinder.cpp @@ -70,27 +70,36 @@ bool KShortestPathsFinder::computeShortestPath(VertexRef const& start, VertexRef std::unordered_set const& forbiddenVertices, std::unordered_set const& forbiddenEdges, Path& result) { - bool found = false; Ball left(start, FORWARD); Ball right(end, BACKWARD); VertexRef join; result.clear(); - while (!left._frontier.empty() && !right._frontier.empty() && !found) { + auto currentBest = boost::optional{}; + + // We will not improve anymore if we have found a best path and the smallest + // combined distance between left and right is bigger than that path + while (!left._frontier.empty() && !right._frontier.empty() && + !(currentBest.has_value() && + (left._closest + right._closest > currentBest.value()))) { _options.isQueryKilledCallback(); // Choose the smaller frontier to expand. if (left._frontier.size() < right._frontier.size()) { - found = advanceFrontier(left, right, forbiddenVertices, forbiddenEdges, join); + advanceFrontier(left, right, forbiddenVertices, forbiddenEdges, join, currentBest); } else { - found = advanceFrontier(right, left, forbiddenVertices, forbiddenEdges, join); + advanceFrontier(right, left, forbiddenVertices, forbiddenEdges, join, currentBest); } } - if (found) { + + if (currentBest.has_value()) { reconstructPath(left, right, join, result); + return true; + } else { + // No path found + return false; } - return found; } void KShortestPathsFinder::computeNeighbourhoodOfVertexCache(VertexRef vertex, @@ -178,10 +187,11 @@ void KShortestPathsFinder::computeNeighbourhoodOfVertex(VertexRef vertex, Direct } } -bool KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target, +void KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target, std::unordered_set const& forbiddenVertices, std::unordered_set const& forbiddenEdges, - VertexRef& join) { + VertexRef& join, + boost::optional& currentBest) { VertexRef vr; DijkstraInfo *v, *w; std::vector* neighbours; @@ -190,7 +200,7 @@ bool KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target, TRI_ASSERT(v != nullptr); TRI_ASSERT(vr == v->_vertex); if (!success) { - return false; + return; } computeNeighbourhoodOfVertexCache(vr, source._direction, neighbours); @@ -217,14 +227,17 @@ bool KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target, } } v->_done = true; + source._closest = v->_weight; w = target._frontier.find(v->_vertex); if (w != nullptr && w->_done) { - join = v->_vertex; - return true; + // The total weight of the found path + double totalWeight = v->_weight + w->_weight; + if (!currentBest.has_value() || totalWeight < currentBest.value()) { + join = v->_vertex; + currentBest = v->_weight + w->_weight; + } } - - return false; } void KShortestPathsFinder::reconstructPath(Ball const& left, Ball const& right, diff --git a/arangod/Graph/KShortestPathsFinder.h b/arangod/Graph/KShortestPathsFinder.h index 36986cb2d5..4bc84114fe 100644 --- a/arangod/Graph/KShortestPathsFinder.h +++ b/arangod/Graph/KShortestPathsFinder.h @@ -34,6 +34,8 @@ #include +#include + namespace arangodb { namespace velocypack { @@ -141,9 +143,9 @@ class KShortestPathsFinder : public ShortestPathFinder { void setWeight(double weight) { _weight = weight; } DijkstraInfo(VertexRef const& vertex, Edge const&& edge, VertexRef const& pred, double weight) - : _vertex(vertex), _edge(std::move(edge)), _pred(pred), _weight(weight), _done(false) {} + : _vertex(vertex), _edge(std::move(edge)), _pred(pred), _weight(weight), _done(false) {} explicit DijkstraInfo(VertexRef const& vertex) - : _vertex(vertex), _weight(0), _done(true) {} + : _vertex(vertex), _weight(0), _done(true) {} }; typedef ShortestPathPriorityQueue Frontier; @@ -154,10 +156,13 @@ class KShortestPathsFinder : public ShortestPathFinder { VertexRef _centre; Direction _direction; Frontier _frontier; + // The distance of the last node that has been fully expanded + // from _centre + double _closest; Ball() {} Ball(VertexRef const& centre, Direction direction) - : _centre(centre), _direction(direction) { + : _centre(centre), _direction(direction), _closest(0) { _frontier.insert(centre, std::make_unique(centre)); } ~Ball() {} @@ -193,7 +198,7 @@ class KShortestPathsFinder : public ShortestPathFinder { std::vector _paths; explicit FoundVertex(VertexRef const& vertex) - : _vertex(vertex), _hasCachedOutNeighbours(false), _hasCachedInNeighbours(false) {} + : _vertex(vertex), _hasCachedOutNeighbours(false), _hasCachedInNeighbours(false) {} }; // Contains the vertices that were found while searching // for a shortest path between start and end together with @@ -246,9 +251,10 @@ class KShortestPathsFinder : public ShortestPathFinder { void computeNeighbourhoodOfVertex(VertexRef vertex, Direction direction, std::vector& steps); - bool advanceFrontier(Ball& source, Ball const& target, + void advanceFrontier(Ball& source, Ball const& target, std::unordered_set const& forbiddenVertices, - std::unordered_set const& forbiddenEdges, VertexRef& join); + std::unordered_set const& forbiddenEdges, + VertexRef& join, boost::optional& currentBest); private: bool _pathAvailable; diff --git a/arangod/IResearch/IResearchAnalyzerFeature.cpp b/arangod/IResearch/IResearchAnalyzerFeature.cpp index 84f373911f..bf4bc1fe8b 100644 --- a/arangod/IResearch/IResearchAnalyzerFeature.cpp +++ b/arangod/IResearch/IResearchAnalyzerFeature.cpp @@ -45,10 +45,12 @@ #include "Aql/QueryString.h" #include "Basics/StaticStrings.h" #include "Basics/StringUtils.h" +#include "Basics/VelocyPackHelper.h" #include "Cluster/ClusterComm.h" #include "Cluster/ClusterInfo.h" #include "Cluster/ServerState.h" #include "IResearchAnalyzerFeature.h" +#include "IResearchLink.h" #include "IResearchCommon.h" #include "Logger/LogMacros.h" #include "RestHandler/RestVocbaseBaseHandler.h" @@ -65,6 +67,7 @@ #include "VelocyPackHelper.h" #include "VocBase/LocalDocumentId.h" #include "VocBase/LogicalCollection.h" +#include "VocBase/LogicalView.h" #include "VocBase/ManagedDocumentResult.h" #include "VocBase/vocbase.h" #include "VocBase/Methods/Collections.h" @@ -76,14 +79,13 @@ static const irs::text_format::type_id VPACK("vpack"); const type_id& vpack_t() { return VPACK; } -} -} +} // iresearch +} // text_format namespace { using namespace std::literals::string_literals; -static std::string const ANALYZER_COLLECTION_NAME("_analyzers"); static char const ANALYZER_PREFIX_DELIM = ':'; // name prefix delimiter (2 chars) static size_t const ANALYZER_PROPERTIES_SIZE_MAX = 1024 * 1024; // arbitrary value static size_t const DEFAULT_POOL_SIZE = 8; // arbitrary value @@ -98,7 +100,7 @@ bool normalize(std::string& out, // in ArangoSearch we don't allow to have analyzers with empty type string return false; } - + // for API consistency we only support analyzers configurable via jSON return irs::analysis::analyzers::normalize( out, type, @@ -220,7 +222,7 @@ bool parse_ngram_vpack_config(const irs::string_ref& args, irs::analysis::ngram_ auto slice = arangodb::iresearch::slice(args); if (!slice.isObject()) { - LOG_TOPIC("c0168", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("c0168", WARN, arangodb::iresearch::TOPIC) << "Not a jSON object passed while constructing ngram_token_stream, " "arguments: " << slice.toString(); return false; @@ -229,8 +231,8 @@ bool parse_ngram_vpack_config(const irs::string_ref& args, irs::analysis::ngram_ uint64_t min = 0, max = 0; bool seen = false; if (!arangodb::iresearch::getNumber(min, slice, MIN_PARAM_NAME, seen, min) || !seen) { - LOG_TOPIC("7b706", WARN, arangodb::iresearch::TOPIC) - << "Failed to read '" << MIN_PARAM_NAME + LOG_TOPIC("7b706", WARN, arangodb::iresearch::TOPIC) + << "Failed to read '" << MIN_PARAM_NAME << "' attribute as number while constructing " "ngram_token_stream from jSON arguments: " << slice.toString(); @@ -307,9 +309,9 @@ bool text_vpack_normalizer(const irs::string_ref& args, std::string& out) noexce std::string tmp; auto slice = arangodb::iresearch::slice(args); - if (!slice.isNone() && + if (!slice.isNone() && irs::analysis::analyzers::normalize(tmp, "text", irs::text_format::json, - slice.toString(), + slice.toString(), false)) { auto vpack = VPackParser::fromJson(tmp); out.assign(vpack->slice().startAs(), vpack->slice().byteSize()); @@ -330,14 +332,14 @@ namespace stem_vpack { return irs::analysis::analyzers::get("stem", irs::text_format::json, slice.toString(), false); - } + } return nullptr; } bool stem_vpack_normalizer(const irs::string_ref& args, std::string& out) noexcept { std::string tmp; auto slice = arangodb::iresearch::slice(args); - if (!slice.isNone() && + if (!slice.isNone() && irs::analysis::analyzers::normalize(tmp, "stem", irs::text_format::json, slice.toString(), false)) { auto vpack = VPackParser::fromJson(tmp); @@ -524,13 +526,13 @@ bool equalAnalyzer( if (type != pool.type() || features != pool.features()) { return false; } - + // this check is not final as old-normalized definition may be present in database! if (arangodb::basics::VelocyPackHelper::equal(arangodb::iresearch::slice(normalizedProperties), pool.properties(), false)) { return true; - } - + } + // Here could be analyzer definition with old-normalized properties (see Issue #9652) // To make sure properties really differ, let`s re-normalize and re-check std::string reNormalizedProperties; @@ -557,216 +559,250 @@ bool equalAnalyzer( /// @note cannot use arangodb::CollectionNameResolver::getCollection(...) since /// it will try to resolve via vocbase for the case of db-server //////////////////////////////////////////////////////////////////////////////// -std::shared_ptr getAnalyzerCollection( // get collection - TRI_vocbase_t const& vocbase // collection vocbase -) { +std::shared_ptr getAnalyzerCollection( + TRI_vocbase_t const& vocbase) { if (arangodb::ServerState::instance()->isSingleServer()) { - return vocbase.lookupCollection(ANALYZER_COLLECTION_NAME); + return vocbase.lookupCollection(arangodb::StaticStrings::AnalyzersCollection); } - try { - auto* ci = arangodb::ClusterInfo::instance(); + auto* ci = arangodb::ClusterInfo::instance(); - if (ci) { - return ci->getCollectionNT(vocbase.name(), ANALYZER_COLLECTION_NAME); - } - - LOG_TOPIC("00001", WARN, arangodb::iresearch::TOPIC) - << "failure to find 'ClusterInfo' instance while looking up Analyzer collection '" << ANALYZER_COLLECTION_NAME << "' in vocbase '" << vocbase.name() << "'"; - } catch (arangodb::basics::Exception& e) { - LOG_TOPIC("00002", WARN, arangodb::iresearch::TOPIC) - << "caught exception while looking up Analyzer collection '" << ANALYZER_COLLECTION_NAME << "' in vocbase '" << vocbase.name() << "': " << e.code() << " " << e.what(); - IR_LOG_EXCEPTION(); - } catch (std::exception& e) { - LOG_TOPIC("00003", WARN, arangodb::iresearch::TOPIC) - << "caught exception while looking up Analyzer collection '" << ANALYZER_COLLECTION_NAME << "' in vocbase '" << vocbase.name() << "': " << e.what(); - IR_LOG_EXCEPTION(); - } catch (...) { - LOG_TOPIC("00004", WARN, arangodb::iresearch::TOPIC) - << "caught exception while looking up Analyzer collection '" << ANALYZER_COLLECTION_NAME << "' in vocbase '" << vocbase.name() << "'"; - IR_LOG_EXCEPTION(); + if (ci) { + return ci->getCollectionNT(vocbase.name(), arangodb::StaticStrings::AnalyzersCollection); } return nullptr; } -std::string normalizedAnalyzerName( - std::string database, // database - irs::string_ref const& analyzer // analyzer -) { - return database.append(2, ANALYZER_PREFIX_DELIM).append(analyzer); -} - //////////////////////////////////////////////////////////////////////////////// /// @brief read analyzers from vocbase /// @return visitation completed fully //////////////////////////////////////////////////////////////////////////////// -arangodb::Result visitAnalyzers( // visit analyzers - TRI_vocbase_t& vocbase, // vocbase to visit - std::function const& visitor // visitor -) { +arangodb::Result visitAnalyzers( + TRI_vocbase_t& vocbase, + std::function const& visitor) { static const auto resultVisitor = []( - std::function const& visitor, // visitor - TRI_vocbase_t const& vocbase, // vocbase - arangodb::velocypack::Slice const& slice // slice to visit - )->arangodb::Result { + std::function const& visitor, + TRI_vocbase_t const& vocbase, + VPackSlice const& slice) -> arangodb::Result { if (!slice.isArray()) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failed to parse contents of collection '") + ANALYZER_COLLECTION_NAME + "' in database '" + vocbase.name() + " while visiting analyzers" - ); + return { + TRI_ERROR_INTERNAL, + "failed to parse contents of collection '" + arangodb::StaticStrings::AnalyzersCollection + + "' in database '" + vocbase.name() + " while visiting analyzers" }; } - for (arangodb::velocypack::ArrayIterator itr(slice); itr.valid(); ++itr) { - auto res = visitor(itr.value().resolveExternal()); + for (VPackArrayIterator itr(slice); itr.valid(); ++itr) { + auto const res = visitor(itr.value().resolveExternal()); if (!res.ok()) { return res; } } - return arangodb::Result(); + return {}; }; - // FIXME TODO find a better way to query a cluster collection - // workaround for aql::Query failing to execute on a cluster collection + static const auto queryString = arangodb::aql::QueryString( + "FOR d IN " + arangodb::StaticStrings::AnalyzersCollection + " RETURN d"); + if (arangodb::ServerState::instance()->isDBServer()) { auto cc = arangodb::ClusterComm::instance(); if (!cc) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find 'ClusterComm' instance while visiting Analyzer collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + "'" - ); + return { + TRI_ERROR_INTERNAL, + "failure to find 'ClusterComm' instance while visiting Analyzer collection '" + + arangodb::StaticStrings::AnalyzersCollection + + "' in vocbase '" + vocbase.name() + "'" + }; } - auto collection = getAnalyzerCollection(vocbase); + auto ci = arangodb::ClusterInfo::instance(); - if (!collection) { - return arangodb::Result(); // nothing to load + if (!ci) { + return { + TRI_ERROR_INTERNAL, + "failure to find 'ClusterInfo' instance while visiting Analyzer collection '" + + arangodb::StaticStrings::AnalyzersCollection + + "' in vocbase '" + vocbase.name() + "'" + }; } - static const std::string body("{}"); // RestSimpleQueryHandler::allDocuments() expects opbject (calls get() on slice) - std::vector requests; + static const std::string BODY("{ \"query\" : \"" + queryString.string() + "\" }"); + auto const coords = ci->getCurrentCoordinators(); - // create a request for every shard - //for (auto& entry: collection->errorNum()) { - for (auto& entry: *(collection->shardIds())) { - auto& shardId = entry.first; - auto url = // url - "/_db/" + arangodb::basics::StringUtils::urlEncode(vocbase.name()) - + arangodb::RestVocbaseBaseHandler::SIMPLE_QUERY_ALL_PATH - + "?collection=" + shardId; + std::vector requests(1); + auto& request = requests[0]; + request.path = "/_api/cursor"; + request.body = std::shared_ptr(std::shared_ptr(), &BODY); + request.requestType = arangodb::rest::RequestType::POST; - requests.emplace_back( // add shard request - "shard:" + shardId, // shard - arangodb::rest::RequestType::PUT, // request type as per SimpleQueryHandker - url, // request url - std::shared_ptr(&body, [](std::string const*)->void {}) // body - ); - } + arangodb::Result res; + for (auto const& coord : coords) { + request.destination = "server:" + coord; - // same timeout as in ClusterMethods::getDocumentOnCoordinator() - cc->performRequests( // execute requests - requests, 120.0, arangodb::iresearch::TOPIC, false, false // args - ); + // same timeout as in ClusterMethods::getDocumentOnCoordinator() + static double const CL_DEFAULT_TIMEOUT = 120.0; + cc->performRequests(requests, CL_DEFAULT_TIMEOUT, arangodb::iresearch::TOPIC, false, false); - for (auto& request: requests) { if (TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND == request.result.errorCode) { - continue; // treat missing collection as if there are no analyzers + // no "_analyzers" collection => not an error + return {}; + } + + if (TRI_ERROR_CLUSTER_TIMEOUT == request.result.errorCode) { + res = { request.result.errorCode, request.result.errorMessage }; + continue; // try another coordinator } if (TRI_ERROR_NO_ERROR != request.result.errorCode) { - return arangodb::Result( // result - request.result.errorCode, request.result.errorMessage // args - ); + return { request.result.errorCode, request.result.errorMessage }; } if (!request.result.answer) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failed to get answer from 'ClusterComm' instance while visiting Analyzer collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + "'" - ); + return { TRI_ERROR_INTERNAL, + "failed to get answer from 'ClusterComm' instance while visiting Analyzer collection '" + + arangodb::StaticStrings::AnalyzersCollection + "' in vocbase '" + vocbase.name() + "'" }; } auto slice = request.result.answer->payload(); if (!slice.hasKey("result")) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failed to parse result from 'ClusterComm' instance while visiting Analyzer collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase.name() + "'" - ); + return { TRI_ERROR_INTERNAL, + "failed to parse result from 'ClusterComm' instance while visiting Analyzer collection '" + + arangodb::StaticStrings::AnalyzersCollection + "' in vocbase '" + vocbase.name() + "'" }; } - auto res = resultVisitor(visitor, vocbase, slice.get("result")); + res = resultVisitor(visitor, vocbase, slice.get("result")); - if (!res.ok()) { - return res; - } + break; } - return arangodb::Result(); - } - - if (arangodb::ServerState::instance()->isClusterRole()) { - if (!getAnalyzerCollection(vocbase)) { - return arangodb::Result(); // treat missing collection as if there are no analyzers - } - - static const auto queryString = arangodb::aql::QueryString( // query to execute - std::string("FOR d IN ") + ANALYZER_COLLECTION_NAME + " RETURN d" // query - ); - arangodb::aql::Query query( // query - false, vocbase, queryString, nullptr, nullptr, arangodb::aql::PART_MAIN // args - ); - auto* queryRegistry = arangodb::QueryRegistryFeature::registry(); - auto result = query.executeSync(queryRegistry); - - if (TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND == result.result.errorNumber()) { - return arangodb::Result(); // treat missing collection as if there are no analyzers - } - - if (result.result.fail()) { - return result.result; - } - - auto slice = result.data->slice(); - - return resultVisitor(visitor, vocbase, slice); - } - - if (!vocbase.lookupCollection(ANALYZER_COLLECTION_NAME)) { - return arangodb::Result(); // treat missing collection as if there are no analyzers - } - - arangodb::OperationOptions options; - arangodb::SingleCollectionTransaction trx( // transaction - arangodb::transaction::StandaloneContext::Create(vocbase), // context - ANALYZER_COLLECTION_NAME, // collection - arangodb::AccessMode::Type::READ // access more - ); - auto res = trx.begin(); - - if (!res.ok()) { return res; } - auto commit = irs::make_finally([&trx]()->void { trx.commit(); }); // end read-only transaction - auto result = trx.all(ANALYZER_COLLECTION_NAME, 0, 0, options); + arangodb::aql::Query query(false, vocbase, queryString, + nullptr, nullptr, arangodb::aql::PART_MAIN); - if (!result.result.ok()) { + auto* queryRegistry = arangodb::QueryRegistryFeature::registry(); + + if (!queryRegistry) { + return { TRI_ERROR_INTERNAL, "QueryRegistry is not set" }; + } + + auto result = query.executeSync(queryRegistry); + + if (TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND == result.result.errorNumber()) { + return {}; // treat missing collection as if there are no analyzers + } + + if (result.result.fail()) { return result.result; } - auto slice = arangodb::velocypack::Slice(result.buffer->data()); + auto slice = result.data->slice(); return resultVisitor(visitor, vocbase, slice); } +inline std::string normalizedAnalyzerName( + std::string database, + irs::string_ref const& analyzer) { + return database.append(2, ANALYZER_PREFIX_DELIM).append(analyzer); +} + +bool analyzerInUse(irs::string_ref const& dbName, + arangodb::iresearch::AnalyzerPool::ptr const& analyzerPtr) { + using arangodb::application_features::ApplicationServer; + + TRI_ASSERT(analyzerPtr); + + if (analyzerPtr.use_count() > 1) { // +1 for ref in '_analyzers' + return true; + } + + auto* server = arangodb::ServerState::instance(); + + if (!server) { + // no server state + return false; + } + + auto checkDatabase = [server, analyzer = analyzerPtr.get()](TRI_vocbase_t* vocbase) { + if (!vocbase) { + // no database + return false; + } + + std::vector> collections; + + if (server->isCoordinator()) { + auto* ci = arangodb::ClusterInfo::instance(); + + if (ci) { + collections = ci->getCollections(vocbase->name()); + } + } else { + collections = vocbase->collections(false); + } + + for (auto const& collection : collections) { + for (auto const& index : collection->getIndexes()) { + if (!index || arangodb::Index::TRI_IDX_TYPE_IRESEARCH_LINK != index->type()) { + continue; // not an IResearchLink + } + + // TODO FIXME find a better way to retrieve an iResearch Link + // cannot use static_cast/reinterpret_cast since Index is not related to + // IResearchLink + auto link = std::dynamic_pointer_cast(index); + + if (!link) { + continue; + } + + if (nullptr != link->findAnalyzer(*analyzer)) { + // found referenced analyzer + return true; + } + } + } + + return false; + }; + + TRI_vocbase_t* vocbase{}; + + // check analyzer database + auto* dbFeature = ApplicationServer::lookupFeature("Database"); + + if (dbFeature) { + vocbase = dbFeature->lookupDatabase(dbName); + + if (checkDatabase(vocbase)) { + return true; + } + } + + // check system database if necessary + auto* sysDbFeature = ApplicationServer::lookupFeature("SystemDatabase"); + + if (sysDbFeature) { + auto sysVocbase = sysDbFeature->use(); + + if (sysVocbase.get() != vocbase && checkDatabase(sysVocbase.get())) { + return true; + } + } + + return false; +} + typedef irs::async_utils::read_write_mutex::read_mutex ReadMutex; typedef irs::async_utils::read_write_mutex::write_mutex WriteMutex; -} // namespace +} // namespace namespace arangodb { namespace iresearch { @@ -775,18 +811,15 @@ void AnalyzerPool::toVelocyPack( VPackBuilder& builder, irs::string_ref const& name) { TRI_ASSERT(builder.isOpenObject()); -// if (forPersistence) { -// arangodb::iresearch::addStringRef(builder, arangodb::StaticStrings::KeyString, name); -// } - arangodb::iresearch::addStringRef(builder, StaticStrings::AnalyzerNameField, name); - arangodb::iresearch::addStringRef(builder, StaticStrings::AnalyzerTypeField, type()); + addStringRef(builder, StaticStrings::AnalyzerNameField, name); + addStringRef(builder, StaticStrings::AnalyzerTypeField, type()); builder.add(StaticStrings::AnalyzerPropertiesField, properties()); // add features VPackArrayBuilder featuresScope(&builder, StaticStrings::AnalyzerFeaturesField); for (auto& feature: features()) { TRI_ASSERT(feature); // has to be non-nullptr - arangodb::iresearch::addStringRef(builder, feature->name()); + addStringRef(builder, feature->name()); } } @@ -811,11 +844,16 @@ void AnalyzerPool::toVelocyPack(VPackBuilder& builder, void AnalyzerPool::toVelocyPack(VPackBuilder& builder, bool forPersistence /*= false*/) { irs::string_ref name = this->name(); - if (forPersistence) { - name = IResearchAnalyzerFeature::splitAnalyzerName(name).second; - } VPackObjectBuilder rootScope(&builder); + + if (forPersistence) { + name = IResearchAnalyzerFeature::splitAnalyzerName(name).second; + + // ensure names are unique + addStringRef(builder, arangodb::StaticStrings::KeyString, name); + } + toVelocyPack(builder, name); } @@ -838,6 +876,13 @@ AnalyzerPool::AnalyzerPool(irs::string_ref const& name) _name(name) { } +bool AnalyzerPool::operator==(AnalyzerPool const& rhs) const { + return _name == rhs._name && + _type == rhs._type && + _features == rhs._features && + basics::VelocyPackHelper::equal(_properties, rhs._properties, true); +} + bool AnalyzerPool::init( irs::string_ref const& type, VPackSlice const properties, @@ -864,14 +909,14 @@ bool AnalyzerPool::init( // ensure no reallocations will happen _config.reserve(_config.size() + type.size()); - auto instance = _cache.emplace(type, arangodb::iresearch::slice(_config)); + auto instance = _cache.emplace(type, iresearch::slice(_config)); if (instance) { _properties = VPackSlice::noneSlice(); _type = irs::string_ref::NIL; _key = irs::string_ref::NIL; - _properties = arangodb::iresearch::slice(_config); + _properties = iresearch::slice(_config); if (!type.null()) { _config.append(type); @@ -882,18 +927,18 @@ bool AnalyzerPool::init( return true; } - } catch (arangodb::basics::Exception& e) { - LOG_TOPIC("62062", WARN, arangodb::iresearch::TOPIC) + } catch (basics::Exception& e) { + LOG_TOPIC("62062", WARN, iresearch::TOPIC) << "caught exception while initializing an arangosearch analizer type '" << _type << "' properties '" << _properties << "': " << e.code() << " " << e.what(); IR_LOG_EXCEPTION(); } catch (std::exception& e) { - LOG_TOPIC("a9196", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("a9196", WARN, iresearch::TOPIC) << "caught exception while initializing an arangosearch analizer type '" << _type << "' properties '" << _properties << "': " << e.what(); IR_LOG_EXCEPTION(); } catch (...) { - LOG_TOPIC("7524a", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("7524a", WARN, iresearch::TOPIC) << "caught exception while initializing an arangosearch analizer type '" << _type << "' properties '" << _properties << "'"; IR_LOG_EXCEPTION(); @@ -920,7 +965,7 @@ void AnalyzerPool::setKey(irs::string_ref const& key) { // After reallocation all methods of Slice will not work! const auto propertiesIsNone = _properties.isNone(); const auto propertiesByteSize = propertiesIsNone ? 0 : _properties.byteSize(); - + const auto keyOffset = _config.size(); _config.append(key.c_str(), key.size()); @@ -928,7 +973,7 @@ void AnalyzerPool::setKey(irs::string_ref const& key) { // append(...) if (!propertiesIsNone) { TRI_ASSERT(propertiesByteSize <= _config.size()); - _properties = arangodb::iresearch::slice(_config); + _properties = iresearch::slice(_config); } // update '_type' since '_config' might have been reallocated during @@ -945,21 +990,21 @@ irs::analysis::analyzer::ptr AnalyzerPool::get() const noexcept { try { // FIXME do not use shared_ptr return _cache.emplace(_type, _properties).release(); - } catch (arangodb::basics::Exception const& e) { - LOG_TOPIC("c9256", WARN, arangodb::iresearch::TOPIC) + } catch (basics::Exception const& e) { + LOG_TOPIC("c9256", WARN, iresearch::TOPIC) << "caught exception while instantiating an arangosearch analizer type " "'" << _type << "' properties '" << _properties << "': " << e.code() << " " << e.what(); IR_LOG_EXCEPTION(); } catch (std::exception& e) { - LOG_TOPIC("93baf", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("93baf", WARN, iresearch::TOPIC) << "caught exception while instantiating an arangosearch analizer type " "'" << _type << "' properties '" << _properties << "': " << e.what(); IR_LOG_EXCEPTION(); } catch (...) { - LOG_TOPIC("08db9", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("08db9", WARN, iresearch::TOPIC) << "caught exception while instantiating an arangosearch analizer type " "'" << _type << "' properties '" << _properties << "'"; @@ -969,7 +1014,7 @@ irs::analysis::analyzer::ptr AnalyzerPool::get() const noexcept { return nullptr; } -IResearchAnalyzerFeature::IResearchAnalyzerFeature(arangodb::application_features::ApplicationServer& server) +IResearchAnalyzerFeature::IResearchAnalyzerFeature(application_features::ApplicationServer& server) : ApplicationFeature(server, IResearchAnalyzerFeature::name()) { setOptional(true); startsAfter("V8Phase"); @@ -986,21 +1031,21 @@ IResearchAnalyzerFeature::IResearchAnalyzerFeature(arangodb::application_feature /*static*/ bool IResearchAnalyzerFeature::canUse( // check permissions TRI_vocbase_t const& vocbase, // analyzer vocbase - arangodb::auth::Level const& level // access level + auth::Level const& level // access level ) { - auto* ctx = arangodb::ExecContext::CURRENT; + auto* ctx = ExecContext::CURRENT; return !ctx // authentication not enabled || (ctx->canUseDatabase(vocbase.name(), level) // can use vocbase - && (ctx->canUseCollection(vocbase.name(), ANALYZER_COLLECTION_NAME, level)) // can use analyzers + && (ctx->canUseCollection(vocbase.name(), arangodb::StaticStrings::AnalyzersCollection, level)) // can use analyzers ); } /*static*/ bool IResearchAnalyzerFeature::canUse( // check permissions irs::string_ref const& name, // analyzer name (already normalized) - arangodb::auth::Level const& level // access level + auth::Level const& level // access level ) { - auto* ctx = arangodb::ExecContext::CURRENT; + auto* ctx = ExecContext::CURRENT; if (!ctx) { return true; // authentication not enabled @@ -1016,37 +1061,34 @@ IResearchAnalyzerFeature::IResearchAnalyzerFeature(arangodb::application_feature return split.first.null() // static analyzer (always allowed) || (ctx->canUseDatabase(split.first, level) // can use vocbase - && ctx->canUseCollection(split.first, ANALYZER_COLLECTION_NAME, level) // can use analyzers + && ctx->canUseCollection(split.first, arangodb::StaticStrings::AnalyzersCollection, level) // can use analyzers ); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief validate analyzer parameters and emplace into map -//////////////////////////////////////////////////////////////////////////////// -arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace - EmplaceAnalyzerResult& result, // emplacement result on success (out-param) - Analyzers& analyzers, + +/*static*/ Result IResearchAnalyzerFeature::createAnalyzerPool( + AnalyzerPool::ptr& pool, irs::string_ref const& name, irs::string_ref const& type, VPackSlice const properties, irs::flags const& features) { - // check type available if (!irs::analysis::analyzers::exists(type, irs::text_format::vpack, false)) { - return arangodb::Result( + return { TRI_ERROR_NOT_IMPLEMENTED, - "Not implemented analyzer type '" + std::string(type) + "'."); + "Not implemented analyzer type '" + std::string(type) + "'." + }; } // validate analyzer name auto split = splitAnalyzerName(name); if (!TRI_vocbase_t::IsAllowedName(false, velocypack::StringRef(split.second.c_str(), split.second.size()))) { - return arangodb::Result( // result - TRI_ERROR_BAD_PARAMETER, // code + return { + TRI_ERROR_BAD_PARAMETER, std::string("invalid characters in analyzer name '") + std::string(split.second) + "'" - ); -} + }; + } // validate that features are supported by arangod an ensure that their // dependencies are met @@ -1057,24 +1099,101 @@ arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace // no extra validation required } else if (&irs::position::type() == feature) { if (!features.check(irs::frequency::type())) { - return arangodb::Result( + return { TRI_ERROR_BAD_PARAMETER, "missing feature '" + std::string(irs::frequency::type().name()) + - "' required when '" + std::string(feature->name()) + "' feature is specified"); + "' required when '" + std::string(feature->name()) + "' feature is specified" + }; } } else if (feature) { - return arangodb::Result( + return { TRI_ERROR_BAD_PARAMETER, - "unsupported analyzer feature '" + std::string(feature->name()) + "'"); + "unsupported analyzer feature '" + std::string(feature->name()) + "'" + }; } } // limit the maximum size of analyzer properties if (ANALYZER_PROPERTIES_SIZE_MAX < properties.byteSize()) { - return arangodb::Result( + return { TRI_ERROR_BAD_PARAMETER, "analyzer properties size of '" + std::to_string(properties.byteSize()) + - "' exceeds the maximum allowed limit of '" + std::to_string(ANALYZER_PROPERTIES_SIZE_MAX) + "'"); + "' exceeds the maximum allowed limit of '" + std::to_string(ANALYZER_PROPERTIES_SIZE_MAX) + "'" + }; + } + + auto analyzerPool = std::make_shared(name); + + if (!analyzerPool->init(type, properties, features)) { + return { + TRI_ERROR_BAD_PARAMETER, + "Failure initializing an arangosearch analyzer instance for name '" + std::string(name) + + "' type '" + std::string(type) + "'." + + (properties.isNone() ? + std::string(" Init without properties") + : std::string(" Properties '") + properties.toString() + "'") + + " was rejected by analyzer. Please check documentation for corresponding analyzer type." + }; + } + + // noexcept + pool = analyzerPool; + return {}; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief validate analyzer parameters and emplace into map +//////////////////////////////////////////////////////////////////////////////// +Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace + EmplaceAnalyzerResult& result, // emplacement result on success (out-param) + Analyzers& analyzers, + irs::string_ref const& name, + irs::string_ref const& type, + VPackSlice const properties, + irs::flags const& features) { + // check type available + if (!irs::analysis::analyzers::exists(type, irs::text_format::vpack, false)) { + return { + TRI_ERROR_NOT_IMPLEMENTED, + "Not implemented analyzer type '" + std::string(type) + "'." }; + } + + // validate analyzer name + auto split = splitAnalyzerName(name); + + if (!TRI_vocbase_t::IsAllowedName(false, velocypack::StringRef(split.second.c_str(), split.second.size()))) { + return { + TRI_ERROR_BAD_PARAMETER, + std::string("invalid characters in analyzer name '") + std::string(split.second) + "'" }; + } + + // validate that features are supported by arangod an ensure that their + // dependencies are met + for(auto& feature: features) { + if (&irs::frequency::type() == feature) { + // no extra validation required + } else if (&irs::norm::type() == feature) { + // no extra validation required + } else if (&irs::position::type() == feature) { + if (!features.check(irs::frequency::type())) { + return { + TRI_ERROR_BAD_PARAMETER, + "missing feature '" + std::string(irs::frequency::type().name()) + + "' required when '" + std::string(feature->name()) + "' feature is specified" }; + } + } else if (feature) { + return { + TRI_ERROR_BAD_PARAMETER, + "unsupported analyzer feature '" + std::string(feature->name()) + "'" }; + } + } + + // limit the maximum size of analyzer properties + if (ANALYZER_PROPERTIES_SIZE_MAX < properties.byteSize()) { + return { + TRI_ERROR_BAD_PARAMETER, + "analyzer properties size of '" + std::to_string(properties.byteSize()) + + "' exceeds the maximum allowed limit of '" + std::to_string(ANALYZER_PROPERTIES_SIZE_MAX) + "'" }; } static const auto generator = []( // key + value generator @@ -1093,10 +1212,10 @@ arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace auto analyzer = itr.first->second; if (!analyzer) { - return arangodb::Result( + return { TRI_ERROR_BAD_PARAMETER, "failure creating an arangosearch analyzer instance for name '" + std::string(name) + - "' type '" + std::string(type) + "' properties '" + properties.toString() + "'"); + "' type '" + std::string(type) + "' properties '" + properties.toString() + "'" }; } // new analyzer creation, validate @@ -1108,15 +1227,15 @@ arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace } }); - if (!analyzer->init(type, properties, features)) { - return arangodb::Result( + if (!analyzer->init(type, properties, features)) { + return { TRI_ERROR_BAD_PARAMETER, "Failure initializing an arangosearch analyzer instance for name '" + std::string(name) + - "' type '" + std::string(type) + "'." + - (properties.isNone() ? + "' type '" + std::string(type) + "'." + + (properties.isNone() ? std::string(" Init without properties") : std::string(" Properties '") + properties.toString() + "'") + - " was rejected by analyzer. Please check documentation for corresponding analyzer type."); + " was rejected by analyzer. Please check documentation for corresponding analyzer type." }; } erase = false; @@ -1128,7 +1247,7 @@ arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace if (!properties.isNone()) { errorText << " properties:'" << properties.toString() << "'\n"; } - errorText << " features: [\n"; + errorText << " features: [\n"; for (auto feature = std::begin(features); feature != std::end(features);) { errorText << " '" << (*feature)->name() << "'"; ++feature; @@ -1141,47 +1260,32 @@ arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace analyzer->toVelocyPack(existingDefinition, false); errorText << " ]\n}\nPrevious definition was:\n" << existingDefinition.toString(); - return arangodb::Result(TRI_ERROR_BAD_PARAMETER, errorText.str()); - + return {TRI_ERROR_BAD_PARAMETER, errorText.str() }; + } result = itr; - return arangodb::Result(); + return {}; } -arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence if possible - EmplaceResult& result, // emplacement result on success (out-param) - irs::string_ref const& name, // analyzer name - irs::string_ref const& type, // analyzer type - VPackSlice const properties, // analyzer properties - irs::flags const& features, // analyzer features - bool isEmplace) { - try { - auto split = splitAnalyzerName(name); +Result IResearchAnalyzerFeature::emplace( + EmplaceResult& result, + irs::string_ref const& name, + irs::string_ref const& type, + VPackSlice const properties, + irs::flags const& features) { + auto const split = splitAnalyzerName(name); + try { WriteMutex mutex(_mutex); SCOPED_LOCK(mutex); if (!split.first.null()) { // do not trigger load for static-analyzer requests - // do not trigger load of analyzers on coordinator or db-server to avoid - // recursive lock aquisition in ClusterInfo::loadPlan() if called due to - // IResearchLink creation, - // also avoids extra cluster calls if it can be helped (optimization) - if (!isEmplace && arangodb::ServerState::instance()->isClusterRole()) { - auto itr = _analyzers.find( // find analyzer previous definition - irs::make_hashed_ref(name, std::hash()) - ); + auto res = loadAnalyzers(split.first); - if (itr != _analyzers.end()) { - _analyzers.erase(itr); // remove old definition instead of reloading all - } - } else { // trigger analyzer load - auto res = loadAnalyzers(split.first); - - if (!res.ok()) { - return res; - } + if (!res.ok()) { + return res; } } @@ -1193,7 +1297,7 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence return res; } - auto* engine = arangodb::EngineSelectorFeature::ENGINE; + auto* engine = EngineSelectorFeature::ENGINE; bool erase = itr.second; // an insertion took place auto cleanup = irs::make_finally([&erase, this, &itr]()->void { if (erase) { @@ -1205,17 +1309,17 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence // new pool creation if (itr.second) { if (!pool) { - return arangodb::Result( + return { TRI_ERROR_INTERNAL, "failure creating an arangosearch analyzer instance for name '" + std::string(name) + "' type '" + std::string(type) + - "' properties '" + properties.toString() + "'"); + "' properties '" + properties.toString() + "'" }; } // persist only on coordinator and single-server while not in recovery if ((!engine || !engine->inRecovery()) // do not persist during recovery - && (arangodb::ServerState::instance()->isCoordinator() // coordinator - || arangodb::ServerState::instance()->isSingleServer())) {// single-server + && (ServerState::instance()->isCoordinator() // coordinator + || ServerState::instance()->isSingleServer())) {// single-server res = storeAnalyzer(*pool); } @@ -1228,56 +1332,45 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence } result = std::make_pair(pool, itr.second); - } catch (arangodb::basics::Exception const& e) { - return arangodb::Result( // result - e.code(), // code + } catch (basics::Exception const& e) { + return { + e.code(), "caught exception while registering an arangosearch analizer name '" + std::string(name) + "' type '" + std::string(type) + "' properties '" + properties.toString() + - "': " + std::to_string(e.code()) + " " + e.what()); + "': " + std::to_string(e.code()) + " " + e.what() }; } catch (std::exception const& e) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code + return { + TRI_ERROR_INTERNAL, "caught exception while registering an arangosearch analizer name '" + std::string(name) + "' type '" + std::string(type) + - "' properties '" + properties.toString() + "': " + e.what()); + "' properties '" + properties.toString() + "': " + e.what() }; } catch (...) { - return arangodb::Result( // result + return { TRI_ERROR_INTERNAL, // code "caught exception while registering an arangosearch analizer name '" + std::string(name) + "' type '" + std::string(type) + - "' properties '" + properties.toString() + "'"); + "' properties '" + properties.toString() + "'" }; } - return arangodb::Result(); + return {}; } -AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer - irs::string_ref const& name, // analyzer name - TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name - TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix - bool onlyCached /*= false*/ // check only locally cached analyzers -) const noexcept { +AnalyzerPool::ptr IResearchAnalyzerFeature::get( + irs::string_ref const& normalizedName, + AnalyzerName const& name, + bool onlyCached) const noexcept { try { - auto const normalizedName = normalize(name, activeVocbase, systemVocbase, true); - - auto const split = splitAnalyzerName(normalizedName); - - // FIXME deduplicate code below, see get(irs::string, bool) - - if (!split.first.null()) { // check if analyzer is static - if (split.first != activeVocbase.name() && split.first != systemVocbase.name()) { - // accessing local analyzer from within another database - return nullptr; - } - + if (!name.first.null()) { // check if analyzer is static if (!onlyCached) { // load analyzers for database - auto res = const_cast(this)->loadAnalyzers(split.first); + auto const res = const_cast(this)->loadAnalyzers(name.first); if (!res.ok()) { - LOG_TOPIC("36062", WARN, arangodb::iresearch::TOPIC) - << "failure to load analyzers for database '" << split.first << "' while getting analyzer '" << name << "': " << res.errorNumber() << " " << res.errorMessage(); + LOG_TOPIC("36062", WARN, iresearch::TOPIC) + << "failure to load analyzers for database '" << name.first + << "' while getting analyzer '" << name + << "': " << res.errorNumber() << " " << res.errorMessage(); TRI_set_errno(res.errorNumber()); return nullptr; @@ -1287,14 +1380,11 @@ AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer ReadMutex mutex(_mutex); SCOPED_LOCK(mutex); - auto itr = _analyzers.find( - irs::make_hashed_ref(static_cast(normalizedName), - std::hash()) - ); + auto itr = _analyzers.find(irs::make_hashed_ref(normalizedName, std::hash())); if (itr == _analyzers.end()) { - LOG_TOPIC("4049c", WARN, arangodb::iresearch::TOPIC) - << "failure to find arangosearch analyzer name '" << name << "'"; + LOG_TOPIC("4049c", WARN, iresearch::TOPIC) + << "failure to find arangosearch analyzer name '" << normalizedName << "'"; return nullptr; } @@ -1305,88 +1395,43 @@ AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer return pool; } - LOG_TOPIC("1a29c", WARN, arangodb::iresearch::TOPIC) - << "failure to get arangosearch analyzer name '" << name << "'"; + LOG_TOPIC("1a29c", WARN, iresearch::TOPIC) + << "failure to get arangosearch analyzer name '" << normalizedName << "'"; TRI_set_errno(TRI_ERROR_INTERNAL); - } catch (arangodb::basics::Exception& e) { - LOG_TOPIC("29eff", WARN, arangodb::iresearch::TOPIC) + } catch (basics::Exception const& e) { + LOG_TOPIC("29eff", WARN, iresearch::TOPIC) << "caught exception while retrieving an arangosearch analizer name '" - << name << "': " << e.code() << " " << e.what(); - IR_LOG_EXCEPTION(); - } catch (std::exception& e) { - LOG_TOPIC("ce8d5", WARN, arangodb::iresearch::TOPIC) + << normalizedName << "': " << e.code() << " " << e.what(); + } catch (std::exception const& e) { + LOG_TOPIC("ce8d5", WARN, iresearch::TOPIC) << "caught exception while retrieving an arangosearch analizer name '" - << name << "': " << e.what(); - IR_LOG_EXCEPTION(); + << normalizedName << "': " << e.what(); } catch (...) { - LOG_TOPIC("5505f", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("5505f", WARN, iresearch::TOPIC) << "caught exception while retrieving an arangosearch analizer name '" - << name << "'"; - IR_LOG_EXCEPTION(); + << normalizedName << "'"; } return nullptr; } -AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer - irs::string_ref const& name, // analyzer name - bool onlyCached /*= false*/ // check only locally cached analyzers -) const noexcept { - try { - auto const split = splitAnalyzerName(name); +AnalyzerPool::ptr IResearchAnalyzerFeature::get( + irs::string_ref const& name, + TRI_vocbase_t const& activeVocbase, + TRI_vocbase_t const& systemVocbase, + bool onlyCached /*= false*/) const { + auto const normalizedName = normalize(name, activeVocbase, systemVocbase, true); - if (!split.first.null() && !onlyCached) { // do not trigger load for static-analyzer requests - auto res = // load analyzers for database - const_cast(this)->loadAnalyzers(split.first); + auto const split = splitAnalyzerName(normalizedName); - if (!res.ok()) { - LOG_TOPIC("36068", WARN, arangodb::iresearch::TOPIC) - << "failure to load analyzers for database '" << split.first << "' while getting analyzer '" << name << "': " << res.errorNumber() << " " << res.errorMessage(); - TRI_set_errno(res.errorNumber()); - - return nullptr; - } - } - - ReadMutex mutex(_mutex); - SCOPED_LOCK(mutex); - auto itr = - _analyzers.find(irs::make_hashed_ref(name, std::hash())); - - if (itr == _analyzers.end()) { - LOG_TOPIC("4049d", WARN, arangodb::iresearch::TOPIC) - << "failure to find arangosearch analyzer name '" << name << "'"; - - return nullptr; - } - - auto pool = itr->second; - - if (pool) { - return pool; - } - - LOG_TOPIC("826db", WARN, arangodb::iresearch::TOPIC) - << "failure to get arangosearch analyzer name '" << name << "'"; - TRI_set_errno(TRI_ERROR_INTERNAL); - } catch (arangodb::basics::Exception& e) { - LOG_TOPIC("89eff", WARN, arangodb::iresearch::TOPIC) - << "caught exception while retrieving an arangosearch analizer name '" - << name << "': " << e.code() << " " << e.what(); - IR_LOG_EXCEPTION(); - } catch (std::exception& e) { - LOG_TOPIC("ce8d9", WARN, arangodb::iresearch::TOPIC) - << "caught exception while retrieving an arangosearch analizer name '" - << name << "': " << e.what(); - IR_LOG_EXCEPTION(); - } catch (...) { - LOG_TOPIC("55050", WARN, arangodb::iresearch::TOPIC) - << "caught exception while retrieving an arangosearch analizer name '" - << name << "'"; - IR_LOG_EXCEPTION(); + if (!split.first.null() && + split.first != activeVocbase.name() && + split.first != systemVocbase.name()) { + // accessing local analyzer from within another database + return nullptr; } - return nullptr; + return get(normalizedName, split, onlyCached); } //////////////////////////////////////////////////////////////////////////////// @@ -1407,7 +1452,7 @@ AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer if (!pool || !pool->init(IdentityAnalyzer::type().name(), VPackSlice::emptyObjectSlice(), extraFeatures)) { - LOG_TOPIC("26de1", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("26de1", WARN, iresearch::TOPIC) << "failure creating an arangosearch static analyzer instance " "for name '" << name << "'"; @@ -1457,7 +1502,7 @@ AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer auto pool = std::make_shared(name); if (!pool->init(type, properties.slice(), extraFeatures)) { - LOG_TOPIC("e25f5", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("e25f5", WARN, iresearch::TOPIC) << "failure creating an arangosearch static analyzer instance " "for name '" << name << "'"; @@ -1498,45 +1543,38 @@ AnalyzerPool::ptr IResearchAnalyzerFeature::get( // find analyzer return identity.instance; } -arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( +Result IResearchAnalyzerFeature::loadAnalyzers( irs::string_ref const& database /*= irs::string_ref::NIL*/) { + auto* dbFeature = application_features::ApplicationServer::lookupFeature("Database"); + + if (!dbFeature) { + return { + TRI_ERROR_INTERNAL, + "failure to find feature 'Database' while loading analyzers for database '" + std::string(database)+ "'" + }; + } + try { - auto* dbFeature = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - arangodb::DatabaseFeature // feature type - >("Database"); - - if (!dbFeature) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find feature 'Database' while loading analyzers for database '") + std::string(database)+ "'" - ); - } - WriteMutex mutex(_mutex); SCOPED_LOCK(mutex); // '_analyzers'/'_lastLoad' can be asynchronously read // load all databases if (database.null()) { - arangodb::Result res; + Result res; std::unordered_set seen; auto visitor = [this, &res, &seen](TRI_vocbase_t& vocbase)->void { - if (!vocbase.lookupCollection(ANALYZER_COLLECTION_NAME)) { - return; // skip databases lacking 'ANALYZER_COLLECTION_NAME' (no analyzers there, not an error) - } - - auto name = irs::make_hashed_ref( // vocbase name - irs::string_ref(vocbase.name()), std::hash() // args - ); + auto name = irs::make_hashed_ref(irs::string_ref(vocbase.name()), + std::hash()); auto result = loadAnalyzers(name); auto itr = _lastLoad.find(name); if (itr != _lastLoad.end()) { seen.insert(name); } else if (res.ok()) { // load errors take precedence - res = arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - "failure to find database last load timestamp after loading analyzers" // message - ); + res = { + TRI_ERROR_INTERNAL, + "failure to find database last load timestamp after loading analyzers" + }; } if (!result.ok()) { @@ -1550,9 +1588,8 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( // remove unseen databases from timestamp list for (auto itr = _lastLoad.begin(), end = _lastLoad.end(); itr != end;) { - auto name = irs::make_hashed_ref( // vocbase name - irs::string_ref(itr->first), std::hash() // args - ); + auto name = irs::make_hashed_ref(irs::string_ref(itr->first), + std::hash()); auto seenItr = seen.find(name); if (seenItr == seen.end()) { @@ -1566,8 +1603,8 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( // remove no longer valid analyzers (force remove) for (auto itr = _analyzers.begin(), end = _analyzers.end(); itr != end;) { auto split = splitAnalyzerName(itr->first); - auto unseenItr = // ignore static analyzers - split.first.null() ? unseen.end() : unseen.find(split.first); + // ignore static analyzers + auto unseenItr = split.first.null() ? unseen.end() : unseen.find(split.first); if (unseenItr != unseen.end()) { itr = _analyzers.erase(itr); @@ -1586,40 +1623,39 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( auto currentTimestamp = std::chrono::system_clock::now(); auto databaseKey = // database key used in '_lastLoad' irs::make_hashed_ref(database, std::hash()); - auto* engine = arangodb::EngineSelectorFeature::ENGINE; + auto* engine = EngineSelectorFeature::ENGINE; auto itr = _lastLoad.find(databaseKey); // find last update timestamp if (!engine || engine->inRecovery()) { // always load if inRecovery since collection contents might have changed // unless on db-server which does not store analyzer definitions in collections - if (arangodb::ServerState::instance()->isDBServer()) { - return arangodb::Result(); // db-server should not access cluster during inRecovery + if (ServerState::instance()->isDBServer()) { + return {}; // db-server should not access cluster during inRecovery } - } else if (arangodb::ServerState::instance()->isSingleServer()) { // single server + } else if (ServerState::instance()->isSingleServer()) { // single server if(itr != _lastLoad.end()) { - return arangodb::Result(); // do not reload on single-server + return {}; // do not reload on single-server } } else if (itr != _lastLoad.end() // had a previous load - && itr->second + RELOAD_INTERVAL > currentTimestamp // timeout not reached - ) { - return arangodb::Result(); // reload interval not reached + && itr->second + RELOAD_INTERVAL > currentTimestamp) { // timeout not reached + return {}; // reload interval not reached } auto* vocbase = dbFeature->lookupDatabase(database); if (!vocbase) { if (engine && engine->inRecovery()) { - return arangodb::Result(); // database might not have come up yet + return {}; // database might not have come up yet } if (itr != _lastLoad.end()) { cleanupAnalyzers(database); // cleanup any analyzers for 'database' _lastLoad.erase(itr); } - return arangodb::Result( // result - TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, // code - std::string("failed to find database '") + std::string(database) + "' while loading analyzers" - ); + return { + TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, + "failed to find database '" + std::string(database) + "' while loading analyzers" + }; } if (!getAnalyzerCollection(*vocbase)) { @@ -1628,18 +1664,19 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( } _lastLoad[databaseKey] = currentTimestamp; // update timestamp - return arangodb::Result(); // no collection means nothing to load + return {}; // no collection means nothing to load } Analyzers analyzers; - auto visitor = [this, &analyzers, &vocbase](arangodb::velocypack::Slice const& slice)->arangodb::Result { + auto visitor = [this, &analyzers, &vocbase](velocypack::Slice const& slice)-> Result { if (!slice.isObject()) { - LOG_TOPIC("5c7a5", ERR, arangodb::iresearch::TOPIC) - << "failed to find an object value for analyzer definition while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + LOG_TOPIC("5c7a5", ERR, iresearch::TOPIC) + << "failed to find an object value for analyzer definition while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } irs::flags features; @@ -1651,36 +1688,38 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( if (!slice.hasKey(arangodb::StaticStrings::KeyString) // no such field (required) || !slice.get(arangodb::StaticStrings::KeyString).isString()) { - LOG_TOPIC("1dc56", ERR, arangodb::iresearch::TOPIC) + LOG_TOPIC("1dc56", ERR, iresearch::TOPIC) << "failed to find a string value for analyzer '" << arangodb::StaticStrings::KeyString - << "' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + << "' while loading analyzer form collection '" << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } key = getStringRef(slice.get(arangodb::StaticStrings::KeyString)); if (!slice.hasKey("name") // no such field (required) || !(slice.get("name").isString() || slice.get("name").isNull())) { - LOG_TOPIC("f5920", ERR, arangodb::iresearch::TOPIC) - << "failed to find a string value for analyzer 'name' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + LOG_TOPIC("f5920", ERR, iresearch::TOPIC) + << "failed to find a string value for analyzer 'name' while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } name = getStringRef(slice.get("name")); if (!slice.hasKey("type") // no such field (required) || !(slice.get("type").isString() || slice.get("name").isNull())) { - LOG_TOPIC("9f5c8", ERR, arangodb::iresearch::TOPIC) - << "failed to find a string value for analyzer 'type' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + LOG_TOPIC("9f5c8", ERR, iresearch::TOPIC) + << "failed to find a string value for analyzer 'type' while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } type = getStringRef(slice.get("type")); @@ -1692,11 +1731,12 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( properties = subSlice; // format as a jSON encoded string } else { LOG_TOPIC("a297e", ERR, arangodb::iresearch::TOPIC) - << "failed to find a string value for analyzer 'properties' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + << "failed to find a string value for analyzer 'properties' while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } } @@ -1705,38 +1745,41 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( if (!subSlice.isArray()) { LOG_TOPIC("7ec8a", ERR, arangodb::iresearch::TOPIC) - << "failed to find an array value for analyzer 'features' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + << "failed to find an array value for analyzer 'features' while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } - for (arangodb::velocypack::ArrayIterator subItr(subSlice); + for (velocypack::ArrayIterator subItr(subSlice); subItr.valid(); ++subItr ) { auto subEntry = *subItr; if (!subEntry.isString() && !subSlice.isNull()) { - LOG_TOPIC("7620d", ERR, arangodb::iresearch::TOPIC) - << "failed to find a string value for an entry in analyzer 'features' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + LOG_TOPIC("7620d", ERR, iresearch::TOPIC) + << "failed to find a string value for an entry in analyzer 'features' while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } auto featureName = getStringRef(subEntry); auto* feature = irs::attribute::type_id::get(featureName); if (!feature) { - LOG_TOPIC("4fedc", ERR, arangodb::iresearch::TOPIC) - << "failed to find feature '" << featureName << "' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME + LOG_TOPIC("4fedc", ERR, iresearch::TOPIC) + << "failed to find feature '" << featureName << "' while loading analyzer form collection '" + << arangodb::StaticStrings::AnalyzersCollection << "' in database '" << vocbase->name() << "', skipping it: " << slice.toString(); - return arangodb::Result(); // skip analyzer + return {}; // skip analyzer } features.add(*feature); @@ -1755,7 +1798,7 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( result.first->second->setKey(key); // update key } - return arangodb::Result(); + return {}; }; auto res = visitAnalyzers(*vocbase, visitor); @@ -1777,15 +1820,19 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( if (!result.second) { // existing entry if (result.first->second // valid new entry - && !equalAnalyzer(*(entry.second), result.first->second->type(), result.first->second->properties(), result.first->second->features())) { - return arangodb::Result( // result - TRI_ERROR_BAD_PARAMETER, // code - "name collision detected while re-registering a duplicate arangosearch analizer name '" + std::string(result.first->second->name()) + + && !equalAnalyzer(*(entry.second), + result.first->second->type(), + result.first->second->properties(), + result.first->second->features())) { + return { + TRI_ERROR_BAD_PARAMETER, + "name collision detected while re-registering a duplicate arangosearch analizer name '" + + std::string(result.first->second->name()) + "' type '" + std::string(result.first->second->type()) + "' properties '" + result.first->second->properties().toString() + "', previous registration type '" + std::string(entry.second->type()) + "' properties '" + entry.second->properties().toString() + "'" - ); + }; } result.first->second = entry.second; // reuse old analyzer pool to avoid duplicates in memmory @@ -1802,15 +1849,19 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( } if (itr->second // valid new entry - && !equalAnalyzer(*(entry.second), itr->second->type(), itr->second->properties(), itr->second->features())) { - return arangodb::Result( // result - TRI_ERROR_BAD_PARAMETER, // code - "name collision detected while registering a duplicate arangosearch analizer name '" + std::string(itr->second->name()) + + && !equalAnalyzer(*(entry.second), + itr->second->type(), + itr->second->properties(), + itr->second->features())) { + return { + TRI_ERROR_BAD_PARAMETER, + "name collision detected while registering a duplicate arangosearch analizer name '" + + std::string(itr->second->name()) + "' type '" + std::string(itr->second->type()) + "' properties '" + itr->second->properties().toString() + "', previous registration type '" + std::string(entry.second->type()) + "' properties '" + entry.second->properties().toString() + "'" - ); + }; } itr->second = entry.second; // reuse old analyzer pool to avoid duplicates in memmory @@ -1819,38 +1870,29 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( _lastLoad[databaseKey] = currentTimestamp; // update timestamp _analyzers = std::move(analyzers); // update mappings - } catch (arangodb::basics::Exception const& e) { - return arangodb::Result( // result - e.code(), // code - std::string("caught exception while loading configuration for arangosearch analyzers from database '") + std::string(database) + "': " + std::to_string(e.code()) + " "+ e.what() - ); + } catch (basics::Exception const& e) { + return { e.code(), + "caught exception while loading configuration for arangosearch analyzers from database '" + + std::string(database) + "': " + std::to_string(e.code()) + " "+ e.what() }; } catch (std::exception const& e) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("caught exception while loading configuration for arangosearch analyzers from database '") + std::string(database) + "': " + e.what() - ); + return { TRI_ERROR_INTERNAL, + "caught exception while loading configuration for arangosearch analyzers from database '" + + std::string(database) + "': " + e.what() }; } catch (...) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("caught exception while loading configuration for arangosearch analyzers from database '") + std::string(database) + "'" - ); + return { TRI_ERROR_INTERNAL, + "caught exception while loading configuration for arangosearch analyzers from database '" + + std::string(database) + "'" }; } - return arangodb::Result(); + return {}; } /*static*/ std::string const& IResearchAnalyzerFeature::name() noexcept { return FEATURE_NAME; } -/*static*/ irs::string_ref IResearchAnalyzerFeature::extractVocbaseName( - irs::string_ref const& name) noexcept {// analyzer name (normalized) - auto split = splitAnalyzerName(name); - return split.first; -} - /*static*/ bool IResearchAnalyzerFeature::analyzerReachableFromDb( - irs::string_ref const& dbNameFromAnalyzer, + irs::string_ref const& dbNameFromAnalyzer, irs::string_ref const& currentDbName, bool forGetters) noexcept { TRI_ASSERT(!currentDbName.empty()); if (dbNameFromAnalyzer.null()) { // NULL means local db name = always reachable @@ -1862,11 +1904,11 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( } return currentDbName == arangodb::StaticStrings::SystemDatabase; } - return currentDbName == dbNameFromAnalyzer || + return currentDbName == dbNameFromAnalyzer || (forGetters && dbNameFromAnalyzer == arangodb::StaticStrings::SystemDatabase); } -/*static*/ std::pair IResearchAnalyzerFeature::splitAnalyzerName( +/*static*/ std::pair IResearchAnalyzerFeature::splitAnalyzerName( irs::string_ref const& analyzer) noexcept { // search for vocbase prefix ending with '::' for (size_t i = 1, count = analyzer.size(); i < count; ++i) { @@ -1887,11 +1929,10 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( } /*static*/ std::string IResearchAnalyzerFeature::normalize( // normalize name - irs::string_ref const& name, // analyzer name - TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name - TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix - bool expandVocbasePrefix /*= true*/ // use full vocbase name as prefix for active/system v.s. EMPTY/'::' -) { + irs::string_ref const& name, // analyzer name + TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name + TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix + bool expandVocbasePrefix /*= true*/) { // use full vocbase name as prefix for active/system v.s. EMPTY/'::' auto& staticAnalyzers = getStaticAnalyzers(); if (staticAnalyzers.find(irs::make_hashed_ref(name, std::hash())) != staticAnalyzers.end()) { @@ -1913,7 +1954,9 @@ arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( // normalize vocbase such that active vocbase takes precedence over system // vocbase i.e. prefer NIL over EMPTY // ......................................................................... - if (&systemVocbase == &activeVocbase || split.first.null() || (split.first == activeVocbase.name())) { // active vocbase + if (&systemVocbase == &activeVocbase || + split.first.null() || + (split.first == activeVocbase.name())) { // active vocbase return split.second; } @@ -1937,31 +1980,26 @@ void IResearchAnalyzerFeature::prepare() { _analyzers = getStaticAnalyzers(); } -arangodb::Result IResearchAnalyzerFeature::remove( // remove analyzer - irs::string_ref const& name, // analyzer name - bool force /*= false*/ // force removal -) { +Result IResearchAnalyzerFeature::remove( + irs::string_ref const& name, + bool force /*= false*/) { try { auto split = splitAnalyzerName(name); if (split.first.null()) { - return arangodb::Result( // result - TRI_ERROR_FORBIDDEN, // code - "built-in analyzers cannot be removed" // message - ); + return { TRI_ERROR_FORBIDDEN, "built-in analyzers cannot be removed" }; } WriteMutex mutex(_mutex); SCOPED_LOCK(mutex); - auto itr = // find analyzer - _analyzers.find(irs::make_hashed_ref(name, std::hash())); + auto itr = _analyzers.find(irs::make_hashed_ref(name, std::hash())); if (itr == _analyzers.end()) { - return arangodb::Result( // result - TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, // code - std::string("failure to find analyzer while removing arangosearch analyzer '") + std::string(name)+ "'" - ); + return { + TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, + "failure to find analyzer while removing arangosearch analyzer '" + std::string(name)+ "'" + }; } auto& pool = itr->second; @@ -1970,25 +2008,25 @@ arangodb::Result IResearchAnalyzerFeature::remove( // remove analyzer // will make the state consistent again if (!pool) { _analyzers.erase(itr); - return arangodb::Result( // result - TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, // code - std::string("failure to find a valid analyzer while removing arangosearch analyzer '") + std::string(name)+ "'" - ); + return { + TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, + "failure to find a valid analyzer while removing arangosearch analyzer '" + std::string(name)+ "'" + }; } - if (!force && pool.use_count() > 1) { // +1 for ref in '_analyzers' - return arangodb::Result( // result - TRI_ERROR_ARANGO_CONFLICT, // code - std::string("analyzer in-use while removing arangosearch analyzer '") + std::string(name)+ "'" - ); + if (!force && analyzerInUse(split.first, pool)) { // +1 for ref in '_analyzers' + return { + TRI_ERROR_ARANGO_CONFLICT, + "analyzer in-use while removing arangosearch analyzer '" + std::string(name)+ "'" + }; } // on db-server analyzers are not persisted // allow removal even inRecovery() - if (arangodb::ServerState::instance()->isDBServer()) { + if (ServerState::instance()->isDBServer()) { _analyzers.erase(itr); - return arangodb::Result(); + return {}; } // ......................................................................... @@ -1998,123 +2036,116 @@ arangodb::Result IResearchAnalyzerFeature::remove( // remove analyzer // this should never happen since non-static analyzers should always have a // valid '_key' after if (pool->_key.null()) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find '") + arangodb::StaticStrings::KeyString + "' while removing arangosearch analyzer '" + std::string(name)+ "'" - ); + return { + TRI_ERROR_INTERNAL, + "failure to find '" + arangodb::StaticStrings::KeyString + + "' while removing arangosearch analyzer '" + std::string(name)+ "'" }; } - auto* engine = arangodb::EngineSelectorFeature::ENGINE; + auto* engine = EngineSelectorFeature::ENGINE; // do not allow persistence while in recovery if (engine && engine->inRecovery()) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to remove arangosearch analyzer '") + std::string(name)+ "' configuration while storage engine in recovery" - ); + return { TRI_ERROR_INTERNAL, + "failure to remove arangosearch analyzer '" + std::string(name) + + "' configuration while storage engine in recovery" }; } - auto* dbFeature = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - arangodb::DatabaseFeature // feature type + auto* dbFeature = application_features::ApplicationServer::lookupFeature< + DatabaseFeature >("Database"); if (!dbFeature) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find feature 'Database' while removing arangosearch analyzer '") + std::string(name)+ "'" - ); + return { + TRI_ERROR_INTERNAL, + "failure to find feature 'Database' while removing arangosearch analyzer '" + std::string(name)+ "'" + }; } auto* vocbase = dbFeature->useDatabase(split.first); if (!vocbase) { - return arangodb::Result( // result - TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, // code - std::string("failure to find vocbase while removing arangosearch analyzer '") + std::string(name)+ "'" - ); + return { + TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, + "failure to find vocbase while removing arangosearch analyzer '" + std::string(name)+ "'" + }; } - std::shared_ptr collection; - auto collectionCallback = [&collection]( // store collection - std::shared_ptr const& col // args - )->void { + std::shared_ptr collection; + auto collectionCallback = [&collection]( + std::shared_ptr const& col)->void { collection = col; }; - arangodb::methods::Collections::lookup( // find collection - *vocbase, ANALYZER_COLLECTION_NAME, collectionCallback // args + methods::Collections::lookup( // find collection + *vocbase, arangodb::StaticStrings::AnalyzersCollection, collectionCallback // args ); if (!collection) { - return arangodb::Result( // result - TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, // code - std::string("failure to find collection while removing arangosearch analyzer '") + std::string(name)+ "'" - ); + return { + TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, + "failure to find collection while removing arangosearch analyzer '" + std::string(name)+ "'" + }; } - arangodb::SingleCollectionTransaction trx( // transaction - arangodb::transaction::StandaloneContext::Create(*vocbase), // transaction context - ANALYZER_COLLECTION_NAME, // collection name - arangodb::AccessMode::Type::WRITE // collection access type - ); + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(*vocbase), + arangodb::StaticStrings::AnalyzersCollection, + AccessMode::Type::WRITE); auto res = trx.begin(); if (!res.ok()) { return res; } - arangodb::velocypack::Builder builder; - arangodb::OperationOptions options; + velocypack::Builder builder; + OperationOptions options; builder.openObject(); addStringRef(builder, arangodb::StaticStrings::KeyString, pool->_key); builder.close(); auto result = // remove - trx.remove(ANALYZER_COLLECTION_NAME, builder.slice(), options); + trx.remove(arangodb::StaticStrings::AnalyzersCollection, builder.slice(), options); if (!result.ok()) { trx.abort(); return result.result; } - + auto commitResult = trx.commit(); if (!commitResult.ok()) { return commitResult; } _analyzers.erase(itr); - } catch (arangodb::basics::Exception const& e) { - return arangodb::Result( // result - e.code(), // code - std::string("caught exception while removing configuration for arangosearch analyzer name '") + std::string(name) + "': " + std::to_string(e.code()) + " "+ e.what() - ); + } catch (basics::Exception const& e) { + return { e.code(), + "caught exception while removing configuration for arangosearch analyzer name '" + + std::string(name) + "': " + std::to_string(e.code()) + " "+ e.what() }; } catch (std::exception const& e) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("caught exception while removing configuration for arangosearch analyzer name '") + std::string(name) + "': " + e.what() - ); + return { TRI_ERROR_INTERNAL, + "caught exception while removing configuration for arangosearch analyzer name '" + + std::string(name) + "': " + e.what() }; } catch (...) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("caught exception while removing configuration for arangosearch analyzer name '") + std::string(name) + "'" - ); + return { TRI_ERROR_INTERNAL, + "caught exception while removing configuration for arangosearch analyzer name '" + + std::string(name) + "'" }; } - return arangodb::Result(); + return {}; } void IResearchAnalyzerFeature::start() { if (!isEnabled()) { return; } - + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE // sanity check: we rely on this condition is true internally { auto sysDbFeature = - arangodb::application_features::ApplicationServer::lookupFeature( + application_features::ApplicationServer::lookupFeature( "SystemDatabase"); if (sysDbFeature && sysDbFeature->use()) { // feature/db may be absent in some unit-test enviroment TRI_ASSERT(sysDbFeature->use()->name() == arangodb::StaticStrings::SystemDatabase); @@ -2124,23 +2155,17 @@ void IResearchAnalyzerFeature::start() { // register analyzer functions { auto* functions = - arangodb::application_features::ApplicationServer::lookupFeature( + application_features::ApplicationServer::lookupFeature( "AQLFunctions"); if (functions) { addFunctions(*functions); } else { - LOG_TOPIC("7dcd9", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("7dcd9", WARN, iresearch::TOPIC) << "failure to find feature 'AQLFunctions' while registering " "IResearch functions"; } } - - auto res = loadAnalyzers(); - - if (!res.ok()) { - THROW_ARANGO_EXCEPTION(res); - } } void IResearchAnalyzerFeature::stop() { @@ -2156,76 +2181,71 @@ void IResearchAnalyzerFeature::stop() { } } -arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) { - auto* dbFeature = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - arangodb::DatabaseFeature // feature type +Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) { + auto* dbFeature = application_features::ApplicationServer::lookupFeature< // find feature + DatabaseFeature // feature type >("Database"); if (!dbFeature) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find feature 'Database' while persising arangosearch analyzer '") + pool.name()+ "'" - ); + return { TRI_ERROR_INTERNAL, + "failure to find feature 'Database' while persising arangosearch analyzer '" + + pool.name()+ "'" }; } if (pool.type().null()) { - return arangodb::Result( // result - TRI_ERROR_BAD_PARAMETER, // code - std::string("failure to persist arangosearch analyzer '") + pool.name()+ "' configuration with 'null' type" - ); + return { TRI_ERROR_BAD_PARAMETER, + "failure to persist arangosearch analyzer '" + + pool.name() + "' configuration with 'null' type" }; } - auto* engine = arangodb::EngineSelectorFeature::ENGINE; + auto* engine = EngineSelectorFeature::ENGINE; // do not allow persistence while in recovery if (engine && engine->inRecovery()) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to persist arangosearch analyzer '") + pool.name()+ "' configuration while storage engine in recovery" - ); + return { TRI_ERROR_INTERNAL, + "failure to persist arangosearch analyzer '" + pool.name() + + "' configuration while storage engine in recovery" }; } auto split = splitAnalyzerName(pool.name()); auto* vocbase = dbFeature->useDatabase(split.first); if (!vocbase) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find vocbase while persising arangosearch analyzer '") + pool.name()+ "'" - ); + return { TRI_ERROR_INTERNAL, + "failure to find vocbase while persising arangosearch analyzer '" + + pool.name() + "'" }; } try { auto collection = getAnalyzerCollection(*vocbase); + if (!collection) { - return arangodb::Result( // result - TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, // code - std::string("failure to find collection '") + - ANALYZER_COLLECTION_NAME + - "' in vocbase '" + vocbase->name() + - "' vocbase while persising arangosearch analyzer '" + pool.name()+ "'" - ); + return { + TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, + "failure to find collection '" + + arangodb::StaticStrings::AnalyzersCollection + + "' in vocbase '" + vocbase->name() + + "' vocbase while persising arangosearch analyzer '" + pool.name()+ "'" }; } - arangodb::SingleCollectionTransaction trx( // transaction - arangodb::transaction::StandaloneContext::Create(*vocbase), // transaction context - ANALYZER_COLLECTION_NAME, // collection name - arangodb::AccessMode::Type::WRITE // collection access type - ); + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(*vocbase), + arangodb::StaticStrings::AnalyzersCollection, + AccessMode::Type::WRITE); auto res = trx.begin(); if (!res.ok()) { return res; } - arangodb::velocypack::Builder builder; - // for storing in db analyzers collection - store only analyzer name + velocypack::Builder builder; + // for storing in db analyzers collection - store only analyzer name pool.toVelocyPack(builder, true); - arangodb::OperationOptions options; + OperationOptions options; options.waitForSync = true; - auto result = trx.insert(ANALYZER_COLLECTION_NAME, builder.slice(), options); + auto result = trx.insert(arangodb::StaticStrings::AnalyzersCollection, + builder.slice(), options); if (!result.ok()) { trx.abort(); @@ -2236,19 +2256,17 @@ arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) { auto slice = result.slice(); if (!slice.isObject()) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to parse result as a JSON object while persisting configuration for arangosearch analyzer name '") + pool.name() + "'" - ); + return { TRI_ERROR_INTERNAL, + "failure to parse result as a JSON object while persisting configuration for arangosearch analyzer name '" + + pool.name() + "'" }; } auto key = slice.get(arangodb::StaticStrings::KeyString); if (!key.isString()) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("failure to find the resulting key field while persisting configuration for arangosearch analyzer name '") + pool.name() + "'" - ); + return { TRI_ERROR_INTERNAL, + "failure to find the resulting key field while persisting configuration for arangosearch analyzer name '" + + pool.name() + "'" }; } res = trx.commit(); @@ -2260,24 +2278,21 @@ arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) { } pool.setKey(getStringRef(key)); - } catch (arangodb::basics::Exception const& e) { - return arangodb::Result( // result - e.code(), // code - std::string("caught exception while persisting configuration for arangosearch analyzer name '") + pool.name() + "': " + std::to_string(e.code()) + " "+ e.what() - ); + } catch (basics::Exception const& e) { + return { e.code(), + "caught exception while persisting configuration for arangosearch analyzer name '" + + pool.name() + "': " + std::to_string(e.code()) + " "+ e.what() }; } catch (std::exception const& e) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("caught exception while persisting configuration for arangosearch analyzer name '") + pool.name() + "': " + e.what() - ); + return { TRI_ERROR_INTERNAL, + "caught exception while persisting configuration for arangosearch analyzer name '" + + pool.name() + "': " + e.what() }; } catch (...) { - return arangodb::Result( // result - TRI_ERROR_INTERNAL, // code - std::string("caught exception while persisting configuration for arangosearch analyzer name '") + pool.name() + "'" - ); + return { TRI_ERROR_INTERNAL, + "caught exception while persisting configuration for arangosearch analyzer name '" + + pool.name() + "'" }; } - return arangodb::Result(); + return {}; } bool IResearchAnalyzerFeature::visit( // visit all analyzers @@ -2320,7 +2335,7 @@ bool IResearchAnalyzerFeature::visit( // visit analyzers ); if (!res.ok()) { - LOG_TOPIC("73695", WARN, arangodb::iresearch::TOPIC) + LOG_TOPIC("73695", WARN, iresearch::TOPIC) << "failure to load analyzers while visiting database '" << vocbase->name() << "': " << res.errorNumber() << " " << res.errorMessage(); TRI_set_errno(res.errorNumber()); @@ -2364,10 +2379,10 @@ void IResearchAnalyzerFeature::cleanupAnalyzers(irs::string_ref const& database) void IResearchAnalyzerFeature::invalidate(const TRI_vocbase_t& vocbase) { WriteMutex mutex(_mutex); - SCOPED_LOCK(mutex); + SCOPED_LOCK(mutex); auto database = irs::string_ref(vocbase.name()); auto itr = _lastLoad.find( - irs::make_hashed_ref(database, std::hash())); + irs::make_hashed_ref(database, std::hash())); if (itr != _lastLoad.end()) { cleanupAnalyzers(database); _lastLoad.erase(itr); diff --git a/arangod/IResearch/IResearchAnalyzerFeature.h b/arangod/IResearch/IResearchAnalyzerFeature.h index 1cc00c9589..685cedd74b 100644 --- a/arangod/IResearch/IResearchAnalyzerFeature.h +++ b/arangod/IResearch/IResearchAnalyzerFeature.h @@ -70,15 +70,21 @@ class AnalyzerPool : private irs::util::noncopyable { irs::string_ref const& type() const noexcept { return _type; } // definition to be stored in _analyzers collection or shown to the end user - void toVelocyPack(arangodb::velocypack::Builder& builder, + void toVelocyPack(velocypack::Builder& builder, bool forPersistence = false); // definition to be stored/shown in a link definition - void toVelocyPack(arangodb::velocypack::Builder& builder, + void toVelocyPack(velocypack::Builder& builder, TRI_vocbase_t const* vocbase = nullptr); + bool operator==(AnalyzerPool const& rhs) const; + bool operator!=(AnalyzerPool const& rhs) const { + return !(*this == rhs); + } + private: - friend class IResearchAnalyzerFeature; // required for calling AnalyzerPool::init(...) and AnalyzerPool::setKey(...) + // required for calling AnalyzerPool::init(...) and AnalyzerPool::setKey(...) + friend class IResearchAnalyzerFeature; // 'make(...)' method wrapper for irs::analysis::analyzer types struct Builder { @@ -86,7 +92,7 @@ class AnalyzerPool : private irs::util::noncopyable { DECLARE_FACTORY(irs::string_ref const& type, VPackSlice properties); }; - void toVelocyPack(arangodb::velocypack::Builder& builder, + void toVelocyPack(velocypack::Builder& builder, irs::string_ref const& name); bool init(irs::string_ref const& type, @@ -94,14 +100,15 @@ class AnalyzerPool : private irs::util::noncopyable { irs::flags const& features = irs::flags::empty_instance()); void setKey(irs::string_ref const& type); - mutable irs::unbounded_object_pool _cache; // cache of irs::analysis::analyzer (constructed via - // AnalyzerBuilder::make(...)) - std::string _config; // non-null type + non-null properties + key - irs::flags _features; // cached analyzer features - irs::string_ref _key; // the key of the persisted configuration for this pool, null == static analyzer - std::string _name; // ArangoDB alias for an IResearch analyzer configuration + mutable irs::unbounded_object_pool _cache; // cache of irs::analysis::analyzer + // (constructed via AnalyzerBuilder::make(...)) + std::string _config; // non-null type + non-null properties + key + irs::flags _features; // cached analyzer features + irs::string_ref _key; // the key of the persisted configuration for this pool, + // null == static analyzer + std::string _name; // ArangoDB alias for an IResearch analyzer configuration VPackSlice _properties; // IResearch analyzer configuration - irs::string_ref _type; // IResearch analyzer name + irs::string_ref _type; // IResearch analyzer name }; // AnalyzerPool //////////////////////////////////////////////////////////////////////////////// @@ -111,26 +118,95 @@ class AnalyzerPool : private irs::util::noncopyable { /// invalidates all AnalyzerPool instances previously provided /// by the deallocated feature instance //////////////////////////////////////////////////////////////////////////////// -class IResearchAnalyzerFeature final : public arangodb::application_features::ApplicationFeature { +class IResearchAnalyzerFeature final + : public application_features::ApplicationFeature { public: - explicit IResearchAnalyzerFeature(arangodb::application_features::ApplicationServer& server); + /// first == vocbase name, second == analyzer name + /// EMPTY == system vocbase + /// NIL == unprefixed analyzer name, i.e. active vocbase + using AnalyzerName = std::pair; + + explicit IResearchAnalyzerFeature(application_features::ApplicationServer& server); ////////////////////////////////////////////////////////////////////////////// + /// @brief check permissions + /// @param vocbase analyzer vocbase + /// @param level access level /// @return analyzers in the specified vocbase are granted 'level' access ////////////////////////////////////////////////////////////////////////////// - static bool canUse( // check permissions - TRI_vocbase_t const& vocbase, // analyzer vocbase - arangodb::auth::Level const& level // access level - ); + static bool canUse(TRI_vocbase_t const& vocbase, auth::Level const& level); ////////////////////////////////////////////////////////////////////////////// + /// @brief check permissions + /// @param name analyzer name (already normalized) + /// @param level access level /// @return analyzer with the given prefixed name (or unprefixed and resides /// in defaultVocbase) is granted 'level' access ////////////////////////////////////////////////////////////////////////////// - static bool canUse( // check permissions - irs::string_ref const& name, // analyzer name (already normalized) - arangodb::auth::Level const& level // access level - ); + static bool canUse(irs::string_ref const& name, auth::Level const& level); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief create new analyzer pool + /// @param analyzer created analyzer + /// @param name analyzer name (already normalized) + /// @param type the underlying IResearch analyzer type + /// @param properties the configuration for the underlying IResearch type + /// @param features the expected features the analyzer should produce + /// @return success + ////////////////////////////////////////////////////////////////////////////// + static Result createAnalyzerPool(AnalyzerPool::ptr& analyzer, + irs::string_ref const& name, + irs::string_ref const& type, + VPackSlice const properties, + irs::flags const& features); + + static AnalyzerPool::ptr identity() noexcept; // the identity analyzer + static std::string const& name() noexcept; + + ////////////////////////////////////////////////////////////////////////////// + /// @param name analyzer name + /// @param activeVocbase fallback vocbase if not part of name + /// @param systemVocbase the system vocbase for use with empty prefix + /// @param expandVocbasePrefix use full vocbase name as prefix for + /// active/system v.s. EMPTY/'::' + /// @return normalized analyzer name, i.e. with vocbase prefix + ////////////////////////////////////////////////////////////////////////////// + static std::string normalize(irs::string_ref const& name, + TRI_vocbase_t const& activeVocbase, + TRI_vocbase_t const& systemVocbase, + bool expandVocbasePrefix = true); + + ////////////////////////////////////////////////////////////////////////////// + /// @param name analyzer name (normalized) + /// @return vocbase prefix extracted from normalized analyzer name + /// EMPTY == system vocbase + /// NIL == analyzer name have had no db name prefix + /// @see analyzerReachableFromDb + ////////////////////////////////////////////////////////////////////////////// + static irs::string_ref extractVocbaseName(irs::string_ref const& name) noexcept { + return splitAnalyzerName(name).first; + } + + ////////////////////////////////////////////////////////////////////////////// + /// Checks if analyzer db (identified by db name prefix extracted from analyzer + /// name) could be reached from specified db. + /// Properly handles special cases (e.g. NIL and EMPTY) + /// @param dbNameFromAnalyzer database name extracted from analyzer name + /// @param currentDbName database name to check against (should not be empty!) + /// @param forGetters check special case for getting analyzer (not creating/removing) + /// @return true if analyzer is reachable + static bool analyzerReachableFromDb(irs::string_ref const& dbNameFromAnalyzer, + irs::string_ref const& currentDbName, + bool forGetters = false) noexcept; + + //////////////////////////////////////////////////////////////////////////////// + /// @brief split the analyzer name into the vocbase part and analyzer part + /// @param name analyzer name + /// @return pair of first == vocbase name, second == analyzer name + /// EMPTY == system vocbase + /// NIL == unprefixed analyzer name, i.e. active vocbase + //////////////////////////////////////////////////////////////////////////////// + static AnalyzerName splitAnalyzerName(irs::string_ref const& analyzer) noexcept; ////////////////////////////////////////////////////////////////////////////// /// @brief emplace an analyzer as per the specified parameters @@ -149,134 +225,70 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap /// allowed during recovery ////////////////////////////////////////////////////////////////////////////// typedef std::pair EmplaceResult; - arangodb::Result emplace( - EmplaceResult& result, - irs::string_ref const& name, - irs::string_ref const& type, - VPackSlice const properties, - irs::flags const& features = irs::flags::empty_instance()) { - return ensure(result, name, type, properties, features, true); - } - - ////////////////////////////////////////////////////////////////////////////// - /// @brief find analyzer - /// @param result the result of the successful emplacement (out-param) - /// first - the emplaced pool - /// second - if an insertion of an new analyzer occured - /// @param name analyzer name (already normalized) - /// @param type the underlying IResearch analyzer type - /// @param properties the configuration for the underlying IResearch type - /// @param features the expected features the analyzer should produce - /// @return analyzer matching the specified parameters or nullptr - /// @note will construct and cache the analyzer if missing only on db-server - /// persistence in the cases of inRecovery or !storage engine will fail - ////////////////////////////////////////////////////////////////////////////// - arangodb::Result get( - EmplaceResult& result, - irs::string_ref const& name, - irs::string_ref const& type, - VPackSlice const properties, - irs::flags const& features) { - return ensure(result, name, type, properties, features, false); - } + Result emplace(EmplaceResult& result, + irs::string_ref const& name, + irs::string_ref const& type, + VPackSlice const properties, + irs::flags const& features = irs::flags::empty_instance()); ////////////////////////////////////////////////////////////////////////////// /// @brief find analyzer /// @param name analyzer name (already normalized) + /// @param onlyCached check only locally cached analyzers /// @return analyzer with the specified name or nullptr ////////////////////////////////////////////////////////////////////////////// - AnalyzerPool::ptr get( // find analyzer - irs::string_ref const& name, // analyzer name - bool onlyCached = false // check only locally cached analyzers - ) const noexcept; + AnalyzerPool::ptr get(irs::string_ref const& name, + bool onlyCached = false) const noexcept { + return get(name, splitAnalyzerName(name), onlyCached); + } ////////////////////////////////////////////////////////////////////////////// /// @brief find analyzer - /// @return analyzer with the specified name or nullptr - ////////////////////////////////////////////////////////////////////////////// - AnalyzerPool::ptr get( // find analyzer - irs::string_ref const& name, // analyzer name - TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name - TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix - bool onlyCached = false // check only locally cached analyzers - ) const noexcept; - - static AnalyzerPool::ptr identity() noexcept; // the identity analyzer - static std::string const& name() noexcept; - - ////////////////////////////////////////////////////////////////////////////// - /// @return normalized analyzer name, i.e. with vocbase prefix - ////////////////////////////////////////////////////////////////////////////// - static std::string normalize( // normalize name - irs::string_ref const& name, // analyzer name - TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name - TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix - bool expandVocbasePrefix = true // use full vocbase name as prefix for active/system v.s. EMPTY/'::' - ); - - ////////////////////////////////////////////////////////////////////////////// - /// @param name analyzer name (normalized) - /// @return vocbase prefix extracted from normalized analyzer name - /// EMPTY == system vocbase - /// NIL == analyzer name have had no db name prefix - /// @see analyzerReachableFromDb - ////////////////////////////////////////////////////////////////////////////// - static irs::string_ref extractVocbaseName(irs::string_ref const& name) noexcept; - - ////////////////////////////////////////////////////////////////////////////// - /// Checks if analyzer db (identified by db name prefix extracted from analyzer - /// name) could be reached from specified db. - /// Properly handles special cases (e.g. NIL and EMPTY) - /// @param dbNameFromAnalyzer database name extracted from analyzer name - /// @param currentDbName database name to check against (should not be empty!) - /// @param forGetters check special case for getting analyzer (not creating/removing) - /// @return true if analyzer is reachable - static bool analyzerReachableFromDb(irs::string_ref const& dbNameFromAnalyzer, - irs::string_ref const& currentDbName, - bool forGetters = false) noexcept; - - //////////////////////////////////////////////////////////////////////////////// - /// @brief split the analyzer name into the vocbase part and analyzer part /// @param name analyzer name - /// @return pair of first == vocbase name, second == analyzer name - /// EMPTY == system vocbase - /// NIL == unprefixed analyzer name, i.e. active vocbase - //////////////////////////////////////////////////////////////////////////////// - static std::pair splitAnalyzerName(irs::string_ref const& analyzer) noexcept; - - void prepare() override; + /// @param activeVocbase fallback vocbase if not part of name + /// @param systemVocbase the system vocbase for use with empty prefix + /// @param onlyCached check only locally cached analyzers + /// @return analyzer with the specified name or nullptr + ////////////////////////////////////////////////////////////////////////////// + AnalyzerPool::ptr get(irs::string_ref const& name, + TRI_vocbase_t const& activeVocbase, + TRI_vocbase_t const& systemVocbase, + bool onlyCached = false) const; ////////////////////////////////////////////////////////////////////////////// /// @brief remove the specified analyzer /// @param name analyzer name (already normalized) /// @param force remove even if the analyzer is actively referenced ////////////////////////////////////////////////////////////////////////////// - arangodb::Result remove(irs::string_ref const& name, bool force = false); + Result remove(irs::string_ref const& name, bool force = false); - void start() override; - void stop() override; ////////////////////////////////////////////////////////////////////////////// /// @brief visit all analyzers for the specified vocbase /// @param vocbase only visit analysers for this vocbase (nullptr == static) /// @return visitation compleated fully ////////////////////////////////////////////////////////////////////////////// - bool visit(std::function const& visitor) const; - bool visit(std::function const& visitor, + bool visit(std::function const& visitor) const; + bool visit(std::function const& visitor, TRI_vocbase_t const* vocbase) const; /////////////////////////////////////////////////////////////////////////////// - // @brief removes analyzers for specified database from cache - // @param vocbase database to invalidate analyzers + /// @brief removes analyzers for specified database from cache + /// @param vocbase database to invalidate analyzers /////////////////////////////////////////////////////////////////////////////// void invalidate(const TRI_vocbase_t& vocbase); + virtual void prepare() override; + virtual void start() override; + virtual void stop() override; + private: // map of caches of irs::analysis::analyzer pools indexed by analyzer name and // their associated metas typedef std::unordered_map Analyzers; - Analyzers _analyzers; // all analyzers known to this feature (including static) (names are stored with expanded vocbase prefixes) + Analyzers _analyzers; // all analyzers known to this feature (including static) + // (names are stored with expanded vocbase prefixes) std::unordered_map _lastLoad; // last time a database was loaded mutable irs::async_utils::read_write_mutex _mutex; // for use with member '_analyzers', '_lastLoad' @@ -286,42 +298,17 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap /// @brief validate analyzer parameters and emplace into map ////////////////////////////////////////////////////////////////////////////// typedef std::pair EmplaceAnalyzerResult; - arangodb::Result emplaceAnalyzer( // emplace + Result emplaceAnalyzer( // emplace EmplaceAnalyzerResult& result, // emplacement result on success (out-param) - arangodb::iresearch::IResearchAnalyzerFeature::Analyzers& analyzers, // analyzers + iresearch::IResearchAnalyzerFeature::Analyzers& analyzers, // analyzers irs::string_ref const& name, // analyzer name irs::string_ref const& type, // analyzer type VPackSlice const properties, // analyzer properties irs::flags const& features // analyzer features ); - ////////////////////////////////////////////////////////////////////////////// - /// @brief ensure an analyzer as per the specified parameters exists if - /// possible - /// @param result the result of the successful emplacement (out-param) - /// first - the emplaced pool - /// second - if an insertion of an new analyzer occured - /// @param name analyzer name (already normalized) - /// @param type the underlying IResearch analyzer type - /// @param properties the configuration for the underlying IResearch type - /// @param features the expected features the analyzer should produce - /// @param isEmplace request coming from emplace(...) - /// @return success - /// @note ensure while inRecovery() will not allow new analyzer persistance - /// valid because for existing links the analyzer definition should - /// already have been persisted and feature administration is not - /// allowed during recovery - /// @note for db-server analyzers are not persisted - /// valid because the authoritative analyzer source is from coordinators - ////////////////////////////////////////////////////////////////////////////// - arangodb::Result ensure( // ensure analyzer existence if possible - EmplaceResult& result, // emplacement result on success (out-param) - irs::string_ref const& name, // analyzer name - irs::string_ref const& type, // analyzer type - VPackSlice const properties, // analyzer properties - irs::flags const& features, // analyzer features - bool isEmplace - ); + AnalyzerPool::ptr get(irs::string_ref const& normalizedName, + AnalyzerName const& name, bool onlyCached) const noexcept; ////////////////////////////////////////////////////////////////////////////// /// @brief load the analyzers for the specific database, analyzers read from @@ -330,9 +317,7 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap /// @note on coordinator and db-server reload is also done if the database has /// not been reloaded in 'timeout' seconds ////////////////////////////////////////////////////////////////////////////// - arangodb::Result loadAnalyzers( // load analyzers - irs::string_ref const& database = irs::string_ref::NIL // database to load - ); + Result loadAnalyzers(irs::string_ref const& database = irs::string_ref::NIL); //////////////////////////////////////////////////////////////////////////////// /// removes analyzers for database from feature cache @@ -345,7 +330,7 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap /// vocbase /// @note on success will modify the '_key' of the pool ////////////////////////////////////////////////////////////////////////////// - arangodb::Result storeAnalyzer(AnalyzerPool& pool); + Result storeAnalyzer(AnalyzerPool& pool); }; } // namespace iresearch diff --git a/arangod/IResearch/IResearchDocument.cpp b/arangod/IResearch/IResearchDocument.cpp index 0b103a6546..747c562172 100644 --- a/arangod/IResearch/IResearchDocument.cpp +++ b/arangod/IResearch/IResearchDocument.cpp @@ -158,8 +158,9 @@ inline bool keyFromSlice(VPackSlice keySlice, irs::string_ref& key) { } } -inline bool canHandleValue(std::string const& key, VPackSlice const& value, - arangodb::iresearch::IResearchLinkMeta const& context) noexcept { +inline bool canHandleValue(std::string const& key, + VPackSlice const& value, + arangodb::iresearch::FieldMeta const& context) noexcept { switch (value.type()) { case VPackValueType::None: case VPackValueType::Illegal: @@ -190,8 +191,9 @@ inline bool canHandleValue(std::string const& key, VPackSlice const& value, } // returns 'context' in case if can't find the specified 'field' -inline arangodb::iresearch::IResearchLinkMeta const* findMeta( - irs::string_ref const& key, arangodb::iresearch::IResearchLinkMeta const* context) { +inline arangodb::iresearch::FieldMeta const* findMeta( + irs::string_ref const& key, + arangodb::iresearch::FieldMeta const* context) { TRI_ASSERT(context); auto const* meta = context->_fields.findPtr(key); @@ -199,7 +201,7 @@ inline arangodb::iresearch::IResearchLinkMeta const* findMeta( } inline bool inObjectFiltered(std::string& buffer, - arangodb::iresearch::IResearchLinkMeta const*& context, + arangodb::iresearch::FieldMeta const*& context, arangodb::iresearch::IteratorValue const& value) { irs::string_ref key; @@ -220,7 +222,7 @@ inline bool inObjectFiltered(std::string& buffer, } inline bool inObject(std::string& buffer, - arangodb::iresearch::IResearchLinkMeta const*& context, + arangodb::iresearch::FieldMeta const*& context, arangodb::iresearch::IteratorValue const& value) { irs::string_ref key; @@ -235,7 +237,7 @@ inline bool inObject(std::string& buffer, } inline bool inArrayOrdered(std::string& buffer, - arangodb::iresearch::IResearchLinkMeta const*& context, + arangodb::iresearch::FieldMeta const*& context, arangodb::iresearch::IteratorValue const& value) { buffer += arangodb::iresearch::NESTING_LIST_OFFSET_PREFIX; append(buffer, value.pos); @@ -245,36 +247,36 @@ inline bool inArrayOrdered(std::string& buffer, } inline bool inArray(std::string& buffer, - arangodb::iresearch::IResearchLinkMeta const*& context, + arangodb::iresearch::FieldMeta const*& context, arangodb::iresearch::IteratorValue const& value) noexcept { return canHandleValue(buffer, value.value, *context); } typedef bool (*Filter)(std::string& buffer, - arangodb::iresearch::IResearchLinkMeta const*& context, + arangodb::iresearch::FieldMeta const*& context, arangodb::iresearch::IteratorValue const& value); Filter const valueAcceptors[] = { - &inObjectFiltered, // type == Object, nestListValues == false, - // includeAllValues == false - &inObject, // type == Object, nestListValues == false, includeAllValues == - // true - &inObjectFiltered, // type == Object, nestListValues == true , - // includeAllValues == false - &inObject, // type == Object, nestListValues == true , includeAllValues == - // true - &inArray, // type == Array , nestListValues == flase, includeAllValues == - // false - &inArray, // type == Array , nestListValues == flase, includeAllValues == - // true - &inArrayOrdered, // type == Array , nestListValues == true, - // includeAllValues == false - &inArrayOrdered // type == Array , nestListValues == true, includeAllValues - // == true + // type == Object, nestListValues == false, // includeAllValues == false + &inObjectFiltered, + // type == Object, nestListValues == false, includeAllValues == // true + &inObject, + // type == Object, nestListValues == true , // includeAllValues == false + &inObjectFiltered, + // type == Object, nestListValues == true , includeAllValues == // true + &inObject, + // type == Array , nestListValues == flase, includeAllValues == // false + &inArray, + // type == Array , nestListValues == flase, includeAllValues == // true + &inArray, + // type == Array , nestListValues == true, // includeAllValues == false + &inArrayOrdered, + // type == Array , nestListValues == true, includeAllValues // == true + &inArrayOrdered }; inline Filter getFilter(VPackSlice value, - arangodb::iresearch::IResearchLinkMeta const& meta) noexcept { + arangodb::iresearch::FieldMeta const& meta) noexcept { TRI_ASSERT(arangodb::iresearch::isArrayOrObject(value)); return valueAcceptors[4 * value.isArray() + 2 * meta._trackListPositions + meta._includeAllFields]; @@ -292,7 +294,7 @@ namespace iresearch { /*static*/ void Field::setPkValue(Field& field, LocalDocumentId::BaseType const& pk) { field._name = PK_COLUMN; field._features = &irs::flags::empty_instance(); - field._storeValues = ValueStorage::FULL; + field._storeValues = ValueStorage::VALUE; field._value = irs::bytes_ref(reinterpret_cast(&pk), sizeof(pk)); field._analyzer = StringStreamPool.emplace().release(); // FIXME don't use shared_ptr @@ -342,7 +344,7 @@ std::string& FieldIterator::valueBuffer() { return *_valueBuffer; } -void FieldIterator::reset(VPackSlice const& doc, IResearchLinkMeta const& linkMeta) { +void FieldIterator::reset(VPackSlice const& doc, FieldMeta const& linkMeta) { // set surrogate analyzers _begin = nullptr; _end = 1 + _begin; @@ -419,13 +421,11 @@ void FieldIterator::setNullValue(VPackSlice const value) { } bool FieldIterator::setStringValue( - arangodb::velocypack::Slice const value, // value - IResearchLinkMeta::Analyzer const& valueAnalyzer // analyzer to use -) { + arangodb::velocypack::Slice const value, + FieldMeta::Analyzer const& valueAnalyzer) { TRI_ASSERT( // assert (value.isCustom() && nameBuffer() == arangodb::StaticStrings::IdString) // custom string - || value.isString() // verbatim string - ); + || value.isString()); // verbatim string irs::string_ref valueRef; @@ -483,7 +483,7 @@ bool FieldIterator::setStringValue( return true; } -bool FieldIterator::pushAndSetValue(VPackSlice slice, IResearchLinkMeta const*& context) { +bool FieldIterator::pushAndSetValue(VPackSlice slice, FieldMeta const*& context) { auto& name = nameBuffer(); while (isArrayOrObject(slice)) { @@ -522,7 +522,7 @@ bool FieldIterator::pushAndSetValue(VPackSlice slice, IResearchLinkMeta const*& return setAttributeValue(*context); } -bool FieldIterator::setAttributeValue(IResearchLinkMeta const& context) { +bool FieldIterator::setAttributeValue(FieldMeta const& context) { auto const value = topValue().value; _value._storeValues = context._storeValues; @@ -580,7 +580,7 @@ void FieldIterator::next() { } } - IResearchLinkMeta const* context; + FieldMeta const* context; auto& name = nameBuffer(); @@ -651,4 +651,4 @@ void FieldIterator::next() { // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/arangod/IResearch/IResearchDocument.h b/arangod/IResearch/IResearchDocument.h index a45c4371a7..cd0f08d0b2 100644 --- a/arangod/IResearch/IResearchDocument.h +++ b/arangod/IResearch/IResearchDocument.h @@ -141,7 +141,7 @@ class FieldIterator : public std::iterator const TypeHandlers{ // any string {irs::string_ref("string"), - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool { kludge::mangleAnalyzer(name); return true; // a prefix match }}, // any non-string type {irs::string_ref("type"), - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool { kludge::mangleType(name); return true; // a prefix match }}, // concrete analyzer from the context {TypeAnalyzer, - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const& analyzer)->bool { kludge::mangleStringField(name, analyzer); return false; // not a prefix match }}, {irs::string_ref("numeric"), - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool { kludge::mangleNumeric(name); return false; // not a prefix match }}, {irs::string_ref("bool"), - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool { kludge::mangleBool(name); return false; // not a prefix match }}, {irs::string_ref("boolean"), - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool { kludge::mangleBool(name); return false; // not a prefix match }}, {irs::string_ref("null"), - [](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool { + [](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool { kludge::mangleNull(name); return false; // not a prefix match }}}; @@ -2082,7 +2082,7 @@ namespace iresearch { arangodb::aql::AstNode const& node) { // The analyzer is referenced in the FilterContext and used during the // following ::filter() call, so may not be a temporary. - IResearchLinkMeta::Analyzer analyzer = IResearchLinkMeta::Analyzer(); + FieldMeta::Analyzer analyzer = FieldMeta::Analyzer(); FilterContext const filterCtx(analyzer, irs::no_boost()); return ::filter(filter, ctx, filterCtx, node); diff --git a/arangod/IResearch/IResearchKludge.cpp b/arangod/IResearch/IResearchKludge.cpp index 5d3cc0545c..66a50f03f3 100644 --- a/arangod/IResearch/IResearchKludge.cpp +++ b/arangod/IResearch/IResearchKludge.cpp @@ -52,18 +52,16 @@ void mangleNumeric(std::string& name) { name.append(NUMERIC_SUFFIX.c_str(), NUMERIC_SUFFIX.size()); } -void mangleStringField( // mangle string field - std::string& name, // field name - arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply -) { +void mangleStringField( + std::string& name, + arangodb::iresearch::FieldMeta::Analyzer const& analyzer) { name += ANALYZER_DELIMITER; name += analyzer._shortName; } -void demangleStringField( // demangle string field - std::string& name, // field name - arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply -) { +void demangleStringField( + std::string& name, + arangodb::iresearch::FieldMeta::Analyzer const& analyzer) { // +1 for preceding '\1' auto const suffixSize = 1 + analyzer._shortName.size(); @@ -77,4 +75,4 @@ void demangleStringField( // demangle string field // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/arangod/IResearch/IResearchKludge.h b/arangod/IResearch/IResearchKludge.h index a414ca0ecb..89a8a6d2d4 100644 --- a/arangod/IResearch/IResearchKludge.h +++ b/arangod/IResearch/IResearchKludge.h @@ -44,17 +44,16 @@ void mangleNull(std::string& name); void mangleBool(std::string& name); void mangleNumeric(std::string& name); -void mangleStringField( // mangle string field - std::string& name, // field name - arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply -); -void demangleStringField( // demangle string field - std::string& name, // field name - arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply -); +void mangleStringField( + std::string& name, + arangodb::iresearch::FieldMeta::Analyzer const& analyzer); + +void demangleStringField( + std::string& name, + arangodb::iresearch::FieldMeta::Analyzer const& analyzer); } // namespace kludge } // namespace iresearch } // namespace arangodb -#endif \ No newline at end of file +#endif diff --git a/arangod/IResearch/IResearchLink.cpp b/arangod/IResearch/IResearchLink.cpp index 330a2fd49e..8753b9317e 100644 --- a/arangod/IResearch/IResearchLink.cpp +++ b/arangod/IResearch/IResearchLink.cpp @@ -31,7 +31,6 @@ #include "Aql/QueryCache.h" #include "Basics/LocalTaskQueue.h" #include "Basics/StaticStrings.h" -#include "Basics/VelocyPackHelper.h" #include "Cluster/ClusterInfo.h" #include "MMFiles/MMFilesCollection.h" #include "RestServer/DatabaseFeature.h" @@ -1749,6 +1748,25 @@ Result IResearchLink::unload() { return {}; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookup referenced analyzer +//////////////////////////////////////////////////////////////////////////////// +AnalyzerPool::ptr IResearchLink::findAnalyzer(AnalyzerPool const& analyzer) const { + auto const it = _meta._analyzerDefinitions.find(irs::string_ref(analyzer.name())); + + if (it == _meta._analyzerDefinitions.end()) { + return nullptr; + } + + auto const pool = *it; + + if (pool && analyzer == *pool) { + return pool; + } + + return nullptr; +} + } // namespace iresearch } // namespace arangodb diff --git a/arangod/IResearch/IResearchLink.h b/arangod/IResearch/IResearchLink.h index 0748030f35..082e879361 100644 --- a/arangod/IResearch/IResearchLink.h +++ b/arangod/IResearch/IResearchLink.h @@ -218,6 +218,11 @@ class IResearchLink { //////////////////////////////////////////////////////////////////////////////// Result unload(); + //////////////////////////////////////////////////////////////////////////////// + /// @brief lookup referenced analyzer + //////////////////////////////////////////////////////////////////////////////// + AnalyzerPool::ptr findAnalyzer(AnalyzerPool const& analyzer) const; + protected: typedef std::function InitCallback; diff --git a/arangod/IResearch/IResearchLinkHelper.cpp b/arangod/IResearch/IResearchLinkHelper.cpp index f6e4f11e3c..2009dfed0a 100644 --- a/arangod/IResearch/IResearchLinkHelper.cpp +++ b/arangod/IResearch/IResearchLinkHelper.cpp @@ -63,8 +63,8 @@ arangodb::Result canUseAnalyzers( // validate >(); auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr; - for (auto& entry: meta._analyzers) { - if (!entry._pool) { + for (auto& pool: meta._analyzerDefinitions) { + if (!pool) { continue; // skip invalid entries } @@ -73,35 +73,35 @@ arangodb::Result canUseAnalyzers( // validate if (sysVocbase) { result = arangodb::iresearch::IResearchAnalyzerFeature::canUse( // validate arangodb::iresearch::IResearchAnalyzerFeature::normalize( // normalize - entry._pool->name(), defaultVocbase, *sysVocbase // args + pool->name(), defaultVocbase, *sysVocbase // args ), // analyzer arangodb::auth::Level::RO // auth level ); } else { result = arangodb::iresearch::IResearchAnalyzerFeature::canUse( // validate - entry._pool->name(), arangodb::auth::Level::RO // args + pool->name(), arangodb::auth::Level::RO // args ); } if (!result) { - return arangodb::Result( // result - TRI_ERROR_FORBIDDEN, // code - std::string("read access is forbidden to arangosearch analyzer '") + entry._pool->name() + "'" - ); + return { + TRI_ERROR_FORBIDDEN, + std::string("read access is forbidden to arangosearch analyzer '") + pool->name() + "'" + }; } } - for (auto& field: meta._fields) { - TRI_ASSERT(field.value().get()); // ensured by UniqueHeapInstance constructor - auto& entry = field.value(); - auto res = canUseAnalyzers(*entry, defaultVocbase); +// for (auto& field: meta._fields) { +// TRI_ASSERT(field.value().get()); // ensured by UniqueHeapInstance constructor +// auto& entry = field.value(); +// auto res = canUseAnalyzers(*entry, defaultVocbase); +// +// if (!res.ok()) { +// return res; +// } +// } - if (!res.ok()) { - return res; - } - } - - return arangodb::Result(); + return {}; } arangodb::Result createLink( // create link @@ -795,17 +795,22 @@ namespace iresearch { // analyzer should be either from same database as view (and collection) or from system database { const auto& currentVocbase = vocbase.name(); - for (const auto& analyzer : meta._analyzers) { - TRI_ASSERT(analyzer._pool); // should be checked in meta init - if (ADB_UNLIKELY(!analyzer._pool)) { + for (const auto& analyzer : meta._analyzerDefinitions) { + TRI_ASSERT(analyzer); // should be checked in meta init + if (ADB_UNLIKELY(!analyzer)) { continue; } - auto analyzerVocbase = IResearchAnalyzerFeature::extractVocbaseName(analyzer._pool->name()); + auto* pool = analyzer.get(); + auto analyzerVocbase = IResearchAnalyzerFeature::extractVocbaseName(pool->name()); + if (!IResearchAnalyzerFeature::analyzerReachableFromDb(analyzerVocbase, currentVocbase, true)) { - return arangodb::Result( - TRI_ERROR_BAD_PARAMETER, - std::string("Analyzer '").append(analyzer._pool->name()) - .append("' is not accessible from database '").append(currentVocbase).append("'")); + return { + TRI_ERROR_BAD_PARAMETER, + std::string("Analyzer '") + .append(pool->name()) + .append("' is not accessible from database '") + .append(currentVocbase).append("'") + }; } } } diff --git a/arangod/IResearch/IResearchLinkMeta.cpp b/arangod/IResearch/IResearchLinkMeta.cpp index 21382112b1..a606b394d7 100644 --- a/arangod/IResearch/IResearchLinkMeta.cpp +++ b/arangod/IResearch/IResearchLinkMeta.cpp @@ -32,6 +32,7 @@ #include "Cluster/ServerState.h" #include "RestServer/SystemDatabaseFeature.h" #include "VelocyPackHelper.h" +#include "Basics/VelocyPackHelper.h" #include "velocypack/Builder.h" #include "velocypack/Iterator.h" #include "IResearchLinkMeta.h" @@ -39,8 +40,8 @@ namespace { -bool equalAnalyzers(arangodb::iresearch::IResearchLinkMeta::Analyzers const& lhs, - arangodb::iresearch::IResearchLinkMeta::Analyzers const& rhs) noexcept { +bool equalAnalyzers(std::vector const& lhs, + std::vector const& rhs) noexcept { if (lhs.size() != rhs.size()) { return false; } @@ -70,32 +71,41 @@ bool equalAnalyzers(arangodb::iresearch::IResearchLinkMeta::Analyzers const& lhs } // namespace +bool operator<(arangodb::iresearch::FieldMeta::Analyzer const& lhs, + irs::string_ref const& rhs) noexcept { + return lhs._pool->name() < rhs; +} + +bool operator<(irs::string_ref const& lhs, + arangodb::iresearch::FieldMeta::Analyzer const& rhs) noexcept { + return lhs < rhs._pool->name(); +} + namespace arangodb { namespace iresearch { -IResearchLinkMeta::Analyzer::Analyzer() +// ----------------------------------------------------------------------------- +// --SECTION-- FieldMeta::Analyzer +// ----------------------------------------------------------------------------- + +FieldMeta::Analyzer::Analyzer() : _pool(IResearchAnalyzerFeature::identity()) { if (_pool) { _shortName = _pool->name(); // static analyzers are used verbatim } } -IResearchLinkMeta::Mask::Mask(bool mask /*= false*/) noexcept - : _analyzerDefinitions(mask), - _analyzers(mask), - _fields(mask), - _includeAllFields(mask), - _trackListPositions(mask), - _storeValues(mask), - _sort(mask) { +// ----------------------------------------------------------------------------- +// --SECTION-- IResearchLinkMeta::FieldMeta +// ----------------------------------------------------------------------------- + +/*static*/ const FieldMeta& FieldMeta::DEFAULT() { + static const FieldMeta meta; + + return meta; } -IResearchLinkMeta::IResearchLinkMeta() - : //_fields(), // no fields to index by default - _includeAllFields(false), // true to match all encountered fields, false - // match only fields in '_fields' - _trackListPositions(false), // treat '_trackListPositions' as SQL-IN - _storeValues(ValueStorage::NONE) { // do not track values at all +FieldMeta::FieldMeta() { Analyzer analyzer; // identity analyzer // identity-only tokenization @@ -104,16 +114,16 @@ IResearchLinkMeta::IResearchLinkMeta() } } -bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcept { - if (!equalAnalyzers(_analyzers, other._analyzers)) { +bool FieldMeta::operator==(FieldMeta const& rhs) const noexcept { + if (!equalAnalyzers(_analyzers, rhs._analyzers)) { return false; // values do not match } - if (_fields.size() != other._fields.size()) { + if (_fields.size() != rhs._fields.size()) { return false; // values do not match } - auto itr = other._fields.begin(); + auto itr = rhs._fields.begin(); for (auto& entry : _fields) { if (itr.key() != entry.key() || itr.value() != entry.value()) { @@ -123,27 +133,423 @@ bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcep ++itr; } - if (_includeAllFields != other._includeAllFields) { + if (_includeAllFields != rhs._includeAllFields) { return false; // values do not match } - if (_trackListPositions != other._trackListPositions) { + if (_trackListPositions != rhs._trackListPositions) { return false; // values do not match } - if (_storeValues != other._storeValues) { - return false; // values do not match - } - - if (_sort != other._sort) { + if (_storeValues != rhs._storeValues) { return false; // values do not match } return true; } -bool IResearchLinkMeta::operator!=(IResearchLinkMeta const& other) const noexcept { - return !(*this == other); +bool FieldMeta::init(velocypack::Slice const& slice, + std::string& errorField, + TRI_vocbase_t const* defaultVocbase /*= nullptr*/, + FieldMeta const& defaults /*= DEFAULT()*/, + Mask* mask /*= nullptr*/, + std::set* referencedAnalyzers /*= nullptr*/) { + if (!slice.isObject()) { + return false; + } + + Mask tmpMask; + + if (!mask) { + mask = &tmpMask; + } + + { + // optional string list + static const std::string fieldName("analyzers"); + + mask->_analyzers = slice.hasKey(fieldName); + + if (!mask->_analyzers) { + _analyzers = defaults._analyzers; + } else { + auto* analyzers = application_features::ApplicationServer::lookupFeature< // find feature + IResearchAnalyzerFeature // feature type + >(); + auto* sysDatabase = application_features::ApplicationServer::lookupFeature< // find feature + SystemDatabaseFeature // feature type + >(); + auto field = slice.get(fieldName); + + if (!field.isArray()) { + errorField = fieldName; + + return false; + } + + _analyzers.clear(); // reset to match read values exactly + std::unordered_set uniqueGuard; // deduplicate analyzers + + for (velocypack::ArrayIterator itr(field); itr.valid(); ++itr) { + auto value = *itr; + + if (!value.isString()) { + errorField = fieldName + "[" + std::to_string(itr.index()) + "]"; + + return false; + } + + auto name = value.copyString(); + auto shortName = name; + + if (defaultVocbase) { + auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr; + + if (sysVocbase) { + name = IResearchAnalyzerFeature::normalize( + name, *defaultVocbase, *sysVocbase); + shortName = IResearchAnalyzerFeature::normalize( + name, *defaultVocbase, *sysVocbase, false); + } + } + + AnalyzerPool::ptr analyzer; + bool found = false; + + if (referencedAnalyzers) { + auto it = referencedAnalyzers->find(irs::string_ref(name)); + + if (it != referencedAnalyzers->end()) { + analyzer = *it; + found = static_cast(analyzer); + + if (ADB_UNLIKELY(!found)) { + TRI_ASSERT(false); // should not happen + referencedAnalyzers->erase(it); // remove null analyzer + } + } + } + + if (!found && analyzers) { + // for cluster only check cache to avoid ClusterInfo locking issues + // analyzer should have been populated via 'analyzerDefinitions' above + analyzer = analyzers->get(name, ServerState::instance()->isClusterRole()); + } + + if (!analyzer) { + errorField = fieldName + "." + value.copyString(); // original (non-normalized) 'name' value + + return false; + } + + if (!found && referencedAnalyzers) { + // save in referencedAnalyzers + referencedAnalyzers->emplace(analyzer); + } + + // avoid adding same analyzer twice + if (uniqueGuard.emplace(analyzer->name()).second) { + _analyzers.emplace_back(analyzer, std::move(shortName)); + } + } + } + } + + { + // optional bool + static const std::string fieldName("includeAllFields"); + + mask->_includeAllFields = slice.hasKey(fieldName); + + if (!mask->_includeAllFields) { + _includeAllFields = defaults._includeAllFields; + } else { + auto field = slice.get(fieldName); + + if (!field.isBool()) { + errorField = fieldName; + + return false; + } + + _includeAllFields = field.getBool(); + } + } + + { + // optional bool + static const std::string fieldName("trackListPositions"); + + mask->_trackListPositions = slice.hasKey(fieldName); + + if (!mask->_trackListPositions) { + _trackListPositions = defaults._trackListPositions; + } else { + auto field = slice.get(fieldName); + + if (!field.isBool()) { + errorField = fieldName; + + return false; + } + + _trackListPositions = field.getBool(); + } + } + + { + // optional string enum + static const std::string fieldName("storeValues"); + + mask->_storeValues = slice.hasKey(fieldName); + + if (!mask->_storeValues) { + _storeValues = defaults._storeValues; + } else { + auto field = slice.get(fieldName); + + if (!field.isString()) { + errorField = fieldName; + + return false; + } + + static const std::unordered_map policies = { + {"none", ValueStorage::NONE}, + {"id", ValueStorage::ID}, + {"value", ValueStorage::VALUE} + }; + + auto name = field.copyString(); + auto itr = policies.find(name); + + if (itr == policies.end()) { + errorField = fieldName + "." + name; + + return false; + } + + _storeValues = itr->second; + } + } + + // ............................................................................. + // process fields last since children inherit from parent + // ............................................................................. + + { + // optional string map + static const std::string fieldName("fields"); + + mask->_fields = slice.hasKey(fieldName); + + if (!mask->_fields) { + _fields = defaults._fields; + } else { + auto field = slice.get(fieldName); + + if (!field.isObject()) { + errorField = fieldName; + + return false; + } + + auto subDefaults = *this; + + subDefaults._fields.clear(); // do not inherit fields and overrides from this field + _fields.clear(); // reset to match either defaults or read values exactly + + for (velocypack::ObjectIterator itr(field); itr.valid(); ++itr) { + auto key = itr.key(); + auto value = itr.value(); + + if (!key.isString()) { + errorField = fieldName + "[" + + basics::StringUtils::itoa(itr.index()) + "]"; + + return false; + } + + auto name = key.copyString(); + + if (!value.isObject()) { + errorField = fieldName + "." + name; + + return false; + } + + std::string childErrorField; + + if (!_fields[name]->init(value, childErrorField, defaultVocbase, + subDefaults, nullptr, referencedAnalyzers)) { + errorField = fieldName + "." + name + "." + childErrorField; + + return false; + } + } + } + } + + return true; +} + +bool FieldMeta::json(velocypack::Builder& builder, + FieldMeta const* ignoreEqual /*= nullptr*/, + TRI_vocbase_t const* defaultVocbase /*= nullptr*/, + Mask const* mask /*= nullptr*/) const { + if (!builder.isOpenObject()) { + return false; + } + + std::map analyzers; + + if ((!ignoreEqual || !equalAnalyzers(_analyzers, ignoreEqual->_analyzers)) && + (!mask || mask->_analyzers)) { + velocypack::Builder analyzersBuilder; + + analyzersBuilder.openArray(); + + for (auto& entry : _analyzers) { + if (!entry._pool) { + continue; // skip null analyzers + } + + std::string name; + + if (defaultVocbase) { + auto* sysDatabase = application_features::ApplicationServer::lookupFeature< // find feature + SystemDatabaseFeature // feature type + >(); + auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr; + + if (!sysVocbase) { + return false; + } + + // @note: DBServerAgencySync::getLocalCollections(...) generates + // 'forPersistence' definitions that are then compared in + // Maintenance.cpp:compareIndexes(...) via + // arangodb::Index::Compare(...) without access to + // 'defaultVocbase', hence the generated definitions must not + // rely on 'defaultVocbase' + // hence must use 'expandVocbasePrefix==true' if + // 'writeAnalyzerDefinition==true' for normalize + // for 'writeAnalyzerDefinition==false' must use + // 'expandVocbasePrefix==false' so that dump/restore an restore + // definitions into differently named databases + name = IResearchAnalyzerFeature::normalize( // normalize + entry._pool->name(), // analyzer name + *defaultVocbase, // active vocbase + *sysVocbase, // system vocbase + false // expand vocbase prefix + ); + } else { + name = entry._pool->name(); // verbatim (assume already normalized) + } + + analyzers.emplace(name, entry._pool); + analyzersBuilder.add(velocypack::Value(std::move(name))); + } + + analyzersBuilder.close(); + builder.add("analyzers", analyzersBuilder.slice()); + } + + if (!mask || mask->_fields) { // fields are not inherited from parent + velocypack::Builder fieldsBuilder; + Mask fieldMask(true); // output all non-matching fields + auto subDefaults = *this; // make modifable copy + + subDefaults._fields.clear(); // do not inherit fields and overrides from this field + fieldsBuilder.openObject(); + + for (auto& entry : _fields) { + fieldMask._fields = !entry.value()->_fields.empty(); // do not output empty fields on subobjects + fieldsBuilder.add( // add sub-object + entry.key(), // field name + velocypack::Value(velocypack::ValueType::Object) + ); + + if (!entry.value()->json(fieldsBuilder, &subDefaults, defaultVocbase, &fieldMask)) { + return false; + } + + fieldsBuilder.close(); + } + + fieldsBuilder.close(); + builder.add("fields", fieldsBuilder.slice()); + } + + if ((!ignoreEqual || _includeAllFields != ignoreEqual->_includeAllFields) && + (!mask || mask->_includeAllFields)) { + builder.add("includeAllFields", velocypack::Value(_includeAllFields)); + } + + if ((!ignoreEqual || _trackListPositions != ignoreEqual->_trackListPositions) && + (!mask || mask->_trackListPositions)) { + builder.add("trackListPositions", velocypack::Value(_trackListPositions)); + } + + if ((!ignoreEqual || _storeValues != ignoreEqual->_storeValues) && + (!mask || mask->_storeValues)) { + static_assert(adjacencyChecker::checkAdjacency(), + "Values are not adjacent"); + + static const std::string policies[]{ + "none", // ValueStorage::NONE + "id", // ValueStorage::ID + "value" // ValueStorage::VALUE + }; + + auto const policyIdx = + static_cast::type>(_storeValues); + + if (policyIdx >= IRESEARCH_COUNTOF(policies)) { + return false; // unsupported value storage policy + } + + builder.add("storeValues", velocypack::Value(policies[policyIdx])); + } + + return true; +} + +size_t FieldMeta::memory() const noexcept { + auto size = sizeof(FieldMeta); + + size += _analyzers.size() * sizeof(decltype(_analyzers)::value_type); + size += _fields.size() * sizeof(decltype(_fields)::value_type); + + for (auto& entry : _fields) { + size += entry.key().size(); + size += entry.value()->memory(); + } + + return size; +} + +// ----------------------------------------------------------------------------- +// --SECTION-- IResearchLinkMeta +// ----------------------------------------------------------------------------- + +IResearchLinkMeta::IResearchLinkMeta() { + // add default analyzers + for (auto& analyzer : _analyzers) { + _analyzerDefinitions.emplace(analyzer._pool); + } +} + +bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcept { + if (FieldMeta::operator!=(other)) { + return false; + } + + if (_sort != other._sort) { + return false; + } + + return true; } /*static*/ const IResearchLinkMeta& IResearchLinkMeta::DEFAULT() { @@ -152,14 +558,12 @@ bool IResearchLinkMeta::operator!=(IResearchLinkMeta const& other) const noexcep return meta; } -bool IResearchLinkMeta::init( // initialize meta - arangodb::velocypack::Slice const& slice, // definition - bool readAnalyzerDefinition, // allow analyzer definitions - std::string& errorField, // field causing error (out-param) - TRI_vocbase_t const* defaultVocbase /*= nullptr*/, // fallback vocbase - IResearchLinkMeta const& defaults /*= DEFAULT()*/, // inherited defaults - Mask* mask /*= nullptr*/ // initialized fields (out-param) -) { +bool IResearchLinkMeta::init(velocypack::Slice const& slice, + bool readAnalyzerDefinition, + std::string& errorField, + TRI_vocbase_t const* defaultVocbase /*= nullptr*/, + FieldMeta const& defaults /*= DEFAULT()*/, + Mask* mask /*= nullptr*/) { if (!slice.isObject()) { return false; } @@ -185,6 +589,9 @@ bool IResearchLinkMeta::init( // initialize meta } { + // clear existing definitions + _analyzerDefinitions.clear(); + // optional object list static const std::string fieldName("analyzerDefinitions"); @@ -193,11 +600,11 @@ bool IResearchLinkMeta::init( // initialize meta // load analyzer definitions if requested (used on cluster) // @note must load definitions before loading 'analyzers' to ensure presence if (readAnalyzerDefinition && mask->_analyzerDefinitions) { - auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature< // find feature + auto* analyzers = application_features::ApplicationServer::lookupFeature< // find feature IResearchAnalyzerFeature // featue type >(); - auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - arangodb::SystemDatabaseFeature // featue type + auto* sysDatabase = application_features::ApplicationServer::lookupFeature< // find feature + SystemDatabaseFeature // featue type >(); auto field = slice.get(fieldName); @@ -207,7 +614,7 @@ bool IResearchLinkMeta::init( // initialize meta return false; } - for (arangodb::velocypack::ArrayIterator itr(field); itr.valid(); ++itr) { + for (velocypack::ArrayIterator itr(field); itr.valid(); ++itr) { auto value = *itr; if (!value.isObject()) { @@ -292,7 +699,7 @@ bool IResearchLinkMeta::init( // initialize meta return false; } - for (arangodb::velocypack::ArrayIterator subItr(subField); + for (velocypack::ArrayIterator subItr(subField); subItr.valid(); ++subItr) { auto subValue = *subItr; @@ -317,226 +724,32 @@ bool IResearchLinkMeta::init( // initialize meta } } - // get analyzer potentially creating it (e.g. on cluster) - // @note do not use emplace(...) since it'll trigger loadAnalyzers(...) - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult emplaceResult; - auto const res = analyzers->get(emplaceResult, name, type, properties, features); - if (!res.ok()) { + AnalyzerPool::ptr analyzer; + auto const res = IResearchAnalyzerFeature::createAnalyzerPool(analyzer, name, type, properties, features); + + if (res.fail() || !analyzer) { errorField = fieldName + "[" + std::to_string(itr.index()) + "]"; - return false; - } - } - } - } - - { - // optional string list - static const std::string fieldName("analyzers"); - - mask->_analyzers = slice.hasKey(fieldName); - - if (!mask->_analyzers) { - _analyzers = defaults._analyzers; - } else { - auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - IResearchAnalyzerFeature // feature type - >(); - auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - arangodb::SystemDatabaseFeature // feature type - >(); - auto field = slice.get(fieldName); - - if (!analyzers || !field.isArray()) { - errorField = fieldName; - - return false; - } - - _analyzers.clear(); // reset to match read values exactly - std::unordered_set uniqueGuard; - - for (arangodb::velocypack::ArrayIterator itr(field); itr.valid(); ++itr) { - auto value = *itr; - - if (!value.isString()) { - errorField = fieldName + "[" + std::to_string(itr.index()) + "]"; - - return false; - } - - auto name = value.copyString(); - auto shortName = name; - - if (defaultVocbase) { - auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr; - - if (sysVocbase) { - name = IResearchAnalyzerFeature::normalize( - name, *defaultVocbase, *sysVocbase); - shortName = IResearchAnalyzerFeature::normalize( - name, *defaultVocbase, *sysVocbase, false); + if (res.fail()) { + errorField.append(": ").append(res.errorMessage()); } - } - - // for cluster only check cache to avoid ClusterInfo locking issues - // analyzer should have been populated via 'analyzerDefinitions' above - auto analyzer = analyzers->get(name, arangodb::ServerState::instance()->isClusterRole()); - - if (!analyzer) { - errorField = fieldName + "." + value.copyString(); // original (non-normalized) 'name' value - - return false; - } - // avoid adding same analyzer twice - if (uniqueGuard.emplace(analyzer->name()).second) - _analyzers.emplace_back(analyzer, std::move(shortName)); - } - } - } - - { - // optional bool - static const std::string fieldName("includeAllFields"); - - mask->_includeAllFields = slice.hasKey(fieldName); - - if (!mask->_includeAllFields) { - _includeAllFields = defaults._includeAllFields; - } else { - auto field = slice.get(fieldName); - - if (!field.isBool()) { - errorField = fieldName; - - return false; - } - - _includeAllFields = field.getBool(); - } - } - - { - // optional bool - static const std::string fieldName("trackListPositions"); - - mask->_trackListPositions = slice.hasKey(fieldName); - - if (!mask->_trackListPositions) { - _trackListPositions = defaults._trackListPositions; - } else { - auto field = slice.get(fieldName); - - if (!field.isBool()) { - errorField = fieldName; - - return false; - } - - _trackListPositions = field.getBool(); - } - } - - { - // optional string enum - static const std::string fieldName("storeValues"); - - mask->_storeValues = slice.hasKey(fieldName); - - if (!mask->_storeValues) { - _storeValues = defaults._storeValues; - } else { - auto field = slice.get(fieldName); - - if (!field.isString()) { - errorField = fieldName; - - return false; - } - - static const std::unordered_map policies = { - {"none", ValueStorage::NONE}, {"id", ValueStorage::ID}, {"full", ValueStorage::FULL}}; - auto name = field.copyString(); - auto itr = policies.find(name); - - if (itr == policies.end()) { - errorField = fieldName + "." + name; - - return false; - } - - _storeValues = itr->second; - } - } - - // ............................................................................. - // process fields last since children inherit from parent - // ............................................................................. - - { - // optional string map - static const std::string fieldName("fields"); - - mask->_fields = slice.hasKey(fieldName); - - if (!mask->_fields) { - _fields = defaults._fields; - } else { - auto field = slice.get(fieldName); - - if (!field.isObject()) { - errorField = fieldName; - - return false; - } - - auto subDefaults = *this; - - subDefaults._fields.clear(); // do not inherit fields and overrides from this field - _fields.clear(); // reset to match either defaults or read values exactly - - for (arangodb::velocypack::ObjectIterator itr(field); itr.valid(); ++itr) { - auto key = itr.key(); - auto value = itr.value(); - - if (!key.isString()) { - errorField = fieldName + "[" + - arangodb::basics::StringUtils::itoa(itr.index()) + "]"; return false; } - auto name = key.copyString(); - - if (!value.isObject()) { - errorField = fieldName + "." + name; - - return false; - } - - std::string childErrorField; - - // false == do not read 'analyzerDefinitions' from child elements - if (!_fields[name]->init(value, false, childErrorField, defaultVocbase, subDefaults)) { - errorField = fieldName + "." + name + "." + childErrorField; - - return false; - } + _analyzerDefinitions.emplace(analyzer); } } } - return true; + return FieldMeta::init(slice, errorField, defaultVocbase, defaults, mask, &_analyzerDefinitions); } -bool IResearchLinkMeta::json( // append meta jSON - arangodb::velocypack::Builder& builder, // output buffer (out-param) - bool writeAnalyzerDefinition, // output fill analyzer definition instead of just name - IResearchLinkMeta const* ignoreEqual /*= nullptr*/, // values to ignore if equal - TRI_vocbase_t const* defaultVocbase /*= nullptr*/, // fallback vocbase - Mask const* mask /*= nullptr*/, // values to ignore always - std::map* usedAnalyzers /*= nullptr*/ // append analyzers used in definition -) const { +bool IResearchLinkMeta::json(velocypack::Builder& builder, + bool writeAnalyzerDefinition, + IResearchLinkMeta const* ignoreEqual /*= nullptr*/, + TRI_vocbase_t const* defaultVocbase /*= nullptr*/, + Mask const* mask /*= nullptr*/) const { if (!builder.isOpenObject()) { return false; } @@ -550,154 +763,29 @@ bool IResearchLinkMeta::json( // append meta jSON } } - std::map analyzers; - - if ((!ignoreEqual || !equalAnalyzers(_analyzers, ignoreEqual->_analyzers)) && - (!mask || mask->_analyzers)) { - arangodb::velocypack::Builder analyzersBuilder; - - analyzersBuilder.openArray(); - - for (auto& entry : _analyzers) { - if (!entry._pool) { - continue; // skip null analyzers - } - - std::string name; - - if (defaultVocbase) { - auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature - arangodb::SystemDatabaseFeature // feature type - >(); - auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr; - - if (!sysVocbase) { - return false; - } - - // @note: DBServerAgencySync::getLocalCollections(...) generates - // 'forPersistence' definitions that are then compared in - // Maintenance.cpp:compareIndexes(...) via - // arangodb::Index::Compare(...) without access to - // 'defaultVocbase', hence the generated definitions must not - // rely on 'defaultVocbase' - // hence must use 'expandVocbasePrefix==true' if - // 'writeAnalyzerDefinition==true' for normalize - // for 'writeAnalyzerDefinition==false' must use - // 'expandVocbasePrefix==false' so that dump/restore an restore - // definitions into differently named databases - name = IResearchAnalyzerFeature::normalize( // normalize - entry._pool->name(), // analyzer name - *defaultVocbase, // active vocbase - *sysVocbase, // system vocbase - false // expand vocbase prefix - ); - } else { - name = entry._pool->name(); // verbatim (assume already normalized) - } - - analyzers.emplace(name, entry._pool); - analyzersBuilder.add(arangodb::velocypack::Value(std::move(name))); - } - - analyzersBuilder.close(); - builder.add("analyzers", analyzersBuilder.slice()); - } - - if (!mask || mask->_fields) { // fields are not inherited from parent - arangodb::velocypack::Builder fieldsBuilder; - Mask fieldMask(true); // output all non-matching fields - auto subDefaults = *this; // make modifable copy - - subDefaults._fields.clear(); // do not inherit fields and overrides from this field - fieldsBuilder.openObject(); - fieldMask._analyzerDefinitions = false; // do not output analyzer definitions in children - - for (auto& entry : _fields) { - fieldMask._fields = !entry.value()->_fields.empty(); // do not output empty fields on subobjects - fieldsBuilder.add( // add sub-object - entry.key(), // field name - arangodb::velocypack::Value(arangodb::velocypack::ValueType::Object) - ); - - if (!entry.value()->json(fieldsBuilder, writeAnalyzerDefinition, &subDefaults, defaultVocbase, &fieldMask, &analyzers)) { - return false; - } - - fieldsBuilder.close(); - } - - fieldsBuilder.close(); - builder.add("fields", fieldsBuilder.slice()); - } - - if ((!ignoreEqual || _includeAllFields != ignoreEqual->_includeAllFields) && - (!mask || mask->_includeAllFields)) { - builder.add("includeAllFields", arangodb::velocypack::Value(_includeAllFields)); - } - - if ((!ignoreEqual || _trackListPositions != ignoreEqual->_trackListPositions) && - (!mask || mask->_trackListPositions)) { - builder.add("trackListPositions", arangodb::velocypack::Value(_trackListPositions)); - } - - if ((!ignoreEqual || _storeValues != ignoreEqual->_storeValues) && - (!mask || mask->_storeValues)) { - static_assert(adjacencyChecker::checkAdjacency(), - "Values are not adjacent"); - - static const std::string policies[]{ - "none", // ValueStorage::NONE - "id", // ValueStorage::ID - "full" // ValueStorage::FULL - }; - - auto const policyIdx = - static_cast::type>(_storeValues); - - if (policyIdx >= IRESEARCH_COUNTOF(policies)) { - return false; // unsupported value storage policy - } - - builder.add("storeValues", arangodb::velocypack::Value(policies[policyIdx])); - } - // output definitions if 'writeAnalyzerDefinition' requested and not maked // this should be the case for the default top-most call if (writeAnalyzerDefinition && (!mask || mask->_analyzerDefinitions)) { VPackArrayBuilder arrayScope(&builder, "analyzerDefinitions"); - for (auto& entry: analyzers) { - TRI_ASSERT(entry.second); // ensured by emplace into 'analyzers' above - entry.second->toVelocyPack(builder, defaultVocbase); + for (auto& entry: _analyzerDefinitions) { + TRI_ASSERT(entry); // ensured by emplace into 'analyzers' above + entry->toVelocyPack(builder, defaultVocbase); } } - if (usedAnalyzers) { - usedAnalyzers->insert(analyzers.begin(), analyzers.end()); - } - - return true; + return FieldMeta::json(builder, ignoreEqual, defaultVocbase, mask); } size_t IResearchLinkMeta::memory() const noexcept { auto size = sizeof(IResearchLinkMeta); size += _analyzers.size() * sizeof(decltype(_analyzers)::value_type); - size += _fields.size() * sizeof(decltype(_fields)::value_type); size += _sort.memory(); - - for (auto& entry : _fields) { - size += entry.key().size(); - size += entry.value()->memory(); - } + size += FieldMeta::memory(); return size; } } // namespace iresearch } // namespace arangodb - -// ----------------------------------------------------------------------------- -// --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- diff --git a/arangod/IResearch/IResearchLinkMeta.h b/arangod/IResearch/IResearchLinkMeta.h index 01ec9bf8d5..ac53771c88 100644 --- a/arangod/IResearch/IResearchLinkMeta.h +++ b/arangod/IResearch/IResearchLinkMeta.h @@ -48,58 +48,147 @@ class Slice; // forward declarations namespace arangodb { namespace iresearch { -// ----------------------------------------------------------------------------- -// --SECTION-- public -// types -// ----------------------------------------------------------------------------- - //////////////////////////////////////////////////////////////////////////////// /// @brief enum of possible ways to store values in the view //////////////////////////////////////////////////////////////////////////////// enum class ValueStorage : uint32_t { NONE = 0, // do not store values in the view ID, // only store value existance - FULL, // store full value in the view + VALUE // store full value in the view }; -//////////////////////////////////////////////////////////////////////////////// -/// @brief metadata describing how to process a field in a collection -//////////////////////////////////////////////////////////////////////////////// -struct IResearchLinkMeta { +struct FieldMeta { + // can't use FieldMeta as value type since it's incomplete type so far + typedef UnorderedRefKeyMap> Fields; + struct Analyzer { - AnalyzerPool::ptr _pool; - std::string _shortName; // vocbase-dependent short analyzer name Analyzer(); // identity analyzer - Analyzer( // constructor - AnalyzerPool::ptr const& pool, // pool - std::string&& shortName // short name (cached for use during insert(...)) - ) noexcept: _pool(pool), _shortName(std::move(shortName)) {} + Analyzer(AnalyzerPool::ptr const& pool, + std::string&& shortName) noexcept + : _pool(pool), + _shortName(std::move(shortName)) { + } operator bool() const noexcept { return false == !_pool; } + + AnalyzerPool::ptr _pool; + std::string _shortName; // vocbase-independent short analyzer name }; + + struct AnalyzerComparer { + using is_transparent = void; + + bool operator()(AnalyzerPool::ptr const& lhs, AnalyzerPool::ptr const& rhs) const noexcept { + return lhs->name() < rhs->name(); + } + + bool operator()(AnalyzerPool::ptr const& lhs, irs::string_ref const& rhs) const noexcept { + return lhs->name() < rhs; + } + + bool operator()(irs::string_ref const& lhs, AnalyzerPool::ptr const& rhs) const noexcept { + return lhs < rhs->name(); + } + }; + struct Mask { - bool _analyzerDefinitions; + explicit Mask(bool mask = false) noexcept + : _analyzers(mask), + _fields(mask), + _includeAllFields(mask), + _trackListPositions(mask), + _storeValues(mask) { + } + bool _analyzers; bool _fields; bool _includeAllFields; bool _trackListPositions; bool _storeValues; - bool _sort; - explicit Mask(bool mask = false) noexcept; }; - typedef std::vector Analyzers; + //////////////////////////////////////////////////////////////////////////////// + /// @brief return default IResearchLinkMeta values + //////////////////////////////////////////////////////////////////////////////// + static const FieldMeta& DEFAULT(); - // can't use IResearchLinkMeta as value type since it's incomplete type so far - typedef UnorderedRefKeyMap> Fields; + FieldMeta(); + FieldMeta(FieldMeta const&) = default; + FieldMeta(FieldMeta&&) = default; - IResearchViewSort _sort; // sort condition associated with the link - Analyzers _analyzers; // analyzers to apply to every field + FieldMeta& operator=(FieldMeta const&) = default; + FieldMeta& operator=(FieldMeta&&) = default; + + bool operator==(FieldMeta const& rhs) const noexcept; + bool operator!=(FieldMeta const& rhs) const noexcept { + return !(*this == rhs); + } + + //////////////////////////////////////////////////////////////////////////////// + /// @brief initialize FieldMeta with values from a JSON description + /// return success or set 'errorField' to specific field with error + /// on failure state is undefined + /// @param slice input definition + /// @param errorField field causing error (out-param) + /// @param defaultVocbase fallback vocbase for analyzer name normalization + /// nullptr == do not normalize + /// @param defaults inherited defaults + /// @param mask if set reflects which fields were initialized from JSON + /// @param analyzers analyzers referenced in this link + //////////////////////////////////////////////////////////////////////////////// + bool init(velocypack::Slice const& slice, + std::string& errorField, + TRI_vocbase_t const* defaultVocbase = nullptr, + FieldMeta const& defaults = DEFAULT(), + Mask* mask = nullptr, + std::set* analyzers = nullptr); + + //////////////////////////////////////////////////////////////////////////////// + /// @brief fill and return a JSON description of a FieldMeta object + /// do not fill values identical to ones available in 'ignoreEqual' + /// or (if 'mask' != nullptr) values in 'mask' that are set to false + /// elements are appended to an existing object + /// return success or set TRI_set_errno(...) and return false + /// @param builder output buffer + /// @param defaultVocbase fallback vocbase for analyzer name normalization + /// nullptr == do not normalize + /// @param ignoreEqual values to ignore if equal + /// @param defaultVocbase fallback vocbase + /// @param mask if set reflects which fields were initialized from JSON + //////////////////////////////////////////////////////////////////////////////// + bool json(arangodb::velocypack::Builder& builder, + FieldMeta const* ignoreEqual = nullptr, + TRI_vocbase_t const* defaultVocbase = nullptr, + Mask const* mask = nullptr) const; + + //////////////////////////////////////////////////////////////////////////////// + /// @brief amount of memory in bytes occupied by this FieldMeta + //////////////////////////////////////////////////////////////////////////////// + size_t memory() const noexcept; + + std::vector _analyzers; // analyzers to apply to every field Fields _fields; // explicit list of fields to be indexed with optional overrides - bool _includeAllFields; // include all fields or only fields listed in - // '_fields' - bool _trackListPositions; // append relative offset in list to attribute name - // (as opposed to without offset) - ValueStorage _storeValues; // how values should be stored inside the view + ValueStorage _storeValues{ ValueStorage::NONE }; // how values should be stored inside the view + bool _includeAllFields{ false }; // include all fields or only fields listed in '_fields' + bool _trackListPositions{ false }; // append relative offset in list to attribute name (as opposed to without offset) +}; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief metadata describing how to process a field in a collection +//////////////////////////////////////////////////////////////////////////////// +struct IResearchLinkMeta : public FieldMeta { + struct Mask : public FieldMeta::Mask { + explicit Mask(bool mask = false) noexcept + : FieldMeta::Mask(mask), + _analyzerDefinitions(mask), + _sort(mask) { + } + + bool _analyzerDefinitions; + bool _sort; + }; + + std::set _analyzerDefinitions; + IResearchViewSort _sort; // sort condition associated with the link // NOTE: if adding fields don't forget to modify the comparison operator !!! // NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask !!! // NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask constructor !!! @@ -114,8 +203,10 @@ struct IResearchLinkMeta { IResearchLinkMeta& operator=(IResearchLinkMeta&& other) = default; IResearchLinkMeta& operator=(IResearchLinkMeta const& other) = default; - bool operator==(IResearchLinkMeta const& other) const noexcept; - bool operator!=(IResearchLinkMeta const& other) const noexcept; + bool operator==(IResearchLinkMeta const& rhs) const noexcept; + bool operator!=(IResearchLinkMeta const& rhs) const noexcept { + return !(*this == rhs); + } //////////////////////////////////////////////////////////////////////////////// /// @brief return default IResearchLinkMeta values @@ -126,18 +217,21 @@ struct IResearchLinkMeta { /// @brief initialize IResearchLinkMeta with values from a JSON description /// return success or set 'errorField' to specific field with error /// on failure state is undefined + /// @param slice definition + /// @param readAnalyzerDefinition allow reading analyzer definitions instead + /// of just name + /// @param erroField field causing error (out-param) /// @param defaultVocbase fallback vocbase for analyzer name normalization /// nullptr == do not normalize + /// @param defaults inherited defaults /// @param mask if set reflects which fields were initialized from JSON //////////////////////////////////////////////////////////////////////////////// - bool init( // initialize meta - arangodb::velocypack::Slice const& slice, // definition - bool readAnalyzerDefinition, // allow reading analyzer definitions instead of just name - std::string& errorField, // field causing error (out-param) - TRI_vocbase_t const* defaultVocbase = nullptr, // fallback vocbase - IResearchLinkMeta const& defaults = DEFAULT(), // inherited defaults - Mask* mask = nullptr // initialized fields (out-param) - ); + bool init(velocypack::Slice const& slice, + bool readAnalyzerDefinition, + std::string& errorField, + TRI_vocbase_t const* defaultVocbase = nullptr, + FieldMeta const& defaults = DEFAULT(), + Mask* mask = nullptr); //////////////////////////////////////////////////////////////////////////////// /// @brief fill and return a JSON description of a IResearchLinkMeta object @@ -145,21 +239,21 @@ struct IResearchLinkMeta { /// or (if 'mask' != nullptr) values in 'mask' that are set to false /// elements are appended to an existing object /// return success or set TRI_set_errno(...) and return false + /// @param builder output buffer (out-param) + /// @param writeAnalyzerDefinition output full analyzer definition instead of just name + /// @param ignoreEqual values to ignore if equal /// @param defaultVocbase fallback vocbase for analyzer name normalization /// nullptr == do not normalize - /// @param usedAnalyzers add to this map analyzers used in meta, + /// @param mask if set reflects which fields were initialized from JSON //////////////////////////////////////////////////////////////////////////////// - bool json( // append meta jSON - arangodb::velocypack::Builder& builder, // output buffer (out-param) - bool writeAnalyzerDefinition, // output full analyzer definition instead of just name - IResearchLinkMeta const* ignoreEqual = nullptr, // values to ignore if equal - TRI_vocbase_t const* defaultVocbase = nullptr, // fallback vocbase - Mask const* mask = nullptr, // values to ignore always - std::map* usedAnalyzers = nullptr // append analyzers used in definition - ) const; + bool json(velocypack::Builder& builder, + bool writeAnalyzerDefinition, + IResearchLinkMeta const* ignoreEqual = nullptr, + TRI_vocbase_t const* defaultVocbase = nullptr, + Mask const* mask = nullptr) const; //////////////////////////////////////////////////////////////////////////////// - /// @brief amount of memory in bytes occupied by this iResearch Link meta + /// @brief amount of memory in bytes occupied by this IResearchLinkMeta //////////////////////////////////////////////////////////////////////////////// size_t memory() const noexcept; }; // IResearchLinkMeta diff --git a/arangod/Replication/utilities.cpp b/arangod/Replication/utilities.cpp index 86545c6748..61edfaa460 100644 --- a/arangod/Replication/utilities.cpp +++ b/arangod/Replication/utilities.cpp @@ -640,6 +640,8 @@ Result parseResponse(velocypack::Builder& builder, velocypack::Parser parser(builder); parser.parse(response->getBody().begin(), response->getBody().length()); return Result(); + } catch (VPackException const& e) { + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, e.what()); } catch (...) { return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE); } diff --git a/arangod/RocksDBEngine/RocksDBIncrementalSync.cpp b/arangod/RocksDBEngine/RocksDBIncrementalSync.cpp index 8bf94ace70..78e9a63dff 100644 --- a/arangod/RocksDBEngine/RocksDBIncrementalSync.cpp +++ b/arangod/RocksDBEngine/RocksDBIncrementalSync.cpp @@ -228,7 +228,7 @@ Result syncChunkRocksDB(DatabaseInitialSyncer& syncer, SingleCollectionTransacti if (r.fail()) { return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + - syncer._state.master.endpoint + ": response is no array"); + syncer._state.master.endpoint + ": " + r.errorMessage()); } VPackSlice const responseBody = builder.slice(); @@ -425,7 +425,7 @@ Result syncChunkRocksDB(DatabaseInitialSyncer& syncer, SingleCollectionTransacti return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + syncer._state.master.endpoint + - ": response is no array"); + ": " + r.errorMessage()); } VPackSlice const slice = docsBuilder->slice(); @@ -613,7 +613,7 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer, if (r.fail()) { return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + - syncer._state.master.endpoint + ": response is no array"); + syncer._state.master.endpoint + ": " + r.errorMessage()); } VPackSlice const chunkSlice = builder.slice(); diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index 66fb48fa2a..6794a824cd 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -1019,8 +1019,8 @@ std::shared_ptr TRI_vocbase_t::lookupCollectionByUu } /// @brief looks up a data-source by identifier -std::shared_ptr TRI_vocbase_t::lookupDataSource(TRI_voc_cid_t id) const - noexcept { +std::shared_ptr TRI_vocbase_t::lookupDataSource( + TRI_voc_cid_t id) const noexcept { RECURSIVE_READ_LOCKER(_dataSourceLock, _dataSourceLockWriteOwner); auto itr = _dataSourceById.find(id); diff --git a/js/actions/api-cluster.js b/js/actions/api-cluster.js index 067ad9bf51..eb8fdd4626 100644 --- a/js/actions/api-cluster.js +++ b/js/actions/api-cluster.js @@ -1365,9 +1365,10 @@ actions.defineHttp({ } // simon: RO is sufficient to rebalance shards for current db - if (req.database !== '_system'/* || !req.isAdminUser*/) { + if (!req.isAdminUser && + users.permission(req.user, req.database) !== 'rw') { actions.resultError(req, res, actions.HTTP_FORBIDDEN, 0, - 'only allowed for admins on the _system database'); + 'only allowed for admins on the database'); return; } diff --git a/js/server/modules/@arangodb/foxx/manager.js b/js/server/modules/@arangodb/foxx/manager.js index b729c8827f..368726b5ff 100644 --- a/js/server/modules/@arangodb/foxx/manager.js +++ b/js/server/modules/@arangodb/foxx/manager.js @@ -284,50 +284,12 @@ function cleanupOrphanedServices (knownServicePaths, knownBundlePaths) { } function startup () { - if (isFoxxmaster()) { - const db = require('internal').db; - const dbName = db._name(); - try { - db._useDatabase('_system'); - const databases = db._databases(); - for (const name of databases) { - try { - db._useDatabase(name); - upsertSystemServices(); - } catch (e) { - console.warnStack(e); - } - } - } finally { - db._useDatabase(dbName); - } - } if (global.ArangoServerState.role() === 'SINGLE') { commitLocalState(true); } selfHealAll(); } -function upsertSystemServices () { - const serviceDefinitions = new Map(); - for (const mount of SYSTEM_SERVICE_MOUNTS) { - try { - const serviceDefinition = utils.getServiceDefinition(mount) || {mount}; - const service = FoxxService.create(serviceDefinition); - serviceDefinitions.set(mount, service.toJSON()); - } catch (e) { - console.errorStack(e); - } - } - db._query(aql` - FOR item IN ${Array.from(serviceDefinitions)} - UPSERT {mount: item[0]} - INSERT item[1] - REPLACE item[1] - IN ${utils.getStorage()} - `); -} - function commitLocalState (replace) { let modified = false; const rootPath = FoxxService.rootPath(); diff --git a/tests/Graph/GraphTestTools.h b/tests/Graph/GraphTestTools.h index 0bd00476b5..f19ac5d6b4 100644 --- a/tests/Graph/GraphTestTools.h +++ b/tests/Graph/GraphTestTools.h @@ -50,6 +50,8 @@ #include "../Mocks/StorageEngineMock.h" #include "IResearch/common.h" +#include "boost/optional.hpp" + using namespace arangodb; using namespace arangodb::aql; using namespace arangodb::graph; @@ -184,9 +186,18 @@ struct MockGraphDatabase { EXPECT_TRUE(vertices->type()); } + struct EdgeDef { + EdgeDef(size_t from, size_t to) : _from(from), _to(to){}; + EdgeDef(size_t from, size_t to, double weight) + : _from(from), _to(to), _weight(weight){}; + size_t _from; + size_t _to; + boost::optional _weight; + }; + // Create a collection with name of edges given by void addEdgeCollection(std::string name, std::string vertexCollection, - std::vector> edgedef) { + std::vector edgedef) { std::shared_ptr edges; auto createJson = velocypack::Parser::fromJson("{ \"name\": \"" + name + "\", \"type\": 3 }"); @@ -205,10 +216,18 @@ struct MockGraphDatabase { for (auto& p : edgedef) { // std::cout << "edge: " << vertexCollection << " " << p.first << " -> " // << p.second << std::endl; - auto docJson = velocypack::Parser::fromJson( - "{ \"_from\": \"" + vertexCollection + "/" + std::to_string(p.first) + - "\"" + ", \"_to\": \"" + vertexCollection + "/" + - std::to_string(p.second) + "\" }"); + // This is moderately horrible + auto docJson = + p._weight.has_value() + ? velocypack::Parser::fromJson( + "{ \"_from\": \"" + vertexCollection + "/" + + std::to_string(p._from) + "\"" + ", \"_to\": \"" + + vertexCollection + "/" + std::to_string(p._to) + + "\", \"cost\": " + std::to_string(p._weight.value()) + "}") + : velocypack::Parser::fromJson( + "{ \"_from\": \"" + vertexCollection + "/" + + std::to_string(p._from) + "\"" + ", \"_to\": \"" + + vertexCollection + "/" + std::to_string(p._to) + "\" }"); docs.emplace_back(docJson); } @@ -282,12 +301,13 @@ struct MockGraphDatabase { spos.emplace_back(spo); return spo; } -}; +}; // namespace graph -bool checkPath(ShortestPathOptions* spo, ShortestPathResult result, std::vector vertices, +bool checkPath(ShortestPathOptions* spo, ShortestPathResult result, + std::vector vertices, std::vector> edges, std::string& msgs); -} -} -} +} // namespace graph +} // namespace tests +} // namespace arangodb #endif diff --git a/tests/Graph/KShortestPathsFinder.cpp b/tests/Graph/KShortestPathsFinder.cpp index f6a3b299d2..9d358fb44d 100644 --- a/tests/Graph/KShortestPathsFinder.cpp +++ b/tests/Graph/KShortestPathsFinder.cpp @@ -212,6 +212,52 @@ TEST_F(KShortestPathsFinderTest, many_edges_between_two_nodes) { ASSERT_TRUE(false == finder->getNextPathShortestPathResult(result)); } +class KShortestPathsFinderTestWeights : public ::testing::Test { + protected: + GraphTestSetup s; + MockGraphDatabase gdb; + + arangodb::aql::Query* query; + arangodb::graph::ShortestPathOptions* spo; + + KShortestPathsFinder* finder; + + KShortestPathsFinderTestWeights() : gdb("testVocbase") { + gdb.addVertexCollection("v", 10); + gdb.addEdgeCollection("e", "v", + {{1, 2, 10}, + {1, 3, 10}, + {1, 10, 100}, + {2, 4, 10}, + {3, 4, 20}, + {7, 3, 10}, + {8, 3, 10}, + {9, 3, 10}}); + + query = gdb.getQuery("RETURN 1"); + spo = gdb.getShortestPathOptions(query); + spo->weightAttribute = "cost"; + + finder = new KShortestPathsFinder(*spo); + } + + ~KShortestPathsFinderTestWeights() { delete finder; } +}; + +TEST_F(KShortestPathsFinderTestWeights, diamond_path) { + auto start = velocypack::Parser::fromJson("\"v/1\""); + auto end = velocypack::Parser::fromJson("\"v/4\""); + ShortestPathResult result; + std::string msgs; + + finder->startKShortestPathsTraversal(start->slice(), end->slice()); + + ASSERT_TRUE(finder->getNextPathShortestPathResult(result)); + auto cpr = checkPath(spo, result, {"1", "2", "4"}, + {{}, {"v/1", "v/2"}, {"v/2", "v/4"}}, msgs); + ASSERT_TRUE(cpr) << msgs; +} + } // namespace graph } // namespace tests } // namespace arangodb diff --git a/tests/IResearch/IResearchAnalyzerFeature-test.cpp b/tests/IResearch/IResearchAnalyzerFeature-test.cpp index d69cf60a81..7311c3d3b2 100644 --- a/tests/IResearch/IResearchAnalyzerFeature-test.cpp +++ b/tests/IResearch/IResearchAnalyzerFeature-test.cpp @@ -67,8 +67,8 @@ #include "RestServer/SystemDatabaseFeature.h" #include "RestServer/TraverserEngineRegistryFeature.h" #include "RestServer/UpgradeFeature.h" -#include "RestServer/ViewTypesFeature.h" #include "RestServer/VocbaseContext.h" +#include "RestServer/ViewTypesFeature.h" #include "Scheduler/SchedulerFeature.h" #include "Sharding/ShardingFeature.h" #include "StorageEngine/EngineSelectorFeature.h" @@ -189,7 +189,6 @@ class TestAnalyzer : public irs::analysis::analyzer { _attrs.emplace(_attr); _attrs.emplace(_increment); // required by field_data::invert(...) } - virtual irs::attribute_view const& attributes() const noexcept override { return _attrs; } @@ -405,7 +404,7 @@ class IResearchAnalyzerFeatureTest : public ::testing::Test { _agencyStore); // need 2 connections or Agency callbacks will fail arangodb::AgencyCommManager::MANAGER.reset(agencyCommManager); // required for Coordinator tests - arangodb::tests::init(); + arangodb::tests::init(true); // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead @@ -413,6 +412,9 @@ class IResearchAnalyzerFeatureTest : public ::testing::Test { arangodb::LogLevel::ERR); // setup required application features + features.emplace_back(new arangodb::TraverserEngineRegistryFeature(server), false); // must be before AqlFeature + features.emplace_back(new arangodb::AqlFeature(server), true); + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(new arangodb::AuthenticationFeature(server), true); features.emplace_back(new arangodb::DatabaseFeature(server), false); features.emplace_back(new arangodb::ShardingFeature(server), false); @@ -883,43 +885,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_emplace_add_static_analyzer) { ASSERT_NE(analyzer, nullptr); } -TEST_F(IResearchAnalyzerFeatureTest, test_get_parameter_match) { - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - auto res = feature.emplace(result, analyzerName(), "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice(), - {irs::frequency::type()}); - EXPECT_TRUE(res.ok()); - ASSERT_FALSE(feature.get(result, - analyzerName(), "identity", - VPackParser::fromJson("\"abc\"")->slice(), - {irs::frequency::type()}).ok()); -} - -TEST_F(IResearchAnalyzerFeatureTest, test_get_properties_mismatch) { - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - auto res = feature.emplace(result, analyzerName(), "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice(), - {irs::frequency::type()}); - EXPECT_TRUE(res.ok()); - ASSERT_FALSE(feature.get(result, analyzerName(), "TestAnalyzer", - VPackParser::fromJson("\"abcd\"")->slice(), - {irs::frequency::type()}).ok()); -} - -TEST_F(IResearchAnalyzerFeatureTest, test_get_feature_mismatch) { - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - auto res = feature.emplace(result, analyzerName(), "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice(), - {irs::frequency::type()}); - EXPECT_TRUE(res.ok()); - ASSERT_FALSE(feature.get(result, analyzerName(), "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice(), - {irs::position::type()}).ok()); -} - TEST_F(IResearchAnalyzerFeatureTest, test_renormalize_for_equal) { arangodb::iresearch::IResearchAnalyzerFeature feature(server); { @@ -951,7 +916,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_renormalize_for_equal) { class IResearchAnalyzerFeatureGetTest : public IResearchAnalyzerFeatureTest { protected: - arangodb::AqlFeature aqlFeature; arangodb::iresearch::IResearchAnalyzerFeature analyzerFeature; std::string dbName; @@ -963,7 +927,6 @@ class IResearchAnalyzerFeatureGetTest : public IResearchAnalyzerFeatureTest { protected: IResearchAnalyzerFeatureGetTest() : IResearchAnalyzerFeatureTest(), - aqlFeature(server), analyzerFeature(server), dbName("testVocbase") {} @@ -971,8 +934,6 @@ class IResearchAnalyzerFeatureGetTest : public IResearchAnalyzerFeatureTest { // Need Setup inorder to alow ASSERTs void SetUp() override { - // required for Query::Query(...), must not call ~AqlFeature() for the duration of the test - aqlFeature.start(); _dbFeature = arangodb::application_features::ApplicationServer::lookupFeature( "Database"); @@ -1161,28 +1122,6 @@ TEST_F(IResearchAnalyzerFeatureGetTest, test_get_static_analyzer_adding_vocbases ASSERT_NE(analyzer, nullptr); } -TEST_F(IResearchAnalyzerFeatureGetTest, test_get_failure_specfic_type_and_properties_mismatch) { - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - ASSERT_FALSE(feature().get(result, specificName(), "TestAnalyzer", - VPackParser::fromJson("{\"args\":\"abc\"}")->slice(), - {irs::frequency::type()}).ok()); -} - -TEST_F(IResearchAnalyzerFeatureGetTest, test_get_db_server) { - auto before = arangodb::ServerState::instance()->getRole(); - arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER); - auto restore = irs::make_finally([&before]() -> void { - arangodb::ServerState::instance()->setRole(before); - }); - - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - ASSERT_TRUE(feature.get(result,"testVocbase::test_analyzer", "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice(), - {irs::frequency::type()}).ok()); - ASSERT_NE(nullptr, result.first); -} - // ----------------------------------------------------------------------------- // --SECTION-- coordinator test suite // ----------------------------------------------------------------------------- @@ -1251,7 +1190,6 @@ class IResearchAnalyzerFeatureCoordinatorTest : public ::testing::Test { // setup required application features buildFeatureEntry(new arangodb::V8DealerFeature(server), false); - buildFeatureEntry(new arangodb::ViewTypesFeature(server), true); buildFeatureEntry(tmpFeature = new arangodb::QueryRegistryFeature(server), false); arangodb::application_features::ApplicationServer::server->addFeature(tmpFeature); // need QueryRegistryFeature feature to be added now in order to create the system database _system = irs::memory::make_unique(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, @@ -1273,6 +1211,7 @@ class IResearchAnalyzerFeatureCoordinatorTest : public ::testing::Test { buildFeatureEntry(new arangodb::ClusterFeature(server), false); buildFeatureEntry(new arangodb::ShardingFeature(server), false); buildFeatureEntry(new arangodb::iresearch::IResearchAnalyzerFeature(server), true); + buildFeatureEntry(new arangodb::ViewTypesFeature(server), true); #if USE_ENTERPRISE buildFeatureEntry(new arangodb::LdapFeature(server), @@ -1446,10 +1385,6 @@ TEST_F(IResearchAnalyzerFeatureCoordinatorTest, test_ensure_index) { arangodb::application_features::ApplicationServer::lookupFeature(); EXPECT_TRUE((feature)); ci->invalidatePlan(); // invalidate plan to test recursive lock aquisition in ClusterInfo::loadPlan() - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - EXPECT_TRUE(!feature->get(result, arangodb::StaticStrings::SystemDatabase + "::missing", - "TestAnalyzer", VPackSlice::noneSlice(), - irs::flags()).ok()); return std::make_shared(id, collection, definition); } @@ -1910,7 +1845,7 @@ TEST_F(IResearchAnalyzerFeatureTest, test_persistence) { } arangodb::iresearch::IResearchAnalyzerFeature feature(server); - EXPECT_ANY_THROW((feature.start())); + EXPECT_NO_THROW(feature.start()); } // read valid configuration (different parameter options) @@ -1962,259 +1897,255 @@ TEST_F(IResearchAnalyzerFeatureTest, test_persistence) { trx.commit(); } - std::map> expected = { - {arangodb::StaticStrings::SystemDatabase + "::valid0", - {"identity", "{\n}"}}, - {arangodb::StaticStrings::SystemDatabase + "::valid2", - {"identity", "{\n}"}}, - {arangodb::StaticStrings::SystemDatabase + "::valid4", - {"identity", "{\n}"}}, - {arangodb::StaticStrings::SystemDatabase + "::valid5", - {"identity", "{\n}"}}, - }; arangodb::iresearch::IResearchAnalyzerFeature feature(server); + feature.start(); // feature doesn't load persisted analyzers - feature.start(); // load persisted analyzers - - feature.visit( - [&expected](arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { - if (staticAnalyzers().find(analyzer->name()) != staticAnalyzers().end()) { - return true; // skip static analyzers - } - - auto itr = expected.find(analyzer->name()); - EXPECT_TRUE((itr != expected.end())); - EXPECT_EQ(itr->second.first, analyzer->type()); - EXPECT_EQ(itr->second.second, analyzer->properties().toString()); - expected.erase(itr); - return true; - }); - EXPECT_TRUE((expected.empty())); + EXPECT_TRUE(feature.visit([](arangodb::iresearch::AnalyzerPool::ptr const&) { + return false; + })); } // add new records - {{arangodb::OperationOptions options; - arangodb::ManagedDocumentResult result; - auto collection = vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName); - arangodb::transaction::Methods trx(arangodb::transaction::StandaloneContext::Create(*vocbase), - EMPTY, EMPTY, EMPTY, - arangodb::transaction::Options()); - EXPECT_TRUE((collection->truncate(trx, options).ok())); -} - -{ - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - - EXPECT_TRUE( - (feature - .emplace(result, arangodb::StaticStrings::SystemDatabase + "::valid", - "identity", VPackParser::fromJson("{\"args\":\"abc\"}")->slice()) - .ok())); - EXPECT_TRUE((result.first)); - EXPECT_TRUE((result.second)); -} - -{ - std::map> expected = { - {arangodb::StaticStrings::SystemDatabase + "::valid", - {"identity", "{\n}"}}, - }; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - - feature.start(); // load persisted analyzers - - feature.visit( - [&expected](arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { - if (staticAnalyzers().find(analyzer->name()) != staticAnalyzers().end()) { - return true; // skip static analyzers - } - - auto itr = expected.find(analyzer->name()); - EXPECT_TRUE((itr != expected.end())); - EXPECT_EQ(itr->second.first, analyzer->type()); - EXPECT_EQ(itr->second.second, analyzer->properties().toString()); - expected.erase(itr); - return true; - }); - EXPECT_TRUE((expected.empty())); -} -} - -// remove existing records -{{std::string collection(arangodb::tests::AnalyzerCollectionName); -arangodb::OperationOptions options; -arangodb::SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(*vocbase), - collection, arangodb::AccessMode::Type::WRITE); -trx.begin(); -trx.truncate(collection, options); -trx.insert( - collection, - VPackParser::fromJson( - "{\"name\": \"valid\", \"type\": \"identity\", \"properties\": {}}") - ->slice(), - options); -trx.commit(); -} - -{ - std::map> expected = { - {"identity", {"identity", "{\n}"}}, - {"text_de", - {"text", - "{ \"locale\": \"de.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_en", - {"text", - "{ \"locale\": \"en.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_es", - {"text", - "{ \"locale\": \"es.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_fi", - {"text", - "{ \"locale\": \"fi.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_fr", - {"text", - "{ \"locale\": \"fr.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_it", - {"text", - "{ \"locale\": \"it.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_nl", - {"text", - "{ \"locale\": \"nl.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_no", - {"text", - "{ \"locale\": \"no.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_pt", - {"text", - "{ \"locale\": \"pt.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_ru", - {"text", - "{ \"locale\": \"ru.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_sv", - {"text", - "{ \"locale\": \"sv.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {"text_zh", - {"text", - "{ \"locale\": \"zh.UTF-8\", \"caseConvert\": \"lower\", " - "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, - {arangodb::StaticStrings::SystemDatabase + "::valid", {"identity", "{}"}}, - }; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - - feature.prepare(); // load static analyzers - feature.start(); // load persisted analyzers - - feature.visit([&expected](arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { - if ((analyzer->name() != "identity" && - !irs::starts_with(irs::string_ref(analyzer->name()), "text_")) && - staticAnalyzers().find(analyzer->name()) != staticAnalyzers().end()) { - return true; // skip static analyzers + { + { + arangodb::OperationOptions options; + arangodb::ManagedDocumentResult result; + auto collection = vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName); + arangodb::transaction::Methods trx(arangodb::transaction::StandaloneContext::Create(*vocbase), + EMPTY, EMPTY, EMPTY, + arangodb::transaction::Options()); + EXPECT_TRUE(collection->truncate(trx, options).ok()); } - auto itr = expected.find(analyzer->name()); - EXPECT_TRUE((itr != expected.end())); - EXPECT_TRUE((itr->second.first == analyzer->type())); + { + arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; + arangodb::iresearch::IResearchAnalyzerFeature feature(server); - std::string expectedProperties; - EXPECT_TRUE(irs::analysis::analyzers::normalize( - expectedProperties, analyzer->type(), irs::text_format::vpack, - arangodb::iresearch::ref(VPackParser::fromJson(itr->second.second)->slice()), - false)); + EXPECT_TRUE(feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::valid", "identity", + VPackParser::fromJson("{\"args\":\"abc\"}")->slice()).ok()); + EXPECT_TRUE(result.first); + EXPECT_TRUE(result.second); + } - EXPECT_EQUAL_SLICES(arangodb::iresearch::slice(expectedProperties), - analyzer->properties()); - expected.erase(itr); - return true; - }); + { + arangodb::iresearch::IResearchAnalyzerFeature feature(server); - EXPECT_TRUE((expected.empty())); - EXPECT_TRUE( - (true == - feature.remove(arangodb::StaticStrings::SystemDatabase + "::valid").ok())); - EXPECT_TRUE((false == feature.remove("identity").ok())); -} + feature.start(); // feature doesn't load persisted analyzers -{ - std::map> expected = {}; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); + EXPECT_TRUE(feature.visit([](arangodb::iresearch::AnalyzerPool::ptr const&) { + return false; + })); + } + } - feature.start(); // load persisted analyzers + // remove existing records + { + { + std::string collection(arangodb::tests::AnalyzerCollectionName); + arangodb::OperationOptions options; + arangodb::SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(*vocbase), + collection, arangodb::AccessMode::Type::WRITE); + trx.begin(); + trx.truncate(collection, options); + trx.insert( + collection, + VPackParser::fromJson( + "{\"name\": \"valid\", \"type\": \"identity\", \"properties\": {}}") + ->slice(), + options); + trx.commit(); + } - feature.visit( - [&expected](arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { - if (staticAnalyzers().find(analyzer->name()) != staticAnalyzers().end()) { - return true; // skip static analyzers - } + { + std::map> expected = { + {"identity", {"identity", "{\n}"}}, + {"text_de", + {"text", + "{ \"locale\": \"de.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_en", + {"text", + "{ \"locale\": \"en.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_es", + {"text", + "{ \"locale\": \"es.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_fi", + {"text", + "{ \"locale\": \"fi.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_fr", + {"text", + "{ \"locale\": \"fr.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_it", + {"text", + "{ \"locale\": \"it.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_nl", + {"text", + "{ \"locale\": \"nl.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_no", + {"text", + "{ \"locale\": \"no.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_pt", + {"text", + "{ \"locale\": \"pt.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_ru", + {"text", + "{ \"locale\": \"ru.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_sv", + {"text", + "{ \"locale\": \"sv.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + {"text_zh", + {"text", + "{ \"locale\": \"zh.UTF-8\", \"caseConvert\": \"lower\", " + "\"stopwords\": [ ], \"noAccent\": true, \"noStrem\": false }"}}, + }; + arangodb::iresearch::IResearchAnalyzerFeature feature(server); + feature.prepare(); // load static analyzers + feature.start(); // doesn't load persisted analyzers + + feature.visit([&expected](arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { auto itr = expected.find(analyzer->name()); - EXPECT_TRUE((itr != expected.end())); - EXPECT_TRUE((itr->second.first == analyzer->type())); - EXPECT_TRUE((itr->second.second == analyzer->properties().toString())); + EXPECT_NE(itr, expected.end()); + EXPECT_TRUE(itr->second.first == analyzer->type()); + + std::string expectedProperties; + EXPECT_TRUE(irs::analysis::analyzers::normalize( + expectedProperties, analyzer->type(), irs::text_format::vpack, + arangodb::iresearch::ref(VPackParser::fromJson(itr->second.second)->slice()), + false)); + + EXPECT_EQUAL_SLICES(arangodb::iresearch::slice(expectedProperties), + analyzer->properties()); expected.erase(itr); return true; }); - EXPECT_TRUE((expected.empty())); -} + + EXPECT_TRUE(expected.empty()); + EXPECT_FALSE(feature.remove(arangodb::StaticStrings::SystemDatabase + "::valid").ok()); + EXPECT_FALSE(feature.remove("identity").ok()); + } + + { + std::map> expected = {}; + arangodb::iresearch::IResearchAnalyzerFeature feature(server); + + feature.start(); // doesn't load persisted analyzers + + EXPECT_TRUE(feature.visit([](arangodb::iresearch::AnalyzerPool::ptr const&) { + return false; + })); + } + } + + // emplace on single-server (should persist) + { + // clear collection + { + std::string collection(arangodb::tests::AnalyzerCollectionName); + arangodb::OperationOptions options; + arangodb::SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(*vocbase), + collection, arangodb::AccessMode::Type::WRITE); + trx.begin(); + trx.truncate(collection, options); + trx.commit(); + } + + arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; + arangodb::iresearch::IResearchAnalyzerFeature feature(server); + EXPECT_TRUE(feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzerA", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice(), + {irs::frequency::type()}).ok()); + EXPECT_TRUE(result.first); + EXPECT_TRUE(feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzerA")); + EXPECT_TRUE(vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName)); + arangodb::OperationOptions options; + arangodb::SingleCollectionTransaction trx( + arangodb::transaction::StandaloneContext::Create(*vocbase), + arangodb::tests::AnalyzerCollectionName, arangodb::AccessMode::Type::WRITE); + EXPECT_TRUE((trx.begin().ok())); + auto queryResult = trx.all(arangodb::tests::AnalyzerCollectionName, 0, 2, options); + EXPECT_TRUE((true == queryResult.ok())); + auto slice = arangodb::velocypack::Slice(queryResult.buffer->data()); + EXPECT_TRUE(slice.isArray()); + ASSERT_EQ(1, slice.length()); + slice = slice.at(0); + EXPECT_TRUE(slice.isObject()); + EXPECT_TRUE(slice.hasKey("_key") && slice.get("_key").isString() && + std::string("test_analyzerA") == slice.get("_key").copyString()); + EXPECT_TRUE(slice.hasKey("name") && slice.get("name").isString() && + std::string("test_analyzerA") == slice.get("name").copyString()); + EXPECT_TRUE(slice.hasKey("type") && slice.get("type").isString() && + std::string("TestAnalyzer") == slice.get("type").copyString()); + EXPECT_TRUE(slice.hasKey("properties") && slice.get("properties").isObject() && + VPackParser::fromJson("{\"args\":\"abc\"}")->slice().toString() == + slice.get("properties").toString()); + EXPECT_TRUE(slice.hasKey("features") && slice.get("features").isArray() && + 1 == slice.get("features").length() && + slice.get("features").at(0).isString() && + std::string("frequency") == slice.get("features").at(0).copyString()); + EXPECT_TRUE(trx.truncate(arangodb::tests::AnalyzerCollectionName, options).ok()); + EXPECT_TRUE(trx.commit().ok()); + } } -// emplace on single-server (should persist) -{ - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - arangodb::iresearch::IResearchAnalyzerFeature feature(server); - EXPECT_TRUE( - (true == feature - .emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzerA", - "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice(), - {irs::frequency::type()}) - .ok())); - EXPECT_TRUE((false == !result.first)); - EXPECT_TRUE((false == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzerA"))); - EXPECT_TRUE((false == !vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName))); - arangodb::OperationOptions options; - arangodb::SingleCollectionTransaction trx( - arangodb::transaction::StandaloneContext::Create(*vocbase), - arangodb::tests::AnalyzerCollectionName, arangodb::AccessMode::Type::WRITE); - EXPECT_TRUE((trx.begin().ok())); - auto queryResult = trx.all(arangodb::tests::AnalyzerCollectionName, 0, 2, options); - EXPECT_TRUE((true == queryResult.ok())); - auto slice = arangodb::velocypack::Slice(queryResult.buffer->data()); - EXPECT_TRUE((slice.isArray() && 1 == slice.length())); - slice = slice.at(0); - EXPECT_TRUE((slice.isObject())); - EXPECT_TRUE((slice.hasKey("name") && slice.get("name").isString() && - std::string("test_analyzerA") == slice.get("name").copyString())); - EXPECT_TRUE((slice.hasKey("type") && slice.get("type").isString() && - std::string("TestAnalyzer") == slice.get("type").copyString())); - EXPECT_TRUE((slice.hasKey("properties") && slice.get("properties").isObject() && - VPackParser::fromJson("{\"args\":\"abc\"}")->slice().toString() == - slice.get("properties").toString())); - EXPECT_TRUE((slice.hasKey("features") && slice.get("features").isArray() && - 1 == slice.get("features").length() && - slice.get("features").at(0).isString() && - std::string("frequency") == slice.get("features").at(0).copyString())); - EXPECT_TRUE((trx.truncate(arangodb::tests::AnalyzerCollectionName, options).ok())); - EXPECT_TRUE((trx.commit().ok())); -} +TEST_F(IResearchAnalyzerFeatureTest, test_analyzer_equality) { + arangodb::iresearch::AnalyzerPool::ptr lhs; + ASSERT_TRUE(arangodb::iresearch::IResearchAnalyzerFeature::createAnalyzerPool( + lhs, "test", "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice(), irs::flags()).ok()); + ASSERT_NE(nullptr, lhs); + ASSERT_EQ(*lhs, *lhs); + + // different name + { + arangodb::iresearch::AnalyzerPool::ptr rhs; + ASSERT_TRUE(arangodb::iresearch::IResearchAnalyzerFeature::createAnalyzerPool( + rhs, "test1", "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice(), irs::flags()).ok()); + ASSERT_NE(nullptr, rhs); + ASSERT_NE(lhs, rhs); + } + + // different type + { + arangodb::iresearch::AnalyzerPool::ptr rhs; + ASSERT_TRUE(arangodb::iresearch::IResearchAnalyzerFeature::createAnalyzerPool( + rhs, "test", "ReNormalizingAnalyzer", VPackParser::fromJson("\"abc\"")->slice(), irs::flags()).ok()); + ASSERT_NE(nullptr, rhs); + ASSERT_NE(lhs, rhs); + } + + // different properties + { + arangodb::iresearch::AnalyzerPool::ptr rhs; + ASSERT_TRUE(arangodb::iresearch::IResearchAnalyzerFeature::createAnalyzerPool( + rhs, "test", "TestAnalyzer", VPackParser::fromJson("\"abcd\"")->slice(), irs::flags()).ok()); + ASSERT_NE(nullptr, rhs); + ASSERT_NE(lhs, rhs); + } + + // different features + { + arangodb::iresearch::AnalyzerPool::ptr rhs; + ASSERT_TRUE(arangodb::iresearch::IResearchAnalyzerFeature::createAnalyzerPool( + rhs, "test", "TestAnalyzer", VPackParser::fromJson("\"abcd\"")->slice(), irs::flags{ irs::frequency::type()}).ok()); + ASSERT_NE(nullptr, rhs); + ASSERT_NE(lhs, rhs); + } } TEST_F(IResearchAnalyzerFeatureTest, test_remove) { - auto* dbFeature = - arangodb::application_features::ApplicationServer::lookupFeature( - "Database"); - ASSERT_TRUE((false == !dbFeature)); - arangodb::AqlFeature aqlFeature(server); - aqlFeature.start(); // required for Query::Query(...), must not call ~AqlFeature() for the duration of the test + auto* dbFeature = arangodb::application_features::ApplicationServer::lookupFeature< + arangodb::DatabaseFeature>("Database"); + ASSERT_NE(nullptr, dbFeature); // remove existing { @@ -2224,19 +2155,13 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { // add analyzer { arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - ASSERT_TRUE( - (true == feature - .emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer0", - "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()) - .ok())); - ASSERT_TRUE((false == !feature.get(arangodb::StaticStrings::SystemDatabase + - "::test_analyzer0"))); + ASSERT_TRUE(feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer0", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()) .ok()); + ASSERT_NE(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0")); } - EXPECT_TRUE((true == feature - .remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0") - .ok())); - EXPECT_TRUE((true == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0"))); + EXPECT_TRUE(feature.remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0").ok()); + EXPECT_EQ(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0")); } // remove existing (inRecovery) single-server @@ -2246,13 +2171,9 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { // add analyzer { arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - ASSERT_TRUE( - (true == feature - .emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer0", - "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()) - .ok())); - ASSERT_TRUE((false == !feature.get(arangodb::StaticStrings::SystemDatabase + - "::test_analyzer0"))); + ASSERT_TRUE(feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer0", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok()); + ASSERT_NE(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0")); } auto before = StorageEngineMock::recoveryStateResult; @@ -2261,10 +2182,8 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { StorageEngineMock::recoveryStateResult = before; }); - EXPECT_TRUE((false == feature - .remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0") - .ok())); - EXPECT_TRUE((false == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0"))); + EXPECT_FALSE(feature.remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0").ok()); + EXPECT_NE(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer0")); } // remove existing (dbserver) @@ -2297,10 +2216,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { server.addFeature(new arangodb::application_features::CommunicationFeaturePhase( server)); // required for SimpleHttpClient::doRequest() server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); @@ -2332,21 +2247,14 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { // add analyzer { arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - ASSERT_TRUE((true == !feature->get(arangodb::StaticStrings::SystemDatabase + - "::test_analyzer2"))); - ASSERT_TRUE((true == feature - ->emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer2", - "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice()) - .ok())); - ASSERT_TRUE((false == !feature->get(arangodb::StaticStrings::SystemDatabase + - "::test_analyzer2"))); + ASSERT_EQ(nullptr, feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2")); + ASSERT_TRUE(feature->emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer2", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok()); + ASSERT_NE(nullptr, feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2")); } - EXPECT_TRUE((true == feature - ->remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2") - .ok())); - EXPECT_TRUE((true == !feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2"))); + EXPECT_TRUE(feature->remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2").ok()); + EXPECT_EQ(nullptr, feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2")); } // remove existing (inRecovery) dbserver @@ -2379,10 +2287,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { server.addFeature(new arangodb::application_features::CommunicationFeaturePhase( server)); // required for SimpleHttpClient::doRequest() server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); @@ -2414,15 +2318,10 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { // add analyzer { arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - ASSERT_TRUE((true == !feature->get(arangodb::StaticStrings::SystemDatabase + - "::test_analyzer2"))); - ASSERT_TRUE((true == feature - ->emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer2", - "TestAnalyzer", - VPackParser::fromJson("\"abc\"")->slice()) - .ok())); - ASSERT_TRUE((false == !feature->get(arangodb::StaticStrings::SystemDatabase + - "::test_analyzer2"))); + ASSERT_EQ(nullptr, feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2")); + ASSERT_TRUE(feature->emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer2", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok()); + ASSERT_NE(nullptr, feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2")); } auto before = StorageEngineMock::recoveryStateResult; @@ -2431,40 +2330,31 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { StorageEngineMock::recoveryStateResult = before; }); - EXPECT_TRUE((true == feature - ->remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2") - .ok())); - EXPECT_TRUE((true == !feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2"))); + EXPECT_TRUE(feature->remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2").ok()); + EXPECT_EQ(nullptr, feature->get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer2")); } // remove existing (in-use) { arangodb::iresearch::IResearchAnalyzerFeature feature(server); arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; // will keep reference - ASSERT_TRUE( - (true == feature - .emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer3", - "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()) - .ok())); - ASSERT_TRUE((false == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3"))); + ASSERT_TRUE(feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer3", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok()); + ASSERT_NE(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3")); - EXPECT_TRUE((false == feature - .remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3", false) - .ok())); - EXPECT_TRUE((false == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3"))); - EXPECT_TRUE((true == feature - .remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3", true) - .ok())); - EXPECT_TRUE((true == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3"))); + EXPECT_FALSE(feature.remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3", false).ok()); + EXPECT_NE(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3")); + EXPECT_TRUE(feature.remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3", true).ok()); + EXPECT_EQ(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer3")); } // remove missing (no vocbase) { arangodb::iresearch::IResearchAnalyzerFeature feature(server); - ASSERT_TRUE((nullptr == dbFeature->lookupDatabase("testVocbase"))); + ASSERT_EQ(nullptr, dbFeature->lookupDatabase("testVocbase")); - EXPECT_TRUE((true == !feature.get("testVocbase::test_analyzer"))); - EXPECT_TRUE((false == feature.remove("testVocbase::test_analyzer").ok())); + EXPECT_EQ(nullptr, feature.get("testVocbase::test_analyzer")); + EXPECT_FALSE(feature.remove("testVocbase::test_analyzer").ok()); } // remove missing (no collection) @@ -2472,28 +2362,26 @@ TEST_F(IResearchAnalyzerFeatureTest, test_remove) { arangodb::iresearch::IResearchAnalyzerFeature feature(server); TRI_vocbase_t* vocbase; ASSERT_TRUE(dbFeature->createDatabase(1, "testVocbase", vocbase).ok()); - ASSERT_TRUE((nullptr != dbFeature->lookupDatabase("testVocbase"))); + ASSERT_NE(nullptr, dbFeature->lookupDatabase("testVocbase")); - EXPECT_TRUE((true == !feature.get("testVocbase::test_analyzer"))); - EXPECT_TRUE((false == feature.remove("testVocbase::test_analyzer").ok())); + EXPECT_EQ(nullptr, feature.get("testVocbase::test_analyzer")); + EXPECT_FALSE(feature.remove("testVocbase::test_analyzer").ok()); } // remove invalid { arangodb::iresearch::IResearchAnalyzerFeature feature(server); - EXPECT_TRUE((true == !feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer"))); - EXPECT_TRUE((false == feature - .remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer") - .ok())); + EXPECT_EQ(nullptr, feature.get(arangodb::StaticStrings::SystemDatabase + "::test_analyzer")); + EXPECT_FALSE(feature.remove(arangodb::StaticStrings::SystemDatabase + "::test_analyzer").ok()); } // remove static analyzer { arangodb::iresearch::IResearchAnalyzerFeature feature(server); feature.prepare(); // add static analyzers - EXPECT_TRUE((false == !feature.get("identity"))); - EXPECT_TRUE((false == feature.remove("identity").ok())); - EXPECT_TRUE((false == !feature.get("identity"))); + EXPECT_NE(nullptr, feature.get("identity")); + EXPECT_FALSE(feature.remove("identity").ok()); + EXPECT_NE(nullptr, feature.get("identity")); } } @@ -2545,7 +2433,7 @@ TEST_F(IResearchAnalyzerFeatureTest, test_start) { } collection = vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName); - EXPECT_TRUE((nullptr == collection)); + EXPECT_EQ(nullptr, collection); } auto before = StorageEngineMock::recoveryStateResult; @@ -2597,11 +2485,8 @@ TEST_F(IResearchAnalyzerFeatureTest, test_start) { arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; arangodb::iresearch::IResearchAnalyzerFeature feature(server); arangodb::methods::Collections::createSystem(*vocbase, arangodb::tests::AnalyzerCollectionName, false); - EXPECT_TRUE( - (true == feature - .emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer", - "identity", VPackParser::fromJson("\"abc\"")->slice()) - .ok())); + EXPECT_TRUE(feature.emplace(result, arangodb::StaticStrings::SystemDatabase + "::test_analyzer", + "identity", VPackParser::fromJson("\"abc\"")->slice()).ok()); EXPECT_TRUE((false == !result.first)); collection = vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName); EXPECT_TRUE((nullptr != collection)); @@ -2614,15 +2499,11 @@ TEST_F(IResearchAnalyzerFeatureTest, test_start) { }); arangodb::iresearch::IResearchAnalyzerFeature feature(server); feature.prepare(); // add static analyzers - feature.start(); // load persisted analyzers + feature.start(); // doesn't load persisted analyzers EXPECT_TRUE((nullptr != vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName))); auto expected = staticAnalyzers(); - auto expectedAnalyzer = - arangodb::StaticStrings::SystemDatabase + "::test_analyzer"; - expected.emplace(std::piecewise_construct, std::forward_as_tuple(expectedAnalyzer), - std::forward_as_tuple("identity", "\"abc\"")); feature.visit( [&expected, &feature]( arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { @@ -2642,7 +2523,7 @@ TEST_F(IResearchAnalyzerFeatureTest, test_start) { expected.erase(itr); return true; }); - EXPECT_TRUE((expected.empty())); + EXPECT_TRUE(expected.empty()); } // test feature start load configuration (no configuration collection) @@ -2660,7 +2541,7 @@ TEST_F(IResearchAnalyzerFeatureTest, test_start) { } arangodb::iresearch::IResearchAnalyzerFeature feature(server); feature.prepare(); // add static analyzers - feature.start(); // load persisted analyzers + feature.start(); // doesn't load persisted analyzers EXPECT_TRUE((nullptr == vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName))); auto expected = staticAnalyzers(); @@ -2715,15 +2596,11 @@ TEST_F(IResearchAnalyzerFeatureTest, test_start) { arangodb::iresearch::IResearchAnalyzerFeature feature(server); feature.prepare(); // add static analyzers - feature.start(); // load persisted analyzers + feature.start(); // doesn't load persisted analyzers EXPECT_TRUE((nullptr != vocbase->lookupCollection(arangodb::tests::AnalyzerCollectionName))); auto expected = staticAnalyzers(); - auto expectedAnalyzer = - arangodb::StaticStrings::SystemDatabase + "::test_analyzer"; - expected.emplace(std::piecewise_construct, std::forward_as_tuple(expectedAnalyzer), - std::forward_as_tuple("identity", "{}")); feature.visit( [&expected, &feature]( arangodb::iresearch::AnalyzerPool::ptr const& analyzer) -> bool { @@ -2923,8 +2800,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { auto collectionId = std::to_string(42); auto legacyCollectionId = std::to_string(43); auto versionJson = VPackParser::fromJson("{ \"version\": 0, \"tasks\": {} }"); - arangodb::AqlFeature aqlFeature(server); - aqlFeature.start(); // required for Query::Query(...), must not call ~AqlFeature() for the duration of the test // test no system, no analyzer collection (single-server) { @@ -2950,10 +2825,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); feature->start(); // register upgrade tasks @@ -3011,10 +2882,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); feature->start(); // register upgrade tasks @@ -3094,10 +2961,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); feature->start(); // register upgrade tasks auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); @@ -3157,10 +3020,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); feature->start(); // register upgrade tasks auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); @@ -3245,10 +3104,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); feature->start(); // register upgrade tasks auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); @@ -3323,10 +3178,6 @@ TEST_F(IResearchAnalyzerFeatureTest, test_upgrade_static_legacy) { server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(feature = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - arangodb::aql::OptimizerRulesFeature(this->server).prepare(); // required for Query::preparePlan(...) - auto clearOptimizerRules = irs::make_finally([this]() -> void { - arangodb::aql::OptimizerRulesFeature(this->server).unprepare(); - }); feature->start(); // register upgrade tasks auto cleanup = arangodb::scopeGuard([dbFeature]() { dbFeature->unprepare(); }); @@ -3754,6 +3605,7 @@ TEST_F(IResearchAnalyzerFeatureTest, custom_analyzers_toVelocyPack) { sysDatabase->start(); // get system database from DatabaseFeature auto vocbase = dbFeature->useDatabase(arangodb::StaticStrings::SystemDatabase); arangodb::methods::Collections::createSystem(*vocbase, arangodb::tests::AnalyzerCollectionName, false); + EXPECT_NE(nullptr, sysDatabase->use()); } arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; @@ -3768,7 +3620,7 @@ TEST_F(IResearchAnalyzerFeatureTest, custom_analyzers_toVelocyPack) { // for persistence { auto expectedVpack = VPackParser::fromJson( - "{ \"name\": \"test_norm_analyzer4\", \"type\": \"norm\", " + "{ \"_key\": \"test_norm_analyzer4\", \"name\": \"test_norm_analyzer4\", \"type\": \"norm\", " "\"properties\":{\"locale\":\"ru_RU.utf-8\",\"case\":\"upper\",\"accent\":true}, " "\"features\": [] }"); @@ -3799,7 +3651,7 @@ TEST_F(IResearchAnalyzerFeatureTest, custom_analyzers_toVelocyPack) { "\"features\": [] }"); VPackBuilder builder; - result.first->toVelocyPack(builder, sysDatabase); + result.first->toVelocyPack(builder, sysDatabase->use().get()); EXPECT_EQUAL_SLICES(expectedVpack->slice(), builder.slice()); } diff --git a/tests/IResearch/IResearchDocument-test.cpp b/tests/IResearch/IResearchDocument-test.cpp index 81527ec0b9..65a940c609 100644 --- a/tests/IResearch/IResearchDocument-test.cpp +++ b/tests/IResearch/IResearchDocument-test.cpp @@ -46,6 +46,9 @@ #include "RestServer/DatabaseFeature.h" #include "RestServer/QueryRegistryFeature.h" #include "RestServer/SystemDatabaseFeature.h" +#include "RestServer/TraverserEngineRegistryFeature.h" +#include "RestServer/AqlFeature.h" +#include "Aql/OptimizerRulesFeature.h" #include "Sharding/ShardingFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "Transaction/Methods.h" @@ -165,6 +168,9 @@ class IResearchDocumentTest : public ::testing::Test { arangodb::LogLevel::ERR); // setup required application features + features.emplace_back(new arangodb::TraverserEngineRegistryFeature(server), false); // must be before AqlFeature + features.emplace_back(new arangodb::AqlFeature(server), true); + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(new arangodb::AuthenticationFeature(server), true); features.emplace_back(new arangodb::DatabaseFeature(server), false); features.emplace_back(new arangodb::QueryRegistryFeature(server), false); // required for constructing TRI_vocbase_t @@ -722,7 +728,7 @@ TEST_F(IResearchDocumentTest, FieldIterator_traverse_complex_object_ordered_chec auto const slice = json->slice(); arangodb::iresearch::IResearchLinkMeta linkMeta; - linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( + linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer( analyzers->get(arangodb::StaticStrings::SystemDatabase + "::iresearch-document-empty"), "iresearch-document-empty")); // add analyzer @@ -1549,12 +1555,12 @@ TEST_F(IResearchDocumentTest, FieldIterator_nullptr_analyzer) { // last analyzer invalid { arangodb::iresearch::IResearchLinkMeta linkMeta; - linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( + linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer( analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"), "empty")); // add analyzer InvalidAnalyzer::returnNullFromMake = false; - linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( + linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer( analyzers.get(arangodb::StaticStrings::SystemDatabase + "::invalid"), "invalid")); // add analyzer linkMeta._includeAllFields = true; // include all fields @@ -1620,10 +1626,10 @@ TEST_F(IResearchDocumentTest, FieldIterator_nullptr_analyzer) { linkMeta._analyzers.clear(); InvalidAnalyzer::returnNullFromMake = false; - linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( + linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer( analyzers.get(arangodb::StaticStrings::SystemDatabase + "::invalid"), "invalid")); // add analyzer - linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( + linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer( analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"), "empty")); // add analyzer linkMeta._includeAllFields = true; // include all fields diff --git a/tests/IResearch/IResearchFilterBoolean-test.cpp b/tests/IResearch/IResearchFilterBoolean-test.cpp index e6253ad046..66d86c3459 100644 --- a/tests/IResearch/IResearchFilterBoolean-test.cpp +++ b/tests/IResearch/IResearchFilterBoolean-test.cpp @@ -36,6 +36,7 @@ #include "Aql/ExecutionPlan.h" #include "Aql/ExpressionContext.h" #include "Aql/Query.h" +#include "Aql/OptimizerRulesFeature.h" #include "Cluster/ClusterFeature.h" #include "GeneralServer/AuthenticationFeature.h" #include "IResearch/AqlHelper.h" @@ -108,6 +109,7 @@ class IResearchFilterBooleanTest : public ::testing::Test { features.emplace_back(new arangodb::V8DealerFeature(server), false); // required for DatabaseFeature::createDatabase(...) features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(new arangodb::AqlFeature(server), true); features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server), true); // required for IResearchAnalyzerFeature diff --git a/tests/IResearch/IResearchFilterCompare-test.cpp b/tests/IResearch/IResearchFilterCompare-test.cpp index 10519e73fb..cfd757c228 100644 --- a/tests/IResearch/IResearchFilterCompare-test.cpp +++ b/tests/IResearch/IResearchFilterCompare-test.cpp @@ -36,6 +36,7 @@ #include "Aql/ExecutionPlan.h" #include "Aql/ExpressionContext.h" #include "Aql/Query.h" +#include "Aql/OptimizerRulesFeature.h" #include "Cluster/ClusterFeature.h" #include "GeneralServer/AuthenticationFeature.h" #include "IResearch/AqlHelper.h" @@ -109,6 +110,7 @@ class IResearchFilterCompareTest : public ::testing::Test { features.emplace_back(new arangodb::V8DealerFeature(server), false); // required for DatabaseFeature::createDatabase(...) features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(new arangodb::AqlFeature(server), true); features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server), true); // required for IResearchAnalyzerFeature diff --git a/tests/IResearch/IResearchFilterFunction-test.cpp b/tests/IResearch/IResearchFilterFunction-test.cpp index d23eacf8e1..f1174b3ce8 100644 --- a/tests/IResearch/IResearchFilterFunction-test.cpp +++ b/tests/IResearch/IResearchFilterFunction-test.cpp @@ -36,6 +36,7 @@ #include "Aql/ExecutionPlan.h" #include "Aql/ExpressionContext.h" #include "Aql/Query.h" +#include "Aql/OptimizerRulesFeature.h" #include "Cluster/ClusterFeature.h" #include "GeneralServer/AuthenticationFeature.h" #include "IResearch/AqlHelper.h" @@ -111,6 +112,7 @@ class IResearchFilterFunctionTest : public ::testing::Test { features.emplace_back(new arangodb::V8DealerFeature(server), false); // required for DatabaseFeature::createDatabase(...) features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(new arangodb::AqlFeature(server), true); features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server), true); // required for IResearchAnalyzerFeature diff --git a/tests/IResearch/IResearchFilterIn-test.cpp b/tests/IResearch/IResearchFilterIn-test.cpp index 534e90e0d4..d2ab9e2c13 100644 --- a/tests/IResearch/IResearchFilterIn-test.cpp +++ b/tests/IResearch/IResearchFilterIn-test.cpp @@ -36,6 +36,7 @@ #include "Aql/ExecutionPlan.h" #include "Aql/ExpressionContext.h" #include "Aql/Query.h" +#include "Aql/OptimizerRulesFeature.h" #include "Cluster/ClusterFeature.h" #include "GeneralServer/AuthenticationFeature.h" #include "IResearch/AqlHelper.h" @@ -109,6 +110,7 @@ class IResearchFilterInTest : public ::testing::Test { false); // required for DatabaseFeature::createDatabase(...) features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature features.emplace_back(new arangodb::AqlFeature(server), true); + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server), true); // required for IResearchAnalyzerFeature features.emplace_back(new arangodb::iresearch::IResearchAnalyzerFeature(server), true); diff --git a/tests/IResearch/IResearchLinkHelper-test.cpp b/tests/IResearch/IResearchLinkHelper-test.cpp index 133e719c23..5504299cea 100644 --- a/tests/IResearch/IResearchLinkHelper-test.cpp +++ b/tests/IResearch/IResearchLinkHelper-test.cpp @@ -232,10 +232,39 @@ TEST_F(IResearchLinkHelperTestSingle, test_validate_cross_db_analyzer) { TEST_F(IResearchLinkHelperTestSingle, test_normalize_single) { auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature(); - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; TRI_vocbase_t& sysVocbase = server.getSystemDatabase(); - // analyzer single-server + // analyzer single-server, for creation + { + auto json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"testAnalyzer0\", \"type\": \"identity\" } ], \ + \"analyzers\": [\"testAnalyzer0\" ] \ + }"); + arangodb::velocypack::Builder builder; + builder.openObject(); + EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize( + builder, json->slice(), true, sysVocbase).ok()); + builder.close(); + EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0")); + + auto expected_json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"type\":\"arangosearch\", \ + \"primarySort\":[], \ + \"fields\":{}, \ + \"includeAllFields\": false, \ + \"trackListPositions\": false, \ + \"storeValues\": \"none\", \ + \"analyzerDefinitions\": [ \ + { \"name\": \"testAnalyzer0\", \"type\": \"identity\", \"properties\":{}, \"features\":[] } \ + ], \ + \"analyzers\": [\"testAnalyzer0\" ] \ + }"); + EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice()); + } + + // analyzer single-server, user definition { auto json = arangodb::velocypack::Parser::fromJson( "{ \ @@ -246,11 +275,50 @@ TEST_F(IResearchLinkHelperTestSingle, test_normalize_single) { builder.openObject(); EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize( builder, json->slice(), false, sysVocbase).ok()); - EXPECT_TRUE((true == !analyzers->get(arangodb::StaticStrings::SystemDatabase + - "::testAnalyzer1"))); + builder.close(); + EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0")); + + auto expected_json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"type\":\"arangosearch\", \ + \"fields\":{}, \ + \"includeAllFields\": false, \ + \"trackListPositions\": false, \ + \"storeValues\": \"none\", \ + \"analyzers\": [\"testAnalyzer0\" ] \ + }"); + EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice()); } - // analyzer single-server (inRecovery) fail persist in recovery + // analyzer single-server, not for creation, missing "testAanalyzer0" + { + auto json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"analyzers\": [\"testAnalyzer0\" ] \ + }"); + arangodb::velocypack::Builder builder; + builder.openObject(); + EXPECT_FALSE(arangodb::iresearch::IResearchLinkHelper::normalize( + builder, json->slice(), false, sysVocbase).ok()); + builder.close(); + EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0")); + } + + // analyzer single-server, for creation, missing "testAanalyzer0" + { + auto json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"analyzers\": [\"testAnalyzer0\" ] \ + }"); + arangodb::velocypack::Builder builder; + builder.openObject(); + EXPECT_FALSE(arangodb::iresearch::IResearchLinkHelper::normalize( + builder, json->slice(), false, sysVocbase).ok()); + builder.close(); + EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0")); + } + + // analyzer single-server (inRecovery), for creation { auto json = arangodb::velocypack::Parser::fromJson( "{ \ @@ -263,11 +331,55 @@ TEST_F(IResearchLinkHelperTestSingle, test_normalize_single) { [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); arangodb::velocypack::Builder builder; builder.openObject(); - EXPECT_TRUE((false == arangodb::iresearch::IResearchLinkHelper::normalize( - builder, json->slice(), false, sysVocbase) - .ok())); - EXPECT_TRUE((true == !analyzers->get(arangodb::StaticStrings::SystemDatabase + - "::testAnalyzer2"))); + EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize( + builder, json->slice(), true, sysVocbase).ok()); + builder.close(); + EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1")); + + auto expected_json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"type\":\"arangosearch\", \ + \"primarySort\":[], \ + \"fields\":{}, \ + \"includeAllFields\": false, \ + \"trackListPositions\": false, \ + \"storeValues\": \"none\", \ + \"analyzerDefinitions\": [ \ + { \"name\": \"testAnalyzer1\", \"type\": \"identity\", \"properties\":{}, \"features\":[] } \ + ], \ + \"analyzers\": [\"testAnalyzer1\" ] \ + }"); + EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice()); + } + + // analyzer single-server (inRecovery), not for creation + { + auto json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"testAnalyzer1\", \"type\": \"identity\" } ], \ + \"analyzers\": [\"testAnalyzer1\" ] \ + }"); + auto before = StorageEngineMock::recoveryStateResult; + StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS; + auto restore = irs::make_finally( + [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); + arangodb::velocypack::Builder builder; + builder.openObject(); + EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize( + builder, json->slice(), false, sysVocbase).ok()); + builder.close(); + EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1")); + + auto expected_json = arangodb::velocypack::Parser::fromJson( + "{ \ + \"type\":\"arangosearch\", \ + \"fields\":{}, \ + \"includeAllFields\": false, \ + \"trackListPositions\": false, \ + \"storeValues\": \"none\", \ + \"analyzers\": [\"testAnalyzer1\" ] \ + }"); + EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice()); } } diff --git a/tests/IResearch/IResearchLinkMeta-test.cpp b/tests/IResearch/IResearchLinkMeta-test.cpp index 84090ddcd7..ed50019d15 100644 --- a/tests/IResearch/IResearchLinkMeta-test.cpp +++ b/tests/IResearch/IResearchLinkMeta-test.cpp @@ -37,6 +37,7 @@ #include "ApplicationFeatures/GreetingsPhase.h" #include "ApplicationFeatures/V8Phase.h" #include "Aql/AqlFunctionFeature.h" +#include "Aql/OptimizerRulesFeature.h" #include "Cluster/ClusterFeature.h" #if USE_ENTERPRISE @@ -48,6 +49,8 @@ #include "IResearch/IResearchFeature.h" #include "IResearch/IResearchLinkMeta.h" #include "IResearch/VelocyPackHelper.h" +#include "RestServer/AqlFeature.h" +#include "RestServer/TraverserEngineRegistryFeature.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/QueryRegistryFeature.h" #include "RestServer/SystemDatabaseFeature.h" @@ -137,6 +140,9 @@ class IResearchLinkMetaTest : public ::testing::Test { arangodb::LogLevel::ERR); // setup required application features + features.emplace_back(new arangodb::TraverserEngineRegistryFeature(server), false); // must be before AqlFeature + features.emplace_back(new arangodb::AqlFeature(server), true); + features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true); features.emplace_back(new arangodb::AuthenticationFeature(server), true); features.emplace_back(new arangodb::DatabaseFeature(server), false); features.emplace_back(new arangodb::ShardingFeature(server), false); @@ -237,16 +243,25 @@ class IResearchLinkMetaTest : public ::testing::Test { TEST_F(IResearchLinkMetaTest, test_defaults) { arangodb::iresearch::IResearchLinkMeta meta; + { + ASSERT_EQ(1, meta._analyzerDefinitions.size()); + EXPECT_TRUE("identity" == (*meta._analyzerDefinitions.begin())->name()); + EXPECT_TRUE(irs::flags({irs::norm::type(), irs::frequency::type()}) == + (*meta._analyzerDefinitions.begin())->features()); + EXPECT_NE(nullptr, meta._analyzerDefinitions.begin()->get()); + } + + EXPECT_TRUE(meta._sort.empty()); EXPECT_TRUE(true == meta._fields.empty()); EXPECT_TRUE(false == meta._includeAllFields); EXPECT_TRUE(false == meta._trackListPositions); - EXPECT_TRUE((arangodb::iresearch::ValueStorage::NONE == meta._storeValues)); + EXPECT_TRUE(arangodb::iresearch::ValueStorage::NONE == meta._storeValues); EXPECT_TRUE(1U == meta._analyzers.size()); - EXPECT_TRUE((*(meta._analyzers.begin()))); - EXPECT_TRUE(("identity" == meta._analyzers.begin()->_pool->name())); - EXPECT_TRUE(("identity" == meta._analyzers.begin()->_shortName)); - EXPECT_TRUE((irs::flags({irs::norm::type(), irs::frequency::type()}) == - meta._analyzers.begin()->_pool->features())); + EXPECT_TRUE(*(meta._analyzers.begin())); + EXPECT_TRUE("identity" == meta._analyzers.begin()->_pool->name()); + EXPECT_TRUE("identity" == meta._analyzers.begin()->_shortName); + EXPECT_TRUE(irs::flags({irs::norm::type(), irs::frequency::type()}) == + meta._analyzers.begin()->_pool->features()); EXPECT_TRUE(false == !meta._analyzers.begin()->_pool->get()); } @@ -260,14 +275,15 @@ TEST_F(IResearchLinkMetaTest, test_inheritDefaults) { analyzers.start(); - defaults._fields["abc"] = arangodb::iresearch::IResearchLinkMeta(); + defaults._fields["abc"] = arangodb::iresearch::FieldMeta(); defaults._includeAllFields = true; defaults._trackListPositions = true; - defaults._storeValues = arangodb::iresearch::ValueStorage::FULL; + defaults._storeValues = arangodb::iresearch::ValueStorage::VALUE; defaults._analyzers.clear(); defaults._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( analyzers.get("testVocbase::empty"), "empty")); defaults._fields["abc"]->_fields["xyz"] = arangodb::iresearch::IResearchLinkMeta(); + defaults._sort.emplace_back(std::vector{{"foo",false}}, true); auto json = VPackParser::fromJson("{}"); EXPECT_TRUE(true == meta.init(json->slice(), false, tmpString, nullptr, defaults)); @@ -301,7 +317,7 @@ TEST_F(IResearchLinkMetaTest, test_inheritDefaults) { EXPECT_TRUE(true == expectedFields.empty()); EXPECT_TRUE(true == meta._includeAllFields); EXPECT_TRUE(true == meta._trackListPositions); - EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == meta._storeValues)); + EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == meta._storeValues)); EXPECT_TRUE(1U == meta._analyzers.size()); EXPECT_TRUE((*(meta._analyzers.begin()))); @@ -310,6 +326,8 @@ TEST_F(IResearchLinkMetaTest, test_inheritDefaults) { EXPECT_TRUE((irs::flags({irs::frequency::type()}) == meta._analyzers.begin()->_pool->features())); EXPECT_TRUE(false == !meta._analyzers.begin()->_pool->get()); + + EXPECT_EQ(0, meta._sort.size()); } TEST_F(IResearchLinkMetaTest, test_readDefaults) { @@ -363,7 +381,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) { \"c\": { \ \"fields\": { \ \"default\": { \"fields\": {}, \"includeAllFields\": false, \"trackListPositions\": false, \"storeValues\": \"none\", \"analyzers\": [ \"identity\" ] }, \ - \"all\": { \"fields\": {\"d\": {}, \"e\": {}}, \"includeAllFields\": true, \"trackListPositions\": true, \"storeValues\": \"full\", \"analyzers\": [ \"empty\" ] }, \ + \"all\": { \"fields\": {\"d\": {}, \"e\": {}}, \"includeAllFields\": true, \"trackListPositions\": true, \"storeValues\": \"value\", \"analyzers\": [ \"empty\" ] }, \ \"some\": { \"trackListPositions\": true, \"storeValues\": \"id\" }, \ \"none\": {} \ } \ @@ -371,7 +389,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) { }, \ \"includeAllFields\": true, \ \"trackListPositions\": true, \ - \"storeValues\": \"full\", \ + \"storeValues\": \"value\", \ \"analyzers\": [ \"empty\", \"identity\" ] \ }"); @@ -392,8 +410,8 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) { "testVocbase"); arangodb::iresearch::IResearchLinkMeta meta; std::string tmpString; - EXPECT_TRUE((true == meta.init(json->slice(), false, tmpString, &vocbase))); - EXPECT_TRUE((3U == meta._fields.size())); + EXPECT_TRUE(meta.init(json->slice(), false, tmpString, &vocbase)); + EXPECT_TRUE(3U == meta._fields.size()); for (auto& field : meta._fields) { EXPECT_TRUE((1U == expectedFields.erase(field.key()))); @@ -421,7 +439,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) { EXPECT_TRUE((true == (actual._fields.find("e") != actual._fields.end()))); EXPECT_TRUE((true == actual._includeAllFields)); EXPECT_TRUE((true == actual._trackListPositions)); - EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == actual._storeValues)); + EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == actual._storeValues)); EXPECT_TRUE((1U == actual._analyzers.size())); EXPECT_TRUE((*(actual._analyzers.begin()))); EXPECT_TRUE(("testVocbase::empty" == actual._analyzers.begin()->_pool->name())); @@ -452,7 +470,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) { EXPECT_TRUE((true == actual._fields.empty())); // not inherited EXPECT_TRUE((true == actual._includeAllFields)); // inherited EXPECT_TRUE((true == actual._trackListPositions)); // inherited - EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == actual._storeValues)); + EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == actual._storeValues)); auto itr = actual._analyzers.begin(); EXPECT_TRUE((*itr)); EXPECT_TRUE(("testVocbase::empty" == itr->_pool->name())); @@ -474,7 +492,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) { EXPECT_TRUE((true == expectedFields.empty())); EXPECT_TRUE((true == meta._includeAllFields)); EXPECT_TRUE((true == meta._trackListPositions)); - EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == meta._storeValues)); + EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == meta._storeValues)); auto itr = meta._analyzers.begin(); EXPECT_TRUE((*itr)); EXPECT_TRUE(("testVocbase::empty" == itr->_pool->name())); @@ -644,13 +662,16 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) { meta._includeAllFields = true; meta._trackListPositions = true; - meta._storeValues = arangodb::iresearch::ValueStorage::FULL; + meta._storeValues = arangodb::iresearch::ValueStorage::VALUE; + meta._analyzerDefinitions.clear(); + meta._analyzerDefinitions.emplace(analyzers.get("identity")); + meta._analyzerDefinitions.emplace(analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty")); meta._analyzers.clear(); meta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( analyzers.get("identity"), "identity")); meta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer( analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"), - "enmpty")); + "empty")); meta._fields["a"] = meta; // copy from meta meta._fields["a"]->_fields.clear(); // do not inherit fields to match jSon inheritance meta._fields["b"] = meta; // copy from meta @@ -779,7 +800,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) { tmpSlice = slice.get("trackListPositions"); EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool())); tmpSlice = slice.get("storeValues"); - EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString())); + EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString())); tmpSlice = slice.get("analyzers"); EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length())); @@ -894,7 +915,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) { tmpSlice = slice.get("trackListPositions"); EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool())); tmpSlice = slice.get("storeValues"); - EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString())); + EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString())); tmpSlice = slice.get("analyzers"); EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length())); @@ -1036,7 +1057,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) { tmpSlice = slice.get("trackListPositions"); EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool())); tmpSlice = slice.get("storeValues"); - EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString())); + EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString())); tmpSlice = slice.get("analyzers"); EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length())); @@ -1149,7 +1170,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) { tmpSlice = slice.get("trackListPositions"); EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool())); tmpSlice = slice.get("storeValues"); - EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString())); + EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString())); tmpSlice = slice.get("analyzers"); EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length())); @@ -1219,7 +1240,7 @@ TEST_F(IResearchLinkMetaTest, test_readMaskAll) { \"fields\": { \"a\": {} }, \ \"includeAllFields\": true, \ \"trackListPositions\": true, \ - \"storeValues\": \"full\", \ + \"storeValues\": \"value\", \ \"analyzers\": [] \ }"); EXPECT_TRUE(true == meta.init(json->slice(), false, tmpString, nullptr, @@ -1380,6 +1401,31 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { EXPECT_EQ(std::string("analyzerDefinitions[0].type"), errorField); } + // missing analyzer definition single-server + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzers\": [ \"missing0\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing0", errorField); + } + + // missing analyzer (full) single-server (ignore analyzer definition) + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"missing0\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \ + \"analyzers\": [ \"missing0\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing0", errorField); + } + // missing analyzer (full) single-server { auto json = VPackParser::fromJson( @@ -1397,7 +1443,133 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { meta._analyzers[0]._pool->properties()); EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); - EXPECT_EQ(std::string("missing0"), meta._analyzers[0]._shortName); + EXPECT_EQ("missing0", meta._analyzers[0]._shortName); + } + + // complex definition on single-server + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ("testVocbase::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ("_system::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + } + + // complex definition on single-server + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + ASSERT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ("testVocbase::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + // definition from cache since it's not presented "analyzerDefinitions" + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ("_system::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + } + + // missing analyzer definition coordinator + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzers\": [ \"missing1\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing1", errorField); + } + + // missing analyzer definition coordinator + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"fields\" : { \"field\": { \"analyzers\": [ \"missing1\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ("fields.field.analyzers.missing1", errorField); } // missing analyzer (full) coordinator @@ -1415,8 +1587,138 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { }"); arangodb::iresearch::IResearchLinkMeta meta; std::string errorField; - EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase))); - EXPECT_EQ(std::string("analyzerDefinitions[0]"), errorField); + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::missing1", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + meta._analyzers[0]._pool->properties()); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("missing1", meta._analyzers[0]._shortName); + } + + // missing analyzer (full) coordinator (ignore analyzer definition) + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"missing1\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \ + \"analyzers\": [ \"missing1\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing1", errorField); + } + + // complex definition on coordinator + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ(std::string("testVocbase::empty"), analyzer._pool->name()); + EXPECT_EQ(std::string("empty"), analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ(std::string("_system::empty"), analyzer._pool->name()); + EXPECT_EQ(std::string("empty"), analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + } + + // complex definition on coordinator + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ("testVocbase::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + // definition from cache since it's not presented "analyzerDefinitions" + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ("_system::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } } // missing analyzer (full) db-server @@ -1434,16 +1736,156 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { }"); arangodb::iresearch::IResearchLinkMeta meta; std::string errorField; - EXPECT_TRUE((true == meta.init(json->slice(), true, errorField, &vocbase))); - EXPECT_TRUE((1 == meta._analyzers.size())); - EXPECT_TRUE( - (std::string("testVocbase::missing2") == meta._analyzers[0]._pool->name())); - EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._pool->type())); + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::missing2", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), meta._analyzers[0]._pool->properties()); - EXPECT_TRUE((1 == meta._analyzers[0]._pool->features().size())); - EXPECT_TRUE((true == meta._analyzers[0]._pool->features().check(irs::frequency::type()))); - EXPECT_TRUE((std::string("missing2") == meta._analyzers[0]._shortName)); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("missing2", meta._analyzers[0]._shortName); + } + + // missing analyzer (full) db-server (ignore analyzer definition) + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"missing2\", \"type\": \"empty\", \"properties\": { \"args\": \"ru\" }, \"features\": [ \"frequency\" ] } ], \ + \"analyzers\": [ \"missing2\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing2", errorField); + } + + // missing analyzer definition db-server + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzers\": [ \"missing2\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing2", errorField); + } + + // complex definition on db-server + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ(std::string("testVocbase::empty"), analyzer._pool->name()); + EXPECT_EQ(std::string("empty"), analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ(std::string("_system::empty"), analyzer._pool->name()); + EXPECT_EQ(std::string("empty"), analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + } + + // complex definition on db-server + { + auto before = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER); + auto restore = irs::make_finally([&before]() -> void { + arangodb::ServerState::instance()->setRole(before); + }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ("testVocbase::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + // definition from cache since it's not presented "analyzerDefinitions" + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ("_system::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } } // missing analyzer (full) inRecovery @@ -1459,8 +1901,48 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); arangodb::iresearch::IResearchLinkMeta meta; std::string errorField; - EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase))); - EXPECT_EQ(std::string("analyzers.missing3"), errorField); // not in the persisted collection + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::missing3", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + meta._analyzers[0]._pool->properties()); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("missing3", meta._analyzers[0]._shortName); + } + + // missing analyzer (full) inRecovery (ignore analyzer definition) + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"missing3\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \ + \"analyzers\": [ \"missing3\" ] \ + }"); + auto before = StorageEngineMock::recoveryStateResult; + StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS; + auto restore = irs::make_finally( + [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing3", errorField); + } + + // missing analyzer definition inRecovery + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzers\": [ \"missing3\" ] \ + }"); + auto before = StorageEngineMock::recoveryStateResult; + StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS; + auto restore = irs::make_finally( + [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ("analyzers.missing3", errorField); } // existing analyzer (name only) @@ -1507,6 +1989,108 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._shortName)); } + // complex definition inRecovery + { + auto before = StorageEngineMock::recoveryStateResult; + StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS; + auto restore = irs::make_finally( + [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ(std::string("testVocbase::empty"), analyzer._pool->name()); + EXPECT_EQ(std::string("empty"), analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ(std::string("_system::empty"), analyzer._pool->name()); + EXPECT_EQ(std::string("empty"), analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + } + + // complex definition inRecovery + { + auto before = StorageEngineMock::recoveryStateResult; + StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS; + auto restore = irs::make_finally( + [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); + + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ \ + { \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \ + ], \ + \"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(2, meta._analyzerDefinitions.size()); + EXPECT_EQ(1, meta._analyzers.size()); + { + auto& analyzer = meta._analyzers[0]; + EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool); + } + + ASSERT_EQ(1, meta._fields.size()); + ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size()); + { + auto& analyzer = meta._fields.begin().value()->_analyzers[0]; + EXPECT_EQ("testVocbase::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + // definition from cache since it's not presented "analyzerDefinitions" + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + { + auto& analyzer = meta._fields.begin().value()->_analyzers[1]; + EXPECT_EQ("_system::empty", analyzer._pool->name()); + EXPECT_EQ("empty", analyzer._pool->type()); + EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + analyzer._pool->properties()); + EXPECT_EQ(1, analyzer._pool->features().size()); + EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type())); + EXPECT_EQ("::empty", analyzer._shortName); + EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool); + } + } + // existing analyzer (full) analyzer creation not allowed (pass) { auto json = VPackParser::fromJson( @@ -1575,16 +2159,16 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); arangodb::iresearch::IResearchLinkMeta meta; std::string errorField; - EXPECT_TRUE((true == meta.init(json->slice(), true, errorField, &vocbase))); - EXPECT_TRUE((1 == meta._analyzers.size())); - EXPECT_TRUE((std::string("testVocbase::empty") == meta._analyzers[0]._pool->name())); - EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._pool->type())); + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); EXPECT_EQUAL_SLICES( VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), meta._analyzers[0]._pool->properties()); - EXPECT_TRUE((1 == meta._analyzers[0]._pool->features().size())); - EXPECT_TRUE((true == meta._analyzers[0]._pool->features().check(irs::frequency::type()))); - EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._shortName)); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", meta._analyzers[0]._shortName); } // existing analyzer (definition mismatch) @@ -1596,8 +2180,38 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { }"); arangodb::iresearch::IResearchLinkMeta meta; std::string errorField; - EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase))); - EXPECT_EQ(std::string("analyzerDefinitions[0]"), errorField); + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); + EXPECT_EQUAL_SLICES( + VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + meta._analyzers[0]._pool->properties()); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", meta._analyzers[0]._shortName); + } + + // existing analyzer (definition mismatch), don't read definitions + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \ + \"analyzers\": [ \"empty\" ] \ + }"); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), false, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); + // read definition from cache since we ignore "analyzerDefinitions" + EXPECT_EQUAL_SLICES( + VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), + meta._analyzers[0]._pool->properties()); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", meta._analyzers[0]._shortName); } // existing analyzer (definition mismatch) inRecovery @@ -1613,8 +2227,43 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) { [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); arangodb::iresearch::IResearchLinkMeta meta; std::string errorField; - EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase))); - EXPECT_EQ(std::string("analyzerDefinitions[0]"), errorField); + EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); + // read definition from cache since we ignore "analyzerDefinitions" + EXPECT_EQUAL_SLICES( + VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(), + meta._analyzers[0]._pool->properties()); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", meta._analyzers[0]._shortName); + } + + // existing analyzer (definition mismatch) inRecovery, don't read definitions + { + auto json = VPackParser::fromJson( + "{ \ + \"analyzerDefinitions\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \ + \"analyzers\": [ \"empty\" ] \ + }"); + auto before = StorageEngineMock::recoveryStateResult; + StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS; + auto restore = irs::make_finally( + [&before]() -> void { StorageEngineMock::recoveryStateResult = before; }); + arangodb::iresearch::IResearchLinkMeta meta; + std::string errorField; + EXPECT_TRUE(meta.init(json->slice(), false, errorField, &vocbase)); + EXPECT_EQ(1, meta._analyzers.size()); + EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name()); + EXPECT_EQ("empty", meta._analyzers[0]._pool->type()); + // read definition from cache since we ignore "analyzerDefinitions" + EXPECT_EQUAL_SLICES( + VPackParser::fromJson("{\"args\" : \"de\"}")->slice(), + meta._analyzers[0]._pool->properties()); + EXPECT_EQ(1, meta._analyzers[0]._pool->features().size()); + EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type())); + EXPECT_EQ("empty", meta._analyzers[0]._shortName); } } @@ -1658,7 +2307,7 @@ TEST_F(IResearchLinkMetaTest, test_addNonUniqueAnalyzers) { "{ \ \"includeAllFields\": true, \ \"trackListPositions\": true, \ - \"storeValues\": \"full\",\ + \"storeValues\": \"value\",\ \"analyzers\": [ \"identity\",\"identity\"" // two built-in analyzers ", \"" + analyzerCustomInTestVocbase + "\"" + // local analyzer by full name diff --git a/tests/IResearch/IResearchView-test.cpp b/tests/IResearch/IResearchView-test.cpp index 2fbde6325c..25452a6ad0 100644 --- a/tests/IResearch/IResearchView-test.cpp +++ b/tests/IResearch/IResearchView-test.cpp @@ -28,6 +28,7 @@ #include "ExpressionContextMock.h" #include "analysis/token_attributes.hpp" +#include "analysis/analyzers.hpp" #include "search/scorers.hpp" #include "utils/locale_utils.hpp" #include "utils/log.hpp" @@ -39,6 +40,7 @@ #include "Aql/AstNode.h" #include "Aql/Function.h" #include "Aql/SortCondition.h" +#include "Aql/OptimizerRulesFeature.h" #include "Basics/ArangoGlobalContext.h" #include "Basics/error.h" #include "Basics/files.h" @@ -130,6 +132,88 @@ struct DocIdScorer: public irs::sort { REGISTER_SCORER_TEXT(DocIdScorer, DocIdScorer::make); +struct TestAttribute : public irs::attribute { + DECLARE_ATTRIBUTE_TYPE(); +}; + +DEFINE_ATTRIBUTE_TYPE(TestAttribute); +REGISTER_ATTRIBUTE(TestAttribute); // required to open reader on segments with analized fields + +struct TestTermAttribute : public irs::term_attribute { + public: + void value(irs::bytes_ref const& value) { value_ = value; } +}; + +class TestAnalyzer : public irs::analysis::analyzer { + public: + DECLARE_ANALYZER_TYPE(); + + TestAnalyzer() : irs::analysis::analyzer(TestAnalyzer::type()) { + _attrs.emplace(_term); + _attrs.emplace(_attr); + _attrs.emplace(_increment); // required by field_data::invert(...) + } + virtual irs::attribute_view const& attributes() const noexcept override { + return _attrs; + } + + static ptr make(irs::string_ref const& args) { + auto slice = arangodb::iresearch::slice(args); + if (slice.isNull()) throw std::exception(); + if (slice.isNone()) return nullptr; + PTR_NAMED(TestAnalyzer, ptr); + return ptr; + } + + static bool normalize(irs::string_ref const& args, std::string& definition) { + // same validation as for make, + // as normalize usually called to sanitize data before make + auto slice = arangodb::iresearch::slice(args); + if (slice.isNull()) throw std::exception(); + if (slice.isNone()) return false; + + arangodb::velocypack::Builder builder; + if (slice.isString()) { + VPackObjectBuilder scope(&builder); + arangodb::iresearch::addStringRef(builder, "args", + arangodb::iresearch::getStringRef(slice)); + } else if (slice.isObject() && slice.hasKey("args") && slice.get("args").isString()) { + VPackObjectBuilder scope(&builder); + arangodb::iresearch::addStringRef(builder, "args", + arangodb::iresearch::getStringRef(slice.get("args"))); + } else { + return false; + } + + definition = builder.buffer()->toString(); + + return true; + } + + virtual bool next() override { + if (_data.empty()) return false; + + _term.value(irs::bytes_ref(_data.c_str(), 1)); + _data = irs::bytes_ref(_data.c_str() + 1, _data.size() - 1); + return true; + } + + virtual bool reset(irs::string_ref const& data) override { + _data = irs::ref_cast(data); + return true; + } + + private: + irs::attribute_view _attrs; + irs::bytes_ref _data; + irs::increment _increment; + TestTermAttribute _term; + TestAttribute _attr; +}; + +DEFINE_ANALYZER_TYPE_NAMED(TestAnalyzer, "TestAnalyzer"); +REGISTER_ANALYZER_VPACK(TestAnalyzer, TestAnalyzer::make, TestAnalyzer::normalize); + } // ----------------------------------------------------------------------------- @@ -191,6 +275,7 @@ protected: buildFeatureEntry(new arangodb::DatabaseFeature(server), false); buildFeatureEntry(new arangodb::DatabasePathFeature(server), false); buildFeatureEntry(new arangodb::TraverserEngineRegistryFeature(server), false); + buildFeatureEntry(new arangodb::aql::OptimizerRulesFeature(server), true); buildFeatureEntry(new arangodb::AqlFeature(server), true); buildFeatureEntry(new arangodb::aql::AqlFunctionFeature(server), true); // required for IResearchAnalyzerFeature buildFeatureEntry(new arangodb::iresearch::IResearchAnalyzerFeature(server), true); @@ -6903,3 +6988,177 @@ TEST_F(IResearchViewTest, test_update_partial) { } } } + +TEST_F(IResearchViewTest, test_remove_referenced_analyzer) { + auto* databaseFeature = arangodb::application_features::ApplicationServer::getFeature("Database"); + ASSERT_NE(nullptr, databaseFeature); + + TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature + ASSERT_TRUE(databaseFeature->createDatabase(0, "testDatabase" TOSTRING(__LINE__), vocbase).ok()); + ASSERT_NE(nullptr, vocbase); + + // create _analyzers collection + { + auto createJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"" + arangodb::StaticStrings::AnalyzersCollection + "\", \"isSystem\":true }"); + ASSERT_NE(nullptr, vocbase->createCollection(createJson->slice())); + } + + auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature< + arangodb::iresearch::IResearchAnalyzerFeature>(); + ASSERT_NE(nullptr, analyzers); + + std::shared_ptr view; + std::shared_ptr collection; + + // remove existing (used by link) + { + // add analyzer + { + arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; + ASSERT_TRUE(analyzers->emplace(result, vocbase->name() + "::test_analyzer3", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok()); + ASSERT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + } + + // create collection + { + auto createJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection1\" }"); + collection = vocbase->createCollection(createJson->slice()); + ASSERT_NE(nullptr, collection); + } + + // create view + { + auto createJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"testView\", \"type\": \"arangosearch\" }"); + view = vocbase->createView(createJson->slice()); + ASSERT_NE(nullptr, view); + + auto updateJson = arangodb::velocypack::Parser::fromJson( + "{ \"links\": { \"testCollection1\": { \"includeAllFields\": true, \"analyzers\":[\"test_analyzer3\"] }}}"); + EXPECT_TRUE(view->properties(updateJson->slice(), true).ok()); + } + + EXPECT_FALSE(analyzers->remove(vocbase->name() + "::test_analyzer3", false).ok()); // used by link + EXPECT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + EXPECT_TRUE(analyzers->remove(vocbase->name() + "::test_analyzer3", true).ok()); + EXPECT_EQ(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + + auto cleanup = arangodb::scopeGuard([vocbase, &view, &collection]() { + if (view) { + EXPECT_TRUE(vocbase->dropView(view->id(), false).ok()); + view = nullptr; + } + + if (collection) { + EXPECT_TRUE(vocbase->dropCollection(collection->id(), false, 1.0).ok()); + collection = nullptr; + } + }); + } + + // remove existing (used by link) + { + // add analyzer + { + arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; + ASSERT_TRUE(analyzers->emplace(result, vocbase->name() + "::test_analyzer3", + "TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok()); + ASSERT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + } + + // create collection + { + auto createJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection1\" }"); + collection = vocbase->createCollection(createJson->slice()); + ASSERT_NE(nullptr, collection); + } + + // create view + { + auto createJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"testView\", \"type\": \"arangosearch\" }"); + view = vocbase->createView(createJson->slice()); + ASSERT_NE(nullptr, view); + + auto updateJson = arangodb::velocypack::Parser::fromJson( + "{ \"analyzerDefinitions\" : { \ + \"name\":\"test_analyzer3\", \"features\":[], \ + \"type\":\"TestAnalyzer\", \"properties\": {\"args\":\"abc\"} \ + }, \ + \"links\": { \"testCollection1\": { \"includeAllFields\": true, \"analyzers\":[\"test_analyzer3\"] }} \ + }"); + EXPECT_TRUE(view->properties(updateJson->slice(), true).ok()); + } + + EXPECT_FALSE(analyzers->remove(vocbase->name() + "::test_analyzer3", false).ok()); // used by link + EXPECT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + EXPECT_TRUE(analyzers->remove(vocbase->name() + "::test_analyzer3", true).ok()); + EXPECT_EQ(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + + auto cleanup = arangodb::scopeGuard([vocbase, &view, &collection]() { + if (view) { + EXPECT_TRUE(vocbase->dropView(view->id(), false).ok()); + view = nullptr; + } + + if (collection) { + EXPECT_TRUE(vocbase->dropCollection(collection->id(), false, 1.0).ok()); + collection = nullptr; + } + }); + } + + // remove existing (properties don't match + { + // add analyzer + { + arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; + ASSERT_TRUE(analyzers->emplace(result, vocbase->name() + "::test_analyzer3", + "TestAnalyzer", VPackParser::fromJson("\"abcd\"")->slice()).ok()); + ASSERT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + } + + // create collection + { + auto createJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection1\" }"); + collection = vocbase->createCollection(createJson->slice()); + ASSERT_NE(nullptr, collection); + } + + // create view + { + auto createJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"testView\", \"type\": \"arangosearch\" }"); + view = vocbase->createView(createJson->slice()); + ASSERT_NE(nullptr, view); + + auto updateJson = arangodb::velocypack::Parser::fromJson( + "{ \"analyzerDefinitions\" : { \ + \"name\":\"test_analyzer3\", \"features\":[], \ + \"type\":\"TestAnalyzer\", \"properties\": \"abc\" \ + }, \ + \"links\": { \"testCollection1\": { \"includeAllFields\": true, \"analyzers\":[\"test_analyzer3\"] }} \ + }"); + EXPECT_TRUE(view->properties(updateJson->slice(), true).ok()); + } + + EXPECT_FALSE(analyzers->remove(vocbase->name() + "::test_analyzer3", false).ok()); // used by link (we ignore analyzerDefinitions on single-server) + EXPECT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + EXPECT_TRUE(analyzers->remove(vocbase->name() + "::test_analyzer3", true).ok()); + EXPECT_EQ(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3")); + + auto cleanup = arangodb::scopeGuard([vocbase, &view, &collection]() { + if (view) { + EXPECT_TRUE(vocbase->dropView(view->id(), false).ok()); + view = nullptr; + } + + if (collection) { + EXPECT_TRUE(vocbase->dropCollection(collection->id(), false, 1.0).ok()); + collection = nullptr; + } + }); + } +} diff --git a/tests/IResearch/common.cpp b/tests/IResearch/common.cpp index b80120feee..69b1642bce 100644 --- a/tests/IResearch/common.cpp +++ b/tests/IResearch/common.cpp @@ -469,7 +469,7 @@ std::string mangleString(std::string name, std::string suffix) { } std::string mangleStringIdentity(std::string name) { - arangodb::iresearch::kludge::mangleStringField(name, arangodb::iresearch::IResearchLinkMeta::Analyzer() // args + arangodb::iresearch::kludge::mangleStringField(name, arangodb::iresearch::FieldMeta::Analyzer() // args ); return name; diff --git a/tests/V8Server/v8-analyzers-test.cpp b/tests/V8Server/v8-analyzers-test.cpp index b5ae8c2f0a..7f4e573ca4 100644 --- a/tests/V8Server/v8-analyzers-test.cpp +++ b/tests/V8Server/v8-analyzers-test.cpp @@ -27,6 +27,7 @@ #include "../IResearch/common.h" #include "../Mocks/StorageEngineMock.h" +#include "Aql/OptimizerRulesFeature.h" #include "Aql/QueryRegistry.h" #include "gtest/gtest.h" @@ -38,9 +39,11 @@ #include "IResearch/IResearchAnalyzerFeature.h" #include "IResearch/IResearchCommon.h" #include "IResearch/VelocyPackHelper.h" +#include "RestServer/AqlFeature.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/QueryRegistryFeature.h" #include "RestServer/SystemDatabaseFeature.h" +#include "RestServer/TraverserEngineRegistryFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "VocBase/Methods/Collections.h" #include "Utils/ExecContext.h" @@ -203,12 +206,30 @@ TEST_F(V8AnalyzersTest, test_accessors) { arangodb::application_features::ApplicationServer server(nullptr, nullptr); arangodb::iresearch::IResearchAnalyzerFeature* analyzers; arangodb::DatabaseFeature* dbFeature; - server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + arangodb::QueryRegistryFeature* queryRegistry; + arangodb::AqlFeature* aqlFeature; + arangodb::aql::OptimizerRulesFeature* optimizer; + arangodb::TraverserEngineRegistryFeature* traverser; + server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server)); + server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature + server.addFeature(aqlFeature = new arangodb::AqlFeature(server)); server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...) server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task - auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); }); + traverser->prepare(); + aqlFeature->start(); + optimizer->prepare(); + queryRegistry->prepare(); + + auto cleanup = arangodb::scopeGuard([&](){ + dbFeature->unprepare(); + optimizer->unprepare(); + aqlFeature->stop(); + queryRegistry->unprepare(); + traverser->unprepare(); + }); // create system vocbase { @@ -225,10 +246,8 @@ TEST_F(V8AnalyzersTest, test_accessors) { } arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; - ASSERT_TRUE((analyzers - ->emplace(result, arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1", - "identity", VPackSlice::noneSlice()) - .ok())); + ASSERT_TRUE(analyzers->emplace(result, arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1", + "identity", VPackSlice::noneSlice()).ok()); auto analyzer = analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1"); ASSERT_TRUE((false == !analyzer)); @@ -242,8 +261,7 @@ TEST_F(V8AnalyzersTest, test_accessors) { arangodb::ExecContextScope execContextScope(&execContext); auto* authFeature = arangodb::AuthenticationFeature::instance(); auto* userManager = authFeature->userManager(); - arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB() - userManager->setQueryRegistry(&queryRegistry); + userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry()); auto resetUserManager = std::shared_ptr( userManager, [](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); }); @@ -699,13 +717,31 @@ TEST_F(V8AnalyzersTest, test_create) { arangodb::iresearch::IResearchAnalyzerFeature* analyzers; arangodb::DatabaseFeature* dbFeature; arangodb::SystemDatabaseFeature* sysDatabase; - server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + arangodb::QueryRegistryFeature* queryRegistry; + arangodb::AqlFeature* aqlFeature; + arangodb::aql::OptimizerRulesFeature* optimizer; + arangodb::TraverserEngineRegistryFeature* traverser; + server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server)); + server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature + server.addFeature(aqlFeature = new arangodb::AqlFeature(server)); server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...) server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task server.addFeature(sysDatabase = new arangodb::SystemDatabaseFeature(server)); // required for IResearchAnalyzerFeature::start() - auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); }); + traverser->prepare(); + aqlFeature->start(); + optimizer->prepare(); + queryRegistry->prepare(); + + auto cleanup = arangodb::scopeGuard([&](){ + dbFeature->unprepare(); + optimizer->unprepare(); + aqlFeature->stop(); + queryRegistry->unprepare(); + traverser->unprepare(); + }); // create system vocbase { @@ -746,8 +782,7 @@ TEST_F(V8AnalyzersTest, test_create) { arangodb::ExecContextScope execContextScope(&execContext); auto* authFeature = arangodb::AuthenticationFeature::instance(); auto* userManager = authFeature->userManager(); - arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB() - userManager->setQueryRegistry(&queryRegistry); + userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry()); auto resetUserManager = std::shared_ptr( userManager, [](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); }); @@ -1404,13 +1439,32 @@ TEST_F(V8AnalyzersTest, test_get) { arangodb::application_features::ApplicationServer server(nullptr, nullptr); arangodb::iresearch::IResearchAnalyzerFeature* analyzers; arangodb::DatabaseFeature* dbFeature; + arangodb::QueryRegistryFeature* queryRegistry; + arangodb::AqlFeature* aqlFeature; + arangodb::aql::OptimizerRulesFeature* optimizer; + arangodb::TraverserEngineRegistryFeature* traverser; + server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server)); + server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature + server.addFeature(aqlFeature = new arangodb::AqlFeature(server)); server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...) server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task analyzers->prepare(); // add static analyzers - auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); }); + traverser->prepare(); + aqlFeature->start(); + optimizer->prepare(); + queryRegistry->prepare(); + + auto cleanup = arangodb::scopeGuard([&](){ + dbFeature->unprepare(); + optimizer->unprepare(); + aqlFeature->stop(); + queryRegistry->unprepare(); + traverser->unprepare(); + }); // create system vocbase { @@ -1449,8 +1503,7 @@ TEST_F(V8AnalyzersTest, test_get) { arangodb::ExecContextScope execContextScope(&execContext); auto* authFeature = arangodb::AuthenticationFeature::instance(); auto* userManager = authFeature->userManager(); - arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB() - userManager->setQueryRegistry(&queryRegistry); + userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry()); auto resetUserManager = std::shared_ptr( userManager, [](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); }); @@ -2068,12 +2121,32 @@ TEST_F(V8AnalyzersTest, test_list) { arangodb::iresearch::IResearchAnalyzerFeature* analyzers; arangodb::DatabaseFeature* dbFeature; arangodb::SystemDatabaseFeature* sysDatabase; - server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + arangodb::QueryRegistryFeature* queryRegistry; + arangodb::AqlFeature* aqlFeature; + arangodb::aql::OptimizerRulesFeature* optimizer; + arangodb::TraverserEngineRegistryFeature* traverser; + server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server)); + server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature + server.addFeature(aqlFeature = new arangodb::AqlFeature(server)); server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...) server.addFeature(sysDatabase = new arangodb::SystemDatabaseFeature(server)); // required for IResearchAnalyzerFeature::start() server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task + traverser->prepare(); + aqlFeature->start(); + optimizer->prepare(); + queryRegistry->prepare(); + + auto cleanup = arangodb::scopeGuard([&](){ + dbFeature->unprepare(); + optimizer->unprepare(); + aqlFeature->stop(); + queryRegistry->unprepare(); + traverser->unprepare(); + }); + // create system vocbase { auto const databases = arangodb::velocypack::Parser::fromJson( @@ -2094,8 +2167,6 @@ TEST_F(V8AnalyzersTest, test_list) { arangodb::tests::AnalyzerCollectionName, false); } - auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); }); - arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result; ASSERT_TRUE((analyzers ->emplace(result, arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1", @@ -2115,8 +2186,7 @@ TEST_F(V8AnalyzersTest, test_list) { arangodb::ExecContextScope execContextScope(&execContext); auto* authFeature = arangodb::AuthenticationFeature::instance(); auto* userManager = authFeature->userManager(); - arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB() - userManager->setQueryRegistry(&queryRegistry); + userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry()); auto resetUserManager = std::shared_ptr( userManager, [](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); }); @@ -2536,13 +2606,31 @@ TEST_F(V8AnalyzersTest, test_remove) { arangodb::iresearch::IResearchAnalyzerFeature* analyzers; arangodb::DatabaseFeature* dbFeature; arangodb::SystemDatabaseFeature* sysDbFeature(new arangodb::SystemDatabaseFeature(server)); - server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + arangodb::QueryRegistryFeature* queryRegistry; + arangodb::AqlFeature* aqlFeature; + arangodb::aql::OptimizerRulesFeature* optimizer; + arangodb::TraverserEngineRegistryFeature* traverser; + server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t + server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server)); + server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature + server.addFeature(aqlFeature = new arangodb::AqlFeature(server)); server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...) server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...) server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task server.addFeature(sysDbFeature); - auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); }); + traverser->prepare(); + aqlFeature->start(); + optimizer->prepare(); + queryRegistry->prepare(); + + auto cleanup = arangodb::scopeGuard([&](){ + dbFeature->unprepare(); + optimizer->unprepare(); + aqlFeature->stop(); + queryRegistry->unprepare(); + traverser->unprepare(); + }); // create system vocbase { @@ -2601,8 +2689,7 @@ TEST_F(V8AnalyzersTest, test_remove) { arangodb::ExecContextScope execContextScope(&execContext); auto* authFeature = arangodb::AuthenticationFeature::instance(); auto* userManager = authFeature->userManager(); - arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB() - userManager->setQueryRegistry(&queryRegistry); + userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry()); auto resetUserManager = std::shared_ptr( userManager, [](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); }); diff --git a/tests/js/common/aql/aql-view-arangosearch-ddl-cluster.js b/tests/js/common/aql/aql-view-arangosearch-ddl-cluster.js index 5a69bbcb25..6cc8dd00c7 100644 --- a/tests/js/common/aql/aql-view-arangosearch-ddl-cluster.js +++ b/tests/js/common/aql/aql-view-arangosearch-ddl-cluster.js @@ -316,7 +316,7 @@ function IResearchFeatureDDLTestSuite () { var meta = { links: { "TestCollection0": { }, - "TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "full" }, + "TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "value" }, "TestCollection2": { fields: { "b": { fields: { "b1": {} } }, "c": { includeAllFields: true }, @@ -352,7 +352,7 @@ function IResearchFeatureDDLTestSuite () { assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor); assertEqual(true, properties.links.TestCollection1.trackListPositions); assertTrue(String === properties.links.TestCollection1.storeValues.constructor); - assertEqual("full", properties.links.TestCollection1.storeValues); + assertEqual("value", properties.links.TestCollection1.storeValues); assertTrue(Array === properties.links.TestCollection1.analyzers.constructor); assertEqual(1, properties.links.TestCollection1.analyzers.length); assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor); @@ -405,7 +405,7 @@ function IResearchFeatureDDLTestSuite () { assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor); assertEqual(true, properties.links.TestCollection1.trackListPositions); assertTrue(String === properties.links.TestCollection1.storeValues.constructor); - assertEqual("full", properties.links.TestCollection1.storeValues); + assertEqual("value", properties.links.TestCollection1.storeValues); assertTrue(Array === properties.links.TestCollection1.analyzers.constructor); assertEqual(1, properties.links.TestCollection1.analyzers.length); assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor); @@ -1114,7 +1114,47 @@ function IResearchFeatureDDLTestSuite () { assertNull(analyzers.analyzer(analyzerName)); // this should be no name conflict analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"}); - + + db._useDatabase("_system"); + db._dropDatabase(dbName); + }, + + testIndexAnalyzerCollection : function() { + const dbName = "TestNameDroppedDB"; + const analyzerName = "TestAnalyzer"; + db._useDatabase("_system"); + assertNotEqual(null, db._collection("_analyzers")); + try { db._dropDatabase(dbName); } catch (e) {} + try { analyzers.remove(analyzerName); } catch (e) {} + assertEqual(0, db._analyzers.count()); + db._createDatabase(dbName); + db._useDatabase(dbName); + analyzers.save(analyzerName, "identity"); + // recreating database + db._useDatabase("_system"); + db._dropDatabase(dbName); + db._createDatabase(dbName); + db._useDatabase(dbName); + + assertNull(analyzers.analyzer(analyzerName)); + // this should be no name conflict + analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"}); + assertEqual(1, db._analyzers.count()); + + var view = db._createView("analyzersView", "arangosearch", { + links: { + _analyzers : { + includeAllFields:true, + analyzers: [ analyzerName ] + } + } + }); + + var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray(); + assertEqual(1, db._analyzers.count()); + assertEqual(1, res.length); + assertEqual(db._analyzers.toArray()[0], res[0]); + db._useDatabase("_system"); db._dropDatabase(dbName); } diff --git a/tests/js/common/aql/aql-view-arangosearch-ddl-noncluster.js b/tests/js/common/aql/aql-view-arangosearch-ddl-noncluster.js index 4b4f3207ea..f9098e191f 100644 --- a/tests/js/common/aql/aql-view-arangosearch-ddl-noncluster.js +++ b/tests/js/common/aql/aql-view-arangosearch-ddl-noncluster.js @@ -316,7 +316,7 @@ function IResearchFeatureDDLTestSuite () { var meta = { links: { "TestCollection0": { }, - "TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "full" }, + "TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "value" }, "TestCollection2": { fields: { "b": { fields: { "b1": {} } }, "c": { includeAllFields: true }, @@ -352,7 +352,7 @@ function IResearchFeatureDDLTestSuite () { assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor); assertEqual(true, properties.links.TestCollection1.trackListPositions); assertTrue(String === properties.links.TestCollection1.storeValues.constructor); - assertEqual("full", properties.links.TestCollection1.storeValues); + assertEqual("value", properties.links.TestCollection1.storeValues); assertTrue(Array === properties.links.TestCollection1.analyzers.constructor); assertEqual(1, properties.links.TestCollection1.analyzers.length); assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor); @@ -405,7 +405,7 @@ function IResearchFeatureDDLTestSuite () { assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor); assertEqual(true, properties.links.TestCollection1.trackListPositions); assertTrue(String === properties.links.TestCollection1.storeValues.constructor); - assertEqual("full", properties.links.TestCollection1.storeValues); + assertEqual("value", properties.links.TestCollection1.storeValues); assertTrue(Array === properties.links.TestCollection1.analyzers.constructor); assertEqual(1, properties.links.TestCollection1.analyzers.length); assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor); @@ -1114,7 +1114,50 @@ function IResearchFeatureDDLTestSuite () { assertNull(analyzers.analyzer(analyzerName)); // this should be no name conflict analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"}); - + + db._useDatabase("_system"); + db._dropDatabase(dbName); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test link on analyzers collection +//////////////////////////////////////////////////////////////////////////////// + testIndexAnalyzerCollection : function() { + const dbName = "TestNameDroppedDB"; + const analyzerName = "TestAnalyzer"; + db._useDatabase("_system"); + assertNotEqual(null, db._collection("_analyzers")); + try { db._dropDatabase(dbName); } catch (e) {} + try { analyzers.remove(analyzerName); } catch (e) {} + assertEqual(0, db._analyzers.count()); + db._createDatabase(dbName); + db._useDatabase(dbName); + analyzers.save(analyzerName, "identity"); + // recreating database + db._useDatabase("_system"); + db._dropDatabase(dbName); + db._createDatabase(dbName); + db._useDatabase(dbName); + + assertNull(analyzers.analyzer(analyzerName)); + // this should be no name conflict + analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"}); + assertEqual(1, db._analyzers.count()); + + var view = db._createView("analyzersView", "arangosearch", { + links: { + _analyzers : { + includeAllFields:true, + analyzers: [ analyzerName ] + } + } + }); + + var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray(); + assertEqual(1, db._analyzers.count()); + assertEqual(1, res.length); + assertEqual(db._analyzers.toArray()[0], res[0]); + db._useDatabase("_system"); db._dropDatabase(dbName); } diff --git a/tests/js/common/aql/aql-view-arangosearch-feature.js b/tests/js/common/aql/aql-view-arangosearch-feature.js index 743e847064..477068e389 100644 --- a/tests/js/common/aql/aql-view-arangosearch-feature.js +++ b/tests/js/common/aql/aql-view-arangosearch-feature.js @@ -56,38 +56,53 @@ function iResearchFeatureAqlTestSuite () { testAnalyzersInvalidPropertiesDiscarded : function() { { try {analyzers.remove("normPropAnalyzer"); } catch (e) {} + assertEqual(0, db._analyzers.count()); let analyzer = analyzers.save("normPropAnalyzer", "norm", { "locale":"en", "invalid_param":true}); + assertEqual(1, db._analyzers.count()); assertTrue(null != analyzer); assertTrue(null == analyzer.properties.invalid_param); analyzers.remove("normPropAnalyzer", true); + assertEqual(0, db._analyzers.count()); } { try {analyzers.remove("textPropAnalyzer"); } catch (e) {} + assertEqual(0, db._analyzers.count()); let analyzer = analyzers.save("textPropAnalyzer", "text", {"stopwords" : [], "locale":"en", "invalid_param":true}); + assertEqual(1, db._analyzers.count()); assertTrue(null != analyzer); assertTrue(null == analyzer.properties.invalid_param); analyzers.remove("textPropAnalyzer", true); + assertEqual(0, db._analyzers.count()); } { try {analyzers.remove("delimiterPropAnalyzer"); } catch (e) {} + assertEqual(0, db._analyzers.count()); let analyzer = analyzers.save("delimiterPropAnalyzer", "delimiter", { "delimiter":"|", "invalid_param":true}); + assertEqual(1, db._analyzers.count()); assertTrue(null != analyzer); assertTrue(null == analyzer.properties.invalid_param); analyzers.remove("delimiterPropAnalyzer", true); + assertEqual(0, db._analyzers.count()); } { try {analyzers.remove("stemPropAnalyzer"); } catch (e) {} + assertEqual(0, db._analyzers.count()); let analyzer = analyzers.save("stemPropAnalyzer", "stem", { "locale":"en", "invalid_param":true}); + assertEqual(1, db._analyzers.count()); assertTrue(null != analyzer); assertTrue(null == analyzer.properties.invalid_param); analyzers.remove("stemPropAnalyzer", true); + assertEqual(0, db._analyzers.count()); } { try {analyzers.remove("ngramPropAnalyzer"); } catch (e) {} + assertEqual(0, db._analyzers.count()); let analyzer = analyzers.save("ngramPropAnalyzer", "ngram", { "min":1, "max":5, "preserveOriginal":true, "invalid_param":true}); + assertEqual(1, db._analyzers.count()); assertTrue(null != analyzer); assertTrue(null == analyzer.properties.invalid_param); analyzers.remove("ngramPropAnalyzer", true); + assertEqual(0, db._analyzers.count()); } }, testAnalyzerRemovalWithDatabaseName_InSystem: function() { @@ -203,6 +218,8 @@ function iResearchFeatureAqlTestSuite () { let oldListInCollection = db._analyzers.toArray(); assertTrue(Array === oldList.constructor); + assertEqual(0, db._analyzers.count()); + // creation analyzers.save("testAnalyzer", "stem", { "locale":"en"}, [ "frequency" ]); @@ -216,12 +233,27 @@ function iResearchFeatureAqlTestSuite () { assertTrue(Array === analyzer.features().constructor); assertEqual(1, analyzer.features().length); assertEqual([ "frequency" ], analyzer.features()); + + // check persisted analyzer + assertEqual(1, db._analyzers.count()); + let savedAnalyzer = db._analyzers.toArray()[0]; + assertTrue(null !== savedAnalyzer); + assertEqual(7, Object.keys(savedAnalyzer).length); + assertEqual("_analyzers/testAnalyzer", savedAnalyzer._id); + assertEqual("testAnalyzer", savedAnalyzer._key); + assertEqual("testAnalyzer", savedAnalyzer.name); + assertEqual("stem", savedAnalyzer.type); + assertEqual(analyzer.properties(), savedAnalyzer.properties); + assertEqual(analyzer.features(), savedAnalyzer.features); + analyzer = undefined; // release reference // check the analyzers collection in database assertEqual(oldListInCollection.length + 1, db._analyzers.toArray().length); let dbAnalyzer = db._query("FOR d in _analyzers FILTER d.name=='testAnalyzer' RETURN d").toArray(); assertEqual(1, dbAnalyzer.length); + assertEqual("_analyzers/testAnalyzer", dbAnalyzer[0]._id); + assertEqual("testAnalyzer", dbAnalyzer[0]._key); assertEqual("testAnalyzer", dbAnalyzer[0].name); assertEqual("stem", dbAnalyzer[0].type); assertEqual(1, Object.keys(dbAnalyzer[0].properties).length); diff --git a/tests/js/server/aql/aql-modify.js b/tests/js/server/aql/aql-modify.js index 72810dbc4b..c7b180126f 100644 --- a/tests/js/server/aql/aql-modify.js +++ b/tests/js/server/aql/aql-modify.js @@ -1127,6 +1127,39 @@ function aqlUpsertOptionsSuite() { validateDocsAreUpdated(docs, invalid, true); }, + testUpsertEmptyObject: function () { + let NEW, OLD, res; + col.insert({ "value1": "find me" }); + + // before "find me" was found because the empty object + // was ignored now we will insert instead + let q1 = `UPSERT { "value1" : "find me", "value2" : {} } + INSERT { "value1" : "find me", "value2" : {} } + UPDATE { "value1" : "not gonna happen" } + IN ${collectionName} + RETURN [$OLD, $NEW]`; + + res = db._query(q1).toArray(); + OLD = res[0][0]; + NEW = res[0][1]; + assertEqual(NEW.value1, "find me"); + assertEqual(NEW.value2, {}); + + + // now we update the exact doucment + let q2 = `UPSERT { "value1" : "find me", "value2" : {} } + INSERT { "value1" : "not gonna happen" } + UPDATE { "value1" : "update" } + IN ${collectionName} + RETURN [$OLD, $NEW]`; + + res = db._query(q2).toArray(); + OLD = res[0][0]; + NEW = res[0][1]; + assertEqual(NEW.value1, "update"); + assertEqual(NEW.value2, {}); + }, + /* We cannot yet solve this. If you need to ensure _rev value checks put them in the UPDATE {} clause testUpsertSingleWithInvalidRevInMatch : function () { const invalid = genInvalidValue(); diff --git a/tests/js/server/dump/dump-mmfiles-cluster.js b/tests/js/server/dump/dump-mmfiles-cluster.js index db77616e05..ac15145e87 100644 --- a/tests/js/server/dump/dump-mmfiles-cluster.js +++ b/tests/js/server/dump/dump-mmfiles-cluster.js @@ -732,6 +732,16 @@ function dumpTestEnterpriseSuite () { assertEqual(6, p.numberOfShards); }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test link on analyzers collection +//////////////////////////////////////////////////////////////////////////////// + testIndexAnalyzerCollection : function() { + var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray(); + assertEqual(1, db._analyzers.count()); + assertEqual(1, res.length); + assertEqual(db._analyzers.toArray()[0], res[0]); + }, + testViewOnSmartEdgeCollection : function() { try { db._createView("check", "arangosearch", {}); diff --git a/tests/js/server/dump/dump-mmfiles.js b/tests/js/server/dump/dump-mmfiles.js index 6508447f1c..546192aa70 100644 --- a/tests/js/server/dump/dump-mmfiles.js +++ b/tests/js/server/dump/dump-mmfiles.js @@ -488,6 +488,7 @@ function dumpTestSuite () { /// @brief test custom analyzers restoring //////////////////////////////////////////////////////////////////////////////// testAnalyzers: function() { + assertNotEqual(null, db._collection("_analyzers")); assertEqual(1, db._analyzers.count()); // only 1 stored custom analyzer let analyzer = analyzers.analyzer("custom"); @@ -497,8 +498,17 @@ function dumpTestSuite () { assertEqual(" ", analyzer.properties().delimiter); assertEqual(1, analyzer.features().length); assertEqual("frequency", analyzer.features()[0]); - } + }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test link on analyzers collection +//////////////////////////////////////////////////////////////////////////////// + testIndexAnalyzerCollection : function() { + var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray(); + assertEqual(1, db._analyzers.count()); + assertEqual(1, res.length); + assertEqual(db._analyzers.toArray()[0], res[0]); + } }; } diff --git a/tests/js/server/dump/dump-rocksdb.js b/tests/js/server/dump/dump-rocksdb.js index 2c174dd336..dbefb055af 100644 --- a/tests/js/server/dump/dump-rocksdb.js +++ b/tests/js/server/dump/dump-rocksdb.js @@ -453,6 +453,7 @@ function dumpTestSuite () { /// @brief test custom analyzers restoring //////////////////////////////////////////////////////////////////////////////// testAnalyzers: function() { + assertNotEqual(null, db._collection("_analyzers")); assertEqual(1, db._analyzers.count()); // only 1 stored custom analyzer let analyzer = analyzers.analyzer("custom"); @@ -462,8 +463,17 @@ function dumpTestSuite () { assertEqual(" ", analyzer.properties().delimiter); assertEqual(1, analyzer.features().length); assertEqual("frequency", analyzer.features()[0]); - } + }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test link on analyzers collection +//////////////////////////////////////////////////////////////////////////////// + testIndexAnalyzerCollection : function() { + var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray(); + assertEqual(1, db._analyzers.count()); + assertEqual(1, res.length); + assertEqual(db._analyzers.toArray()[0], res[0]); + } }; } diff --git a/tests/js/server/dump/dump-setup-cluster.js b/tests/js/server/dump/dump-setup-cluster.js index 92ec2011a2..b0442e7e40 100644 --- a/tests/js/server/dump/dump-setup-cluster.js +++ b/tests/js/server/dump/dump-setup-cluster.js @@ -287,6 +287,18 @@ function setupSatelliteCollections() { c.save({ value: -1, text: "the red foxx jumps over the pond" }); } catch (err) { } + // setup a view on _analyzers collection + try { + let view = db._createView("analyzersView", "arangosearch", { + links: { + _analyzers : { + includeAllFields:true, + analyzers: [ analyzer.name ] + } + } + }); + } catch (err) { } + setupSmartGraph(); setupSmartArangoSearch(); setupSatelliteCollections(); diff --git a/tests/js/server/dump/dump-setup.js b/tests/js/server/dump/dump-setup.js index cb22d56731..8008fd468b 100644 --- a/tests/js/server/dump/dump-setup.js +++ b/tests/js/server/dump/dump-setup.js @@ -259,6 +259,18 @@ c.save({ value: -1, text: "the red foxx jumps over the pond" }); } catch (err) { } + // setup a view on _analyzers collection + try { + let view = db._createView("analyzersView", "arangosearch", { + links: { + _analyzers : { + includeAllFields: true, + analyzers: [ analyzer.name ] + } + } + }); + } catch (err) { } + // Install Foxx const fs = require('fs'); const SERVICE_PATH = fs.makeAbsolute(fs.join( diff --git a/tests/js/server/replication/sync/replication-sync.js b/tests/js/server/replication/sync/replication-sync.js index 4932a09037..0c2664c4ec 100644 --- a/tests/js/server/replication/sync/replication-sync.js +++ b/tests/js/server/replication/sync/replication-sync.js @@ -930,6 +930,51 @@ function BaseTestConfig () { } }, + testSyncViewOnAnalyzersCollection: function () { + connectToMaster(); + + // create custom analyzer + assertNotEqual(null, db._collection("_analyzers")); + assertEqual(0, db._analyzers.count()); + let analyzer = analyzers.save('custom', 'delimiter', { delimiter: ' ' }, ['frequency']); + assertEqual(1, db._analyzers.count()); + + // create view & collection on master + db._flushCache(); + db._createView('analyzersView', 'arangosearch', { + consolidationIntervalMsec:0, + links: { _analyzers: { analyzers: [ analyzer.name ], includeAllFields:true } } + }); + + db._flushCache(); + connectToSlave(); + internal.wait(0.1, false); + // sync on slave + replication.sync({ endpoint: masterEndpoint }); + + db._flushCache(); + + assertEqual(1, db._analyzers.count()); + var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray(); + assertEqual(1, db._analyzers.count()); + assertEqual(1, res.length); + assertEqual(db._analyzers.toArray()[0], res[0]); + + { + // check state is the same + let view = db._view('analyzersView'); + assertTrue(view !== null); + let props = view.properties(); + assertTrue(props.hasOwnProperty('links')); + assertEqual(0, props.consolidationIntervalMsec); + assertEqual(Object.keys(props.links).length, 1); + assertTrue(props.links.hasOwnProperty('_analyzers')); + assertTrue(props.links['_analyzers'].includeAllFields); + assertTrue(props.links['_analyzers'].analyzers.length, 1); + assertTrue(props.links['_analyzers'].analyzers[0], 'custom'); + } + }, + // ////////////////////////////////////////////////////////////////////////////// // / @brief test with edges // //////////////////////////////////////////////////////////////////////////////