Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch '3.5' of https://github.com/arangodb/arangodb into 3.5.3

This commit is contained in:
KVS85 2019-11-12 20:42:28 +01:00
parent 6910540c78 4ddc94fac9
commit 3da611dee0
45 changed files with 3409 additions and 1906 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
CHANGELOG
View File

@ -1,6 +1,34 @@
v3.5.3 (XXXX-XX-XX)
-------------------
* Fixed UPSERT matching.
Empty objects in the `UPSERT { ... }` expression will not be omitted anymore:
db._collection("col").insert({ "find" : "me" });
db._query(` UPSERT { "find" : "me", "foo" : {} }
UPDATE { "foo" : "not gonna happen" }
INSERT { "find" : "me", "foo" : {} }
INTO col
`)
This will now correctly insert a document instead of updating the existing,
that only partially machtes the upsert-expression.
* Fixed undefined behaviour with creation of ArangoSearch links with custom
analyzers in cluster environment.
* Fixed internal issue #651: analyzer duplication in _analyzers collection.
* Fixed internal issue #4597: rebalanceShards API cannot work on any database
other than the _system database.
* Stop putting system services in _apps on single server, this has never
worked on cluster and was not needed.
* Fixed issue #10371: K_SHORTEST_PATHS LIMIT 1 can not return the shortest path.
Now the shortest path is returned as the first one in such queries.
* Improve killability of some types of cluster AQL queries. Previously, several
cluster queries, especially those containing a `DistributeNode` in their
execution plans, did not respond to a kill instruction.

5
arangod/Aql/Ast.cpp
View File

@ -2665,7 +2665,6 @@ AstNode* Ast::makeConditionFromExample(AstNode const* node) {
TRI_ASSERT(object->type == NODE_TYPE_OBJECT);
auto const n = object->numMembers();
for (size_t i = 0; i < n; ++i) {
auto member = object->getMember(i);
@ -2680,8 +2679,8 @@ AstNode* Ast::makeConditionFromExample(AstNode const* node) {
auto value = member->getMember(0);
if (value->type == NODE_TYPE_OBJECT) {
createCondition(value);
if (value->type == NODE_TYPE_OBJECT && value->numMembers() != 0) {
createCondition(value);
} else {
auto access = variable;
for (auto const& it : attributeParts) {

1
arangod/Aql/QueryString.h
View File

@ -55,6 +55,7 @@ class QueryString {
~QueryString() = default;
public:
std::string const& string() const noexcept { return _queryString; }
char const* data() const { return _queryString.data(); }
size_t size() const { return _queryString.size(); }
size_t length() const { return _queryString.size(); }

39
arangod/Graph/KShortestPathsFinder.cpp
View File

@ -70,27 +70,36 @@ bool KShortestPathsFinder::computeShortestPath(VertexRef const& start, VertexRef
std::unordered_set<VertexRef> const& forbiddenVertices,
std::unordered_set<Edge> const& forbiddenEdges,
Path& result) {
bool found = false;
Ball left(start, FORWARD);
Ball right(end, BACKWARD);
VertexRef join;
result.clear();
while (!left._frontier.empty() && !right._frontier.empty() && !found) {
auto currentBest = boost::optional<double>{};
// We will not improve anymore if we have found a best path and the smallest
// combined distance between left and right is bigger than that path
while (!left._frontier.empty() && !right._frontier.empty() &&
!(currentBest.has_value() &&
(left._closest + right._closest > currentBest.value()))) {
_options.isQueryKilledCallback();
// Choose the smaller frontier to expand.
if (left._frontier.size() < right._frontier.size()) {
found = advanceFrontier(left, right, forbiddenVertices, forbiddenEdges, join);
advanceFrontier(left, right, forbiddenVertices, forbiddenEdges, join, currentBest);
} else {
found = advanceFrontier(right, left, forbiddenVertices, forbiddenEdges, join);
advanceFrontier(right, left, forbiddenVertices, forbiddenEdges, join, currentBest);
}
}
if (found) {
if (currentBest.has_value()) {
reconstructPath(left, right, join, result);
return true;
} else {
// No path found
return false;
}
return found;
}
void KShortestPathsFinder::computeNeighbourhoodOfVertexCache(VertexRef vertex,
@ -178,10 +187,11 @@ void KShortestPathsFinder::computeNeighbourhoodOfVertex(VertexRef vertex, Direct
}
}
bool KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target,
void KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target,
std::unordered_set<VertexRef> const& forbiddenVertices,
std::unordered_set<Edge> const& forbiddenEdges,
VertexRef& join) {
VertexRef& join,
boost::optional<double>& currentBest) {
VertexRef vr;
DijkstraInfo *v, *w;
std::vector<Step>* neighbours;
@ -190,7 +200,7 @@ bool KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target,
TRI_ASSERT(v != nullptr);
TRI_ASSERT(vr == v->_vertex);
if (!success) {
return false;
return;
}
computeNeighbourhoodOfVertexCache(vr, source._direction, neighbours);
@ -217,14 +227,17 @@ bool KShortestPathsFinder::advanceFrontier(Ball& source, Ball const& target,
}
}
v->_done = true;
source._closest = v->_weight;
w = target._frontier.find(v->_vertex);
if (w != nullptr && w->_done) {
join = v->_vertex;
return true;
// The total weight of the found path
double totalWeight = v->_weight + w->_weight;
if (!currentBest.has_value() || totalWeight < currentBest.value()) {
join = v->_vertex;
currentBest = v->_weight + w->_weight;
}
}
return false;
}
void KShortestPathsFinder::reconstructPath(Ball const& left, Ball const& right,

18
arangod/Graph/KShortestPathsFinder.h
View File

@ -34,6 +34,8 @@
#include <list>
#include <boost/optional.hpp>
namespace arangodb {
namespace velocypack {
@ -141,9 +143,9 @@ class KShortestPathsFinder : public ShortestPathFinder {
void setWeight(double weight) { _weight = weight; }
DijkstraInfo(VertexRef const& vertex, Edge const&& edge, VertexRef const& pred, double weight)
: _vertex(vertex), _edge(std::move(edge)), _pred(pred), _weight(weight), _done(false) {}
: _vertex(vertex), _edge(std::move(edge)), _pred(pred), _weight(weight), _done(false) {}
explicit DijkstraInfo(VertexRef const& vertex)
: _vertex(vertex), _weight(0), _done(true) {}
: _vertex(vertex), _weight(0), _done(true) {}
};
typedef ShortestPathPriorityQueue<VertexRef, DijkstraInfo, double> Frontier;
@ -154,10 +156,13 @@ class KShortestPathsFinder : public ShortestPathFinder {
VertexRef _centre;
Direction _direction;
Frontier _frontier;
// The distance of the last node that has been fully expanded
// from _centre
double _closest;
Ball() {}
Ball(VertexRef const& centre, Direction direction)
: _centre(centre), _direction(direction) {
: _centre(centre), _direction(direction), _closest(0) {
_frontier.insert(centre, std::make_unique<DijkstraInfo>(centre));
}
~Ball() {}
@ -193,7 +198,7 @@ class KShortestPathsFinder : public ShortestPathFinder {
std::vector<size_t> _paths;
explicit FoundVertex(VertexRef const& vertex)
: _vertex(vertex), _hasCachedOutNeighbours(false), _hasCachedInNeighbours(false) {}
: _vertex(vertex), _hasCachedOutNeighbours(false), _hasCachedInNeighbours(false) {}
};
// Contains the vertices that were found while searching
// for a shortest path between start and end together with
@ -246,9 +251,10 @@ class KShortestPathsFinder : public ShortestPathFinder {
void computeNeighbourhoodOfVertex(VertexRef vertex, Direction direction,
std::vector<Step>& steps);
bool advanceFrontier(Ball& source, Ball const& target,
void advanceFrontier(Ball& source, Ball const& target,
std::unordered_set<VertexRef> const& forbiddenVertices,
std::unordered_set<Edge> const& forbiddenEdges, VertexRef& join);
std::unordered_set<Edge> const& forbiddenEdges,
VertexRef& join, boost::optional<double>& currentBest);
private:
bool _pathAvailable;

1227
arangod/IResearch/IResearchAnalyzerFeature.cpp
View File

File diff suppressed because it is too large Load Diff

281
arangod/IResearch/IResearchAnalyzerFeature.h
View File

@ -70,15 +70,21 @@ class AnalyzerPool : private irs::util::noncopyable {
irs::string_ref const& type() const noexcept { return _type; }
// definition to be stored in _analyzers collection or shown to the end user
void toVelocyPack(arangodb::velocypack::Builder& builder,
void toVelocyPack(velocypack::Builder& builder,
bool forPersistence = false);
// definition to be stored/shown in a link definition
void toVelocyPack(arangodb::velocypack::Builder& builder,
void toVelocyPack(velocypack::Builder& builder,
TRI_vocbase_t const* vocbase = nullptr);
bool operator==(AnalyzerPool const& rhs) const;
bool operator!=(AnalyzerPool const& rhs) const {
return !(*this == rhs);
}
private:
friend class IResearchAnalyzerFeature; // required for calling AnalyzerPool::init(...) and AnalyzerPool::setKey(...)
// required for calling AnalyzerPool::init(...) and AnalyzerPool::setKey(...)
friend class IResearchAnalyzerFeature;
// 'make(...)' method wrapper for irs::analysis::analyzer types
struct Builder {
@ -86,7 +92,7 @@ class AnalyzerPool : private irs::util::noncopyable {
DECLARE_FACTORY(irs::string_ref const& type, VPackSlice properties);
};
void toVelocyPack(arangodb::velocypack::Builder& builder,
void toVelocyPack(velocypack::Builder& builder,
irs::string_ref const& name);
bool init(irs::string_ref const& type,
@ -94,14 +100,15 @@ class AnalyzerPool : private irs::util::noncopyable {
irs::flags const& features = irs::flags::empty_instance());
void setKey(irs::string_ref const& type);
mutable irs::unbounded_object_pool<Builder> _cache; // cache of irs::analysis::analyzer (constructed via
// AnalyzerBuilder::make(...))
std::string _config; // non-null type + non-null properties + key
irs::flags _features; // cached analyzer features
irs::string_ref _key; // the key of the persisted configuration for this pool, null == static analyzer
std::string _name; // ArangoDB alias for an IResearch analyzer configuration
mutable irs::unbounded_object_pool<Builder> _cache; // cache of irs::analysis::analyzer
// (constructed via AnalyzerBuilder::make(...))
std::string _config; // non-null type + non-null properties + key
irs::flags _features; // cached analyzer features
irs::string_ref _key; // the key of the persisted configuration for this pool,
// null == static analyzer
std::string _name; // ArangoDB alias for an IResearch analyzer configuration
VPackSlice _properties; // IResearch analyzer configuration
irs::string_ref _type; // IResearch analyzer name
irs::string_ref _type; // IResearch analyzer name
}; // AnalyzerPool
////////////////////////////////////////////////////////////////////////////////
@ -111,26 +118,95 @@ class AnalyzerPool : private irs::util::noncopyable {
/// invalidates all AnalyzerPool instances previously provided
/// by the deallocated feature instance
////////////////////////////////////////////////////////////////////////////////
class IResearchAnalyzerFeature final : public arangodb::application_features::ApplicationFeature {
class IResearchAnalyzerFeature final
: public application_features::ApplicationFeature {
public:
explicit IResearchAnalyzerFeature(arangodb::application_features::ApplicationServer& server);
/// first == vocbase name, second == analyzer name
/// EMPTY == system vocbase
/// NIL == unprefixed analyzer name, i.e. active vocbase
using AnalyzerName = std::pair<irs::string_ref, irs::string_ref>;
explicit IResearchAnalyzerFeature(application_features::ApplicationServer& server);
//////////////////////////////////////////////////////////////////////////////
/// @brief check permissions
/// @param vocbase analyzer vocbase
/// @param level access level
/// @return analyzers in the specified vocbase are granted 'level' access
//////////////////////////////////////////////////////////////////////////////
static bool canUse( // check permissions
TRI_vocbase_t const& vocbase, // analyzer vocbase
arangodb::auth::Level const& level // access level
);
static bool canUse(TRI_vocbase_t const& vocbase, auth::Level const& level);
//////////////////////////////////////////////////////////////////////////////
/// @brief check permissions
/// @param name analyzer name (already normalized)
/// @param level access level
/// @return analyzer with the given prefixed name (or unprefixed and resides
/// in defaultVocbase) is granted 'level' access
//////////////////////////////////////////////////////////////////////////////
static bool canUse( // check permissions
irs::string_ref const& name, // analyzer name (already normalized)
arangodb::auth::Level const& level // access level
);
static bool canUse(irs::string_ref const& name, auth::Level const& level);
//////////////////////////////////////////////////////////////////////////////
/// @brief create new analyzer pool
/// @param analyzer created analyzer
/// @param name analyzer name (already normalized)
/// @param type the underlying IResearch analyzer type
/// @param properties the configuration for the underlying IResearch type
/// @param features the expected features the analyzer should produce
/// @return success
//////////////////////////////////////////////////////////////////////////////
static Result createAnalyzerPool(AnalyzerPool::ptr& analyzer,
irs::string_ref const& name,
irs::string_ref const& type,
VPackSlice const properties,
irs::flags const& features);
static AnalyzerPool::ptr identity() noexcept; // the identity analyzer
static std::string const& name() noexcept;
//////////////////////////////////////////////////////////////////////////////
/// @param name analyzer name
/// @param activeVocbase fallback vocbase if not part of name
/// @param systemVocbase the system vocbase for use with empty prefix
/// @param expandVocbasePrefix use full vocbase name as prefix for
/// active/system v.s. EMPTY/'::'
/// @return normalized analyzer name, i.e. with vocbase prefix
//////////////////////////////////////////////////////////////////////////////
static std::string normalize(irs::string_ref const& name,
TRI_vocbase_t const& activeVocbase,
TRI_vocbase_t const& systemVocbase,
bool expandVocbasePrefix = true);
//////////////////////////////////////////////////////////////////////////////
/// @param name analyzer name (normalized)
/// @return vocbase prefix extracted from normalized analyzer name
/// EMPTY == system vocbase
/// NIL == analyzer name have had no db name prefix
/// @see analyzerReachableFromDb
//////////////////////////////////////////////////////////////////////////////
static irs::string_ref extractVocbaseName(irs::string_ref const& name) noexcept {
return splitAnalyzerName(name).first;
}
//////////////////////////////////////////////////////////////////////////////
/// Checks if analyzer db (identified by db name prefix extracted from analyzer
/// name) could be reached from specified db.
/// Properly handles special cases (e.g. NIL and EMPTY)
/// @param dbNameFromAnalyzer database name extracted from analyzer name
/// @param currentDbName database name to check against (should not be empty!)
/// @param forGetters check special case for getting analyzer (not creating/removing)
/// @return true if analyzer is reachable
static bool analyzerReachableFromDb(irs::string_ref const& dbNameFromAnalyzer,
irs::string_ref const& currentDbName,
bool forGetters = false) noexcept;
////////////////////////////////////////////////////////////////////////////////
/// @brief split the analyzer name into the vocbase part and analyzer part
/// @param name analyzer name
/// @return pair of first == vocbase name, second == analyzer name
/// EMPTY == system vocbase
/// NIL == unprefixed analyzer name, i.e. active vocbase
////////////////////////////////////////////////////////////////////////////////
static AnalyzerName splitAnalyzerName(irs::string_ref const& analyzer) noexcept;
//////////////////////////////////////////////////////////////////////////////
/// @brief emplace an analyzer as per the specified parameters
@ -149,134 +225,70 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
/// allowed during recovery
//////////////////////////////////////////////////////////////////////////////
typedef std::pair<AnalyzerPool::ptr, bool> EmplaceResult;
arangodb::Result emplace(
EmplaceResult& result,
irs::string_ref const& name,
irs::string_ref const& type,
VPackSlice const properties,
irs::flags const& features = irs::flags::empty_instance()) {
return ensure(result, name, type, properties, features, true);
}
//////////////////////////////////////////////////////////////////////////////
/// @brief find analyzer
/// @param result the result of the successful emplacement (out-param)
/// first - the emplaced pool
/// second - if an insertion of an new analyzer occured
/// @param name analyzer name (already normalized)
/// @param type the underlying IResearch analyzer type
/// @param properties the configuration for the underlying IResearch type
/// @param features the expected features the analyzer should produce
/// @return analyzer matching the specified parameters or nullptr
/// @note will construct and cache the analyzer if missing only on db-server
/// persistence in the cases of inRecovery or !storage engine will fail
//////////////////////////////////////////////////////////////////////////////
arangodb::Result get(
EmplaceResult& result,
irs::string_ref const& name,
irs::string_ref const& type,
VPackSlice const properties,
irs::flags const& features) {
return ensure(result, name, type, properties, features, false);
}
Result emplace(EmplaceResult& result,
irs::string_ref const& name,
irs::string_ref const& type,
VPackSlice const properties,
irs::flags const& features = irs::flags::empty_instance());
//////////////////////////////////////////////////////////////////////////////
/// @brief find analyzer
/// @param name analyzer name (already normalized)
/// @param onlyCached check only locally cached analyzers
/// @return analyzer with the specified name or nullptr
//////////////////////////////////////////////////////////////////////////////
AnalyzerPool::ptr get( // find analyzer
irs::string_ref const& name, // analyzer name
bool onlyCached = false // check only locally cached analyzers
) const noexcept;
AnalyzerPool::ptr get(irs::string_ref const& name,
bool onlyCached = false) const noexcept {
return get(name, splitAnalyzerName(name), onlyCached);
}
//////////////////////////////////////////////////////////////////////////////
/// @brief find analyzer
/// @return analyzer with the specified name or nullptr
//////////////////////////////////////////////////////////////////////////////
AnalyzerPool::ptr get( // find analyzer
irs::string_ref const& name, // analyzer name
TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name
TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix
bool onlyCached = false // check only locally cached analyzers
) const noexcept;
static AnalyzerPool::ptr identity() noexcept; // the identity analyzer
static std::string const& name() noexcept;
//////////////////////////////////////////////////////////////////////////////
/// @return normalized analyzer name, i.e. with vocbase prefix
//////////////////////////////////////////////////////////////////////////////
static std::string normalize( // normalize name
irs::string_ref const& name, // analyzer name
TRI_vocbase_t const& activeVocbase, // fallback vocbase if not part of name
TRI_vocbase_t const& systemVocbase, // the system vocbase for use with empty prefix
bool expandVocbasePrefix = true // use full vocbase name as prefix for active/system v.s. EMPTY/'::'
);
//////////////////////////////////////////////////////////////////////////////
/// @param name analyzer name (normalized)
/// @return vocbase prefix extracted from normalized analyzer name
/// EMPTY == system vocbase
/// NIL == analyzer name have had no db name prefix
/// @see analyzerReachableFromDb
//////////////////////////////////////////////////////////////////////////////
static irs::string_ref extractVocbaseName(irs::string_ref const& name) noexcept;
//////////////////////////////////////////////////////////////////////////////
/// Checks if analyzer db (identified by db name prefix extracted from analyzer
/// name) could be reached from specified db.
/// Properly handles special cases (e.g. NIL and EMPTY)
/// @param dbNameFromAnalyzer database name extracted from analyzer name
/// @param currentDbName database name to check against (should not be empty!)
/// @param forGetters check special case for getting analyzer (not creating/removing)
/// @return true if analyzer is reachable
static bool analyzerReachableFromDb(irs::string_ref const& dbNameFromAnalyzer,
irs::string_ref const& currentDbName,
bool forGetters = false) noexcept;
////////////////////////////////////////////////////////////////////////////////
/// @brief split the analyzer name into the vocbase part and analyzer part
/// @param name analyzer name
/// @return pair of first == vocbase name, second == analyzer name
/// EMPTY == system vocbase
/// NIL == unprefixed analyzer name, i.e. active vocbase
////////////////////////////////////////////////////////////////////////////////
static std::pair<irs::string_ref, irs::string_ref> splitAnalyzerName(irs::string_ref const& analyzer) noexcept;
void prepare() override;
/// @param activeVocbase fallback vocbase if not part of name
/// @param systemVocbase the system vocbase for use with empty prefix
/// @param onlyCached check only locally cached analyzers
/// @return analyzer with the specified name or nullptr
//////////////////////////////////////////////////////////////////////////////
AnalyzerPool::ptr get(irs::string_ref const& name,
TRI_vocbase_t const& activeVocbase,
TRI_vocbase_t const& systemVocbase,
bool onlyCached = false) const;
//////////////////////////////////////////////////////////////////////////////
/// @brief remove the specified analyzer
/// @param name analyzer name (already normalized)
/// @param force remove even if the analyzer is actively referenced
//////////////////////////////////////////////////////////////////////////////
arangodb::Result remove(irs::string_ref const& name, bool force = false);
Result remove(irs::string_ref const& name, bool force = false);
void start() override;
void stop() override;
//////////////////////////////////////////////////////////////////////////////
/// @brief visit all analyzers for the specified vocbase
/// @param vocbase only visit analysers for this vocbase (nullptr == static)
/// @return visitation compleated fully
//////////////////////////////////////////////////////////////////////////////
bool visit(std::function<bool(AnalyzerPool::ptr const& analyzer)> const& visitor) const;
bool visit(std::function<bool(AnalyzerPool::ptr const& analyzer)> const& visitor,
bool visit(std::function<bool(AnalyzerPool::ptr const&)> const& visitor) const;
bool visit(std::function<bool(AnalyzerPool::ptr const&)> const& visitor,
TRI_vocbase_t const* vocbase) const;
///////////////////////////////////////////////////////////////////////////////
// @brief removes analyzers for specified database from cache
// @param vocbase database to invalidate analyzers
/// @brief removes analyzers for specified database from cache
/// @param vocbase database to invalidate analyzers
///////////////////////////////////////////////////////////////////////////////
void invalidate(const TRI_vocbase_t& vocbase);
virtual void prepare() override;
virtual void start() override;
virtual void stop() override;
private:
// map of caches of irs::analysis::analyzer pools indexed by analyzer name and
// their associated metas
typedef std::unordered_map<irs::hashed_string_ref, AnalyzerPool::ptr> Analyzers;
Analyzers _analyzers; // all analyzers known to this feature (including static) (names are stored with expanded vocbase prefixes)
Analyzers _analyzers; // all analyzers known to this feature (including static)
// (names are stored with expanded vocbase prefixes)
std::unordered_map<std::string, std::chrono::system_clock::time_point> _lastLoad; // last time a database was loaded
mutable irs::async_utils::read_write_mutex _mutex; // for use with member '_analyzers', '_lastLoad'
@ -286,42 +298,17 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
/// @brief validate analyzer parameters and emplace into map
//////////////////////////////////////////////////////////////////////////////
typedef std::pair<Analyzers::iterator, bool> EmplaceAnalyzerResult;
arangodb::Result emplaceAnalyzer( // emplace
Result emplaceAnalyzer( // emplace
EmplaceAnalyzerResult& result, // emplacement result on success (out-param)
arangodb::iresearch::IResearchAnalyzerFeature::Analyzers& analyzers, // analyzers
iresearch::IResearchAnalyzerFeature::Analyzers& analyzers, // analyzers
irs::string_ref const& name, // analyzer name
irs::string_ref const& type, // analyzer type
VPackSlice const properties, // analyzer properties
irs::flags const& features // analyzer features
);
//////////////////////////////////////////////////////////////////////////////
/// @brief ensure an analyzer as per the specified parameters exists if
/// possible
/// @param result the result of the successful emplacement (out-param)
/// first - the emplaced pool
/// second - if an insertion of an new analyzer occured
/// @param name analyzer name (already normalized)
/// @param type the underlying IResearch analyzer type
/// @param properties the configuration for the underlying IResearch type
/// @param features the expected features the analyzer should produce
/// @param isEmplace request coming from emplace(...)
/// @return success
/// @note ensure while inRecovery() will not allow new analyzer persistance
/// valid because for existing links the analyzer definition should
/// already have been persisted and feature administration is not
/// allowed during recovery
/// @note for db-server analyzers are not persisted
/// valid because the authoritative analyzer source is from coordinators
//////////////////////////////////////////////////////////////////////////////
arangodb::Result ensure( // ensure analyzer existence if possible
EmplaceResult& result, // emplacement result on success (out-param)
irs::string_ref const& name, // analyzer name
irs::string_ref const& type, // analyzer type
VPackSlice const properties, // analyzer properties
irs::flags const& features, // analyzer features
bool isEmplace
);
AnalyzerPool::ptr get(irs::string_ref const& normalizedName,
AnalyzerName const& name, bool onlyCached) const noexcept;
//////////////////////////////////////////////////////////////////////////////
/// @brief load the analyzers for the specific database, analyzers read from
@ -330,9 +317,7 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
/// @note on coordinator and db-server reload is also done if the database has
/// not been reloaded in 'timeout' seconds
//////////////////////////////////////////////////////////////////////////////
arangodb::Result loadAnalyzers( // load analyzers
irs::string_ref const& database = irs::string_ref::NIL // database to load
);
Result loadAnalyzers(irs::string_ref const& database = irs::string_ref::NIL);
////////////////////////////////////////////////////////////////////////////////
/// removes analyzers for database from feature cache
@ -345,7 +330,7 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
/// vocbase
/// @note on success will modify the '_key' of the pool
//////////////////////////////////////////////////////////////////////////////
arangodb::Result storeAnalyzer(AnalyzerPool& pool);
Result storeAnalyzer(AnalyzerPool& pool);
};
} // namespace iresearch

74
arangod/IResearch/IResearchDocument.cpp
View File

@ -158,8 +158,9 @@ inline bool keyFromSlice(VPackSlice keySlice, irs::string_ref& key) {
}
}
inline bool canHandleValue(std::string const& key, VPackSlice const& value,
arangodb::iresearch::IResearchLinkMeta const& context) noexcept {
inline bool canHandleValue(std::string const& key,
VPackSlice const& value,
arangodb::iresearch::FieldMeta const& context) noexcept {
switch (value.type()) {
case VPackValueType::None:
case VPackValueType::Illegal:
@ -190,8 +191,9 @@ inline bool canHandleValue(std::string const& key, VPackSlice const& value,
}
// returns 'context' in case if can't find the specified 'field'
inline arangodb::iresearch::IResearchLinkMeta const* findMeta(
irs::string_ref const& key, arangodb::iresearch::IResearchLinkMeta const* context) {
inline arangodb::iresearch::FieldMeta const* findMeta(
irs::string_ref const& key,
arangodb::iresearch::FieldMeta const* context) {
TRI_ASSERT(context);
auto const* meta = context->_fields.findPtr(key);
@ -199,7 +201,7 @@ inline arangodb::iresearch::IResearchLinkMeta const* findMeta(
}
inline bool inObjectFiltered(std::string& buffer,
arangodb::iresearch::IResearchLinkMeta const*& context,
arangodb::iresearch::FieldMeta const*& context,
arangodb::iresearch::IteratorValue const& value) {
irs::string_ref key;
@ -220,7 +222,7 @@ inline bool inObjectFiltered(std::string& buffer,
}
inline bool inObject(std::string& buffer,
arangodb::iresearch::IResearchLinkMeta const*& context,
arangodb::iresearch::FieldMeta const*& context,
arangodb::iresearch::IteratorValue const& value) {
irs::string_ref key;
@ -235,7 +237,7 @@ inline bool inObject(std::string& buffer,
}
inline bool inArrayOrdered(std::string& buffer,
arangodb::iresearch::IResearchLinkMeta const*& context,
arangodb::iresearch::FieldMeta const*& context,
arangodb::iresearch::IteratorValue const& value) {
buffer += arangodb::iresearch::NESTING_LIST_OFFSET_PREFIX;
append(buffer, value.pos);
@ -245,36 +247,36 @@ inline bool inArrayOrdered(std::string& buffer,
}
inline bool inArray(std::string& buffer,
arangodb::iresearch::IResearchLinkMeta const*& context,
arangodb::iresearch::FieldMeta const*& context,
arangodb::iresearch::IteratorValue const& value) noexcept {
return canHandleValue(buffer, value.value, *context);
}
typedef bool (*Filter)(std::string& buffer,
arangodb::iresearch::IResearchLinkMeta const*& context,
arangodb::iresearch::FieldMeta const*& context,
arangodb::iresearch::IteratorValue const& value);
Filter const valueAcceptors[] = {
&inObjectFiltered, // type == Object, nestListValues == false,
// includeAllValues == false
&inObject, // type == Object, nestListValues == false, includeAllValues ==
// true
&inObjectFiltered, // type == Object, nestListValues == true ,
// includeAllValues == false
&inObject, // type == Object, nestListValues == true , includeAllValues ==
// true
&inArray, // type == Array , nestListValues == flase, includeAllValues ==
// false
&inArray, // type == Array , nestListValues == flase, includeAllValues ==
// true
&inArrayOrdered, // type == Array , nestListValues == true,
// includeAllValues == false
&inArrayOrdered // type == Array , nestListValues == true, includeAllValues
// == true
// type == Object, nestListValues == false, // includeAllValues == false
&inObjectFiltered,
// type == Object, nestListValues == false, includeAllValues == // true
&inObject,
// type == Object, nestListValues == true , // includeAllValues == false
&inObjectFiltered,
// type == Object, nestListValues == true , includeAllValues == // true
&inObject,
// type == Array , nestListValues == flase, includeAllValues == // false
&inArray,
// type == Array , nestListValues == flase, includeAllValues == // true
&inArray,
// type == Array , nestListValues == true, // includeAllValues == false
&inArrayOrdered,
// type == Array , nestListValues == true, includeAllValues // == true
&inArrayOrdered
};
inline Filter getFilter(VPackSlice value,
arangodb::iresearch::IResearchLinkMeta const& meta) noexcept {
arangodb::iresearch::FieldMeta const& meta) noexcept {
TRI_ASSERT(arangodb::iresearch::isArrayOrObject(value));
return valueAcceptors[4 * value.isArray() + 2 * meta._trackListPositions + meta._includeAllFields];
@ -292,7 +294,7 @@ namespace iresearch {
/*static*/ void Field::setPkValue(Field& field, LocalDocumentId::BaseType const& pk) {
field._name = PK_COLUMN;
field._features = &irs::flags::empty_instance();
field._storeValues = ValueStorage::FULL;
field._storeValues = ValueStorage::VALUE;
field._value =
irs::bytes_ref(reinterpret_cast<irs::byte_type const*>(&pk), sizeof(pk));
field._analyzer = StringStreamPool.emplace().release(); // FIXME don't use shared_ptr
@ -342,7 +344,7 @@ std::string& FieldIterator::valueBuffer() {
return *_valueBuffer;
}
void FieldIterator::reset(VPackSlice const& doc, IResearchLinkMeta const& linkMeta) {
void FieldIterator::reset(VPackSlice const& doc, FieldMeta const& linkMeta) {
// set surrogate analyzers
_begin = nullptr;
_end = 1 + _begin;
@ -419,13 +421,11 @@ void FieldIterator::setNullValue(VPackSlice const value) {
}
bool FieldIterator::setStringValue(
arangodb::velocypack::Slice const value, // value
IResearchLinkMeta::Analyzer const& valueAnalyzer // analyzer to use
) {
arangodb::velocypack::Slice const value,
FieldMeta::Analyzer const& valueAnalyzer) {
TRI_ASSERT( // assert
(value.isCustom() && nameBuffer() == arangodb::StaticStrings::IdString) // custom string
|| value.isString() // verbatim string
);
|| value.isString()); // verbatim string
irs::string_ref valueRef;
@ -483,7 +483,7 @@ bool FieldIterator::setStringValue(
return true;
}
bool FieldIterator::pushAndSetValue(VPackSlice slice, IResearchLinkMeta const*& context) {
bool FieldIterator::pushAndSetValue(VPackSlice slice, FieldMeta const*& context) {
auto& name = nameBuffer();
while (isArrayOrObject(slice)) {
@ -522,7 +522,7 @@ bool FieldIterator::pushAndSetValue(VPackSlice slice, IResearchLinkMeta const*&
return setAttributeValue(*context);
}
bool FieldIterator::setAttributeValue(IResearchLinkMeta const& context) {
bool FieldIterator::setAttributeValue(FieldMeta const& context) {
auto const value = topValue().value;
_value._storeValues = context._storeValues;
@ -580,7 +580,7 @@ void FieldIterator::next() {
}
}
IResearchLinkMeta const* context;
FieldMeta const* context;
auto& name = nameBuffer();
@ -651,4 +651,4 @@ void FieldIterator::next() {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

36
arangod/IResearch/IResearchDocument.h
View File

@ -141,7 +141,7 @@ class FieldIterator : public std::iterator<std::forward_iterator_tag, Field cons
bool valid() const noexcept { return !_stack.empty(); }
bool operator==(FieldIterator const& rhs) const noexcept {
TRI_ASSERT(_trx == rhs._trx); // compatibility
TRI_ASSERT(_trx == rhs._trx); // compatibility
return _stack == rhs._stack;
}
@ -149,27 +149,25 @@ class FieldIterator : public std::iterator<std::forward_iterator_tag, Field cons
return !(*this == rhs);
}
void reset( // reset
arangodb::velocypack::Slice const& doc, // doc
IResearchLinkMeta const& linkMeta // link meta
);
void reset(velocypack::Slice const& doc,
FieldMeta const& linkMeta);
private:
typedef IResearchLinkMeta::Analyzer const* AnalyzerIterator;
typedef FieldMeta::Analyzer const* AnalyzerIterator;
typedef bool(*Filter)( // filter
std::string& buffer, // buffer
IResearchLinkMeta const*& rootMeta, // root link meta
FieldMeta const*& rootMeta, // root link meta
IteratorValue const& value // value
);
struct Level {
Level( // constructor
arangodb::velocypack::Slice slice, // slice
size_t nameLength, // name length
IResearchLinkMeta const& meta, // link meta
Filter filter // filter
): it(slice), nameLength(nameLength), meta(&meta), filter(filter) {
Level(velocypack::Slice slice,
size_t nameLength,
FieldMeta const& meta,
Filter filter)
: it(slice), nameLength(nameLength),
meta(&meta), filter(filter) {
}
bool operator==(Level const& rhs) const noexcept { return it == rhs.it; }
@ -178,7 +176,7 @@ class FieldIterator : public std::iterator<std::forward_iterator_tag, Field cons
Iterator it;
size_t nameLength; // length of the name at the current level
IResearchLinkMeta const* meta; // metadata
FieldMeta const* meta; // metadata
Filter filter;
}; // Level
@ -201,17 +199,17 @@ class FieldIterator : public std::iterator<std::forward_iterator_tag, Field cons
FieldIterator& operator=(FieldIterator const&) = delete;
void next();
bool pushAndSetValue(arangodb::velocypack::Slice slice, IResearchLinkMeta const*& topMeta);
bool setAttributeValue(IResearchLinkMeta const& context);
bool pushAndSetValue(arangodb::velocypack::Slice slice, FieldMeta const*& topMeta);
bool setAttributeValue(FieldMeta const& context);
bool setStringValue( // set value
arangodb::velocypack::Slice const value, // value
IResearchLinkMeta::Analyzer const& valueAnalyzer // analyzer to use
FieldMeta::Analyzer const& valueAnalyzer // analyzer to use
);
void setNullValue(VPackSlice const value);
void setNumericValue(VPackSlice const value);
void setBoolValue(VPackSlice const value);
void resetAnalyzers(IResearchLinkMeta const& context) noexcept {
void resetAnalyzers(FieldMeta const& context) noexcept {
auto const& analyzers = context._analyzers;
_begin = analyzers.data();
@ -253,4 +251,4 @@ struct DocumentPrimaryKey {
} // namespace iresearch
} // namespace arangodb
#endif
#endif

32
arangod/IResearch/IResearchFilterFactory.cpp
View File

@ -62,7 +62,7 @@ namespace {
struct FilterContext {
FilterContext( // constructor
arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer, // analyzer
arangodb::iresearch::FieldMeta::Analyzer const& analyzer, // analyzer
irs::boost_t boost) noexcept
: analyzer(analyzer), boost(boost) {
TRI_ASSERT(analyzer._pool);
@ -72,7 +72,7 @@ struct FilterContext {
FilterContext& operator=(FilterContext const&) = delete;
// need shared_ptr since pool could be deleted from the feature
arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer;
arangodb::iresearch::FieldMeta::Analyzer const& analyzer;
irs::boost_t boost;
}; // FilterContext
@ -141,10 +141,10 @@ FORCE_INLINE void appendExpression(irs::boolean_filter& filter,
exprFilter.boost(filterCtx.boost);
}
arangodb::iresearch::IResearchLinkMeta::Analyzer extractAnalyzerFromArg(
arangodb::iresearch::FieldMeta::Analyzer extractAnalyzerFromArg(
irs::boolean_filter const* filter, arangodb::aql::AstNode const* analyzerArg,
QueryContext const& ctx, size_t argIdx, irs::string_ref const& functionName) {
static const arangodb::iresearch::IResearchLinkMeta::Analyzer invalid( // invalid analyzer
static const arangodb::iresearch::FieldMeta::Analyzer invalid( // invalid analyzer
nullptr, ""
);
@ -170,7 +170,7 @@ arangodb::iresearch::IResearchLinkMeta::Analyzer extractAnalyzerFromArg(
ScopedAqlValue analyzerValue(*analyzerArg);
if (!filter && !analyzerValue.isConstant()) {
return arangodb::iresearch::IResearchLinkMeta::Analyzer();
return arangodb::iresearch::FieldMeta::Analyzer();
}
if (!analyzerValue.execute(ctx)) {
@ -196,7 +196,7 @@ arangodb::iresearch::IResearchLinkMeta::Analyzer extractAnalyzerFromArg(
return invalid;
}
arangodb::iresearch::IResearchLinkMeta::Analyzer result(nullptr, analyzerId);
arangodb::iresearch::FieldMeta::Analyzer result(nullptr, analyzerId);
auto& analyzer = result._pool;
auto& shortName = result._shortName;
@ -1134,7 +1134,7 @@ arangodb::Result fromFuncAnalyzer(irs::boolean_filter* filter, QueryContext cons
}
ScopedAqlValue analyzerId(*analyzerArg);
arangodb::iresearch::IResearchLinkMeta::Analyzer analyzerValue; // default analyzer
arangodb::iresearch::FieldMeta::Analyzer analyzerValue; // default analyzer
auto& analyzer = analyzerValue._pool;
auto& shortName = analyzerValue._shortName;
@ -1349,44 +1349,44 @@ arangodb::Result fromFuncExists(irs::boolean_filter* filter, QueryContext const&
arangodb::basics::StringUtils::tolowerInPlace(&strArg); // normalize user input
irs::string_ref const TypeAnalyzer("analyzer");
typedef bool (*TypeHandler)(std::string&, arangodb::iresearch::IResearchLinkMeta::Analyzer const&);
typedef bool (*TypeHandler)(std::string&, arangodb::iresearch::FieldMeta::Analyzer const&);
static std::map<irs::string_ref, TypeHandler> const TypeHandlers{
// any string
{irs::string_ref("string"),
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool {
kludge::mangleAnalyzer(name);
return true; // a prefix match
}},
// any non-string type
{irs::string_ref("type"),
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool {
kludge::mangleType(name);
return true; // a prefix match
}},
// concrete analyzer from the context
{TypeAnalyzer,
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const& analyzer)->bool {
kludge::mangleStringField(name, analyzer);
return false; // not a prefix match
}},
{irs::string_ref("numeric"),
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool {
kludge::mangleNumeric(name);
return false; // not a prefix match
}},
{irs::string_ref("bool"),
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool {
kludge::mangleBool(name);
return false; // not a prefix match
}},
{irs::string_ref("boolean"),
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool {
kludge::mangleBool(name);
return false; // not a prefix match
}},
{irs::string_ref("null"),
[](std::string& name, arangodb::iresearch::IResearchLinkMeta::Analyzer const&)->bool {
[](std::string& name, arangodb::iresearch::FieldMeta::Analyzer const&)->bool {
kludge::mangleNull(name);
return false; // not a prefix match
}}};
@ -2082,7 +2082,7 @@ namespace iresearch {
arangodb::aql::AstNode const& node) {
// The analyzer is referenced in the FilterContext and used during the
// following ::filter() call, so may not be a temporary.
IResearchLinkMeta::Analyzer analyzer = IResearchLinkMeta::Analyzer();
FieldMeta::Analyzer analyzer = FieldMeta::Analyzer();
FilterContext const filterCtx(analyzer, irs::no_boost());
return ::filter(filter, ctx, filterCtx, node);

16
arangod/IResearch/IResearchKludge.cpp
View File

@ -52,18 +52,16 @@ void mangleNumeric(std::string& name) {
name.append(NUMERIC_SUFFIX.c_str(), NUMERIC_SUFFIX.size());
}
void mangleStringField( // mangle string field
std::string& name, // field name
arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply
) {
void mangleStringField(
std::string& name,
arangodb::iresearch::FieldMeta::Analyzer const& analyzer) {
name += ANALYZER_DELIMITER;
name += analyzer._shortName;
}
void demangleStringField( // demangle string field
std::string& name, // field name
arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply
) {
void demangleStringField(
std::string& name,
arangodb::iresearch::FieldMeta::Analyzer const& analyzer) {
// +1 for preceding '\1'
auto const suffixSize = 1 + analyzer._shortName.size();
@ -77,4 +75,4 @@ void demangleStringField( // demangle string field
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

17
arangod/IResearch/IResearchKludge.h
View File

@ -44,17 +44,16 @@ void mangleNull(std::string& name);
void mangleBool(std::string& name);
void mangleNumeric(std::string& name);
void mangleStringField( // mangle string field
std::string& name, // field name
arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply
);
void demangleStringField( // demangle string field
std::string& name, // field name
arangodb::iresearch::IResearchLinkMeta::Analyzer const& analyzer // analyzer to apply
);
void mangleStringField(
std::string& name,
arangodb::iresearch::FieldMeta::Analyzer const& analyzer);
void demangleStringField(
std::string& name,
arangodb::iresearch::FieldMeta::Analyzer const& analyzer);
} // namespace kludge
} // namespace iresearch
} // namespace arangodb
#endif
#endif

20
arangod/IResearch/IResearchLink.cpp
View File

@ -31,7 +31,6 @@
#include "Aql/QueryCache.h"
#include "Basics/LocalTaskQueue.h"
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h"
#include "Cluster/ClusterInfo.h"
#include "MMFiles/MMFilesCollection.h"
#include "RestServer/DatabaseFeature.h"
@ -1749,6 +1748,25 @@ Result IResearchLink::unload() {
return {};
}
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup referenced analyzer
////////////////////////////////////////////////////////////////////////////////
AnalyzerPool::ptr IResearchLink::findAnalyzer(AnalyzerPool const& analyzer) const {
auto const it = _meta._analyzerDefinitions.find(irs::string_ref(analyzer.name()));
if (it == _meta._analyzerDefinitions.end()) {
return nullptr;
}
auto const pool = *it;
if (pool && analyzer == *pool) {
return pool;
}
return nullptr;
}
} // namespace iresearch
} // namespace arangodb

5
arangod/IResearch/IResearchLink.h
View File

@ -218,6 +218,11 @@ class IResearchLink {
////////////////////////////////////////////////////////////////////////////////
Result unload();
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup referenced analyzer
////////////////////////////////////////////////////////////////////////////////
AnalyzerPool::ptr findAnalyzer(AnalyzerPool const& analyzer) const;
protected:
typedef std::function<void(irs::directory&)> InitCallback;

57
arangod/IResearch/IResearchLinkHelper.cpp
View File

@ -63,8 +63,8 @@ arangodb::Result canUseAnalyzers( // validate
>();
auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
for (auto& entry: meta._analyzers) {
if (!entry._pool) {
for (auto& pool: meta._analyzerDefinitions) {
if (!pool) {
continue; // skip invalid entries
}
@ -73,35 +73,35 @@ arangodb::Result canUseAnalyzers( // validate
if (sysVocbase) {
result = arangodb::iresearch::IResearchAnalyzerFeature::canUse( // validate
arangodb::iresearch::IResearchAnalyzerFeature::normalize( // normalize
entry._pool->name(), defaultVocbase, *sysVocbase // args
pool->name(), defaultVocbase, *sysVocbase // args
), // analyzer
arangodb::auth::Level::RO // auth level
);
} else {
result = arangodb::iresearch::IResearchAnalyzerFeature::canUse( // validate
entry._pool->name(), arangodb::auth::Level::RO // args
pool->name(), arangodb::auth::Level::RO // args
);
}
if (!result) {
return arangodb::Result( // result
TRI_ERROR_FORBIDDEN, // code
std::string("read access is forbidden to arangosearch analyzer '") + entry._pool->name() + "'"
);
return {
TRI_ERROR_FORBIDDEN,
std::string("read access is forbidden to arangosearch analyzer '") + pool->name() + "'"
};
}
}
for (auto& field: meta._fields) {
TRI_ASSERT(field.value().get()); // ensured by UniqueHeapInstance constructor
auto& entry = field.value();
auto res = canUseAnalyzers(*entry, defaultVocbase);
// for (auto& field: meta._fields) {
// TRI_ASSERT(field.value().get()); // ensured by UniqueHeapInstance constructor
// auto& entry = field.value();
// auto res = canUseAnalyzers(*entry, defaultVocbase);
//
// if (!res.ok()) {
// return res;
// }
// }
if (!res.ok()) {
return res;
}
}
return arangodb::Result();
return {};
}
arangodb::Result createLink( // create link
@ -795,17 +795,22 @@ namespace iresearch {
// analyzer should be either from same database as view (and collection) or from system database
{
const auto& currentVocbase = vocbase.name();
for (const auto& analyzer : meta._analyzers) {
TRI_ASSERT(analyzer._pool); // should be checked in meta init
if (ADB_UNLIKELY(!analyzer._pool)) {
for (const auto& analyzer : meta._analyzerDefinitions) {
TRI_ASSERT(analyzer); // should be checked in meta init
if (ADB_UNLIKELY(!analyzer)) {
continue;
}
auto analyzerVocbase = IResearchAnalyzerFeature::extractVocbaseName(analyzer._pool->name());
auto* pool = analyzer.get();
auto analyzerVocbase = IResearchAnalyzerFeature::extractVocbaseName(pool->name());
if (!IResearchAnalyzerFeature::analyzerReachableFromDb(analyzerVocbase, currentVocbase, true)) {
return arangodb::Result(
TRI_ERROR_BAD_PARAMETER,
std::string("Analyzer '").append(analyzer._pool->name())
.append("' is not accessible from database '").append(currentVocbase).append("'"));
return {
TRI_ERROR_BAD_PARAMETER,
std::string("Analyzer '")
.append(pool->name())
.append("' is not accessible from database '")
.append(currentVocbase).append("'")
};
}
}
}

848
arangod/IResearch/IResearchLinkMeta.cpp
View File

@ -32,6 +32,7 @@
#include "Cluster/ServerState.h"
#include "RestServer/SystemDatabaseFeature.h"
#include "VelocyPackHelper.h"
#include "Basics/VelocyPackHelper.h"
#include "velocypack/Builder.h"
#include "velocypack/Iterator.h"
#include "IResearchLinkMeta.h"
@ -39,8 +40,8 @@
namespace {
bool equalAnalyzers(arangodb::iresearch::IResearchLinkMeta::Analyzers const& lhs,
arangodb::iresearch::IResearchLinkMeta::Analyzers const& rhs) noexcept {
bool equalAnalyzers(std::vector<arangodb::iresearch::FieldMeta::Analyzer> const& lhs,
std::vector<arangodb::iresearch::FieldMeta::Analyzer> const& rhs) noexcept {
if (lhs.size() != rhs.size()) {
return false;
}
@ -70,32 +71,41 @@ bool equalAnalyzers(arangodb::iresearch::IResearchLinkMeta::Analyzers const& lhs
} // namespace
bool operator<(arangodb::iresearch::FieldMeta::Analyzer const& lhs,
irs::string_ref const& rhs) noexcept {
return lhs._pool->name() < rhs;
}
bool operator<(irs::string_ref const& lhs,
arangodb::iresearch::FieldMeta::Analyzer const& rhs) noexcept {
return lhs < rhs._pool->name();
}
namespace arangodb {
namespace iresearch {
IResearchLinkMeta::Analyzer::Analyzer()
// -----------------------------------------------------------------------------
// --SECTION-- FieldMeta::Analyzer
// -----------------------------------------------------------------------------
FieldMeta::Analyzer::Analyzer()
: _pool(IResearchAnalyzerFeature::identity()) {
if (_pool) {
_shortName = _pool->name(); // static analyzers are used verbatim
}
}
IResearchLinkMeta::Mask::Mask(bool mask /*= false*/) noexcept
: _analyzerDefinitions(mask),
_analyzers(mask),
_fields(mask),
_includeAllFields(mask),
_trackListPositions(mask),
_storeValues(mask),
_sort(mask) {
// -----------------------------------------------------------------------------
// --SECTION-- IResearchLinkMeta::FieldMeta
// -----------------------------------------------------------------------------
/*static*/ const FieldMeta& FieldMeta::DEFAULT() {
static const FieldMeta meta;
return meta;
}
IResearchLinkMeta::IResearchLinkMeta()
: //_fields(<empty>), // no fields to index by default
_includeAllFields(false), // true to match all encountered fields, false
// match only fields in '_fields'
_trackListPositions(false), // treat '_trackListPositions' as SQL-IN
_storeValues(ValueStorage::NONE) { // do not track values at all
FieldMeta::FieldMeta() {
Analyzer analyzer; // identity analyzer
// identity-only tokenization
@ -104,16 +114,16 @@ IResearchLinkMeta::IResearchLinkMeta()
}
}
bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcept {
if (!equalAnalyzers(_analyzers, other._analyzers)) {
bool FieldMeta::operator==(FieldMeta const& rhs) const noexcept {
if (!equalAnalyzers(_analyzers, rhs._analyzers)) {
return false; // values do not match
}
if (_fields.size() != other._fields.size()) {
if (_fields.size() != rhs._fields.size()) {
return false; // values do not match
}
auto itr = other._fields.begin();
auto itr = rhs._fields.begin();
for (auto& entry : _fields) {
if (itr.key() != entry.key() || itr.value() != entry.value()) {
@ -123,27 +133,423 @@ bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcep
++itr;
}
if (_includeAllFields != other._includeAllFields) {
if (_includeAllFields != rhs._includeAllFields) {
return false; // values do not match
}
if (_trackListPositions != other._trackListPositions) {
if (_trackListPositions != rhs._trackListPositions) {
return false; // values do not match
}
if (_storeValues != other._storeValues) {
return false; // values do not match
}
if (_sort != other._sort) {
if (_storeValues != rhs._storeValues) {
return false; // values do not match
}
return true;
}
bool IResearchLinkMeta::operator!=(IResearchLinkMeta const& other) const noexcept {
return !(*this == other);
bool FieldMeta::init(velocypack::Slice const& slice,
std::string& errorField,
TRI_vocbase_t const* defaultVocbase /*= nullptr*/,
FieldMeta const& defaults /*= DEFAULT()*/,
Mask* mask /*= nullptr*/,
std::set<AnalyzerPool::ptr, AnalyzerComparer>* referencedAnalyzers /*= nullptr*/) {
if (!slice.isObject()) {
return false;
}
Mask tmpMask;
if (!mask) {
mask = &tmpMask;
}
{
// optional string list
static const std::string fieldName("analyzers");
mask->_analyzers = slice.hasKey(fieldName);
if (!mask->_analyzers) {
_analyzers = defaults._analyzers;
} else {
auto* analyzers = application_features::ApplicationServer::lookupFeature< // find feature
IResearchAnalyzerFeature // feature type
>();
auto* sysDatabase = application_features::ApplicationServer::lookupFeature< // find feature
SystemDatabaseFeature // feature type
>();
auto field = slice.get(fieldName);
if (!field.isArray()) {
errorField = fieldName;
return false;
}
_analyzers.clear(); // reset to match read values exactly
std::unordered_set<irs::string_ref> uniqueGuard; // deduplicate analyzers
for (velocypack::ArrayIterator itr(field); itr.valid(); ++itr) {
auto value = *itr;
if (!value.isString()) {
errorField = fieldName + "[" + std::to_string(itr.index()) + "]";
return false;
}
auto name = value.copyString();
auto shortName = name;
if (defaultVocbase) {
auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
if (sysVocbase) {
name = IResearchAnalyzerFeature::normalize(
name, *defaultVocbase, *sysVocbase);
shortName = IResearchAnalyzerFeature::normalize(
name, *defaultVocbase, *sysVocbase, false);
}
}
AnalyzerPool::ptr analyzer;
bool found = false;
if (referencedAnalyzers) {
auto it = referencedAnalyzers->find(irs::string_ref(name));
if (it != referencedAnalyzers->end()) {
analyzer = *it;
found = static_cast<bool>(analyzer);
if (ADB_UNLIKELY(!found)) {
TRI_ASSERT(false); // should not happen
referencedAnalyzers->erase(it); // remove null analyzer
}
}
}
if (!found && analyzers) {
// for cluster only check cache to avoid ClusterInfo locking issues
// analyzer should have been populated via 'analyzerDefinitions' above
analyzer = analyzers->get(name, ServerState::instance()->isClusterRole());
}
if (!analyzer) {
errorField = fieldName + "." + value.copyString(); // original (non-normalized) 'name' value
return false;
}
if (!found && referencedAnalyzers) {
// save in referencedAnalyzers
referencedAnalyzers->emplace(analyzer);
}
// avoid adding same analyzer twice
if (uniqueGuard.emplace(analyzer->name()).second) {
_analyzers.emplace_back(analyzer, std::move(shortName));
}
}
}
}
{
// optional bool
static const std::string fieldName("includeAllFields");
mask->_includeAllFields = slice.hasKey(fieldName);
if (!mask->_includeAllFields) {
_includeAllFields = defaults._includeAllFields;
} else {
auto field = slice.get(fieldName);
if (!field.isBool()) {
errorField = fieldName;
return false;
}
_includeAllFields = field.getBool();
}
}
{
// optional bool
static const std::string fieldName("trackListPositions");
mask->_trackListPositions = slice.hasKey(fieldName);
if (!mask->_trackListPositions) {
_trackListPositions = defaults._trackListPositions;
} else {
auto field = slice.get(fieldName);
if (!field.isBool()) {
errorField = fieldName;
return false;
}
_trackListPositions = field.getBool();
}
}
{
// optional string enum
static const std::string fieldName("storeValues");
mask->_storeValues = slice.hasKey(fieldName);
if (!mask->_storeValues) {
_storeValues = defaults._storeValues;
} else {
auto field = slice.get(fieldName);
if (!field.isString()) {
errorField = fieldName;
return false;
}
static const std::unordered_map<std::string, ValueStorage> policies = {
{"none", ValueStorage::NONE},
{"id", ValueStorage::ID},
{"value", ValueStorage::VALUE}
};
auto name = field.copyString();
auto itr = policies.find(name);
if (itr == policies.end()) {
errorField = fieldName + "." + name;
return false;
}
_storeValues = itr->second;
}
}
// .............................................................................
// process fields last since children inherit from parent
// .............................................................................
{
// optional string map<name, overrides>
static const std::string fieldName("fields");
mask->_fields = slice.hasKey(fieldName);
if (!mask->_fields) {
_fields = defaults._fields;
} else {
auto field = slice.get(fieldName);
if (!field.isObject()) {
errorField = fieldName;
return false;
}
auto subDefaults = *this;
subDefaults._fields.clear(); // do not inherit fields and overrides from this field
_fields.clear(); // reset to match either defaults or read values exactly
for (velocypack::ObjectIterator itr(field); itr.valid(); ++itr) {
auto key = itr.key();
auto value = itr.value();
if (!key.isString()) {
errorField = fieldName + "[" +
basics::StringUtils::itoa(itr.index()) + "]";
return false;
}
auto name = key.copyString();
if (!value.isObject()) {
errorField = fieldName + "." + name;
return false;
}
std::string childErrorField;
if (!_fields[name]->init(value, childErrorField, defaultVocbase,
subDefaults, nullptr, referencedAnalyzers)) {
errorField = fieldName + "." + name + "." + childErrorField;
return false;
}
}
}
}
return true;
}
bool FieldMeta::json(velocypack::Builder& builder,
FieldMeta const* ignoreEqual /*= nullptr*/,
TRI_vocbase_t const* defaultVocbase /*= nullptr*/,
Mask const* mask /*= nullptr*/) const {
if (!builder.isOpenObject()) {
return false;
}
std::map<std::string, AnalyzerPool::ptr> analyzers;
if ((!ignoreEqual || !equalAnalyzers(_analyzers, ignoreEqual->_analyzers)) &&
(!mask || mask->_analyzers)) {
velocypack::Builder analyzersBuilder;
analyzersBuilder.openArray();
for (auto& entry : _analyzers) {
if (!entry._pool) {
continue; // skip null analyzers
}
std::string name;
if (defaultVocbase) {
auto* sysDatabase = application_features::ApplicationServer::lookupFeature< // find feature
SystemDatabaseFeature // feature type
>();
auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
if (!sysVocbase) {
return false;
}
// @note: DBServerAgencySync::getLocalCollections(...) generates
// 'forPersistence' definitions that are then compared in
// Maintenance.cpp:compareIndexes(...) via
// arangodb::Index::Compare(...) without access to
// 'defaultVocbase', hence the generated definitions must not
// rely on 'defaultVocbase'
// hence must use 'expandVocbasePrefix==true' if
// 'writeAnalyzerDefinition==true' for normalize
// for 'writeAnalyzerDefinition==false' must use
// 'expandVocbasePrefix==false' so that dump/restore an restore
// definitions into differently named databases
name = IResearchAnalyzerFeature::normalize( // normalize
entry._pool->name(), // analyzer name
*defaultVocbase, // active vocbase
*sysVocbase, // system vocbase
false // expand vocbase prefix
);
} else {
name = entry._pool->name(); // verbatim (assume already normalized)
}
analyzers.emplace(name, entry._pool);
analyzersBuilder.add(velocypack::Value(std::move(name)));
}
analyzersBuilder.close();
builder.add("analyzers", analyzersBuilder.slice());
}
if (!mask || mask->_fields) { // fields are not inherited from parent
velocypack::Builder fieldsBuilder;
Mask fieldMask(true); // output all non-matching fields
auto subDefaults = *this; // make modifable copy
subDefaults._fields.clear(); // do not inherit fields and overrides from this field
fieldsBuilder.openObject();
for (auto& entry : _fields) {
fieldMask._fields = !entry.value()->_fields.empty(); // do not output empty fields on subobjects
fieldsBuilder.add( // add sub-object
entry.key(), // field name
velocypack::Value(velocypack::ValueType::Object)
);
if (!entry.value()->json(fieldsBuilder, &subDefaults, defaultVocbase, &fieldMask)) {
return false;
}
fieldsBuilder.close();
}
fieldsBuilder.close();
builder.add("fields", fieldsBuilder.slice());
}
if ((!ignoreEqual || _includeAllFields != ignoreEqual->_includeAllFields) &&
(!mask || mask->_includeAllFields)) {
builder.add("includeAllFields", velocypack::Value(_includeAllFields));
}
if ((!ignoreEqual || _trackListPositions != ignoreEqual->_trackListPositions) &&
(!mask || mask->_trackListPositions)) {
builder.add("trackListPositions", velocypack::Value(_trackListPositions));
}
if ((!ignoreEqual || _storeValues != ignoreEqual->_storeValues) &&
(!mask || mask->_storeValues)) {
static_assert(adjacencyChecker<ValueStorage>::checkAdjacency<ValueStorage::VALUE, ValueStorage::ID, ValueStorage::NONE>(),
"Values are not adjacent");
static const std::string policies[]{
"none", // ValueStorage::NONE
"id", // ValueStorage::ID
"value" // ValueStorage::VALUE
};
auto const policyIdx =
static_cast<std::underlying_type<ValueStorage>::type>(_storeValues);
if (policyIdx >= IRESEARCH_COUNTOF(policies)) {
return false; // unsupported value storage policy
}
builder.add("storeValues", velocypack::Value(policies[policyIdx]));
}
return true;
}
size_t FieldMeta::memory() const noexcept {
auto size = sizeof(FieldMeta);
size += _analyzers.size() * sizeof(decltype(_analyzers)::value_type);
size += _fields.size() * sizeof(decltype(_fields)::value_type);
for (auto& entry : _fields) {
size += entry.key().size();
size += entry.value()->memory();
}
return size;
}
// -----------------------------------------------------------------------------
// --SECTION-- IResearchLinkMeta
// -----------------------------------------------------------------------------
IResearchLinkMeta::IResearchLinkMeta() {
// add default analyzers
for (auto& analyzer : _analyzers) {
_analyzerDefinitions.emplace(analyzer._pool);
}
}
bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcept {
if (FieldMeta::operator!=(other)) {
return false;
}
if (_sort != other._sort) {
return false;
}
return true;
}
/*static*/ const IResearchLinkMeta& IResearchLinkMeta::DEFAULT() {
@ -152,14 +558,12 @@ bool IResearchLinkMeta::operator!=(IResearchLinkMeta const& other) const noexcep
return meta;
}
bool IResearchLinkMeta::init( // initialize meta
arangodb::velocypack::Slice const& slice, // definition
bool readAnalyzerDefinition, // allow analyzer definitions
std::string& errorField, // field causing error (out-param)
TRI_vocbase_t const* defaultVocbase /*= nullptr*/, // fallback vocbase
IResearchLinkMeta const& defaults /*= DEFAULT()*/, // inherited defaults
Mask* mask /*= nullptr*/ // initialized fields (out-param)
) {
bool IResearchLinkMeta::init(velocypack::Slice const& slice,
bool readAnalyzerDefinition,
std::string& errorField,
TRI_vocbase_t const* defaultVocbase /*= nullptr*/,
FieldMeta const& defaults /*= DEFAULT()*/,
Mask* mask /*= nullptr*/) {
if (!slice.isObject()) {
return false;
}
@ -185,6 +589,9 @@ bool IResearchLinkMeta::init( // initialize meta
}
{
// clear existing definitions
_analyzerDefinitions.clear();
// optional object list
static const std::string fieldName("analyzerDefinitions");
@ -193,11 +600,11 @@ bool IResearchLinkMeta::init( // initialize meta
// load analyzer definitions if requested (used on cluster)
// @note must load definitions before loading 'analyzers' to ensure presence
if (readAnalyzerDefinition && mask->_analyzerDefinitions) {
auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
auto* analyzers = application_features::ApplicationServer::lookupFeature< // find feature
IResearchAnalyzerFeature // featue type
>();
auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::SystemDatabaseFeature // featue type
auto* sysDatabase = application_features::ApplicationServer::lookupFeature< // find feature
SystemDatabaseFeature // featue type
>();
auto field = slice.get(fieldName);
@ -207,7 +614,7 @@ bool IResearchLinkMeta::init( // initialize meta
return false;
}
for (arangodb::velocypack::ArrayIterator itr(field); itr.valid(); ++itr) {
for (velocypack::ArrayIterator itr(field); itr.valid(); ++itr) {
auto value = *itr;
if (!value.isObject()) {
@ -292,7 +699,7 @@ bool IResearchLinkMeta::init( // initialize meta
return false;
}
for (arangodb::velocypack::ArrayIterator subItr(subField);
for (velocypack::ArrayIterator subItr(subField);
subItr.valid();
++subItr) {
auto subValue = *subItr;
@ -317,226 +724,32 @@ bool IResearchLinkMeta::init( // initialize meta
}
}
// get analyzer potentially creating it (e.g. on cluster)
// @note do not use emplace(...) since it'll trigger loadAnalyzers(...)
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult emplaceResult;
auto const res = analyzers->get(emplaceResult, name, type, properties, features);
if (!res.ok()) {
AnalyzerPool::ptr analyzer;
auto const res = IResearchAnalyzerFeature::createAnalyzerPool(analyzer, name, type, properties, features);
if (res.fail() || !analyzer) {
errorField = fieldName + "[" + std::to_string(itr.index()) + "]";
return false;
}
}
}
}
{
// optional string list
static const std::string fieldName("analyzers");
mask->_analyzers = slice.hasKey(fieldName);
if (!mask->_analyzers) {
_analyzers = defaults._analyzers;
} else {
auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
IResearchAnalyzerFeature // feature type
>();
auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::SystemDatabaseFeature // feature type
>();
auto field = slice.get(fieldName);
if (!analyzers || !field.isArray()) {
errorField = fieldName;
return false;
}
_analyzers.clear(); // reset to match read values exactly
std::unordered_set<irs::string_ref> uniqueGuard;
for (arangodb::velocypack::ArrayIterator itr(field); itr.valid(); ++itr) {
auto value = *itr;
if (!value.isString()) {
errorField = fieldName + "[" + std::to_string(itr.index()) + "]";
return false;
}
auto name = value.copyString();
auto shortName = name;
if (defaultVocbase) {
auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
if (sysVocbase) {
name = IResearchAnalyzerFeature::normalize(
name, *defaultVocbase, *sysVocbase);
shortName = IResearchAnalyzerFeature::normalize(
name, *defaultVocbase, *sysVocbase, false);
if (res.fail()) {
errorField.append(": ").append(res.errorMessage());
}
}
// for cluster only check cache to avoid ClusterInfo locking issues
// analyzer should have been populated via 'analyzerDefinitions' above
auto analyzer = analyzers->get(name, arangodb::ServerState::instance()->isClusterRole());
if (!analyzer) {
errorField = fieldName + "." + value.copyString(); // original (non-normalized) 'name' value
return false;
}
// avoid adding same analyzer twice
if (uniqueGuard.emplace(analyzer->name()).second)
_analyzers.emplace_back(analyzer, std::move(shortName));
}
}
}
{
// optional bool
static const std::string fieldName("includeAllFields");
mask->_includeAllFields = slice.hasKey(fieldName);
if (!mask->_includeAllFields) {
_includeAllFields = defaults._includeAllFields;
} else {
auto field = slice.get(fieldName);
if (!field.isBool()) {
errorField = fieldName;
return false;
}
_includeAllFields = field.getBool();
}
}
{
// optional bool
static const std::string fieldName("trackListPositions");
mask->_trackListPositions = slice.hasKey(fieldName);
if (!mask->_trackListPositions) {
_trackListPositions = defaults._trackListPositions;
} else {
auto field = slice.get(fieldName);
if (!field.isBool()) {
errorField = fieldName;
return false;
}
_trackListPositions = field.getBool();
}
}
{
// optional string enum
static const std::string fieldName("storeValues");
mask->_storeValues = slice.hasKey(fieldName);
if (!mask->_storeValues) {
_storeValues = defaults._storeValues;
} else {
auto field = slice.get(fieldName);
if (!field.isString()) {
errorField = fieldName;
return false;
}
static const std::unordered_map<std::string, ValueStorage> policies = {
{"none", ValueStorage::NONE}, {"id", ValueStorage::ID}, {"full", ValueStorage::FULL}};
auto name = field.copyString();
auto itr = policies.find(name);
if (itr == policies.end()) {
errorField = fieldName + "." + name;
return false;
}
_storeValues = itr->second;
}
}
// .............................................................................
// process fields last since children inherit from parent
// .............................................................................
{
// optional string map<name, overrides>
static const std::string fieldName("fields");
mask->_fields = slice.hasKey(fieldName);
if (!mask->_fields) {
_fields = defaults._fields;
} else {
auto field = slice.get(fieldName);
if (!field.isObject()) {
errorField = fieldName;
return false;
}
auto subDefaults = *this;
subDefaults._fields.clear(); // do not inherit fields and overrides from this field
_fields.clear(); // reset to match either defaults or read values exactly
for (arangodb::velocypack::ObjectIterator itr(field); itr.valid(); ++itr) {
auto key = itr.key();
auto value = itr.value();
if (!key.isString()) {
errorField = fieldName + "[" +
arangodb::basics::StringUtils::itoa(itr.index()) + "]";
return false;
}
auto name = key.copyString();
if (!value.isObject()) {
errorField = fieldName + "." + name;
return false;
}
std::string childErrorField;
// false == do not read 'analyzerDefinitions' from child elements
if (!_fields[name]->init(value, false, childErrorField, defaultVocbase, subDefaults)) {
errorField = fieldName + "." + name + "." + childErrorField;
return false;
}
_analyzerDefinitions.emplace(analyzer);
}
}
}
return true;
return FieldMeta::init(slice, errorField, defaultVocbase, defaults, mask, &_analyzerDefinitions);
}
bool IResearchLinkMeta::json( // append meta jSON
arangodb::velocypack::Builder& builder, // output buffer (out-param)
bool writeAnalyzerDefinition, // output fill analyzer definition instead of just name
IResearchLinkMeta const* ignoreEqual /*= nullptr*/, // values to ignore if equal
TRI_vocbase_t const* defaultVocbase /*= nullptr*/, // fallback vocbase
Mask const* mask /*= nullptr*/, // values to ignore always
std::map<std::string, AnalyzerPool::ptr>* usedAnalyzers /*= nullptr*/ // append analyzers used in definition
) const {
bool IResearchLinkMeta::json(velocypack::Builder& builder,
bool writeAnalyzerDefinition,
IResearchLinkMeta const* ignoreEqual /*= nullptr*/,
TRI_vocbase_t const* defaultVocbase /*= nullptr*/,
Mask const* mask /*= nullptr*/) const {
if (!builder.isOpenObject()) {
return false;
}
@ -550,154 +763,29 @@ bool IResearchLinkMeta::json( // append meta jSON
}
}
std::map<std::string, AnalyzerPool::ptr> analyzers;
if ((!ignoreEqual || !equalAnalyzers(_analyzers, ignoreEqual->_analyzers)) &&
(!mask || mask->_analyzers)) {
arangodb::velocypack::Builder analyzersBuilder;
analyzersBuilder.openArray();
for (auto& entry : _analyzers) {
if (!entry._pool) {
continue; // skip null analyzers
}
std::string name;
if (defaultVocbase) {
auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
arangodb::SystemDatabaseFeature // feature type
>();
auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
if (!sysVocbase) {
return false;
}
// @note: DBServerAgencySync::getLocalCollections(...) generates
// 'forPersistence' definitions that are then compared in
// Maintenance.cpp:compareIndexes(...) via
// arangodb::Index::Compare(...) without access to
// 'defaultVocbase', hence the generated definitions must not
// rely on 'defaultVocbase'
// hence must use 'expandVocbasePrefix==true' if
// 'writeAnalyzerDefinition==true' for normalize
// for 'writeAnalyzerDefinition==false' must use
// 'expandVocbasePrefix==false' so that dump/restore an restore
// definitions into differently named databases
name = IResearchAnalyzerFeature::normalize( // normalize
entry._pool->name(), // analyzer name
*defaultVocbase, // active vocbase
*sysVocbase, // system vocbase
false // expand vocbase prefix
);
} else {
name = entry._pool->name(); // verbatim (assume already normalized)
}
analyzers.emplace(name, entry._pool);
analyzersBuilder.add(arangodb::velocypack::Value(std::move(name)));
}
analyzersBuilder.close();
builder.add("analyzers", analyzersBuilder.slice());
}
if (!mask || mask->_fields) { // fields are not inherited from parent
arangodb::velocypack::Builder fieldsBuilder;
Mask fieldMask(true); // output all non-matching fields
auto subDefaults = *this; // make modifable copy
subDefaults._fields.clear(); // do not inherit fields and overrides from this field
fieldsBuilder.openObject();
fieldMask._analyzerDefinitions = false; // do not output analyzer definitions in children
for (auto& entry : _fields) {
fieldMask._fields = !entry.value()->_fields.empty(); // do not output empty fields on subobjects
fieldsBuilder.add( // add sub-object
entry.key(), // field name
arangodb::velocypack::Value(arangodb::velocypack::ValueType::Object)
);
if (!entry.value()->json(fieldsBuilder, writeAnalyzerDefinition, &subDefaults, defaultVocbase, &fieldMask, &analyzers)) {
return false;
}
fieldsBuilder.close();
}
fieldsBuilder.close();
builder.add("fields", fieldsBuilder.slice());
}
if ((!ignoreEqual || _includeAllFields != ignoreEqual->_includeAllFields) &&
(!mask || mask->_includeAllFields)) {
builder.add("includeAllFields", arangodb::velocypack::Value(_includeAllFields));
}
if ((!ignoreEqual || _trackListPositions != ignoreEqual->_trackListPositions) &&
(!mask || mask->_trackListPositions)) {
builder.add("trackListPositions", arangodb::velocypack::Value(_trackListPositions));
}
if ((!ignoreEqual || _storeValues != ignoreEqual->_storeValues) &&
(!mask || mask->_storeValues)) {
static_assert(adjacencyChecker<ValueStorage>::checkAdjacency<ValueStorage::FULL, ValueStorage::ID, ValueStorage::NONE>(),
"Values are not adjacent");
static const std::string policies[]{
"none", // ValueStorage::NONE
"id", // ValueStorage::ID
"full" // ValueStorage::FULL
};
auto const policyIdx =
static_cast<std::underlying_type<ValueStorage>::type>(_storeValues);
if (policyIdx >= IRESEARCH_COUNTOF(policies)) {
return false; // unsupported value storage policy
}
builder.add("storeValues", arangodb::velocypack::Value(policies[policyIdx]));
}
// output definitions if 'writeAnalyzerDefinition' requested and not maked
// this should be the case for the default top-most call
if (writeAnalyzerDefinition && (!mask || mask->_analyzerDefinitions)) {
VPackArrayBuilder arrayScope(&builder, "analyzerDefinitions");
for (auto& entry: analyzers) {
TRI_ASSERT(entry.second); // ensured by emplace into 'analyzers' above
entry.second->toVelocyPack(builder, defaultVocbase);
for (auto& entry: _analyzerDefinitions) {
TRI_ASSERT(entry); // ensured by emplace into 'analyzers' above
entry->toVelocyPack(builder, defaultVocbase);
}
}
if (usedAnalyzers) {
usedAnalyzers->insert(analyzers.begin(), analyzers.end());
}
return true;
return FieldMeta::json(builder, ignoreEqual, defaultVocbase, mask);
}
size_t IResearchLinkMeta::memory() const noexcept {
auto size = sizeof(IResearchLinkMeta);
size += _analyzers.size() * sizeof(decltype(_analyzers)::value_type);
size += _fields.size() * sizeof(decltype(_fields)::value_type);
size += _sort.memory();
for (auto& entry : _fields) {
size += entry.key().size();
size += entry.value()->memory();
}
size += FieldMeta::memory();
return size;
}
} // namespace iresearch
} // namespace arangodb
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

192
arangod/IResearch/IResearchLinkMeta.h
View File

@ -48,58 +48,147 @@ class Slice; // forward declarations
namespace arangodb {
namespace iresearch {
// -----------------------------------------------------------------------------
// --SECTION-- public
// types
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief enum of possible ways to store values in the view
////////////////////////////////////////////////////////////////////////////////
enum class ValueStorage : uint32_t {
NONE = 0, // do not store values in the view
ID, // only store value existance
FULL, // store full value in the view
VALUE // store full value in the view
};
////////////////////////////////////////////////////////////////////////////////
/// @brief metadata describing how to process a field in a collection
////////////////////////////////////////////////////////////////////////////////
struct IResearchLinkMeta {
struct FieldMeta {
// can't use FieldMeta as value type since it's incomplete type so far
typedef UnorderedRefKeyMap<char, UniqueHeapInstance<FieldMeta>> Fields;
struct Analyzer {
AnalyzerPool::ptr _pool;
std::string _shortName; // vocbase-dependent short analyzer name
Analyzer(); // identity analyzer
Analyzer( // constructor
AnalyzerPool::ptr const& pool, // pool
std::string&& shortName // short name (cached for use during insert(...))
) noexcept: _pool(pool), _shortName(std::move(shortName)) {}
Analyzer(AnalyzerPool::ptr const& pool,
std::string&& shortName) noexcept
: _pool(pool),
_shortName(std::move(shortName)) {
}
operator bool() const noexcept { return false == !_pool; }
AnalyzerPool::ptr _pool;
std::string _shortName; // vocbase-independent short analyzer name
};
struct AnalyzerComparer {
using is_transparent = void;
bool operator()(AnalyzerPool::ptr const& lhs, AnalyzerPool::ptr const& rhs) const noexcept {
return lhs->name() < rhs->name();
}
bool operator()(AnalyzerPool::ptr const& lhs, irs::string_ref const& rhs) const noexcept {
return lhs->name() < rhs;
}
bool operator()(irs::string_ref const& lhs, AnalyzerPool::ptr const& rhs) const noexcept {
return lhs < rhs->name();
}
};
struct Mask {
bool _analyzerDefinitions;
explicit Mask(bool mask = false) noexcept
: _analyzers(mask),
_fields(mask),
_includeAllFields(mask),
_trackListPositions(mask),
_storeValues(mask) {
}
bool _analyzers;
bool _fields;
bool _includeAllFields;
bool _trackListPositions;
bool _storeValues;
bool _sort;
explicit Mask(bool mask = false) noexcept;
};
typedef std::vector<Analyzer> Analyzers;
////////////////////////////////////////////////////////////////////////////////
/// @brief return default IResearchLinkMeta values
////////////////////////////////////////////////////////////////////////////////
static const FieldMeta& DEFAULT();
// can't use IResearchLinkMeta as value type since it's incomplete type so far
typedef UnorderedRefKeyMap<char, UniqueHeapInstance<IResearchLinkMeta>> Fields;
FieldMeta();
FieldMeta(FieldMeta const&) = default;
FieldMeta(FieldMeta&&) = default;
IResearchViewSort _sort; // sort condition associated with the link
Analyzers _analyzers; // analyzers to apply to every field
FieldMeta& operator=(FieldMeta const&) = default;
FieldMeta& operator=(FieldMeta&&) = default;
bool operator==(FieldMeta const& rhs) const noexcept;
bool operator!=(FieldMeta const& rhs) const noexcept {
return !(*this == rhs);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initialize FieldMeta with values from a JSON description
/// return success or set 'errorField' to specific field with error
/// on failure state is undefined
/// @param slice input definition
/// @param errorField field causing error (out-param)
/// @param defaultVocbase fallback vocbase for analyzer name normalization
/// nullptr == do not normalize
/// @param defaults inherited defaults
/// @param mask if set reflects which fields were initialized from JSON
/// @param analyzers analyzers referenced in this link
////////////////////////////////////////////////////////////////////////////////
bool init(velocypack::Slice const& slice,
std::string& errorField,
TRI_vocbase_t const* defaultVocbase = nullptr,
FieldMeta const& defaults = DEFAULT(),
Mask* mask = nullptr,
std::set<AnalyzerPool::ptr, AnalyzerComparer>* analyzers = nullptr);
////////////////////////////////////////////////////////////////////////////////
/// @brief fill and return a JSON description of a FieldMeta object
/// do not fill values identical to ones available in 'ignoreEqual'
/// or (if 'mask' != nullptr) values in 'mask' that are set to false
/// elements are appended to an existing object
/// return success or set TRI_set_errno(...) and return false
/// @param builder output buffer
/// @param defaultVocbase fallback vocbase for analyzer name normalization
/// nullptr == do not normalize
/// @param ignoreEqual values to ignore if equal
/// @param defaultVocbase fallback vocbase
/// @param mask if set reflects which fields were initialized from JSON
////////////////////////////////////////////////////////////////////////////////
bool json(arangodb::velocypack::Builder& builder,
FieldMeta const* ignoreEqual = nullptr,
TRI_vocbase_t const* defaultVocbase = nullptr,
Mask const* mask = nullptr) const;
////////////////////////////////////////////////////////////////////////////////
/// @brief amount of memory in bytes occupied by this FieldMeta
////////////////////////////////////////////////////////////////////////////////
size_t memory() const noexcept;
std::vector<Analyzer> _analyzers; // analyzers to apply to every field
Fields _fields; // explicit list of fields to be indexed with optional overrides
bool _includeAllFields; // include all fields or only fields listed in
// '_fields'
bool _trackListPositions; // append relative offset in list to attribute name
// (as opposed to without offset)
ValueStorage _storeValues; // how values should be stored inside the view
ValueStorage _storeValues{ ValueStorage::NONE }; // how values should be stored inside the view
bool _includeAllFields{ false }; // include all fields or only fields listed in '_fields'
bool _trackListPositions{ false }; // append relative offset in list to attribute name (as opposed to without offset)
};
////////////////////////////////////////////////////////////////////////////////
/// @brief metadata describing how to process a field in a collection
////////////////////////////////////////////////////////////////////////////////
struct IResearchLinkMeta : public FieldMeta {
struct Mask : public FieldMeta::Mask {
explicit Mask(bool mask = false) noexcept
: FieldMeta::Mask(mask),
_analyzerDefinitions(mask),
_sort(mask) {
}
bool _analyzerDefinitions;
bool _sort;
};
std::set<AnalyzerPool::ptr, FieldMeta::AnalyzerComparer> _analyzerDefinitions;
IResearchViewSort _sort; // sort condition associated with the link
// NOTE: if adding fields don't forget to modify the comparison operator !!!
// NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask !!!
// NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask constructor !!!
@ -114,8 +203,10 @@ struct IResearchLinkMeta {
IResearchLinkMeta& operator=(IResearchLinkMeta&& other) = default;
IResearchLinkMeta& operator=(IResearchLinkMeta const& other) = default;
bool operator==(IResearchLinkMeta const& other) const noexcept;
bool operator!=(IResearchLinkMeta const& other) const noexcept;
bool operator==(IResearchLinkMeta const& rhs) const noexcept;
bool operator!=(IResearchLinkMeta const& rhs) const noexcept {
return !(*this == rhs);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return default IResearchLinkMeta values
@ -126,18 +217,21 @@ struct IResearchLinkMeta {
/// @brief initialize IResearchLinkMeta with values from a JSON description
/// return success or set 'errorField' to specific field with error
/// on failure state is undefined
/// @param slice definition
/// @param readAnalyzerDefinition allow reading analyzer definitions instead
/// of just name
/// @param erroField field causing error (out-param)
/// @param defaultVocbase fallback vocbase for analyzer name normalization
/// nullptr == do not normalize
/// @param defaults inherited defaults
/// @param mask if set reflects which fields were initialized from JSON
////////////////////////////////////////////////////////////////////////////////
bool init( // initialize meta
arangodb::velocypack::Slice const& slice, // definition
bool readAnalyzerDefinition, // allow reading analyzer definitions instead of just name
std::string& errorField, // field causing error (out-param)
TRI_vocbase_t const* defaultVocbase = nullptr, // fallback vocbase
IResearchLinkMeta const& defaults = DEFAULT(), // inherited defaults
Mask* mask = nullptr // initialized fields (out-param)
);
bool init(velocypack::Slice const& slice,
bool readAnalyzerDefinition,
std::string& errorField,
TRI_vocbase_t const* defaultVocbase = nullptr,
FieldMeta const& defaults = DEFAULT(),
Mask* mask = nullptr);
////////////////////////////////////////////////////////////////////////////////
/// @brief fill and return a JSON description of a IResearchLinkMeta object
@ -145,21 +239,21 @@ struct IResearchLinkMeta {
/// or (if 'mask' != nullptr) values in 'mask' that are set to false
/// elements are appended to an existing object
/// return success or set TRI_set_errno(...) and return false
/// @param builder output buffer (out-param)
/// @param writeAnalyzerDefinition output full analyzer definition instead of just name
/// @param ignoreEqual values to ignore if equal
/// @param defaultVocbase fallback vocbase for analyzer name normalization
/// nullptr == do not normalize
/// @param usedAnalyzers add to this map analyzers used in meta,
/// @param mask if set reflects which fields were initialized from JSON
////////////////////////////////////////////////////////////////////////////////
bool json( // append meta jSON
arangodb::velocypack::Builder& builder, // output buffer (out-param)
bool writeAnalyzerDefinition, // output full analyzer definition instead of just name
IResearchLinkMeta const* ignoreEqual = nullptr, // values to ignore if equal
TRI_vocbase_t const* defaultVocbase = nullptr, // fallback vocbase
Mask const* mask = nullptr, // values to ignore always
std::map<std::string, AnalyzerPool::ptr>* usedAnalyzers = nullptr // append analyzers used in definition
) const;
bool json(velocypack::Builder& builder,
bool writeAnalyzerDefinition,
IResearchLinkMeta const* ignoreEqual = nullptr,
TRI_vocbase_t const* defaultVocbase = nullptr,
Mask const* mask = nullptr) const;
////////////////////////////////////////////////////////////////////////////////
/// @brief amount of memory in bytes occupied by this iResearch Link meta
/// @brief amount of memory in bytes occupied by this IResearchLinkMeta
////////////////////////////////////////////////////////////////////////////////
size_t memory() const noexcept;
}; // IResearchLinkMeta

2
arangod/Replication/utilities.cpp
View File

@ -640,6 +640,8 @@ Result parseResponse(velocypack::Builder& builder,
velocypack::Parser parser(builder);
parser.parse(response->getBody().begin(), response->getBody().length());
return Result();
} catch (VPackException const& e) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, e.what());
} catch (...) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE);
}

6
arangod/RocksDBEngine/RocksDBIncrementalSync.cpp
View File

@ -228,7 +228,7 @@ Result syncChunkRocksDB(DatabaseInitialSyncer& syncer, SingleCollectionTransacti
if (r.fail()) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE,
std::string("got invalid response from master at ") +
syncer._state.master.endpoint + ": response is no array");
syncer._state.master.endpoint + ": " + r.errorMessage());
}
VPackSlice const responseBody = builder.slice();
@ -425,7 +425,7 @@ Result syncChunkRocksDB(DatabaseInitialSyncer& syncer, SingleCollectionTransacti
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE,
std::string("got invalid response from master at ") +
syncer._state.master.endpoint +
": response is no array");
": " + r.errorMessage());
}
VPackSlice const slice = docsBuilder->slice();
@ -613,7 +613,7 @@ Result handleSyncKeysRocksDB(DatabaseInitialSyncer& syncer,
if (r.fail()) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE,
std::string("got invalid response from master at ") +
syncer._state.master.endpoint + ": response is no array");
syncer._state.master.endpoint + ": " + r.errorMessage());
}
VPackSlice const chunkSlice = builder.slice();

4
arangod/VocBase/vocbase.cpp
View File

@ -1019,8 +1019,8 @@ std::shared_ptr<arangodb::LogicalCollection> TRI_vocbase_t::lookupCollectionByUu
}
/// @brief looks up a data-source by identifier
std::shared_ptr<arangodb::LogicalDataSource> TRI_vocbase_t::lookupDataSource(TRI_voc_cid_t id) const
noexcept {
std::shared_ptr<arangodb::LogicalDataSource> TRI_vocbase_t::lookupDataSource(
TRI_voc_cid_t id) const noexcept {
RECURSIVE_READ_LOCKER(_dataSourceLock, _dataSourceLockWriteOwner);
auto itr = _dataSourceById.find(id);

5
js/actions/api-cluster.js
View File

@ -1365,9 +1365,10 @@ actions.defineHttp({
}
// simon: RO is sufficient to rebalance shards for current db
if (req.database !== '_system'/* || !req.isAdminUser*/) {
if (!req.isAdminUser &&
users.permission(req.user, req.database) !== 'rw') {
actions.resultError(req, res, actions.HTTP_FORBIDDEN, 0,
'only allowed for admins on the _system database');
'only allowed for admins on the database');
return;
}

38
js/server/modules/@arangodb/foxx/manager.js
View File

@ -284,50 +284,12 @@ function cleanupOrphanedServices (knownServicePaths, knownBundlePaths) {
}
function startup () {
if (isFoxxmaster()) {
const db = require('internal').db;
const dbName = db._name();
try {
db._useDatabase('_system');
const databases = db._databases();
for (const name of databases) {
try {
db._useDatabase(name);
upsertSystemServices();
} catch (e) {
console.warnStack(e);
}
}
} finally {
db._useDatabase(dbName);
}
}
if (global.ArangoServerState.role() === 'SINGLE') {
commitLocalState(true);
}
selfHealAll();
}
function upsertSystemServices () {
const serviceDefinitions = new Map();
for (const mount of SYSTEM_SERVICE_MOUNTS) {
try {
const serviceDefinition = utils.getServiceDefinition(mount) || {mount};
const service = FoxxService.create(serviceDefinition);
serviceDefinitions.set(mount, service.toJSON());
} catch (e) {
console.errorStack(e);
}
}
db._query(aql`
FOR item IN ${Array.from(serviceDefinitions)}
UPSERT {mount: item[0]}
INSERT item[1]
REPLACE item[1]
IN ${utils.getStorage()}
`);
}
function commitLocalState (replace) {
let modified = false;
const rootPath = FoxxService.rootPath();

40
tests/Graph/GraphTestTools.h
View File

@ -50,6 +50,8 @@
#include "../Mocks/StorageEngineMock.h"
#include "IResearch/common.h"
#include "boost/optional.hpp"
using namespace arangodb;
using namespace arangodb::aql;
using namespace arangodb::graph;
@ -184,9 +186,18 @@ struct MockGraphDatabase {
EXPECT_TRUE(vertices->type());
}
struct EdgeDef {
EdgeDef(size_t from, size_t to) : _from(from), _to(to){};
EdgeDef(size_t from, size_t to, double weight)
: _from(from), _to(to), _weight(weight){};
size_t _from;
size_t _to;
boost::optional<double> _weight;
};
// Create a collection with name <name> of edges given by <edges>
void addEdgeCollection(std::string name, std::string vertexCollection,
std::vector<std::pair<size_t, size_t>> edgedef) {
std::vector<EdgeDef> edgedef) {
std::shared_ptr<arangodb::LogicalCollection> edges;
auto createJson = velocypack::Parser::fromJson("{ \"name\": \"" + name +
"\", \"type\": 3 }");
@ -205,10 +216,18 @@ struct MockGraphDatabase {
for (auto& p : edgedef) {
// std::cout << "edge: " << vertexCollection << " " << p.first << " -> "
// << p.second << std::endl;
auto docJson = velocypack::Parser::fromJson(
"{ \"_from\": \"" + vertexCollection + "/" + std::to_string(p.first) +
"\"" + ", \"_to\": \"" + vertexCollection + "/" +
std::to_string(p.second) + "\" }");
// This is moderately horrible
auto docJson =
p._weight.has_value()
? velocypack::Parser::fromJson(
"{ \"_from\": \"" + vertexCollection + "/" +
std::to_string(p._from) + "\"" + ", \"_to\": \"" +
vertexCollection + "/" + std::to_string(p._to) +
"\", \"cost\": " + std::to_string(p._weight.value()) + "}")
: velocypack::Parser::fromJson(
"{ \"_from\": \"" + vertexCollection + "/" +
std::to_string(p._from) + "\"" + ", \"_to\": \"" +
vertexCollection + "/" + std::to_string(p._to) + "\" }");
docs.emplace_back(docJson);
}
@ -282,12 +301,13 @@ struct MockGraphDatabase {
spos.emplace_back(spo);
return spo;
}
};
}; // namespace graph
bool checkPath(ShortestPathOptions* spo, ShortestPathResult result, std::vector<std::string> vertices,
bool checkPath(ShortestPathOptions* spo, ShortestPathResult result,
std::vector<std::string> vertices,
std::vector<std::pair<std::string, std::string>> edges, std::string& msgs);
}
}
}
} // namespace graph
} // namespace tests
} // namespace arangodb
#endif

46
tests/Graph/KShortestPathsFinder.cpp
View File

@ -212,6 +212,52 @@ TEST_F(KShortestPathsFinderTest, many_edges_between_two_nodes) {
ASSERT_TRUE(false == finder->getNextPathShortestPathResult(result));
}
class KShortestPathsFinderTestWeights : public ::testing::Test {
protected:
GraphTestSetup s;
MockGraphDatabase gdb;
arangodb::aql::Query* query;
arangodb::graph::ShortestPathOptions* spo;
KShortestPathsFinder* finder;
KShortestPathsFinderTestWeights() : gdb("testVocbase") {
gdb.addVertexCollection("v", 10);
gdb.addEdgeCollection("e", "v",
{{1, 2, 10},
{1, 3, 10},
{1, 10, 100},
{2, 4, 10},
{3, 4, 20},
{7, 3, 10},
{8, 3, 10},
{9, 3, 10}});
query = gdb.getQuery("RETURN 1");
spo = gdb.getShortestPathOptions(query);
spo->weightAttribute = "cost";
finder = new KShortestPathsFinder(*spo);
}
~KShortestPathsFinderTestWeights() { delete finder; }
};
TEST_F(KShortestPathsFinderTestWeights, diamond_path) {
auto start = velocypack::Parser::fromJson("\"v/1\"");
auto end = velocypack::Parser::fromJson("\"v/4\"");
ShortestPathResult result;
std::string msgs;
finder->startKShortestPathsTraversal(start->slice(), end->slice());
ASSERT_TRUE(finder->getNextPathShortestPathResult(result));
auto cpr = checkPath(spo, result, {"1", "2", "4"},
{{}, {"v/1", "v/2"}, {"v/2", "v/4"}}, msgs);
ASSERT_TRUE(cpr) << msgs;
}
} // namespace graph
} // namespace tests
} // namespace arangodb

714
tests/IResearch/IResearchAnalyzerFeature-test.cpp
View File

File diff suppressed because it is too large Load Diff

16
tests/IResearch/IResearchDocument-test.cpp
View File

@ -46,6 +46,9 @@
#include "RestServer/DatabaseFeature.h"
#include "RestServer/QueryRegistryFeature.h"
#include "RestServer/SystemDatabaseFeature.h"
#include "RestServer/TraverserEngineRegistryFeature.h"
#include "RestServer/AqlFeature.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Sharding/ShardingFeature.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "Transaction/Methods.h"
@ -165,6 +168,9 @@ class IResearchDocumentTest : public ::testing::Test {
arangodb::LogLevel::ERR);
// setup required application features
features.emplace_back(new arangodb::TraverserEngineRegistryFeature(server), false); // must be before AqlFeature
features.emplace_back(new arangodb::AqlFeature(server), true);
features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true);
features.emplace_back(new arangodb::AuthenticationFeature(server), true);
features.emplace_back(new arangodb::DatabaseFeature(server), false);
features.emplace_back(new arangodb::QueryRegistryFeature(server), false); // required for constructing TRI_vocbase_t
@ -722,7 +728,7 @@ TEST_F(IResearchDocumentTest, FieldIterator_traverse_complex_object_ordered_chec
auto const slice = json->slice();
arangodb::iresearch::IResearchLinkMeta linkMeta;
linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer(
analyzers->get(arangodb::StaticStrings::SystemDatabase +
"::iresearch-document-empty"),
"iresearch-document-empty")); // add analyzer
@ -1549,12 +1555,12 @@ TEST_F(IResearchDocumentTest, FieldIterator_nullptr_analyzer) {
// last analyzer invalid
{
arangodb::iresearch::IResearchLinkMeta linkMeta;
linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer(
analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"),
"empty")); // add analyzer
InvalidAnalyzer::returnNullFromMake = false;
linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer(
analyzers.get(arangodb::StaticStrings::SystemDatabase + "::invalid"),
"invalid")); // add analyzer
linkMeta._includeAllFields = true; // include all fields
@ -1620,10 +1626,10 @@ TEST_F(IResearchDocumentTest, FieldIterator_nullptr_analyzer) {
linkMeta._analyzers.clear();
InvalidAnalyzer::returnNullFromMake = false;
linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer(
analyzers.get(arangodb::StaticStrings::SystemDatabase + "::invalid"),
"invalid")); // add analyzer
linkMeta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer(
analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"),
"empty")); // add analyzer
linkMeta._includeAllFields = true; // include all fields

2
tests/IResearch/IResearchFilterBoolean-test.cpp
View File

@ -36,6 +36,7 @@
#include "Aql/ExecutionPlan.h"
#include "Aql/ExpressionContext.h"
#include "Aql/Query.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Cluster/ClusterFeature.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "IResearch/AqlHelper.h"
@ -108,6 +109,7 @@ class IResearchFilterBooleanTest : public ::testing::Test {
features.emplace_back(new arangodb::V8DealerFeature(server),
false); // required for DatabaseFeature::createDatabase(...)
features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature
features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true);
features.emplace_back(new arangodb::AqlFeature(server), true);
features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server),
true); // required for IResearchAnalyzerFeature

2
tests/IResearch/IResearchFilterCompare-test.cpp
View File

@ -36,6 +36,7 @@
#include "Aql/ExecutionPlan.h"
#include "Aql/ExpressionContext.h"
#include "Aql/Query.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Cluster/ClusterFeature.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "IResearch/AqlHelper.h"
@ -109,6 +110,7 @@ class IResearchFilterCompareTest : public ::testing::Test {
features.emplace_back(new arangodb::V8DealerFeature(server),
false); // required for DatabaseFeature::createDatabase(...)
features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature
features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true);
features.emplace_back(new arangodb::AqlFeature(server), true);
features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server),
true); // required for IResearchAnalyzerFeature

2
tests/IResearch/IResearchFilterFunction-test.cpp
View File

@ -36,6 +36,7 @@
#include "Aql/ExecutionPlan.h"
#include "Aql/ExpressionContext.h"
#include "Aql/Query.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Cluster/ClusterFeature.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "IResearch/AqlHelper.h"
@ -111,6 +112,7 @@ class IResearchFilterFunctionTest : public ::testing::Test {
features.emplace_back(new arangodb::V8DealerFeature(server),
false); // required for DatabaseFeature::createDatabase(...)
features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature
features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true);
features.emplace_back(new arangodb::AqlFeature(server), true);
features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server),
true); // required for IResearchAnalyzerFeature

2
tests/IResearch/IResearchFilterIn-test.cpp
View File

@ -36,6 +36,7 @@
#include "Aql/ExecutionPlan.h"
#include "Aql/ExpressionContext.h"
#include "Aql/Query.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Cluster/ClusterFeature.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "IResearch/AqlHelper.h"
@ -109,6 +110,7 @@ class IResearchFilterInTest : public ::testing::Test {
false); // required for DatabaseFeature::createDatabase(...)
features.emplace_back(new arangodb::ViewTypesFeature(server), false); // required for IResearchFeature
features.emplace_back(new arangodb::AqlFeature(server), true);
features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true);
features.emplace_back(functions = new arangodb::aql::AqlFunctionFeature(server),
true); // required for IResearchAnalyzerFeature
features.emplace_back(new arangodb::iresearch::IResearchAnalyzerFeature(server), true);

132
tests/IResearch/IResearchLinkHelper-test.cpp
View File

@ -232,10 +232,39 @@ TEST_F(IResearchLinkHelperTestSingle, test_validate_cross_db_analyzer) {
TEST_F(IResearchLinkHelperTestSingle, test_normalize_single) {
auto* analyzers =
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::iresearch::IResearchAnalyzerFeature>();
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
TRI_vocbase_t& sysVocbase = server.getSystemDatabase();
// analyzer single-server
// analyzer single-server, for creation
{
auto json = arangodb::velocypack::Parser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"testAnalyzer0\", \"type\": \"identity\" } ], \
\"analyzers\": [\"testAnalyzer0\" ] \
}");
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), true, sysVocbase).ok());
builder.close();
EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0"));
auto expected_json = arangodb::velocypack::Parser::fromJson(
"{ \
\"type\":\"arangosearch\", \
\"primarySort\":[], \
\"fields\":{}, \
\"includeAllFields\": false, \
\"trackListPositions\": false, \
\"storeValues\": \"none\", \
\"analyzerDefinitions\": [ \
{ \"name\": \"testAnalyzer0\", \"type\": \"identity\", \"properties\":{}, \"features\":[] } \
], \
\"analyzers\": [\"testAnalyzer0\" ] \
}");
EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice());
}
// analyzer single-server, user definition
{
auto json = arangodb::velocypack::Parser::fromJson(
"{ \
@ -246,11 +275,50 @@ TEST_F(IResearchLinkHelperTestSingle, test_normalize_single) {
builder.openObject();
EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), false, sysVocbase).ok());
EXPECT_TRUE((true == !analyzers->get(arangodb::StaticStrings::SystemDatabase +
"::testAnalyzer1")));
builder.close();
EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0"));
auto expected_json = arangodb::velocypack::Parser::fromJson(
"{ \
\"type\":\"arangosearch\", \
\"fields\":{}, \
\"includeAllFields\": false, \
\"trackListPositions\": false, \
\"storeValues\": \"none\", \
\"analyzers\": [\"testAnalyzer0\" ] \
}");
EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice());
}
// analyzer single-server (inRecovery) fail persist in recovery
// analyzer single-server, not for creation, missing "testAanalyzer0"
{
auto json = arangodb::velocypack::Parser::fromJson(
"{ \
\"analyzers\": [\"testAnalyzer0\" ] \
}");
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_FALSE(arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), false, sysVocbase).ok());
builder.close();
EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0"));
}
// analyzer single-server, for creation, missing "testAanalyzer0"
{
auto json = arangodb::velocypack::Parser::fromJson(
"{ \
\"analyzers\": [\"testAnalyzer0\" ] \
}");
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_FALSE(arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), false, sysVocbase).ok());
builder.close();
EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer0"));
}
// analyzer single-server (inRecovery), for creation
{
auto json = arangodb::velocypack::Parser::fromJson(
"{ \
@ -263,11 +331,55 @@ TEST_F(IResearchLinkHelperTestSingle, test_normalize_single) {
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE((false == arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), false, sysVocbase)
.ok()));
EXPECT_TRUE((true == !analyzers->get(arangodb::StaticStrings::SystemDatabase +
"::testAnalyzer2")));
EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), true, sysVocbase).ok());
builder.close();
EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1"));
auto expected_json = arangodb::velocypack::Parser::fromJson(
"{ \
\"type\":\"arangosearch\", \
\"primarySort\":[], \
\"fields\":{}, \
\"includeAllFields\": false, \
\"trackListPositions\": false, \
\"storeValues\": \"none\", \
\"analyzerDefinitions\": [ \
{ \"name\": \"testAnalyzer1\", \"type\": \"identity\", \"properties\":{}, \"features\":[] } \
], \
\"analyzers\": [\"testAnalyzer1\" ] \
}");
EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice());
}
// analyzer single-server (inRecovery), not for creation
{
auto json = arangodb::velocypack::Parser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"testAnalyzer1\", \"type\": \"identity\" } ], \
\"analyzers\": [\"testAnalyzer1\" ] \
}");
auto before = StorageEngineMock::recoveryStateResult;
StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS;
auto restore = irs::make_finally(
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::velocypack::Builder builder;
builder.openObject();
EXPECT_TRUE(arangodb::iresearch::IResearchLinkHelper::normalize(
builder, json->slice(), false, sysVocbase).ok());
builder.close();
EXPECT_EQ(nullptr, analyzers->get(arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1"));
auto expected_json = arangodb::velocypack::Parser::fromJson(
"{ \
\"type\":\"arangosearch\", \
\"fields\":{}, \
\"includeAllFields\": false, \
\"trackListPositions\": false, \
\"storeValues\": \"none\", \
\"analyzers\": [\"testAnalyzer1\" ] \
}");
EXPECT_EQUAL_SLICES(expected_json->slice(), builder.slice());
}
}

745
tests/IResearch/IResearchLinkMeta-test.cpp
View File

@ -37,6 +37,7 @@
#include "ApplicationFeatures/GreetingsPhase.h"
#include "ApplicationFeatures/V8Phase.h"
#include "Aql/AqlFunctionFeature.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Cluster/ClusterFeature.h"
#if USE_ENTERPRISE
@ -48,6 +49,8 @@
#include "IResearch/IResearchFeature.h"
#include "IResearch/IResearchLinkMeta.h"
#include "IResearch/VelocyPackHelper.h"
#include "RestServer/AqlFeature.h"
#include "RestServer/TraverserEngineRegistryFeature.h"
#include "RestServer/DatabaseFeature.h"
#include "RestServer/QueryRegistryFeature.h"
#include "RestServer/SystemDatabaseFeature.h"
@ -137,6 +140,9 @@ class IResearchLinkMetaTest : public ::testing::Test {
arangodb::LogLevel::ERR);
// setup required application features
features.emplace_back(new arangodb::TraverserEngineRegistryFeature(server), false); // must be before AqlFeature
features.emplace_back(new arangodb::AqlFeature(server), true);
features.emplace_back(new arangodb::aql::OptimizerRulesFeature(server), true);
features.emplace_back(new arangodb::AuthenticationFeature(server), true);
features.emplace_back(new arangodb::DatabaseFeature(server), false);
features.emplace_back(new arangodb::ShardingFeature(server), false);
@ -237,16 +243,25 @@ class IResearchLinkMetaTest : public ::testing::Test {
TEST_F(IResearchLinkMetaTest, test_defaults) {
arangodb::iresearch::IResearchLinkMeta meta;
{
ASSERT_EQ(1, meta._analyzerDefinitions.size());
EXPECT_TRUE("identity" == (*meta._analyzerDefinitions.begin())->name());
EXPECT_TRUE(irs::flags({irs::norm::type(), irs::frequency::type()}) ==
(*meta._analyzerDefinitions.begin())->features());
EXPECT_NE(nullptr, meta._analyzerDefinitions.begin()->get());
}
EXPECT_TRUE(meta._sort.empty());
EXPECT_TRUE(true == meta._fields.empty());
EXPECT_TRUE(false == meta._includeAllFields);
EXPECT_TRUE(false == meta._trackListPositions);
EXPECT_TRUE((arangodb::iresearch::ValueStorage::NONE == meta._storeValues));
EXPECT_TRUE(arangodb::iresearch::ValueStorage::NONE == meta._storeValues);
EXPECT_TRUE(1U == meta._analyzers.size());
EXPECT_TRUE((*(meta._analyzers.begin())));
EXPECT_TRUE(("identity" == meta._analyzers.begin()->_pool->name()));
EXPECT_TRUE(("identity" == meta._analyzers.begin()->_shortName));
EXPECT_TRUE((irs::flags({irs::norm::type(), irs::frequency::type()}) ==
meta._analyzers.begin()->_pool->features()));
EXPECT_TRUE(*(meta._analyzers.begin()));
EXPECT_TRUE("identity" == meta._analyzers.begin()->_pool->name());
EXPECT_TRUE("identity" == meta._analyzers.begin()->_shortName);
EXPECT_TRUE(irs::flags({irs::norm::type(), irs::frequency::type()}) ==
meta._analyzers.begin()->_pool->features());
EXPECT_TRUE(false == !meta._analyzers.begin()->_pool->get());
}
@ -260,14 +275,15 @@ TEST_F(IResearchLinkMetaTest, test_inheritDefaults) {
analyzers.start();
defaults._fields["abc"] = arangodb::iresearch::IResearchLinkMeta();
defaults._fields["abc"] = arangodb::iresearch::FieldMeta();
defaults._includeAllFields = true;
defaults._trackListPositions = true;
defaults._storeValues = arangodb::iresearch::ValueStorage::FULL;
defaults._storeValues = arangodb::iresearch::ValueStorage::VALUE;
defaults._analyzers.clear();
defaults._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
analyzers.get("testVocbase::empty"), "empty"));
defaults._fields["abc"]->_fields["xyz"] = arangodb::iresearch::IResearchLinkMeta();
defaults._sort.emplace_back(std::vector<arangodb::basics::AttributeName>{{"foo",false}}, true);
auto json = VPackParser::fromJson("{}");
EXPECT_TRUE(true == meta.init(json->slice(), false, tmpString, nullptr, defaults));
@ -301,7 +317,7 @@ TEST_F(IResearchLinkMetaTest, test_inheritDefaults) {
EXPECT_TRUE(true == expectedFields.empty());
EXPECT_TRUE(true == meta._includeAllFields);
EXPECT_TRUE(true == meta._trackListPositions);
EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == meta._storeValues));
EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == meta._storeValues));
EXPECT_TRUE(1U == meta._analyzers.size());
EXPECT_TRUE((*(meta._analyzers.begin())));
@ -310,6 +326,8 @@ TEST_F(IResearchLinkMetaTest, test_inheritDefaults) {
EXPECT_TRUE((irs::flags({irs::frequency::type()}) ==
meta._analyzers.begin()->_pool->features()));
EXPECT_TRUE(false == !meta._analyzers.begin()->_pool->get());
EXPECT_EQ(0, meta._sort.size());
}
TEST_F(IResearchLinkMetaTest, test_readDefaults) {
@ -363,7 +381,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) {
\"c\": { \
\"fields\": { \
\"default\": { \"fields\": {}, \"includeAllFields\": false, \"trackListPositions\": false, \"storeValues\": \"none\", \"analyzers\": [ \"identity\" ] }, \
\"all\": { \"fields\": {\"d\": {}, \"e\": {}}, \"includeAllFields\": true, \"trackListPositions\": true, \"storeValues\": \"full\", \"analyzers\": [ \"empty\" ] }, \
\"all\": { \"fields\": {\"d\": {}, \"e\": {}}, \"includeAllFields\": true, \"trackListPositions\": true, \"storeValues\": \"value\", \"analyzers\": [ \"empty\" ] }, \
\"some\": { \"trackListPositions\": true, \"storeValues\": \"id\" }, \
\"none\": {} \
} \
@ -371,7 +389,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) {
}, \
\"includeAllFields\": true, \
\"trackListPositions\": true, \
\"storeValues\": \"full\", \
\"storeValues\": \"value\", \
\"analyzers\": [ \"empty\", \"identity\" ] \
}");
@ -392,8 +410,8 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) {
"testVocbase");
arangodb::iresearch::IResearchLinkMeta meta;
std::string tmpString;
EXPECT_TRUE((true == meta.init(json->slice(), false, tmpString, &vocbase)));
EXPECT_TRUE((3U == meta._fields.size()));
EXPECT_TRUE(meta.init(json->slice(), false, tmpString, &vocbase));
EXPECT_TRUE(3U == meta._fields.size());
for (auto& field : meta._fields) {
EXPECT_TRUE((1U == expectedFields.erase(field.key())));
@ -421,7 +439,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) {
EXPECT_TRUE((true == (actual._fields.find("e") != actual._fields.end())));
EXPECT_TRUE((true == actual._includeAllFields));
EXPECT_TRUE((true == actual._trackListPositions));
EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == actual._storeValues));
EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == actual._storeValues));
EXPECT_TRUE((1U == actual._analyzers.size()));
EXPECT_TRUE((*(actual._analyzers.begin())));
EXPECT_TRUE(("testVocbase::empty" == actual._analyzers.begin()->_pool->name()));
@ -452,7 +470,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) {
EXPECT_TRUE((true == actual._fields.empty())); // not inherited
EXPECT_TRUE((true == actual._includeAllFields)); // inherited
EXPECT_TRUE((true == actual._trackListPositions)); // inherited
EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == actual._storeValues));
EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == actual._storeValues));
auto itr = actual._analyzers.begin();
EXPECT_TRUE((*itr));
EXPECT_TRUE(("testVocbase::empty" == itr->_pool->name()));
@ -474,7 +492,7 @@ TEST_F(IResearchLinkMetaTest, test_readCustomizedValues) {
EXPECT_TRUE((true == expectedFields.empty()));
EXPECT_TRUE((true == meta._includeAllFields));
EXPECT_TRUE((true == meta._trackListPositions));
EXPECT_TRUE((arangodb::iresearch::ValueStorage::FULL == meta._storeValues));
EXPECT_TRUE((arangodb::iresearch::ValueStorage::VALUE == meta._storeValues));
auto itr = meta._analyzers.begin();
EXPECT_TRUE((*itr));
EXPECT_TRUE(("testVocbase::empty" == itr->_pool->name()));
@ -644,13 +662,16 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) {
meta._includeAllFields = true;
meta._trackListPositions = true;
meta._storeValues = arangodb::iresearch::ValueStorage::FULL;
meta._storeValues = arangodb::iresearch::ValueStorage::VALUE;
meta._analyzerDefinitions.clear();
meta._analyzerDefinitions.emplace(analyzers.get("identity"));
meta._analyzerDefinitions.emplace(analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"));
meta._analyzers.clear();
meta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
analyzers.get("identity"), "identity"));
meta._analyzers.emplace_back(arangodb::iresearch::IResearchLinkMeta::Analyzer(
analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"),
"enmpty"));
"empty"));
meta._fields["a"] = meta; // copy from meta
meta._fields["a"]->_fields.clear(); // do not inherit fields to match jSon inheritance
meta._fields["b"] = meta; // copy from meta
@ -779,7 +800,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) {
tmpSlice = slice.get("trackListPositions");
EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool()));
tmpSlice = slice.get("storeValues");
EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString()));
EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString()));
tmpSlice = slice.get("analyzers");
EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length()));
@ -894,7 +915,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) {
tmpSlice = slice.get("trackListPositions");
EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool()));
tmpSlice = slice.get("storeValues");
EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString()));
EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString()));
tmpSlice = slice.get("analyzers");
EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length()));
@ -1036,7 +1057,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) {
tmpSlice = slice.get("trackListPositions");
EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool()));
tmpSlice = slice.get("storeValues");
EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString()));
EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString()));
tmpSlice = slice.get("analyzers");
EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length()));
@ -1149,7 +1170,7 @@ TEST_F(IResearchLinkMetaTest, test_writeCustomizedValues) {
tmpSlice = slice.get("trackListPositions");
EXPECT_TRUE((true == tmpSlice.isBool() && true == tmpSlice.getBool()));
tmpSlice = slice.get("storeValues");
EXPECT_TRUE((true == tmpSlice.isString() && std::string("full") == tmpSlice.copyString()));
EXPECT_TRUE((true == tmpSlice.isString() && std::string("value") == tmpSlice.copyString()));
tmpSlice = slice.get("analyzers");
EXPECT_TRUE((true == tmpSlice.isArray() && 2 == tmpSlice.length()));
@ -1219,7 +1240,7 @@ TEST_F(IResearchLinkMetaTest, test_readMaskAll) {
\"fields\": { \"a\": {} }, \
\"includeAllFields\": true, \
\"trackListPositions\": true, \
\"storeValues\": \"full\", \
\"storeValues\": \"value\", \
\"analyzers\": [] \
}");
EXPECT_TRUE(true == meta.init(json->slice(), false, tmpString, nullptr,
@ -1380,6 +1401,31 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
EXPECT_EQ(std::string("analyzerDefinitions[0].type"), errorField);
}
// missing analyzer definition single-server
{
auto json = VPackParser::fromJson(
"{ \
\"analyzers\": [ \"missing0\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ("analyzers.missing0", errorField);
}
// missing analyzer (full) single-server (ignore analyzer definition)
{
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"missing0\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \
\"analyzers\": [ \"missing0\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase));
EXPECT_EQ("analyzers.missing0", errorField);
}
// missing analyzer (full) single-server
{
auto json = VPackParser::fromJson(
@ -1397,7 +1443,133 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ(std::string("missing0"), meta._analyzers[0]._shortName);
EXPECT_EQ("missing0", meta._analyzers[0]._shortName);
}
// complex definition on single-server
{
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ("testVocbase::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ("_system::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// complex definition on single-server
{
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
ASSERT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ("testVocbase::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
// definition from cache since it's not presented "analyzerDefinitions"
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ("_system::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// missing analyzer definition coordinator
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzers\": [ \"missing1\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ("analyzers.missing1", errorField);
}
// missing analyzer definition coordinator
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"fields\" : { \"field\": { \"analyzers\": [ \"missing1\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ("fields.field.analyzers.missing1", errorField);
}
// missing analyzer (full) coordinator
@ -1415,8 +1587,138 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase)));
EXPECT_EQ(std::string("analyzerDefinitions[0]"), errorField);
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::missing1", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("missing1", meta._analyzers[0]._shortName);
}
// missing analyzer (full) coordinator (ignore analyzer definition)
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"missing1\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \
\"analyzers\": [ \"missing1\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase));
EXPECT_EQ("analyzers.missing1", errorField);
}
// complex definition on coordinator
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ(std::string("testVocbase::empty"), analyzer._pool->name());
EXPECT_EQ(std::string("empty"), analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ(std::string("_system::empty"), analyzer._pool->name());
EXPECT_EQ(std::string("empty"), analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// complex definition on coordinator
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ("testVocbase::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
// definition from cache since it's not presented "analyzerDefinitions"
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ("_system::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// missing analyzer (full) db-server
@ -1434,16 +1736,156 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE((true == meta.init(json->slice(), true, errorField, &vocbase)));
EXPECT_TRUE((1 == meta._analyzers.size()));
EXPECT_TRUE(
(std::string("testVocbase::missing2") == meta._analyzers[0]._pool->name()));
EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._pool->type()));
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::missing2", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_TRUE((1 == meta._analyzers[0]._pool->features().size()));
EXPECT_TRUE((true == meta._analyzers[0]._pool->features().check(irs::frequency::type())));
EXPECT_TRUE((std::string("missing2") == meta._analyzers[0]._shortName));
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("missing2", meta._analyzers[0]._shortName);
}
// missing analyzer (full) db-server (ignore analyzer definition)
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"missing2\", \"type\": \"empty\", \"properties\": { \"args\": \"ru\" }, \"features\": [ \"frequency\" ] } ], \
\"analyzers\": [ \"missing2\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase));
EXPECT_EQ("analyzers.missing2", errorField);
}
// missing analyzer definition db-server
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzers\": [ \"missing2\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ("analyzers.missing2", errorField);
}
// complex definition on db-server
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ(std::string("testVocbase::empty"), analyzer._pool->name());
EXPECT_EQ(std::string("empty"), analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ(std::string("_system::empty"), analyzer._pool->name());
EXPECT_EQ(std::string("empty"), analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// complex definition on db-server
{
auto before = arangodb::ServerState::instance()->getRole();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_DBSERVER);
auto restore = irs::make_finally([&before]() -> void {
arangodb::ServerState::instance()->setRole(before);
});
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ("testVocbase::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
// definition from cache since it's not presented "analyzerDefinitions"
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ("_system::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// missing analyzer (full) inRecovery
@ -1459,8 +1901,48 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase)));
EXPECT_EQ(std::string("analyzers.missing3"), errorField); // not in the persisted collection
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::missing3", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("missing3", meta._analyzers[0]._shortName);
}
// missing analyzer (full) inRecovery (ignore analyzer definition)
{
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"missing3\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \
\"analyzers\": [ \"missing3\" ] \
}");
auto before = StorageEngineMock::recoveryStateResult;
StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS;
auto restore = irs::make_finally(
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), false, errorField, &vocbase));
EXPECT_EQ("analyzers.missing3", errorField);
}
// missing analyzer definition inRecovery
{
auto json = VPackParser::fromJson(
"{ \
\"analyzers\": [ \"missing3\" ] \
}");
auto before = StorageEngineMock::recoveryStateResult;
StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS;
auto restore = irs::make_finally(
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_FALSE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ("analyzers.missing3", errorField);
}
// existing analyzer (name only)
@ -1507,6 +1989,108 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._shortName));
}
// complex definition inRecovery
{
auto before = StorageEngineMock::recoveryStateResult;
StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS;
auto restore = irs::make_finally(
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] }, \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ(std::string("testVocbase::empty"), analyzer._pool->name());
EXPECT_EQ(std::string("empty"), analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ(std::string("_system::empty"), analyzer._pool->name());
EXPECT_EQ(std::string("empty"), analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// complex definition inRecovery
{
auto before = StorageEngineMock::recoveryStateResult;
StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS;
auto restore = irs::make_finally(
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ \
{ \"name\": \"::empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } \
], \
\"fields\" : { \"field\": { \"analyzers\": [ \"testVocbase::empty\", \"empty\", \"_system::empty\", \"::empty\" ] } } \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(2, meta._analyzerDefinitions.size());
EXPECT_EQ(1, meta._analyzers.size());
{
auto& analyzer = meta._analyzers[0];
EXPECT_EQ(arangodb::iresearch::IResearchAnalyzerFeature::identity(), analyzer._pool);
}
ASSERT_EQ(1, meta._fields.size());
ASSERT_EQ(2, meta._fields.begin().value()->_analyzers.size());
{
auto& analyzer = meta._fields.begin().value()->_analyzers[0];
EXPECT_EQ("testVocbase::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
// definition from cache since it's not presented "analyzerDefinitions"
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
{
auto& analyzer = meta._fields.begin().value()->_analyzers[1];
EXPECT_EQ("_system::empty", analyzer._pool->name());
EXPECT_EQ("empty", analyzer._pool->type());
EXPECT_EQUAL_SLICES(VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
analyzer._pool->properties());
EXPECT_EQ(1, analyzer._pool->features().size());
EXPECT_TRUE(analyzer._pool->features().check(irs::frequency::type()));
EXPECT_EQ("::empty", analyzer._shortName);
EXPECT_EQ((*meta._analyzerDefinitions.find(analyzer._pool->name())), analyzer._pool);
}
}
// existing analyzer (full) analyzer creation not allowed (pass)
{
auto json = VPackParser::fromJson(
@ -1575,16 +2159,16 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE((true == meta.init(json->slice(), true, errorField, &vocbase)));
EXPECT_TRUE((1 == meta._analyzers.size()));
EXPECT_TRUE((std::string("testVocbase::empty") == meta._analyzers[0]._pool->name()));
EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._pool->type()));
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
EXPECT_EQUAL_SLICES(
VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_TRUE((1 == meta._analyzers[0]._pool->features().size()));
EXPECT_TRUE((true == meta._analyzers[0]._pool->features().check(irs::frequency::type())));
EXPECT_TRUE((std::string("empty") == meta._analyzers[0]._shortName));
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", meta._analyzers[0]._shortName);
}
// existing analyzer (definition mismatch)
@ -1596,8 +2180,38 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase)));
EXPECT_EQ(std::string("analyzerDefinitions[0]"), errorField);
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
EXPECT_EQUAL_SLICES(
VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", meta._analyzers[0]._shortName);
}
// existing analyzer (definition mismatch), don't read definitions
{
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \
\"analyzers\": [ \"empty\" ] \
}");
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), false, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
// read definition from cache since we ignore "analyzerDefinitions"
EXPECT_EQUAL_SLICES(
VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", meta._analyzers[0]._shortName);
}
// existing analyzer (definition mismatch) inRecovery
@ -1613,8 +2227,43 @@ TEST_F(IResearchLinkMetaTest, test_readAnalyzerDefinitions) {
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE((false == meta.init(json->slice(), true, errorField, &vocbase)));
EXPECT_EQ(std::string("analyzerDefinitions[0]"), errorField);
EXPECT_TRUE(meta.init(json->slice(), true, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
// read definition from cache since we ignore "analyzerDefinitions"
EXPECT_EQUAL_SLICES(
VPackParser::fromJson("{\"args\" : \"ru\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", meta._analyzers[0]._shortName);
}
// existing analyzer (definition mismatch) inRecovery, don't read definitions
{
auto json = VPackParser::fromJson(
"{ \
\"analyzerDefinitions\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": {\"args\":\"ru\"}, \"features\": [ \"frequency\" ] } ], \
\"analyzers\": [ \"empty\" ] \
}");
auto before = StorageEngineMock::recoveryStateResult;
StorageEngineMock::recoveryStateResult = arangodb::RecoveryState::IN_PROGRESS;
auto restore = irs::make_finally(
[&before]() -> void { StorageEngineMock::recoveryStateResult = before; });
arangodb::iresearch::IResearchLinkMeta meta;
std::string errorField;
EXPECT_TRUE(meta.init(json->slice(), false, errorField, &vocbase));
EXPECT_EQ(1, meta._analyzers.size());
EXPECT_EQ("testVocbase::empty", meta._analyzers[0]._pool->name());
EXPECT_EQ("empty", meta._analyzers[0]._pool->type());
// read definition from cache since we ignore "analyzerDefinitions"
EXPECT_EQUAL_SLICES(
VPackParser::fromJson("{\"args\" : \"de\"}")->slice(),
meta._analyzers[0]._pool->properties());
EXPECT_EQ(1, meta._analyzers[0]._pool->features().size());
EXPECT_TRUE(meta._analyzers[0]._pool->features().check(irs::frequency::type()));
EXPECT_EQ("empty", meta._analyzers[0]._shortName);
}
}
@ -1658,7 +2307,7 @@ TEST_F(IResearchLinkMetaTest, test_addNonUniqueAnalyzers) {
"{ \
\"includeAllFields\": true, \
\"trackListPositions\": true, \
\"storeValues\": \"full\",\
\"storeValues\": \"value\",\
\"analyzers\": [ \"identity\",\"identity\"" // two built-in analyzers
", \"" +
analyzerCustomInTestVocbase + "\"" + // local analyzer by full name

259
tests/IResearch/IResearchView-test.cpp
View File

@ -28,6 +28,7 @@
#include "ExpressionContextMock.h"
#include "analysis/token_attributes.hpp"
#include "analysis/analyzers.hpp"
#include "search/scorers.hpp"
#include "utils/locale_utils.hpp"
#include "utils/log.hpp"
@ -39,6 +40,7 @@
#include "Aql/AstNode.h"
#include "Aql/Function.h"
#include "Aql/SortCondition.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Basics/ArangoGlobalContext.h"
#include "Basics/error.h"
#include "Basics/files.h"
@ -130,6 +132,88 @@ struct DocIdScorer: public irs::sort {
REGISTER_SCORER_TEXT(DocIdScorer, DocIdScorer::make);
struct TestAttribute : public irs::attribute {
DECLARE_ATTRIBUTE_TYPE();
};
DEFINE_ATTRIBUTE_TYPE(TestAttribute);
REGISTER_ATTRIBUTE(TestAttribute); // required to open reader on segments with analized fields
struct TestTermAttribute : public irs::term_attribute {
public:
void value(irs::bytes_ref const& value) { value_ = value; }
};
class TestAnalyzer : public irs::analysis::analyzer {
public:
DECLARE_ANALYZER_TYPE();
TestAnalyzer() : irs::analysis::analyzer(TestAnalyzer::type()) {
_attrs.emplace(_term);
_attrs.emplace(_attr);
_attrs.emplace(_increment); // required by field_data::invert(...)
}
virtual irs::attribute_view const& attributes() const noexcept override {
return _attrs;
}
static ptr make(irs::string_ref const& args) {
auto slice = arangodb::iresearch::slice(args);
if (slice.isNull()) throw std::exception();
if (slice.isNone()) return nullptr;
PTR_NAMED(TestAnalyzer, ptr);
return ptr;
}
static bool normalize(irs::string_ref const& args, std::string& definition) {
// same validation as for make,
// as normalize usually called to sanitize data before make
auto slice = arangodb::iresearch::slice(args);
if (slice.isNull()) throw std::exception();
if (slice.isNone()) return false;
arangodb::velocypack::Builder builder;
if (slice.isString()) {
VPackObjectBuilder scope(&builder);
arangodb::iresearch::addStringRef(builder, "args",
arangodb::iresearch::getStringRef(slice));
} else if (slice.isObject() && slice.hasKey("args") && slice.get("args").isString()) {
VPackObjectBuilder scope(&builder);
arangodb::iresearch::addStringRef(builder, "args",
arangodb::iresearch::getStringRef(slice.get("args")));
} else {
return false;
}
definition = builder.buffer()->toString();
return true;
}
virtual bool next() override {
if (_data.empty()) return false;
_term.value(irs::bytes_ref(_data.c_str(), 1));
_data = irs::bytes_ref(_data.c_str() + 1, _data.size() - 1);
return true;
}
virtual bool reset(irs::string_ref const& data) override {
_data = irs::ref_cast<irs::byte_type>(data);
return true;
}
private:
irs::attribute_view _attrs;
irs::bytes_ref _data;
irs::increment _increment;
TestTermAttribute _term;
TestAttribute _attr;
};
DEFINE_ANALYZER_TYPE_NAMED(TestAnalyzer, "TestAnalyzer");
REGISTER_ANALYZER_VPACK(TestAnalyzer, TestAnalyzer::make, TestAnalyzer::normalize);
}
// -----------------------------------------------------------------------------
@ -191,6 +275,7 @@ protected:
buildFeatureEntry(new arangodb::DatabaseFeature(server), false);
buildFeatureEntry(new arangodb::DatabasePathFeature(server), false);
buildFeatureEntry(new arangodb::TraverserEngineRegistryFeature(server), false);
buildFeatureEntry(new arangodb::aql::OptimizerRulesFeature(server), true);
buildFeatureEntry(new arangodb::AqlFeature(server), true);
buildFeatureEntry(new arangodb::aql::AqlFunctionFeature(server), true); // required for IResearchAnalyzerFeature
buildFeatureEntry(new arangodb::iresearch::IResearchAnalyzerFeature(server), true);
@ -6903,3 +6988,177 @@ TEST_F(IResearchViewTest, test_update_partial) {
}
}
}
TEST_F(IResearchViewTest, test_remove_referenced_analyzer) {
auto* databaseFeature = arangodb::application_features::ApplicationServer::getFeature<arangodb::DatabaseFeature>("Database");
ASSERT_NE(nullptr, databaseFeature);
TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature
ASSERT_TRUE(databaseFeature->createDatabase(0, "testDatabase" TOSTRING(__LINE__), vocbase).ok());
ASSERT_NE(nullptr, vocbase);
// create _analyzers collection
{
auto createJson = arangodb::velocypack::Parser::fromJson(
"{ \"name\": \"" + arangodb::StaticStrings::AnalyzersCollection + "\", \"isSystem\":true }");
ASSERT_NE(nullptr, vocbase->createCollection(createJson->slice()));
}
auto* analyzers = arangodb::application_features::ApplicationServer::lookupFeature<
arangodb::iresearch::IResearchAnalyzerFeature>();
ASSERT_NE(nullptr, analyzers);
std::shared_ptr<arangodb::LogicalView> view;
std::shared_ptr<arangodb::LogicalCollection> collection;
// remove existing (used by link)
{
// add analyzer
{
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
ASSERT_TRUE(analyzers->emplace(result, vocbase->name() + "::test_analyzer3",
"TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok());
ASSERT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
}
// create collection
{
auto createJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection1\" }");
collection = vocbase->createCollection(createJson->slice());
ASSERT_NE(nullptr, collection);
}
// create view
{
auto createJson = arangodb::velocypack::Parser::fromJson(
"{ \"name\": \"testView\", \"type\": \"arangosearch\" }");
view = vocbase->createView(createJson->slice());
ASSERT_NE(nullptr, view);
auto updateJson = arangodb::velocypack::Parser::fromJson(
"{ \"links\": { \"testCollection1\": { \"includeAllFields\": true, \"analyzers\":[\"test_analyzer3\"] }}}");
EXPECT_TRUE(view->properties(updateJson->slice(), true).ok());
}
EXPECT_FALSE(analyzers->remove(vocbase->name() + "::test_analyzer3", false).ok()); // used by link
EXPECT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
EXPECT_TRUE(analyzers->remove(vocbase->name() + "::test_analyzer3", true).ok());
EXPECT_EQ(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
auto cleanup = arangodb::scopeGuard([vocbase, &view, &collection]() {
if (view) {
EXPECT_TRUE(vocbase->dropView(view->id(), false).ok());
view = nullptr;
}
if (collection) {
EXPECT_TRUE(vocbase->dropCollection(collection->id(), false, 1.0).ok());
collection = nullptr;
}
});
}
// remove existing (used by link)
{
// add analyzer
{
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
ASSERT_TRUE(analyzers->emplace(result, vocbase->name() + "::test_analyzer3",
"TestAnalyzer", VPackParser::fromJson("\"abc\"")->slice()).ok());
ASSERT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
}
// create collection
{
auto createJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection1\" }");
collection = vocbase->createCollection(createJson->slice());
ASSERT_NE(nullptr, collection);
}
// create view
{
auto createJson = arangodb::velocypack::Parser::fromJson(
"{ \"name\": \"testView\", \"type\": \"arangosearch\" }");
view = vocbase->createView(createJson->slice());
ASSERT_NE(nullptr, view);
auto updateJson = arangodb::velocypack::Parser::fromJson(
"{ \"analyzerDefinitions\" : { \
\"name\":\"test_analyzer3\", \"features\":[], \
\"type\":\"TestAnalyzer\", \"properties\": {\"args\":\"abc\"} \
}, \
\"links\": { \"testCollection1\": { \"includeAllFields\": true, \"analyzers\":[\"test_analyzer3\"] }} \
}");
EXPECT_TRUE(view->properties(updateJson->slice(), true).ok());
}
EXPECT_FALSE(analyzers->remove(vocbase->name() + "::test_analyzer3", false).ok()); // used by link
EXPECT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
EXPECT_TRUE(analyzers->remove(vocbase->name() + "::test_analyzer3", true).ok());
EXPECT_EQ(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
auto cleanup = arangodb::scopeGuard([vocbase, &view, &collection]() {
if (view) {
EXPECT_TRUE(vocbase->dropView(view->id(), false).ok());
view = nullptr;
}
if (collection) {
EXPECT_TRUE(vocbase->dropCollection(collection->id(), false, 1.0).ok());
collection = nullptr;
}
});
}
// remove existing (properties don't match
{
// add analyzer
{
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
ASSERT_TRUE(analyzers->emplace(result, vocbase->name() + "::test_analyzer3",
"TestAnalyzer", VPackParser::fromJson("\"abcd\"")->slice()).ok());
ASSERT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
}
// create collection
{
auto createJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection1\" }");
collection = vocbase->createCollection(createJson->slice());
ASSERT_NE(nullptr, collection);
}
// create view
{
auto createJson = arangodb::velocypack::Parser::fromJson(
"{ \"name\": \"testView\", \"type\": \"arangosearch\" }");
view = vocbase->createView(createJson->slice());
ASSERT_NE(nullptr, view);
auto updateJson = arangodb::velocypack::Parser::fromJson(
"{ \"analyzerDefinitions\" : { \
\"name\":\"test_analyzer3\", \"features\":[], \
\"type\":\"TestAnalyzer\", \"properties\": \"abc\" \
}, \
\"links\": { \"testCollection1\": { \"includeAllFields\": true, \"analyzers\":[\"test_analyzer3\"] }} \
}");
EXPECT_TRUE(view->properties(updateJson->slice(), true).ok());
}
EXPECT_FALSE(analyzers->remove(vocbase->name() + "::test_analyzer3", false).ok()); // used by link (we ignore analyzerDefinitions on single-server)
EXPECT_NE(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
EXPECT_TRUE(analyzers->remove(vocbase->name() + "::test_analyzer3", true).ok());
EXPECT_EQ(nullptr, analyzers->get(vocbase->name() + "::test_analyzer3"));
auto cleanup = arangodb::scopeGuard([vocbase, &view, &collection]() {
if (view) {
EXPECT_TRUE(vocbase->dropView(view->id(), false).ok());
view = nullptr;
}
if (collection) {
EXPECT_TRUE(vocbase->dropCollection(collection->id(), false, 1.0).ok());
collection = nullptr;
}
});
}
}

2
tests/IResearch/common.cpp
View File

@ -469,7 +469,7 @@ std::string mangleString(std::string name, std::string suffix) {
}
std::string mangleStringIdentity(std::string name) {
arangodb::iresearch::kludge::mangleStringField(name, arangodb::iresearch::IResearchLinkMeta::Analyzer() // args
arangodb::iresearch::kludge::mangleStringField(name, arangodb::iresearch::FieldMeta::Analyzer() // args
);
return name;

135
tests/V8Server/v8-analyzers-test.cpp
View File

@ -27,6 +27,7 @@
#include "../IResearch/common.h"
#include "../Mocks/StorageEngineMock.h"
#include "Aql/OptimizerRulesFeature.h"
#include "Aql/QueryRegistry.h"
#include "gtest/gtest.h"
@ -38,9 +39,11 @@
#include "IResearch/IResearchAnalyzerFeature.h"
#include "IResearch/IResearchCommon.h"
#include "IResearch/VelocyPackHelper.h"
#include "RestServer/AqlFeature.h"
#include "RestServer/DatabaseFeature.h"
#include "RestServer/QueryRegistryFeature.h"
#include "RestServer/SystemDatabaseFeature.h"
#include "RestServer/TraverserEngineRegistryFeature.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "VocBase/Methods/Collections.h"
#include "Utils/ExecContext.h"
@ -203,12 +206,30 @@ TEST_F(V8AnalyzersTest, test_accessors) {
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchAnalyzerFeature* analyzers;
arangodb::DatabaseFeature* dbFeature;
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
arangodb::QueryRegistryFeature* queryRegistry;
arangodb::AqlFeature* aqlFeature;
arangodb::aql::OptimizerRulesFeature* optimizer;
arangodb::TraverserEngineRegistryFeature* traverser;
server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server));
server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature
server.addFeature(aqlFeature = new arangodb::AqlFeature(server));
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); });
traverser->prepare();
aqlFeature->start();
optimizer->prepare();
queryRegistry->prepare();
auto cleanup = arangodb::scopeGuard([&](){
dbFeature->unprepare();
optimizer->unprepare();
aqlFeature->stop();
queryRegistry->unprepare();
traverser->unprepare();
});
// create system vocbase
{
@ -225,10 +246,8 @@ TEST_F(V8AnalyzersTest, test_accessors) {
}
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
ASSERT_TRUE((analyzers
->emplace(result, arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"identity", VPackSlice::noneSlice())
.ok()));
ASSERT_TRUE(analyzers->emplace(result, arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
"identity", VPackSlice::noneSlice()).ok());
auto analyzer = analyzers->get(arangodb::StaticStrings::SystemDatabase +
"::testAnalyzer1");
ASSERT_TRUE((false == !analyzer));
@ -242,8 +261,7 @@ TEST_F(V8AnalyzersTest, test_accessors) {
arangodb::ExecContextScope execContextScope(&execContext);
auto* authFeature = arangodb::AuthenticationFeature::instance();
auto* userManager = authFeature->userManager();
arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB()
userManager->setQueryRegistry(&queryRegistry);
userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry());
auto resetUserManager = std::shared_ptr<arangodb::auth::UserManager>(
userManager,
[](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); });
@ -699,13 +717,31 @@ TEST_F(V8AnalyzersTest, test_create) {
arangodb::iresearch::IResearchAnalyzerFeature* analyzers;
arangodb::DatabaseFeature* dbFeature;
arangodb::SystemDatabaseFeature* sysDatabase;
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
arangodb::QueryRegistryFeature* queryRegistry;
arangodb::AqlFeature* aqlFeature;
arangodb::aql::OptimizerRulesFeature* optimizer;
arangodb::TraverserEngineRegistryFeature* traverser;
server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server));
server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature
server.addFeature(aqlFeature = new arangodb::AqlFeature(server));
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
server.addFeature(sysDatabase = new arangodb::SystemDatabaseFeature(server)); // required for IResearchAnalyzerFeature::start()
auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); });
traverser->prepare();
aqlFeature->start();
optimizer->prepare();
queryRegistry->prepare();
auto cleanup = arangodb::scopeGuard([&](){
dbFeature->unprepare();
optimizer->unprepare();
aqlFeature->stop();
queryRegistry->unprepare();
traverser->unprepare();
});
// create system vocbase
{
@ -746,8 +782,7 @@ TEST_F(V8AnalyzersTest, test_create) {
arangodb::ExecContextScope execContextScope(&execContext);
auto* authFeature = arangodb::AuthenticationFeature::instance();
auto* userManager = authFeature->userManager();
arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB()
userManager->setQueryRegistry(&queryRegistry);
userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry());
auto resetUserManager = std::shared_ptr<arangodb::auth::UserManager>(
userManager,
[](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); });
@ -1404,13 +1439,32 @@ TEST_F(V8AnalyzersTest, test_get) {
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
arangodb::iresearch::IResearchAnalyzerFeature* analyzers;
arangodb::DatabaseFeature* dbFeature;
arangodb::QueryRegistryFeature* queryRegistry;
arangodb::AqlFeature* aqlFeature;
arangodb::aql::OptimizerRulesFeature* optimizer;
arangodb::TraverserEngineRegistryFeature* traverser;
server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server));
server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature
server.addFeature(aqlFeature = new arangodb::AqlFeature(server));
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
analyzers->prepare(); // add static analyzers
auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); });
traverser->prepare();
aqlFeature->start();
optimizer->prepare();
queryRegistry->prepare();
auto cleanup = arangodb::scopeGuard([&](){
dbFeature->unprepare();
optimizer->unprepare();
aqlFeature->stop();
queryRegistry->unprepare();
traverser->unprepare();
});
// create system vocbase
{
@ -1449,8 +1503,7 @@ TEST_F(V8AnalyzersTest, test_get) {
arangodb::ExecContextScope execContextScope(&execContext);
auto* authFeature = arangodb::AuthenticationFeature::instance();
auto* userManager = authFeature->userManager();
arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB()
userManager->setQueryRegistry(&queryRegistry);
userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry());
auto resetUserManager = std::shared_ptr<arangodb::auth::UserManager>(
userManager,
[](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); });
@ -2068,12 +2121,32 @@ TEST_F(V8AnalyzersTest, test_list) {
arangodb::iresearch::IResearchAnalyzerFeature* analyzers;
arangodb::DatabaseFeature* dbFeature;
arangodb::SystemDatabaseFeature* sysDatabase;
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
arangodb::QueryRegistryFeature* queryRegistry;
arangodb::AqlFeature* aqlFeature;
arangodb::aql::OptimizerRulesFeature* optimizer;
arangodb::TraverserEngineRegistryFeature* traverser;
server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server));
server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature
server.addFeature(aqlFeature = new arangodb::AqlFeature(server));
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(sysDatabase = new arangodb::SystemDatabaseFeature(server)); // required for IResearchAnalyzerFeature::start()
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
traverser->prepare();
aqlFeature->start();
optimizer->prepare();
queryRegistry->prepare();
auto cleanup = arangodb::scopeGuard([&](){
dbFeature->unprepare();
optimizer->unprepare();
aqlFeature->stop();
queryRegistry->unprepare();
traverser->unprepare();
});
// create system vocbase
{
auto const databases = arangodb::velocypack::Parser::fromJson(
@ -2094,8 +2167,6 @@ TEST_F(V8AnalyzersTest, test_list) {
arangodb::tests::AnalyzerCollectionName, false);
}
auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); });
arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
ASSERT_TRUE((analyzers
->emplace(result, arangodb::StaticStrings::SystemDatabase + "::testAnalyzer1",
@ -2115,8 +2186,7 @@ TEST_F(V8AnalyzersTest, test_list) {
arangodb::ExecContextScope execContextScope(&execContext);
auto* authFeature = arangodb::AuthenticationFeature::instance();
auto* userManager = authFeature->userManager();
arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB()
userManager->setQueryRegistry(&queryRegistry);
userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry());
auto resetUserManager = std::shared_ptr<arangodb::auth::UserManager>(
userManager,
[](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); });
@ -2536,13 +2606,31 @@ TEST_F(V8AnalyzersTest, test_remove) {
arangodb::iresearch::IResearchAnalyzerFeature* analyzers;
arangodb::DatabaseFeature* dbFeature;
arangodb::SystemDatabaseFeature* sysDbFeature(new arangodb::SystemDatabaseFeature(server));
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
arangodb::QueryRegistryFeature* queryRegistry;
arangodb::AqlFeature* aqlFeature;
arangodb::aql::OptimizerRulesFeature* optimizer;
arangodb::TraverserEngineRegistryFeature* traverser;
server.addFeature(queryRegistry = new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
server.addFeature(optimizer = new arangodb::aql::OptimizerRulesFeature(server));
server.addFeature(traverser = new arangodb::TraverserEngineRegistryFeature(server) ); // must be before AqlFeature
server.addFeature(aqlFeature = new arangodb::AqlFeature(server));
server.addFeature(new arangodb::V8DealerFeature(server)); // required for DatabaseFeature::createDatabase(...)
server.addFeature(dbFeature = new arangodb::DatabaseFeature(server)); // required for IResearchAnalyzerFeature::emplace(...)
server.addFeature(analyzers = new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for running upgrade task
server.addFeature(sysDbFeature);
auto cleanup = arangodb::scopeGuard([dbFeature](){ dbFeature->unprepare(); });
traverser->prepare();
aqlFeature->start();
optimizer->prepare();
queryRegistry->prepare();
auto cleanup = arangodb::scopeGuard([&](){
dbFeature->unprepare();
optimizer->unprepare();
aqlFeature->stop();
queryRegistry->unprepare();
traverser->unprepare();
});
// create system vocbase
{
@ -2601,8 +2689,7 @@ TEST_F(V8AnalyzersTest, test_remove) {
arangodb::ExecContextScope execContextScope(&execContext);
auto* authFeature = arangodb::AuthenticationFeature::instance();
auto* userManager = authFeature->userManager();
arangodb::aql::QueryRegistry queryRegistry(0); // required for UserManager::loadFromDB()
userManager->setQueryRegistry(&queryRegistry);
userManager->setQueryRegistry(arangodb::QueryRegistryFeature::registry());
auto resetUserManager = std::shared_ptr<arangodb::auth::UserManager>(
userManager,
[](arangodb::auth::UserManager* ptr) -> void { ptr->removeAllUsers(); });

48
tests/js/common/aql/aql-view-arangosearch-ddl-cluster.js
View File

@ -316,7 +316,7 @@ function IResearchFeatureDDLTestSuite () {
var meta = { links: {
"TestCollection0": { },
"TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "full" },
"TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "value" },
"TestCollection2": { fields: {
"b": { fields: { "b1": {} } },
"c": { includeAllFields: true },
@ -352,7 +352,7 @@ function IResearchFeatureDDLTestSuite () {
assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor);
assertEqual(true, properties.links.TestCollection1.trackListPositions);
assertTrue(String === properties.links.TestCollection1.storeValues.constructor);
assertEqual("full", properties.links.TestCollection1.storeValues);
assertEqual("value", properties.links.TestCollection1.storeValues);
assertTrue(Array === properties.links.TestCollection1.analyzers.constructor);
assertEqual(1, properties.links.TestCollection1.analyzers.length);
assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor);
@ -405,7 +405,7 @@ function IResearchFeatureDDLTestSuite () {
assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor);
assertEqual(true, properties.links.TestCollection1.trackListPositions);
assertTrue(String === properties.links.TestCollection1.storeValues.constructor);
assertEqual("full", properties.links.TestCollection1.storeValues);
assertEqual("value", properties.links.TestCollection1.storeValues);
assertTrue(Array === properties.links.TestCollection1.analyzers.constructor);
assertEqual(1, properties.links.TestCollection1.analyzers.length);
assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor);
@ -1114,7 +1114,47 @@ function IResearchFeatureDDLTestSuite () {
assertNull(analyzers.analyzer(analyzerName));
// this should be no name conflict
analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"});
db._useDatabase("_system");
db._dropDatabase(dbName);
},
testIndexAnalyzerCollection : function() {
const dbName = "TestNameDroppedDB";
const analyzerName = "TestAnalyzer";
db._useDatabase("_system");
assertNotEqual(null, db._collection("_analyzers"));
try { db._dropDatabase(dbName); } catch (e) {}
try { analyzers.remove(analyzerName); } catch (e) {}
assertEqual(0, db._analyzers.count());
db._createDatabase(dbName);
db._useDatabase(dbName);
analyzers.save(analyzerName, "identity");
// recreating database
db._useDatabase("_system");
db._dropDatabase(dbName);
db._createDatabase(dbName);
db._useDatabase(dbName);
assertNull(analyzers.analyzer(analyzerName));
// this should be no name conflict
analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"});
assertEqual(1, db._analyzers.count());
var view = db._createView("analyzersView", "arangosearch", {
links: {
_analyzers : {
includeAllFields:true,
analyzers: [ analyzerName ]
}
}
});
var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray();
assertEqual(1, db._analyzers.count());
assertEqual(1, res.length);
assertEqual(db._analyzers.toArray()[0], res[0]);
db._useDatabase("_system");
db._dropDatabase(dbName);
}

51
tests/js/common/aql/aql-view-arangosearch-ddl-noncluster.js
View File

@ -316,7 +316,7 @@ function IResearchFeatureDDLTestSuite () {
var meta = { links: {
"TestCollection0": { },
"TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "full" },
"TestCollection1": { analyzers: [ "text_en"], includeAllFields: true, trackListPositions: true, storeValues: "value" },
"TestCollection2": { fields: {
"b": { fields: { "b1": {} } },
"c": { includeAllFields: true },
@ -352,7 +352,7 @@ function IResearchFeatureDDLTestSuite () {
assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor);
assertEqual(true, properties.links.TestCollection1.trackListPositions);
assertTrue(String === properties.links.TestCollection1.storeValues.constructor);
assertEqual("full", properties.links.TestCollection1.storeValues);
assertEqual("value", properties.links.TestCollection1.storeValues);
assertTrue(Array === properties.links.TestCollection1.analyzers.constructor);
assertEqual(1, properties.links.TestCollection1.analyzers.length);
assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor);
@ -405,7 +405,7 @@ function IResearchFeatureDDLTestSuite () {
assertTrue(Boolean === properties.links.TestCollection1.trackListPositions.constructor);
assertEqual(true, properties.links.TestCollection1.trackListPositions);
assertTrue(String === properties.links.TestCollection1.storeValues.constructor);
assertEqual("full", properties.links.TestCollection1.storeValues);
assertEqual("value", properties.links.TestCollection1.storeValues);
assertTrue(Array === properties.links.TestCollection1.analyzers.constructor);
assertEqual(1, properties.links.TestCollection1.analyzers.length);
assertTrue(String === properties.links.TestCollection1.analyzers[0].constructor);
@ -1114,7 +1114,50 @@ function IResearchFeatureDDLTestSuite () {
assertNull(analyzers.analyzer(analyzerName));
// this should be no name conflict
analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"});
db._useDatabase("_system");
db._dropDatabase(dbName);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test link on analyzers collection
////////////////////////////////////////////////////////////////////////////////
testIndexAnalyzerCollection : function() {
const dbName = "TestNameDroppedDB";
const analyzerName = "TestAnalyzer";
db._useDatabase("_system");
assertNotEqual(null, db._collection("_analyzers"));
try { db._dropDatabase(dbName); } catch (e) {}
try { analyzers.remove(analyzerName); } catch (e) {}
assertEqual(0, db._analyzers.count());
db._createDatabase(dbName);
db._useDatabase(dbName);
analyzers.save(analyzerName, "identity");
// recreating database
db._useDatabase("_system");
db._dropDatabase(dbName);
db._createDatabase(dbName);
db._useDatabase(dbName);
assertNull(analyzers.analyzer(analyzerName));
// this should be no name conflict
analyzers.save(analyzerName, "text", {"stopwords" : [], "locale":"en"});
assertEqual(1, db._analyzers.count());
var view = db._createView("analyzersView", "arangosearch", {
links: {
_analyzers : {
includeAllFields:true,
analyzers: [ analyzerName ]
}
}
});
var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray();
assertEqual(1, db._analyzers.count());
assertEqual(1, res.length);
assertEqual(db._analyzers.toArray()[0], res[0]);
db._useDatabase("_system");
db._dropDatabase(dbName);
}

32
tests/js/common/aql/aql-view-arangosearch-feature.js
View File

@ -56,38 +56,53 @@ function iResearchFeatureAqlTestSuite () {
testAnalyzersInvalidPropertiesDiscarded : function() {
{
try {analyzers.remove("normPropAnalyzer"); } catch (e) {}
assertEqual(0, db._analyzers.count());
let analyzer = analyzers.save("normPropAnalyzer", "norm", { "locale":"en", "invalid_param":true});
assertEqual(1, db._analyzers.count());
assertTrue(null != analyzer);
assertTrue(null == analyzer.properties.invalid_param);
analyzers.remove("normPropAnalyzer", true);
assertEqual(0, db._analyzers.count());
}
{
try {analyzers.remove("textPropAnalyzer"); } catch (e) {}
assertEqual(0, db._analyzers.count());
let analyzer = analyzers.save("textPropAnalyzer", "text", {"stopwords" : [], "locale":"en", "invalid_param":true});
assertEqual(1, db._analyzers.count());
assertTrue(null != analyzer);
assertTrue(null == analyzer.properties.invalid_param);
analyzers.remove("textPropAnalyzer", true);
assertEqual(0, db._analyzers.count());
}
{
try {analyzers.remove("delimiterPropAnalyzer"); } catch (e) {}
assertEqual(0, db._analyzers.count());
let analyzer = analyzers.save("delimiterPropAnalyzer", "delimiter", { "delimiter":"|", "invalid_param":true});
assertEqual(1, db._analyzers.count());
assertTrue(null != analyzer);
assertTrue(null == analyzer.properties.invalid_param);
analyzers.remove("delimiterPropAnalyzer", true);
assertEqual(0, db._analyzers.count());
}
{
try {analyzers.remove("stemPropAnalyzer"); } catch (e) {}
assertEqual(0, db._analyzers.count());
let analyzer = analyzers.save("stemPropAnalyzer", "stem", { "locale":"en", "invalid_param":true});
assertEqual(1, db._analyzers.count());
assertTrue(null != analyzer);
assertTrue(null == analyzer.properties.invalid_param);
analyzers.remove("stemPropAnalyzer", true);
assertEqual(0, db._analyzers.count());
}
{
try {analyzers.remove("ngramPropAnalyzer"); } catch (e) {}
assertEqual(0, db._analyzers.count());
let analyzer = analyzers.save("ngramPropAnalyzer", "ngram", { "min":1, "max":5, "preserveOriginal":true, "invalid_param":true});
assertEqual(1, db._analyzers.count());
assertTrue(null != analyzer);
assertTrue(null == analyzer.properties.invalid_param);
analyzers.remove("ngramPropAnalyzer", true);
assertEqual(0, db._analyzers.count());
}
},
testAnalyzerRemovalWithDatabaseName_InSystem: function() {
@ -203,6 +218,8 @@ function iResearchFeatureAqlTestSuite () {
let oldListInCollection = db._analyzers.toArray();
assertTrue(Array === oldList.constructor);
assertEqual(0, db._analyzers.count());
// creation
analyzers.save("testAnalyzer", "stem", { "locale":"en"}, [ "frequency" ]);
@ -216,12 +233,27 @@ function iResearchFeatureAqlTestSuite () {
assertTrue(Array === analyzer.features().constructor);
assertEqual(1, analyzer.features().length);
assertEqual([ "frequency" ], analyzer.features());
// check persisted analyzer
assertEqual(1, db._analyzers.count());
let savedAnalyzer = db._analyzers.toArray()[0];
assertTrue(null !== savedAnalyzer);
assertEqual(7, Object.keys(savedAnalyzer).length);
assertEqual("_analyzers/testAnalyzer", savedAnalyzer._id);
assertEqual("testAnalyzer", savedAnalyzer._key);
assertEqual("testAnalyzer", savedAnalyzer.name);
assertEqual("stem", savedAnalyzer.type);
assertEqual(analyzer.properties(), savedAnalyzer.properties);
assertEqual(analyzer.features(), savedAnalyzer.features);
analyzer = undefined; // release reference
// check the analyzers collection in database
assertEqual(oldListInCollection.length + 1, db._analyzers.toArray().length);
let dbAnalyzer = db._query("FOR d in _analyzers FILTER d.name=='testAnalyzer' RETURN d").toArray();
assertEqual(1, dbAnalyzer.length);
assertEqual("_analyzers/testAnalyzer", dbAnalyzer[0]._id);
assertEqual("testAnalyzer", dbAnalyzer[0]._key);
assertEqual("testAnalyzer", dbAnalyzer[0].name);
assertEqual("stem", dbAnalyzer[0].type);
assertEqual(1, Object.keys(dbAnalyzer[0].properties).length);

33
tests/js/server/aql/aql-modify.js
View File

@ -1127,6 +1127,39 @@ function aqlUpsertOptionsSuite() {
validateDocsAreUpdated(docs, invalid, true);
},
testUpsertEmptyObject: function () {
let NEW, OLD, res;
col.insert({ "value1": "find me" });
// before "find me" was found because the empty object
// was ignored now we will insert instead
let q1 = `UPSERT { "value1" : "find me", "value2" : {} }
INSERT { "value1" : "find me", "value2" : {} }
UPDATE { "value1" : "not gonna happen" }
IN ${collectionName}
RETURN [$OLD, $NEW]`;
res = db._query(q1).toArray();
OLD = res[0][0];
NEW = res[0][1];
assertEqual(NEW.value1, "find me");
assertEqual(NEW.value2, {});
// now we update the exact doucment
let q2 = `UPSERT { "value1" : "find me", "value2" : {} }
INSERT { "value1" : "not gonna happen" }
UPDATE { "value1" : "update" }
IN ${collectionName}
RETURN [$OLD, $NEW]`;
res = db._query(q2).toArray();
OLD = res[0][0];
NEW = res[0][1];
assertEqual(NEW.value1, "update");
assertEqual(NEW.value2, {});
},
/* We cannot yet solve this. If you need to ensure _rev value checks put them in the UPDATE {} clause
testUpsertSingleWithInvalidRevInMatch : function () {
const invalid = genInvalidValue();

10
tests/js/server/dump/dump-mmfiles-cluster.js
View File

@ -732,6 +732,16 @@ function dumpTestEnterpriseSuite () {
assertEqual(6, p.numberOfShards);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test link on analyzers collection
////////////////////////////////////////////////////////////////////////////////
testIndexAnalyzerCollection : function() {
var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray();
assertEqual(1, db._analyzers.count());
assertEqual(1, res.length);
assertEqual(db._analyzers.toArray()[0], res[0]);
},
testViewOnSmartEdgeCollection : function() {
try {
db._createView("check", "arangosearch", {});

12
tests/js/server/dump/dump-mmfiles.js
View File

@ -488,6 +488,7 @@ function dumpTestSuite () {
/// @brief test custom analyzers restoring
////////////////////////////////////////////////////////////////////////////////
testAnalyzers: function() {
assertNotEqual(null, db._collection("_analyzers"));
assertEqual(1, db._analyzers.count()); // only 1 stored custom analyzer
let analyzer = analyzers.analyzer("custom");
@ -497,8 +498,17 @@ function dumpTestSuite () {
assertEqual(" ", analyzer.properties().delimiter);
assertEqual(1, analyzer.features().length);
assertEqual("frequency", analyzer.features()[0]);
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test link on analyzers collection
////////////////////////////////////////////////////////////////////////////////
testIndexAnalyzerCollection : function() {
var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray();
assertEqual(1, db._analyzers.count());
assertEqual(1, res.length);
assertEqual(db._analyzers.toArray()[0], res[0]);
}
};
}

12
tests/js/server/dump/dump-rocksdb.js
View File

@ -453,6 +453,7 @@ function dumpTestSuite () {
/// @brief test custom analyzers restoring
////////////////////////////////////////////////////////////////////////////////
testAnalyzers: function() {
assertNotEqual(null, db._collection("_analyzers"));
assertEqual(1, db._analyzers.count()); // only 1 stored custom analyzer
let analyzer = analyzers.analyzer("custom");
@ -462,8 +463,17 @@ function dumpTestSuite () {
assertEqual(" ", analyzer.properties().delimiter);
assertEqual(1, analyzer.features().length);
assertEqual("frequency", analyzer.features()[0]);
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test link on analyzers collection
////////////////////////////////////////////////////////////////////////////////
testIndexAnalyzerCollection : function() {
var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray();
assertEqual(1, db._analyzers.count());
assertEqual(1, res.length);
assertEqual(db._analyzers.toArray()[0], res[0]);
}
};
}

12
tests/js/server/dump/dump-setup-cluster.js
View File

@ -287,6 +287,18 @@ function setupSatelliteCollections() {
c.save({ value: -1, text: "the red foxx jumps over the pond" });
} catch (err) { }
// setup a view on _analyzers collection
try {
let view = db._createView("analyzersView", "arangosearch", {
links: {
_analyzers : {
includeAllFields:true,
analyzers: [ analyzer.name ]
}
}
});
} catch (err) { }
setupSmartGraph();
setupSmartArangoSearch();
setupSatelliteCollections();

12
tests/js/server/dump/dump-setup.js
View File

@ -259,6 +259,18 @@
c.save({ value: -1, text: "the red foxx jumps over the pond" });
} catch (err) { }
// setup a view on _analyzers collection
try {
let view = db._createView("analyzersView", "arangosearch", {
links: {
_analyzers : {
includeAllFields: true,
analyzers: [ analyzer.name ]
}
}
});
} catch (err) { }
// Install Foxx
const fs = require('fs');
const SERVICE_PATH = fs.makeAbsolute(fs.join(

45
tests/js/server/replication/sync/replication-sync.js
View File

@ -930,6 +930,51 @@ function BaseTestConfig () {
}
},
testSyncViewOnAnalyzersCollection: function () {
connectToMaster();
// create custom analyzer
assertNotEqual(null, db._collection("_analyzers"));
assertEqual(0, db._analyzers.count());
let analyzer = analyzers.save('custom', 'delimiter', { delimiter: ' ' }, ['frequency']);
assertEqual(1, db._analyzers.count());
// create view & collection on master
db._flushCache();
db._createView('analyzersView', 'arangosearch', {
consolidationIntervalMsec:0,
links: { _analyzers: { analyzers: [ analyzer.name ], includeAllFields:true } }
});
db._flushCache();
connectToSlave();
internal.wait(0.1, false);
// sync on slave
replication.sync({ endpoint: masterEndpoint });
db._flushCache();
assertEqual(1, db._analyzers.count());
var res = db._query("FOR d IN analyzersView OPTIONS {waitForSync:true} RETURN d").toArray();
assertEqual(1, db._analyzers.count());
assertEqual(1, res.length);
assertEqual(db._analyzers.toArray()[0], res[0]);
{
// check state is the same
let view = db._view('analyzersView');
assertTrue(view !== null);
let props = view.properties();
assertTrue(props.hasOwnProperty('links'));
assertEqual(0, props.consolidationIntervalMsec);
assertEqual(Object.keys(props.links).length, 1);
assertTrue(props.links.hasOwnProperty('_analyzers'));
assertTrue(props.links['_analyzers'].includeAllFields);
assertTrue(props.links['_analyzers'].analyzers.length, 1);
assertTrue(props.links['_analyzers'].analyzers[0], 'custom');
}
},
// //////////////////////////////////////////////////////////////////////////////
// / @brief test with edges
// //////////////////////////////////////////////////////////////////////////////