issue 526.5: add analyzer retrieval functionality (#8592)

arangod/Aql/RemoteExecutor.cpp

@@ -26,6 +26,7 @@
 #include "Aql/ExecutorInfos.h"
 #include "Aql/WakeupQueryCallback.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/RecursiveLocker.h"
 #include "Basics/StringBuffer.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/ClusterComm.h"
@@ -390,7 +391,14 @@ Result ExecutionBlockImpl<RemoteExecutor>::sendAsyncRequest(
 
   // Make sure to cover against the race that this
   // Request is fullfilled before the register has taken place
-  MUTEX_LOCKER(locker, _communicationMutex);
+  // @note the only reason for not using recursive mutext always is due to the
+  //       concern that there might be recursive calls in production
+  #ifdef ARANGODB_USE_CATCH_TESTS
+    RECURSIVE_MUTEX_LOCKER(_communicationMutex, _communicationMutexOwner);
+  #else
+    MUTEX_LOCKER(locker, _communicationMutex);
+  #endif
+
   // We can only track one request at a time.
   // So assert there is no other request in flight!
   TRI_ASSERT(_lastTicketId == 0);
@@ -405,7 +413,14 @@ bool ExecutionBlockImpl<RemoteExecutor>::handleAsyncResult(ClusterCommResult* re
   // So we cannot have the response being produced while sending the request.
   // Make sure to cover against the race that this
   // Request is fullfilled before the register has taken place
-  MUTEX_LOCKER(locker, _communicationMutex);
+  // @note the only reason for not using recursive mutext always is due to the
+  //       concern that there might be recursive calls in production
+  #ifdef ARANGODB_USE_CATCH_TESTS
+    RECURSIVE_MUTEX_LOCKER(_communicationMutex, _communicationMutexOwner);
+  #else
+    MUTEX_LOCKER(locker, _communicationMutex);
+  #endif
+
   if (_lastTicketId == result->operationID) {
     // TODO Handle exceptions thrown while we are in this code
     // Query will not be woken up again.

arangod/Aql/RemoteExecutor.h

@@ -135,6 +135,7 @@ class ExecutionBlockImpl<RemoteExecutor> : public ExecutionBlock {
   /// @brief Mutex to cover against the race, that a getSome request
   ///        is responded before the ticket id is registered.
   arangodb::Mutex _communicationMutex;
+  std::atomic<std::thread::id> _communicationMutexOwner; // current thread owning '_communicationMutex' lock (workaround for non-recusrive MutexLocker)
 
   OperationID _lastTicketId;
 

arangod/Cluster/ClusterComm.h

@@ -504,7 +504,7 @@ class ClusterComm {
   /// @brief submit a single HTTP request to a shard synchronously.
   //////////////////////////////////////////////////////////////////////////////
 
-  std::unique_ptr<ClusterCommResult> syncRequest(
+  TEST_VIRTUAL std::unique_ptr<ClusterCommResult> syncRequest(
       CoordTransactionID const coordTransactionID, std::string const& destination,
       rest::RequestType reqtype, std::string const& path, std::string const& body,
       std::unordered_map<std::string, std::string> const& headerFields,

arangod/Cluster/ClusterInfo.cpp

@@ -27,6 +27,7 @@
 #include "Basics/ConditionLocker.h"
 #include "Basics/Exceptions.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/RecursiveLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
@@ -53,64 +54,6 @@
 #include <velocypack/Slice.h>
 #include <velocypack/velocypack-aliases.h>
 
-namespace {
-
-// identical code to RecursiveWriteLocker in vocbase.cpp except for type
-template <typename T>
-class RecursiveMutexLocker {
- public:
-  RecursiveMutexLocker(T& mutex, std::atomic<std::thread::id>& owner,
-                       arangodb::basics::LockerType type, bool acquire,
-                       char const* file, int line)
-      : _locker(&mutex, type, false, file, line), _owner(owner), _update(noop) {
-    if (acquire) {
-      lock();
-    }
-  }
-
-  ~RecursiveMutexLocker() { unlock(); }
-
-  bool isLocked() { return _locker.isLocked(); }
-
-  void lock() {
-    // recursive locking of the same instance is not yet supported (create a new
-    // instance instead)
-    TRI_ASSERT(_update != owned);
-
-    if (std::this_thread::get_id() != _owner.load()) {  // not recursive
-      _locker.lock();
-      _owner.store(std::this_thread::get_id());
-      _update = owned;
-    }
-  }
-
-  void unlock() { _update(*this); }
-
- private:
-  arangodb::basics::MutexLocker<T> _locker;
-  std::atomic<std::thread::id>& _owner;
-  void (*_update)(RecursiveMutexLocker& locker);
-
-  static void noop(RecursiveMutexLocker&) {}
-  static void owned(RecursiveMutexLocker& locker) {
-    static std::thread::id unowned;
-    locker._owner.store(unowned);
-    locker._locker.unlock();
-    locker._update = noop;
-  }
-};
-
-#define NAME__(name, line) name##line
-#define NAME_EXPANDER__(name, line) NAME__(name, line)
-#define NAME(name) NAME_EXPANDER__(name, __LINE__)
-#define RECURSIVE_MUTEX_LOCKER_NAMED(name, lock, owner, acquire)        \
-  RecursiveMutexLocker<typename std::decay<decltype(lock)>::type> name( \
-      lock, owner, arangodb::basics::LockerType::BLOCKING, acquire, __FILE__, __LINE__)
-#define RECURSIVE_MUTEX_LOCKER(lock, owner) \
-  RECURSIVE_MUTEX_LOCKER_NAMED(NAME(RecursiveLocker), lock, owner, true)
-
-}  // namespace
-
 #ifdef _WIN32
 // turn off warnings about too long type name for debug symbols blabla in MSVC
 // only...

arangod/IResearch/IResearchAnalyzerFeature.cpp

@@ -36,6 +36,8 @@
 #include "ApplicationServerHelper.h"
 #include "Aql/AqlFunctionFeature.h"
 #include "Aql/ExpressionContext.h"
+#include "Aql/Query.h"
+#include "Aql/QueryString.h"
 #include "Basics/StaticStrings.h"
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/ServerState.h"
@@ -43,6 +45,7 @@
 #include "IResearchCommon.h"
 #include "Logger/LogMacros.h"
 #include "RestServer/DatabaseFeature.h"
+#include "RestServer/QueryRegistryFeature.h"
 #include "RestServer/SystemDatabaseFeature.h"
 #include "RestServer/UpgradeFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"
@@ -65,6 +68,7 @@ static char const ANALYZER_PREFIX_DELIM = ':'; // name prefix delimiter (2 chars
 static size_t const DEFAULT_POOL_SIZE = 8;  // arbitrary value
 static std::string const FEATURE_NAME("IResearchAnalyzer");
 static irs::string_ref const IDENTITY_ANALYZER_NAME("identity");
+static auto const RELOAD_INTERVAL = std::chrono::minutes(60); // arbitrary value
 
 struct IdentityValue : irs::term_attribute {
   void value(irs::bytes_ref const& data) noexcept { value_ = data; }
@@ -228,7 +232,7 @@ arangodb::aql::AqlValue aqlFnTokens(arangodb::aql::ExpressionContext* expression
   while (analyzer->next()) {
     auto value = irs::ref_cast<char>(values->value());
 
-    builder.add(arangodb::iresearch::toValuePair(value));
+    arangodb::iresearch::addStringRef(builder, value);
   }
 
   builder.close();
@@ -260,22 +264,17 @@ void addFunctions(arangodb::aql::AqlFunctionFeature& functions) {
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief ensure configuration collection is present in the specified vocbase
+/// @return pool will generate analyzers as per supplied parameters
 ////////////////////////////////////////////////////////////////////////////////
-void ensureConfigCollection(TRI_vocbase_t& vocbase) {
+bool equalAnalyzer(
-  static const std::string json =
+   arangodb::iresearch::IResearchAnalyzerFeature::AnalyzerPool const& pool, // analyzer
-      std::string("{\"isSystem\": true, \"name\": \"") +
+   irs::string_ref const& type, // analyzer type
-      ANALYZER_COLLECTION_NAME + "\", \"type\": 2}";
+   irs::string_ref const& properties, // analyzer properties
-
+   irs::flags const& features // analyzer features
-  if (!arangodb::ServerState::instance()->isCoordinator()) {
+) noexcept {
-    try {
+  return type == pool.type() // same type
-      vocbase.createCollection(arangodb::velocypack::Parser::fromJson(json)->slice());
+         && properties == pool.properties() // same properties
-    } catch (arangodb::basics::Exception& e) {
+         && features == pool.features(); // same features
-      if (TRI_ERROR_ARANGO_DUPLICATE_NAME != e.code()) {
-        throw;
-      }
-    }
-  }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -296,6 +295,13 @@ arangodb::SystemDatabaseFeature::ptr getSystemDatabase() {
   return database->use();
 }
 
+std::string normalizedAnalyzerName(
+    std::string database, // database
+    irs::string_ref const& analyzer // analyzer
+) {
+  return database.append(2, ANALYZER_PREFIX_DELIM).append(analyzer);
+}
+
 bool iresearchAnalyzerLegacyAnalyzers( // upgrade task
     TRI_vocbase_t& vocbase, // upgraded vocbase
     arangodb::velocypack::Slice const& upgradeParams // upgrade params
@@ -368,10 +374,9 @@ bool iresearchAnalyzerLegacyAnalyzers( // upgrade task
     static const irs::string_ref legacyAnalyzerType("text");
     bool success = true;
 
-    // reguster each legacy static analyzer with the current vocbase
+    // register each legacy static analyzer with the current vocbase
     for (auto& entry: legacyAnalzyers) {
-      auto name = // prefix name with vocbase
+      auto name = normalizedAnalyzerName(vocbase.name(), entry.first);
-        std::string(vocbase.name()).append(2, ANALYZER_PREFIX_DELIM).append(entry.first);
       auto& type = legacyAnalyzerType;
       auto& properties = entry.second;
       arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
@@ -468,28 +473,81 @@ std::pair<irs::string_ref, irs::string_ref> splitAnalyzerName( // split name
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-/// @brief validate that features are supported by arangod an ensure that their
+/// @brief read analyzers from vocbase
-///        dependencies are met
+/// @return visitation completed fully
 ////////////////////////////////////////////////////////////////////////////////
-arangodb::Result validateFeatures(irs::flags const& features) {
+arangodb::Result visitAnalyzers( // visit analyzers
-  for(auto& feature: features) {
+  TRI_vocbase_t& vocbase, // vocbase to visit
-    if (&irs::frequency::type() == feature) {
+  std::function<arangodb::Result(arangodb::velocypack::Slice const& slice)> const& visitor // visitor
-      // no extra validation required
+) {
-    } else if (&irs::norm::type() == feature) {
+  if (arangodb::ServerState::instance()->isClusterRole()) {
-      // no extra validation required
+    static const auto queryString = arangodb::aql::QueryString( // query to execute
-    } else if (&irs::position::type() == feature) {
+      std::string("FOR d IN ") + ANALYZER_COLLECTION_NAME + " RETURN d" // query
-      if (!features.check(irs::frequency::type())) {
+    );
-        return arangodb::Result( // result
+    arangodb::aql::Query query( // query
-          TRI_ERROR_BAD_PARAMETER, // code
+      false, vocbase, queryString, nullptr, nullptr, arangodb::aql::PART_MAIN // args
-          std::string("missing feature '") + std::string(irs::frequency::type().name()) +"' required when '" + std::string(feature->name()) + "' feature is specified"
+    );
-        );
+    auto* queryRegistry = arangodb::QueryRegistryFeature::registry();
-      }
+    auto result = query.executeSync(queryRegistry);
-    } else if (feature) {
+
+    if (TRI_ERROR_NO_ERROR != result.code) {
+      return arangodb::Result(result.code, result.details);
+    }
+
+    auto slice = result.result->slice();
+
+    if (!slice.isArray()) {
       return arangodb::Result( // result
-        TRI_ERROR_BAD_PARAMETER, // code
+        TRI_ERROR_INTERNAL, // code
-        std::string("unsupported analyzer feature '") + std::string(feature->name()) + "'" // value
+        std::string("failed to parse contents of collection '") + ANALYZER_COLLECTION_NAME + "' in database '" + vocbase.name() + " while visiting analyzers"
       );
     }
+
+    for (arangodb::velocypack::ArrayIterator itr(slice); itr.valid(); ++itr) {
+      auto res = visitor(itr.value().resolveExternal());
+
+      if (!res.ok()) {
+        return res;
+      }
+    }
+
+    return arangodb::Result();
+  }
+
+  arangodb::OperationOptions options;
+  arangodb::SingleCollectionTransaction trx( // transaction
+    arangodb::transaction::StandaloneContext::Create(vocbase), // context
+    ANALYZER_COLLECTION_NAME, // collection
+    arangodb::AccessMode::Type::READ // access more
+  );
+  auto res = trx.begin();
+
+  if (!res.ok()) {
+    return res;
+  }
+
+  auto commit  = irs::make_finally([&trx]()->void { trx.commit(); }); // end read-only transaction
+  auto result = trx.all(ANALYZER_COLLECTION_NAME, 0, 0, options);
+
+  if (!result.result.ok()) {
+    return result.result;
+  }
+
+  auto slice = arangodb::velocypack::Slice(result.buffer->data());
+
+  if (!slice.isArray()) {
+    return arangodb::Result( // result
+      TRI_ERROR_INTERNAL, // code
+      std::string("failed to parse contents of collection '") + ANALYZER_COLLECTION_NAME + "' in database '" + vocbase.name() + " while visiting analyzers"
+    );
+  }
+
+  for (arangodb::velocypack::ArrayIterator itr(slice); itr.valid(); ++itr) {
+    auto res = visitor(itr.value().resolveExternal());
+
+    if (!res.ok()) {
+      return res;
+    }
   }
 
   return arangodb::Result();
@@ -656,8 +714,7 @@ IResearchAnalyzerFeature::IResearchAnalyzerFeature(arangodb::application_feature
 }
 
 /*static*/ bool IResearchAnalyzerFeature::canUse( // check permissions
-  irs::string_ref const& analyzer, // analyzer name
+  irs::string_ref const& name, // analyzer name (already normalized)
-  TRI_vocbase_t const& defaultVocbase, // fallback vocbase if not part of name
   arangodb::auth::Level const& level // access level
 ) {
   auto* ctx = arangodb::ExecContext::CURRENT;
@@ -668,25 +725,11 @@ IResearchAnalyzerFeature::IResearchAnalyzerFeature(arangodb::application_feature
 
   auto& staticAnalyzers = getStaticAnalyzers();
 
-  if (staticAnalyzers.find(irs::make_hashed_ref(analyzer, std::hash<irs::string_ref>())) != staticAnalyzers.end()) {
+  if (staticAnalyzers.find(irs::make_hashed_ref(name, std::hash<irs::string_ref>())) != staticAnalyzers.end()) {
     return true; // special case for singleton static analyzers (always allowed)
   }
 
-  auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
+  auto split = splitAnalyzerName(name);
-    arangodb::SystemDatabaseFeature // featue type
-  >();
-  auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
-  std::pair<irs::string_ref, irs::string_ref> split;
-
-  if (sysVocbase) {
-    split = splitAnalyzerName( // split analyzer name
-      arangodb::iresearch::IResearchAnalyzerFeature::normalize( // normalize
-        analyzer, defaultVocbase, *sysVocbase // args
-      )
-    );
-  } else {
-    split = splitAnalyzerName(analyzer);
-  }
 
   return !split.first.null() // have a vocbase
     && ctx->canUseDatabase(split.first, level) // can use vocbase
@@ -835,6 +878,90 @@ arangodb::Result IResearchAnalyzerFeature::emplace( // emplace an analyzer
   return ensure(result, name, type, properties, features, true);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief validate analyzer parameters and emplace into map
+////////////////////////////////////////////////////////////////////////////////
+arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace
+  EmplaceAnalyzerResult& result, // emplacement result on success (out-param)
+  Analyzers& analyzers, // analyzers
+  irs::string_ref const& name, // analyzer name
+  irs::string_ref const& type, // analyzer type
+  irs::string_ref const& properties, // analyzer properties
+  irs::flags const& features // analyzer features
+) {
+  // validate that features are supported by arangod an ensure that their
+  // dependencies are met
+  for(auto& feature: features) {
+    if (&irs::frequency::type() == feature) {
+      // no extra validation required
+    } else if (&irs::norm::type() == feature) {
+      // no extra validation required
+    } else if (&irs::position::type() == feature) {
+      if (!features.check(irs::frequency::type())) {
+        return arangodb::Result( // result
+          TRI_ERROR_BAD_PARAMETER, // code
+          std::string("missing feature '") + std::string(irs::frequency::type().name()) +"' required when '" + std::string(feature->name()) + "' feature is specified"
+        );
+      }
+    } else if (feature) {
+      return arangodb::Result( // result
+        TRI_ERROR_BAD_PARAMETER, // code
+        std::string("unsupported analyzer feature '") + std::string(feature->name()) + "'" // value
+      );
+    }
+  }
+
+  static const auto generator = []( // key + value generator
+    irs::hashed_string_ref const& key, // source key
+    AnalyzerPool::ptr const& value // source value
+  )->irs::hashed_string_ref {
+    auto pool = std::make_shared<AnalyzerPool>(key); // allocate pool
+    const_cast<AnalyzerPool::ptr&>(value) = pool; // lazy-instantiate pool to avoid allocation if pool is already present
+    return pool ? irs::hashed_string_ref(key.hash(), pool->name()) : key; // reuse hash but point ref at value in pool
+  };
+  auto itr = irs::map_utils::try_emplace_update_key( // emplace and update key
+    analyzers, // destination
+    generator, // key generator
+    irs::make_hashed_ref(name, std::hash<irs::string_ref>()) // key
+  );
+  auto analyzer = itr.first->second;
+
+  if (!analyzer) {
+    return arangodb::Result( // result
+      TRI_ERROR_BAD_PARAMETER, // code
+      std::string("failure creating an arangosearch analyzer instance for name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "'"
+    );
+  }
+
+  // new analyzer creation, validate
+  if (itr.second) {
+    bool erase = true; // potentially invalid insertion took place
+    auto cleanup = irs::make_finally([&erase, &analyzers, &itr]()->void {
+      if (erase) {
+        analyzers.erase(itr.first); // ensure no broken analyzers are left behind
+      }
+    });
+
+    if (!analyzer->init(type, properties, features)) {
+      return arangodb::Result( // result
+        TRI_ERROR_BAD_PARAMETER, // code
+        std::string("failure initializing an arangosearch analyzer instance for name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "'"
+      );
+    }
+
+    erase = false;
+  } else if (!equalAnalyzer(*analyzer, type, properties, features)) { // duplicate analyzer with different configuration
+    return arangodb::Result( // result
+      TRI_ERROR_BAD_PARAMETER, // code
+      std::string("name collision detected while registering an arangosearch analizer name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "', previous registration type '" + std::string(analyzer->type()) + "' properties '" + std::string(analyzer->properties()) + "'"
+    );
+  }
+
+  result = itr;
+
+  return arangodb::Result();
+}
+
 IResearchAnalyzerFeature::AnalyzerPool::ptr IResearchAnalyzerFeature::ensure( // get analyzer or placeholder
     irs::string_ref const& name // analyzer name
 ) {
@@ -855,28 +982,26 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence
   bool allowCreation
 ) {
   try {
-    static const auto generator = []( // key + value generator
+    WriteMutex mutex(_mutex);
-      irs::hashed_string_ref const& key, // source key
+    SCOPED_LOCK(mutex);
-      AnalyzerPool::ptr const& value // source value
+    auto split = splitAnalyzerName(name);
-    )->irs::hashed_string_ref {
+
-      auto pool = std::make_shared<AnalyzerPool>(key); // allocate pool
+    if (!split.first.null()) { // do not trigger load for static-analyzer requests
-      const_cast<AnalyzerPool::ptr&>(value) = pool; // lazy-instantiate pool to avoid allocation if pool is already present
+      auto res = loadAnalyzers(split.first);
-      return pool ? irs::hashed_string_ref(key.hash(), pool->name()) : key; // reuse hash but point ref at value in pool
+
-    };
+      if (!res.ok()) {
-    auto res = validateFeatures(features);
+        return res;
+      }
+    }
+
+    EmplaceAnalyzerResult itr;
+    auto res = // validate and emplace an analyzer
+      emplaceAnalyzer(itr, _analyzers, name, type, properties, features);
 
     if (!res.ok()) {
       return res;
     }
 
-    WriteMutex mutex(_mutex);
-    SCOPED_LOCK(mutex);
-
-    auto itr = irs::map_utils::try_emplace_update_key( // emplace and update key
-      _analyzers, // destination
-      generator, // key generator
-      irs::make_hashed_ref(name, std::hash<irs::string_ref>()) // key
-    );
     bool erase = itr.second; // an insertion took place
     auto cleanup = irs::make_finally([&erase, this, &itr]()->void {
       if (erase) {
@@ -885,22 +1010,8 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence
     });
     auto pool = itr.first->second;
 
-    if (!pool) {
-      return arangodb::Result( // result
-        TRI_ERROR_BAD_PARAMETER, // code
-        std::string("failure creating an arangosearch analyzer instance for name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "'"
-      );
-    }
-
     // new pool creation
     if (itr.second) {
-      if (!pool->init(type, properties, features)) {
-        return arangodb::Result( // result
-          TRI_ERROR_BAD_PARAMETER, // code
-          std::string("failure initializing an arangosearch analyzer instance for name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "'"
-        );
-      }
-
       if (!allowCreation) {
         return arangodb::Result( // result
           TRI_ERROR_BAD_PARAMETER, // code
@@ -908,6 +1019,13 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence
         );
       }
 
+      if (!pool) {
+        return arangodb::Result( // result
+          TRI_ERROR_INTERNAL, // code
+          std::string("failure creating an arangosearch analyzer instance for name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "'"
+        );
+      }
+
       // persist only on coordinator and single-server
       res = arangodb::ServerState::instance()->isCoordinator() // coordinator
             || arangodb::ServerState::instance()->isSingleServer() // single-server
@@ -921,17 +1039,6 @@ arangodb::Result IResearchAnalyzerFeature::ensure( // ensure analyzer existence
       return res;
     }
 
-    // pool exists but with different configuration
-    if (type != pool->type() // different type
-        || properties != pool->properties() // different properties
-        || features != pool->features() // different features
-       ) {
-      return arangodb::Result( // result
-        TRI_ERROR_BAD_PARAMETER, // code
-        std::string("name collision detected while registering an arangosearch analizer name '") + std::string(name) + "' type '" + std::string(type) + "' properties '" + std::string(properties) + "', previous registration type '" + std::string(pool->type()) + "' properties '" + std::string(pool->properties()) + "'"
-      );
-    }
-
     result = std::make_pair(pool, itr.second);
   } catch (arangodb::basics::Exception const& e) {
     return arangodb::Result( // result
@@ -957,6 +1064,21 @@ IResearchAnalyzerFeature::AnalyzerPool::ptr IResearchAnalyzerFeature::get( // fi
     irs::string_ref const& name // analyzer name
 ) const noexcept {
   try {
+    auto split = splitAnalyzerName(name);
+
+    if (!split.first.null()) { // do not trigger load for static-analyzer requests
+      auto res = // load analyzers for database
+        const_cast<IResearchAnalyzerFeature*>(this)->loadAnalyzers(split.first);
+
+      if (!res.ok()) {
+        LOG_TOPIC("36062", WARN, arangodb::iresearch::TOPIC)
+          << "failure to load analyzers for database '" << split.first << "' while getting analyzer '" << name << "': " << res.errorNumber() << " " << res.errorMessage();
+        TRI_set_errno(res.errorNumber());
+
+        return nullptr;
+      }
+    }
+
     ReadMutex mutex(_mutex);
     SCOPED_LOCK(mutex);
     auto itr =
@@ -1143,6 +1265,357 @@ IResearchAnalyzerFeature::AnalyzerPool::ptr IResearchAnalyzerFeature::get( // fi
   return identity.instance;
 }
 
+arangodb::Result IResearchAnalyzerFeature::loadAnalyzers( // load
+  irs::string_ref const& database /*= irs::string_ref::NIL*/ // database to load
+) {
+  try {
+    auto* dbFeature = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
+      arangodb::DatabaseFeature // feature type
+    >("Database");
+
+    if (!dbFeature) {
+      return arangodb::Result( // result
+        TRI_ERROR_INTERNAL, // code
+        std::string("failure to find feature 'Database' while loading analyzers for database '") + std::string(database)+ "'"
+      );
+    }
+
+    WriteMutex mutex(_mutex);
+    SCOPED_LOCK(mutex); // '_analyzers'/'_lastLoad' can be asynchronously read
+
+    // load all databases
+    if (database.null()) {
+      arangodb::Result res;
+      std::unordered_set<irs::hashed_string_ref> seen;
+      auto visitor = [this, &res, &seen](TRI_vocbase_t& vocbase)->void {
+        if (!vocbase.lookupCollection(ANALYZER_COLLECTION_NAME)) {
+          return; // skip databases lacking 'ANALYZER_COLLECTION_NAME' (no analyzers there, not an error)
+        }
+
+        auto name = irs::make_hashed_ref( // vocbase name
+          irs::string_ref(vocbase.name()), std::hash<irs::string_ref>() // args
+        );
+        auto result = loadAnalyzers(name);
+        auto itr = _lastLoad.find(name);
+
+        if (itr != _lastLoad.end()) {
+          seen.insert(name);
+        } else if (res.ok()) { // load errors take precedence
+          res = arangodb::Result( // result
+            TRI_ERROR_INTERNAL, // code
+            "failure to find database last load timestamp after loading analyzers" // message
+          );
+        }
+
+        if (!result.ok()) {
+          res = result;
+        }
+      };
+
+      dbFeature->enumerateDatabases(visitor);
+
+      std::unordered_set<std::string> unseen; // make copy since original removed
+
+      // remove unseen databases from timestamp list
+      for (auto itr = _lastLoad.begin(), end = _lastLoad.end(); itr != end;) {
+        auto name = irs::make_hashed_ref( // vocbase name
+          irs::string_ref(itr->first), std::hash<irs::string_ref>() // args
+        );
+        auto seenItr = seen.find(name);
+
+        if (seenItr == seen.end()) {
+          unseen.insert(std::move(itr->first)); // reuse key
+          itr = _lastLoad.erase(itr);
+        } else {
+          ++itr;
+        }
+      }
+
+      // remove no longer valid analyzers (force remove)
+      for (auto itr = _analyzers.begin(), end = _analyzers.end(); itr != end;) {
+        auto split = splitAnalyzerName(itr->first);
+        auto unseenItr = // ignore static analyzers
+          split.first.null() ? unseen.end() : unseen.find(split.first);
+
+        if (unseenItr != unseen.end()) {
+          itr = _analyzers.erase(itr);
+        } else {
+          ++itr;
+        }
+      }
+
+      return res;
+    }
+
+    // .........................................................................
+    // after here load analyzers from a specific database
+    // .........................................................................
+
+    auto currentTimestamp = std::chrono::system_clock::now();
+    auto databaseKey = // database key used in '_lastLoad'
+      irs::make_hashed_ref(database, std::hash<irs::string_ref>());
+    auto* engine = arangodb::EngineSelectorFeature::ENGINE;
+    auto itr = _lastLoad.find(databaseKey); // find last update timestamp
+
+    if (!engine || engine->inRecovery()) {
+      // always load if inRecovery since collection contents might have changed
+    } else if (arangodb::ServerState::instance()->isSingleServer()) { // single server
+      if(itr != _lastLoad.end()) {
+        return arangodb::Result(); // do not reload on single-server
+      }
+    } else if (itr != _lastLoad.end() // had a previous load
+               && itr->second + RELOAD_INTERVAL > currentTimestamp // timeout not reached
+              ) {
+      return arangodb::Result(); // reload interval not reached
+    }
+
+    auto* vocbase = dbFeature->lookupDatabase(database);
+    static auto cleanupAnalyzers = []( // remove invalid analyzers
+      IResearchAnalyzerFeature& feature, // analyzer feature
+      decltype(_lastLoad)::iterator& lastLoadItr, // iterator
+      irs::string_ref const& database // database
+    )->void {
+      if (lastLoadItr == feature._lastLoad.end()) {
+        return; // nothing to do (if not in '_lastLoad' then not in '_analyzers')
+      }
+
+      // remove invalid database and analyzers
+      feature._lastLoad.erase(lastLoadItr);
+
+      // remove no longer valid analyzers (force remove)
+      for (auto itr = feature._analyzers.begin(),
+           end = feature._analyzers.end();
+           itr != end;
+          ) {
+        auto split = splitAnalyzerName(itr->first);
+
+        if (split.first == database) {
+          itr = feature._analyzers.erase(itr);
+        } else {
+          ++itr;
+        }
+      }
+    };
+
+    if (!vocbase) {
+      cleanupAnalyzers(*this, itr, database); // cleanup any analyzers for 'database'
+
+      return arangodb::Result( // result
+        TRI_ERROR_ARANGO_DATABASE_NOT_FOUND, // code
+        std::string("failed to find database '") + std::string(database) + " while loading analyzers"
+      );
+    } else {
+      std::shared_ptr<arangodb::LogicalCollection> collection;
+      auto collectionCallback = [&collection]( // store collection
+        std::shared_ptr<arangodb::LogicalCollection> const& col // args
+      )->void {
+        collection = col;
+      };
+
+      arangodb::methods::Collections::lookup( // find collection
+        *vocbase, ANALYZER_COLLECTION_NAME, collectionCallback // args
+      );
+
+      if (!collection) {
+        cleanupAnalyzers(*this, itr, database); // cleanup any analyzers for 'database'
+        _lastLoad[databaseKey] = currentTimestamp; // update timestamp
+
+        return arangodb::Result(); // no collection means nothing to load
+      }
+    }
+
+    Analyzers analyzers;
+    auto visitor = [this, &analyzers, &vocbase]( // visitor
+      arangodb::velocypack::Slice const& slice // analyzer definition
+    )->arangodb::Result {
+      if (!slice.isObject()) {
+        LOG_TOPIC("5c7a5", ERR, arangodb::iresearch::TOPIC)
+          << "failed to find an object value for analyzer definition while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+        return arangodb::Result(); // skip analyzer
+      }
+
+      irs::flags features;
+      irs::string_ref key;
+      irs::string_ref name;
+      irs::string_ref type;
+      irs::string_ref properties;
+      std::string propertiesBuf;
+
+      if (!slice.hasKey(arangodb::StaticStrings::KeyString) // no such field (required)
+          || !slice.get(arangodb::StaticStrings::KeyString).isString()) {
+        LOG_TOPIC("1dc56", ERR, arangodb::iresearch::TOPIC)
+          << "failed to find a string value for analyzer '" << arangodb::StaticStrings::KeyString << "' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+        return arangodb::Result(); // skip analyzer
+      }
+
+      key = getStringRef(slice.get(arangodb::StaticStrings::KeyString));
+
+      if (!slice.hasKey("name") // no such field (required)
+          || !(slice.get("name").isString() || slice.get("name").isNull())) {
+        LOG_TOPIC("f5920", ERR, arangodb::iresearch::TOPIC)
+          << "failed to find a string value for analyzer 'name' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+        return arangodb::Result(); // skip analyzer
+      }
+
+      name = getStringRef(slice.get("name"));
+
+      if (!slice.hasKey("type") // no such field (required)
+          || !(slice.get("type").isString() || slice.get("name").isNull())) {
+        LOG_TOPIC("9f5c8", ERR, arangodb::iresearch::TOPIC)
+          << "failed to find a string value for analyzer 'type' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+        return arangodb::Result(); // skip analyzer
+      }
+
+      type = getStringRef(slice.get("type"));
+
+      if (slice.hasKey("properties")) {
+        auto subSlice = slice.get("properties");
+
+        // encode jSON array/object as a string for analyzers that support jSON
+        if (subSlice.isArray() || subSlice.isObject()) {
+          propertiesBuf = subSlice.toJson(); // format as a jSON encoded string
+          properties = propertiesBuf;
+        } else if (subSlice.isString() || subSlice.isNull()) {
+          properties = getStringRef(subSlice);
+        } else {
+          LOG_TOPIC("a297e", ERR, arangodb::iresearch::TOPIC)
+            << "failed to find a string value for analyzer 'properties' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+          return arangodb::Result(); // skip analyzer
+        }
+      }
+
+      if (slice.hasKey("features")) {
+        auto subSlice = slice.get("features");
+
+        if (!subSlice.isArray()) {
+          LOG_TOPIC("7ec8a", ERR, arangodb::iresearch::TOPIC)
+            << "failed to find an array value for analyzer 'features' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+          return arangodb::Result(); // skip analyzer
+        }
+
+        for (arangodb::velocypack::ArrayIterator subItr(subSlice);
+             subItr.valid();
+             ++subItr
+            ) {
+          auto subEntry = *subItr;
+
+          if (!subEntry.isString() && !subSlice.isNull()) {
+            LOG_TOPIC("7620d", ERR, arangodb::iresearch::TOPIC)
+              << "failed to find a string value for an entry in analyzer 'features' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+            return arangodb::Result(); // skip analyzer
+          }
+
+          auto featureName = getStringRef(subEntry);
+          auto* feature = irs::attribute::type_id::get(featureName);
+
+          if (!feature) {
+            LOG_TOPIC("4fedc", ERR, arangodb::iresearch::TOPIC)
+              << "failed to find feature '" << featureName << "' while loading analyzer form collection '" << ANALYZER_COLLECTION_NAME << "' in database '" << vocbase->name() + "', skipping it: " << slice.toString();
+
+            return arangodb::Result(); // skip analyzer
+          }
+
+          features.add(*feature);
+        }
+      }
+
+      auto normalizedName = normalizedAnalyzerName(vocbase->name(), name);
+      EmplaceAnalyzerResult result;
+      auto res = emplaceAnalyzer( // emplace into map
+        result, analyzers, normalizedName, type, properties, features // args
+      );
+
+      if (!res.ok()) {
+        return res; // caught error emplacing analyzer (abort further processing)
+      }
+
+      if (result.second && result.first->second) {
+        result.first->second->setKey(key); // update key
+      }
+
+      return arangodb::Result();
+    };
+    auto res = visitAnalyzers(*vocbase, visitor);
+
+    if (!res.ok()) {
+      return res;
+    }
+
+    // copy over relevant analyzers from '_analyzers' and validate no duplicates
+    for (auto& entry: _analyzers) {
+      if (!entry.second) {
+        continue; // invalid analyzer (should never happen if insertions done via here)
+      }
+
+      auto split = splitAnalyzerName(entry.first);
+
+      // different database
+      if (split.first != vocbase->name()) {
+        auto result = analyzers.emplace(entry.first, entry.second);
+
+        if (!result.second) { // existing entry
+          if (result.first->second // valid new entry
+              && !equalAnalyzer(*(entry.second), result.first->second->type(), result.first->second->properties(), result.first->second->features())) {
+            return arangodb::Result( // result
+              TRI_ERROR_BAD_PARAMETER, // code
+              std::string("name collision detected while re-registering a duplicate arangosearch analizer name '") + std::string(result.first->second->name()) + "' type '" + std::string(result.first->second->type()) + "' properties '" + std::string(result.first->second->properties()) + "', previous registration type '" + std::string(entry.second->type()) + "' properties '" + std::string(entry.second->properties()) + "'"
+            );
+          }
+
+          result.first->second = entry.second; // reuse old analyzer pool to avoid duplicates in memmory
+          const_cast<Analyzers::key_type&>(result.first->first) = entry.first; // point key at old pool
+        }
+
+        continue; // done with this analyzer
+      }
+
+      auto itr = analyzers.find(entry.first);
+
+      if (itr == analyzers.end()) {
+        continue; // removed analyzer
+      }
+
+      if (itr->second // valid new entry
+          && !equalAnalyzer(*(entry.second), itr->second->type(), itr->second->properties(), itr->second->features())) {
+        return arangodb::Result( // result
+          TRI_ERROR_BAD_PARAMETER, // code
+          std::string("name collision detected while registering a duplicate arangosearch analizer name '") + std::string(itr->second->name()) + "' type '" + std::string(itr->second->type()) + "' properties '" + std::string(itr->second->properties()) + "', previous registration type '" + std::string(entry.second->type()) + "' properties '" + std::string(entry.second->properties()) + "'"
+        );
+      }
+
+      itr->second = entry.second; // reuse old analyzer pool to avoid duplicates in memmory
+      const_cast<Analyzers::key_type&>(itr->first) = entry.first; // point key at old pool
+    }
+
+    _lastLoad[databaseKey] = currentTimestamp; // update timestamp
+    _analyzers = std::move(analyzers); // update mappings
+  } catch (arangodb::basics::Exception const& e) {
+    return arangodb::Result( // result
+      e.code(), // code
+      std::string("caught exception while loading configuration for arangosearch analyzers from database '") + std::string(database) + "': " + std::to_string(e.code()) + " "+ e.what()
+    );
+  } catch (std::exception const& e) {
+    return arangodb::Result( // result
+      TRI_ERROR_INTERNAL, // code
+      std::string("caught exception while loading configuration for arangosearch analyzers from database '") + std::string(database) + "': " + e.what()
+    );
+  } catch (...) {
+    return arangodb::Result( // result
+      TRI_ERROR_INTERNAL, // code
+      std::string("caught exception while loading configuration for arangosearch analyzers from database '") + std::string(database) + "'"
+    );
+  }
+
+  return arangodb::Result();
+}
+
 bool IResearchAnalyzerFeature::loadConfiguration() {
   if (arangodb::ServerState::instance()->isRunningInCluster()) {
     // the following code will not be working in the cluster
@@ -1412,11 +1885,11 @@ bool IResearchAnalyzerFeature::loadConfiguration() {
 
   if (expandVocbasePrefix) {
     if (split.first.null()) {
-      return std::string(activeVocbase.name()).append(2, ANALYZER_PREFIX_DELIM).append(split.second);
+      return normalizedAnalyzerName(activeVocbase.name(), split.second);
     }
 
     if (split.first.empty()) {
-      return std::string(systemVocbase.name()).append(2, ANALYZER_PREFIX_DELIM).append(split.second);
+      return normalizedAnalyzerName(systemVocbase.name(), split.second);
     }
   } else {
     // .........................................................................
@@ -1428,7 +1901,7 @@ bool IResearchAnalyzerFeature::loadConfiguration() {
     }
 
     if (split.first.empty() || split.first == systemVocbase.name()) { // system vocbase
-      return std::string(2, ANALYZER_PREFIX_DELIM).append(split.second);
+      return normalizedAnalyzerName("", split.second);
     }
   }
 
@@ -1573,7 +2046,7 @@ arangodb::Result IResearchAnalyzerFeature::remove( // remove analyzer
     arangodb::OperationOptions options;
 
     builder.openObject();
-    builder.add(arangodb::StaticStrings::KeyString, toValuePair(pool->_key));
+      addStringRef(builder, arangodb::StaticStrings::KeyString, pool->_key);
     builder.close();
 
     auto result = // remove
@@ -1626,92 +2099,11 @@ void IResearchAnalyzerFeature::start() {
 /*FIXME TODO disable until V8 handler is implemented for JavaScript tests
   registerUpgradeTasks(); // register tasks after UpgradeFeature::prepare() has finished
 */
-  // ensure that the configuration collection is present before loading
-  // configuration for the case of inRecovery() if there is no collection then
-  // obviously no custom analyzer configurations were persisted (so missing
-  // analyzer is failure) if there is a configuration collection then just load
-  // analizer configurations
-  {
-    auto vocbase = getSystemDatabase();
 
-    if (!vocbase) {
+  auto res = loadAnalyzers();
-      LOG_TOPIC("a5692", WARN, arangodb::iresearch::TOPIC)
-          << "failure to get system database while starting feature "
-             "'IResearchAnalyzer'";
-      // assume configuration collection exists
-    } else {
-      auto collection = vocbase->lookupCollection(ANALYZER_COLLECTION_NAME);
 
-      if (!collection) {
+  if (!res.ok()) {
-        auto* engine = arangodb::EngineSelectorFeature::ENGINE;
+    THROW_ARANGO_EXCEPTION(res);
-
-        if (!engine) {
-          LOG_TOPIC("f113f", WARN, arangodb::iresearch::TOPIC)
-              << "failure to get storage engine while starting feature "
-                 "'IResearchAnalyzer'";
-          // assume not inRecovery(), create collection immediately
-        } else if (engine->inRecovery()) {
-          auto* feature =
-              arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
-                  "Database");
-
-          if (!feature) {
-            LOG_TOPIC("13812", WARN, arangodb::iresearch::TOPIC)
-                << "failure to find feature 'Database' while starting feature "
-                   "'IResearchAnalyzer'";
-            // can't register post-recovery callback, create collection
-            // immediately
-          } else {
-            std::shared_ptr<TRI_vocbase_t> sharedVocbase(std::move(vocbase));
-
-            feature->registerPostRecoveryCallback([this, sharedVocbase]() -> arangodb::Result {
-              ensureConfigCollection(*sharedVocbase);  // ensure configuration collection exists
-
-              WriteMutex mutex(_mutex);
-              SCOPED_LOCK(mutex);  // '_started' can be asynchronously read
-
-              // ensure all records were initialized
-              if (!_customAnalyzers.empty()) {
-                return arangodb::Result(TRI_ERROR_INTERNAL,
-                                        "uninitialized AnalyzerPool detected "
-                                        "while validating analyzers");
-              }
-
-              _started = true;
-
-              return arangodb::Result();
-            });
-
-            return;  // nothing more to do while inRecovery()
-          }
-        }
-
-        ensureConfigCollection(*vocbase);  // ensure configuration collection exists
-
-        WriteMutex mutex(_mutex);
-        SCOPED_LOCK(mutex);  // '_customAnalyzers' can be asynchronously
-                             // modified, '_started' can be asynchronously read
-
-        // ensure all records were initialized
-        if (!_customAnalyzers.empty()) {
-          THROW_ARANGO_EXCEPTION_MESSAGE(
-              TRI_ERROR_INTERNAL,
-              "uninitialized AnalyzerPool detected while validating analyzers");
-        }
-
-        _started = true;
-
-        return;  // no persisted configurations to load since just created
-                 // collection
-      }
-    }
-
-    // load persisted configuration
-    if (!loadConfiguration()) {
-      THROW_ARANGO_EXCEPTION_MESSAGE(
-          TRI_ERROR_INTERNAL,
-          "uninitialized AnalyzerPool detected while validating analyzers");
-    }
   }
 
   WriteMutex mutex(_mutex);
@@ -1830,17 +2222,9 @@ arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) {
     arangodb::OperationOptions options;
 
     builder.openObject();
-    builder.add("name", toValuePair(split.second));
+    addStringRef(builder, "name", split.second);
-    builder.add("type", toValuePair(pool.type()));
+    addStringRef(builder, "type", pool.type());
-
+    addStringRef(builder, "properties", pool.properties());
-    if (pool.properties().null()) {
-      builder.add( // add value
-        "properties", // name
-        arangodb::velocypack::Value(arangodb::velocypack::ValueType::Null) // value
-      );
-    } else {
-      builder.add("properties", toValuePair(pool.properties()));
-    }
 
     // only add features if there are present
     if (!pool.features().empty()) {
@@ -1859,13 +2243,7 @@ arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) {
           );
         }
 
-        if (feature->name().null()) {
+        addStringRef(builder, feature->name());
-          builder.add( // add value
-            arangodb::velocypack::Value(arangodb::velocypack::ValueType::Null) // value
-          );
-        } else {
-          builder.add(toValuePair(feature->name()));
-        }
       }
 
       builder.close();
@@ -1933,14 +2311,33 @@ arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) {
 bool IResearchAnalyzerFeature::visit( // visit analyzers
     VisitorType const& visitor, // visitor
     TRI_vocbase_t const* vocbase /*= nullptr*/ // analyzers for vocbase
-) {
+) const {
-  ReadMutex mutex(_mutex);
+  if (vocbase) { // do not trigger load for all-databases requests
-  SCOPED_LOCK(mutex);
+    auto res = const_cast<IResearchAnalyzerFeature*>(this)->loadAnalyzers( // load analyzers for database
+      vocbase->name() // args
+    );
 
-  for (auto& entry: _analyzers) {
+    if (!res.ok()) {
+      LOG_TOPIC("73695", WARN, arangodb::iresearch::TOPIC)
+        << "failure to load analyzers while visiting database '" << vocbase->name() << "': " << res.errorNumber() << " " << res.errorMessage();
+      TRI_set_errno(res.errorNumber());
+
+      return false;
+    }
+  }
+
+  Analyzers analyzers;
+
+  {
+    ReadMutex mutex(_mutex);
+    SCOPED_LOCK(mutex);
+    analyzers = _analyzers;
+  }
+
+  for (auto& entry: analyzers) {
     if (entry.second // have entry
         && (!vocbase || splitAnalyzerName(entry.first).first == vocbase->name()) // requested vocbase
-        && !visitor(entry.first, entry.second->_type, entry.second->_properties) // termination request
+        && !visitor(entry.first, entry.second->_type, entry.second->_properties, entry.second->features()) // termination request
        ) {
       return false;
     }

arangod/IResearch/IResearchAnalyzerFeature.h

@@ -104,8 +104,7 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
   ///         in defaultVocbase) is granted 'level' access
   //////////////////////////////////////////////////////////////////////////////
   static bool canUse( // check permissions
-    irs::string_ref const& analyzer, // analyzer name
+    irs::string_ref const& name, // analyzer name (already normalized)
-    TRI_vocbase_t const& defaultVocbase, // fallback vocbase if not part of name
     arangodb::auth::Level const& level // access level
   );
 
@@ -204,11 +203,11 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
   /// @param vocbase only visit analysers for this vocbase (nullptr == all)
   /// @return visitation compleated fully
   //////////////////////////////////////////////////////////////////////////////
-  typedef std::function<bool(irs::string_ref const& analyzer, irs::string_ref const& type, irs::string_ref const& properties)> VisitorType;
+  typedef std::function<bool(irs::string_ref const& analyzer, irs::string_ref const& type, irs::string_ref const& properties, irs::flags const& features)> VisitorType;
   bool visit( // visit analyzers
     VisitorType const& visitor, // visitor
     TRI_vocbase_t const* vocbase = nullptr // analyzers for vocbase
-  );
+  ) const;
 
  private:
   // map of caches of irs::analysis::analyzer pools indexed by analyzer name and
@@ -218,6 +217,7 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
   Analyzers _analyzers; // all analyzers known to this feature (including static) (names are stored with expanded vocbase prefixes)
   Analyzers _customAnalyzers;  // user defined analyzers managed by this feature, a
                                // subset of '_analyzers' (used for removals)
+  std::unordered_map<std::string, std::chrono::system_clock::time_point> _lastLoad; // last time a database was loaded
   mutable irs::async_utils::read_write_mutex _mutex;
   bool _started;
 
@@ -229,6 +229,19 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
 
   static Analyzers const& getStaticAnalyzers();
 
+  //////////////////////////////////////////////////////////////////////////////
+  /// @brief validate analyzer parameters and emplace into map
+  //////////////////////////////////////////////////////////////////////////////
+  typedef std::pair<Analyzers::iterator, bool> EmplaceAnalyzerResult;
+  arangodb::Result emplaceAnalyzer( // emplace
+    EmplaceAnalyzerResult& result, // emplacement result on success (out-param)
+    arangodb::iresearch::IResearchAnalyzerFeature::Analyzers& analyzers, // analyzers
+    irs::string_ref const& name, // analyzer name
+    irs::string_ref const& type, // analyzer type
+    irs::string_ref const& properties, // analyzer properties
+    irs::flags const& features // analyzer features
+  );
+
   //////////////////////////////////////////////////////////////////////////////
   /// @brief ensure an analyzer as per the specified parameters exists if
   ///        possible
@@ -257,6 +270,17 @@ class IResearchAnalyzerFeature final : public arangodb::application_features::Ap
     bool allowCreation
   );
 
+  //////////////////////////////////////////////////////////////////////////////
+  /// @brief load the analyzers for the specific database, analyzers read from
+  ///        the corresponding collection if they have not been loaded yet
+  /// @param database the database to load analizers for (nullptr == all)
+  /// @note on coordinator and db-server reload is also done if the database has
+  ///       not been reloaded in 'timeout' seconds
+  //////////////////////////////////////////////////////////////////////////////
+  arangodb::Result loadAnalyzers( // load analyzers
+      irs::string_ref const& database = irs::string_ref::NIL // database to load
+  );
+
   bool loadConfiguration();
 
   //////////////////////////////////////////////////////////////////////////////

arangod/IResearch/IResearchFeature.cpp

@@ -456,7 +456,7 @@ arangodb::iresearch::IResearchFeature::WalFlushCallback registerRecoveryMarkerSu
     LOG_TOPIC("7007e", WARN, arangodb::iresearch::TOPIC)
       << "failed to find feature 'Flush' while registering recovery subscription";
 
-    return arangodb::iresearch::IResearchFeature::WalFlushCallback();
+    return {}; // it's an std::function so don't use a constructor or ASAN complains
   }
 
   auto& type = arangodb::iresearch::DATA_SOURCE_TYPE.name();
@@ -467,7 +467,7 @@ arangodb::iresearch::IResearchFeature::WalFlushCallback registerRecoveryMarkerSu
     LOG_TOPIC("df64a", WARN, arangodb::iresearch::TOPIC)
       << "failed to find register subscription with  feature 'Flush' while  registering recovery subscription";
 
-    return arangodb::iresearch::IResearchFeature::WalFlushCallback();
+    return {}; // it's an std::function so don't use a constructor or ASAN complains
   }
 
   auto cid = link.collection().id();

arangod/IResearch/IResearchLink.cpp

@@ -659,7 +659,7 @@ arangodb::Result IResearchLink::drop() {
     _asyncFeature->asyncNotify(); // trigger reload of settings for async jobs
   }
 
-  _flushCallback = IResearchFeature::WalFlushCallback(); // reset together with '_asyncSelf'
+  _flushCallback = {}; // reset together with '_asyncSelf', it's an std::function so don't use a constructor or ASAN complains
   _asyncSelf->reset(); // the data-store is being deallocated, link use is no longer valid (wait for all the view users to finish)
 
   try {
@@ -931,7 +931,7 @@ arangodb::Result IResearchLink::initDataStore(InitCallback const& initCallback)
     _asyncFeature->asyncNotify(); // trigger reload of settings for async jobs
   }
 
-  _flushCallback = IResearchFeature::WalFlushCallback(); // reset together with '_asyncSelf'
+  _flushCallback = {}; // reset together with '_asyncSelf', it's an std::function so don't use a constructor or ASAN complains
   _asyncSelf->reset(); // the data-store is being deallocated, link use is no longer valid (wait for all the view users to finish)
 
   auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
@@ -1738,7 +1738,7 @@ arangodb::Result IResearchLink::unload() {
     _asyncFeature->asyncNotify(); // trigger reload of settings for async jobs
   }
 
-  _flushCallback = IResearchFeature::WalFlushCallback(); // reset together with '_asyncSelf'
+  _flushCallback = {}; // reset together with '_asyncSelf', it's an std::function so don't use a constructor or ASAN complains
   _asyncSelf->reset(); // the data-store is being deallocated, link use is no longer valid (wait for all the view users to finish)
 
   try {
@@ -1747,7 +1747,7 @@ arangodb::Result IResearchLink::unload() {
       _dataStore._writer.reset();
       _dataStore._directory.reset();
     }
-  } catch (arangodb::basics::Exception& e) {
+  } catch (arangodb::basics::Exception const& e) {
     return arangodb::Result( // result
       e.code(), // code
       std::string("caught exception while unloading arangosearch link '") + std::to_string(id()) + "': " + e.what()

arangod/IResearch/IResearchLinkHelper.cpp

@@ -32,6 +32,7 @@
 #include "Basics/StaticStrings.h"
 #include "Logger/Logger.h"
 #include "Logger/LogMacros.h"
+#include "RestServer/SystemDatabaseFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "Transaction/Methods.h"
@@ -53,10 +54,32 @@ arangodb::Result canUseAnalyzers( // validate
   arangodb::iresearch::IResearchLinkMeta const& meta, // metadata
   TRI_vocbase_t const& defaultVocbase // default vocbase
 ) {
+  auto* sysDatabase = arangodb::application_features::ApplicationServer::lookupFeature< // find feature
+    arangodb::SystemDatabaseFeature // featue type
+  >();
+  auto sysVocbase = sysDatabase ? sysDatabase->use() : nullptr;
+
   for (auto& entry: meta._analyzers) {
-    if (entry // valid entry
+    if (!entry) {
-        && !arangodb::iresearch::IResearchAnalyzerFeature::canUse(entry->name(), defaultVocbase, arangodb::auth::Level::RO)
+      continue; // skip invalid entries
-       ) {
+    }
+
+    bool result;
+
+    if (sysVocbase) {
+      result = arangodb::iresearch::IResearchAnalyzerFeature::canUse( // validate
+        arangodb::iresearch::IResearchAnalyzerFeature::normalize( // normalize
+          entry->name(), defaultVocbase, *sysVocbase // args
+        ), // analyzer
+        arangodb::auth::Level::RO // auth level
+      );
+    } else {
+      result = arangodb::iresearch::IResearchAnalyzerFeature::canUse( // validate
+        entry->name(), arangodb::auth::Level::RO // args
+      );
+    }
+
+    if (!result) {
       return arangodb::Result( // result
         TRI_ERROR_FORBIDDEN, // code
         std::string("read access is forbidden to arangosearch analyzer '") + entry->name() + "'"

arangod/IResearch/IResearchLinkMeta.cpp

@@ -234,7 +234,7 @@ bool IResearchLinkMeta::init( // initialize meta
           auto analyzer = analyzers->get(name);
 
           if (!analyzer) {
-            errorField = fieldName + "=>" + std::string(name);
+            errorField = fieldName + "=>" + value.copyString(); // original (non-normalized) 'name' valie
 
             return false;
           }
@@ -305,7 +305,7 @@ bool IResearchLinkMeta::init( // initialize meta
           if (value.hasKey(subFieldName)) {
             auto subField = value.get(subFieldName);
 
-            if (!subField.isString()) {
+            if (!subField.isString() && !subField.isNull()) {
               errorField = fieldName + "=>[" + std::to_string(itr.index()) + "]=>" + subFieldName;
 
               return false;
@@ -335,7 +335,7 @@ bool IResearchLinkMeta::init( // initialize meta
                  ++subItr) {
               auto subValue = *subItr;
 
-              if (!subValue.isString()) {
+              if (!subValue.isString() && !subValue.isNull()) {
                 errorField = fieldName + "=>[" + std::to_string(itr.index()) + "]=>" + subFieldName + "=>[" + std::to_string(subItr.index()) +  + "]";
 
                 return false;
@@ -551,8 +551,8 @@ bool IResearchLinkMeta::json( // append meta jSON
 
       analyzersBuilder.openObject();
         analyzersBuilder.add("name", arangodb::velocypack::Value(name));
-        analyzersBuilder.add("type", toValuePair(entry->type()));
+        addStringRef(analyzersBuilder, "type", entry->type());
-        analyzersBuilder.add("properties", toValuePair(entry->properties()));
+        addStringRef(analyzersBuilder, "properties", entry->properties());
         analyzersBuilder.add(
           "features", // key
           arangodb::velocypack::Value(arangodb::velocypack::ValueType::Array) // value
@@ -560,7 +560,7 @@ bool IResearchLinkMeta::json( // append meta jSON
 
           for (auto& feature: entry->features()) {
             TRI_ASSERT(feature); // has to be non-nullptr
-            analyzersBuilder.add(toValuePair(feature->name()));
+            addStringRef(analyzersBuilder, feature->name());
           }
 
         analyzersBuilder.close();

arangod/IResearch/IResearchViewMeta.cpp

@@ -76,7 +76,7 @@ createConsolidationPolicy<irs::index_utils::consolidate_bytes_accum>(
   }
 
   properties.openObject();
-  properties.add("type", arangodb::iresearch::toValuePair(POLICY_BYTES_ACCUM));
+  properties.add("type", arangodb::velocypack::Value(POLICY_BYTES_ACCUM));
   properties.add("threshold", arangodb::velocypack::Value(options.threshold));
   properties.close();
 
@@ -159,7 +159,7 @@ arangodb::iresearch::IResearchViewMeta::ConsolidationPolicy createConsolidationP
   }
 
   properties.openObject();
-  properties.add("type", arangodb::iresearch::toValuePair(POLICY_TIER));
+  properties.add("type", arangodb::velocypack::Value(POLICY_TIER));
   properties.add("lookahead", arangodb::velocypack::Value(size_t(1)));  // FIXME remove in 3.5
   properties.add("segmentsBytesFloor",
                  arangodb::velocypack::Value(options.floor_segment_bytes));

arangod/IResearch/VelocyPackHelper.cpp

@@ -26,9 +26,96 @@
 #include "velocypack/Builder.h"
 #include "velocypack/Iterator.h"
 
+namespace {
+
+inline arangodb::velocypack::ValuePair toValuePair(irs::bytes_ref const& ref) {
+  TRI_ASSERT(!ref.null()); // consumers of ValuePair usually use memcpy(...) which cannot handle nullptr
+  return arangodb::velocypack::ValuePair( // value pair
+    ref.c_str(), ref.size(), arangodb::velocypack::ValueType::Binary // args
+  );
+}
+
+inline arangodb::velocypack::ValuePair toValuePair(irs::string_ref const& ref) {
+  TRI_ASSERT(!ref.null()); // consumers of ValuePair usually use memcpy(...) which cannot handle nullptr
+  return arangodb::velocypack::ValuePair( // value pair
+    ref.c_str(), ref.size(), arangodb::velocypack::ValueType::String // args
+  );
+}
+
+template<typename T>
+arangodb::velocypack::Builder& addRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::basic_string_ref<T> const& value // value
+) {
+  // store nulls verbatim
+  if (value.null()) {
+    builder.add( // add value
+      arangodb::velocypack::Value(arangodb::velocypack::ValueType::Null) // value
+    );
+  } else {
+    builder.add(toValuePair(value));
+  }
+
+  return builder;
+}
+
+template<typename T>
+arangodb::velocypack::Builder& addRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& key, // key
+  irs::basic_string_ref<T> const& value // value
+) {
+  TRI_ASSERT(!key.null()); // Builder uses memcpy(...) which cannot handle nullptr
+
+  // store nulls verbatim
+  if (value.null()) {
+    builder.add( // add value
+      key.c_str(), // key data
+      key.size(), // key size
+      arangodb::velocypack::Value(arangodb::velocypack::ValueType::Null) // value
+    );
+  } else {
+    builder.add(key.c_str(), key.size(), toValuePair(value));
+  }
+
+  return builder;
+}
+
+}
+
 namespace arangodb {
 namespace iresearch {
 
+arangodb::velocypack::Builder& addBytesRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::bytes_ref const& value // value
+) {
+  return addRef(builder, value);
+}
+
+arangodb::velocypack::Builder& addBytesRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& key, // key
+  irs::bytes_ref const& value // value
+) {
+  return addRef(builder, key, value);
+}
+
+arangodb::velocypack::Builder& addStringRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& value // value
+) {
+  return addRef(builder, value);
+}
+
+arangodb::velocypack::Builder& addStringRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& key, // key
+  irs::string_ref const& value // value
+) {
+  return addRef(builder, key, value);
+}
+
 bool mergeSlice(arangodb::velocypack::Builder& builder,
                 arangodb::velocypack::Slice const& slice) {
   if (builder.isOpenArray()) {
@@ -148,5 +235,5 @@ ObjectIterator& ObjectIterator::operator++() {
 }  // namespace arangodb
 
 // -----------------------------------------------------------------------------
-// --SECTION-- END-OF-FILE
+// --SECTION--                                                       END-OF-FILE
 // -----------------------------------------------------------------------------

arangod/IResearch/VelocyPackHelper.h

@@ -46,6 +46,40 @@ namespace iresearch {
 uint8_t const COMPACT_ARRAY = 0x13;
 uint8_t const COMPACT_OBJECT = 0x14;
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief add a string_ref value to the 'builder' (for JSON arrays)
+////////////////////////////////////////////////////////////////////////////////
+arangodb::velocypack::Builder& addBytesRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::bytes_ref const& value // value
+);
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief add a string_ref value to the 'builder' (for JSON objects)
+////////////////////////////////////////////////////////////////////////////////
+arangodb::velocypack::Builder& addBytesRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& key, // key
+  irs::bytes_ref const& value // value
+);
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief add a string_ref value to the 'builder' (for JSON arrays)
+////////////////////////////////////////////////////////////////////////////////
+arangodb::velocypack::Builder& addStringRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& value // value
+);
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief add a string_ref value to the 'builder' (for JSON objects)
+////////////////////////////////////////////////////////////////////////////////
+arangodb::velocypack::Builder& addStringRef( // add a value
+  arangodb::velocypack::Builder& builder, // builder
+  irs::string_ref const& key, // key
+  irs::string_ref const& value // value
+);
+
 inline bool isArrayOrObject(VPackSlice const& slice) {
   auto const type = slice.type();
   return VPackValueType::Array == type || VPackValueType::Object == type;
@@ -197,35 +231,6 @@ bool mergeSliceSkipOffsets(arangodb::velocypack::Builder& builder,
                            arangodb::velocypack::Slice const& slice,
                            std::function<bool(size_t offset)> const& acceptor);
 
-//////////////////////////////////////////////////////////////////////////////
-/// @brief convert an irs::byte_type array to an
-///        arangodb::velocypack::ValuePair
-//////////////////////////////////////////////////////////////////////////////
-inline arangodb::velocypack::ValuePair toValuePair(const irs::byte_type* data, size_t size) {
-  return arangodb::velocypack::ValuePair(data, size, arangodb::velocypack::ValueType::Binary);
-}
-
-//////////////////////////////////////////////////////////////////////////////
-/// @brief convert an irs::bytes_ref to an arangodb::velocypack::ValuePair
-//////////////////////////////////////////////////////////////////////////////
-inline arangodb::velocypack::ValuePair toValuePair(irs::bytes_ref const& ref) {
-  return toValuePair(ref.c_str(), ref.size());
-}
-
-//////////////////////////////////////////////////////////////////////////////
-/// @brief convert a char array to an arangodb::velocypack::ValuePair
-//////////////////////////////////////////////////////////////////////////////
-inline arangodb::velocypack::ValuePair toValuePair(const char* data, size_t size) {
-  return arangodb::velocypack::ValuePair(data, size, arangodb::velocypack::ValueType::String);
-}
-
-//////////////////////////////////////////////////////////////////////////////
-/// @brief convert an irs::string_ref to an arangodb::velocypack::ValuePair
-//////////////////////////////////////////////////////////////////////////////
-inline arangodb::velocypack::ValuePair toValuePair(irs::string_ref const& ref) {
-  return toValuePair(ref.c_str(), ref.size());
-}
-
 ////////////////////////////////////////////////////////////////////////////
 /// @struct IteratorValue
 /// @brief represents of value of the iterator

arangod/VocBase/vocbase.cpp

@@ -38,6 +38,7 @@
 #include "Basics/HybridLogicalClock.h"
 #include "Basics/NumberUtils.h"
 #include "Basics/ReadLocker.h"
+#include "Basics/RecursiveLocker.h"
 #include "Basics/StaticStrings.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
@@ -68,86 +69,6 @@
 
 #include <thread>
 
-namespace {
-
-template <typename T>
-class RecursiveReadLocker {
- public:
-  RecursiveReadLocker(T& mutex, std::atomic<std::thread::id>& owner, char const* file, int line)
-      : _locker(&mutex, arangodb::basics::LockerType::TRY, true, file, line) {
-    if (!_locker.isLocked() && owner.load() != std::this_thread::get_id()) {
-      _locker.lock();
-    }
-  }
-
- private:
-  arangodb::basics::ReadLocker<T> _locker;
-};
-
-template <typename T>
-class RecursiveWriteLocker {
- public:
-  RecursiveWriteLocker(T& mutex, std::atomic<std::thread::id>& owner,
-                       arangodb::basics::LockerType type, bool acquire,
-                       char const* file, int line)
-      : _locked(false), _locker(&mutex, type, false, file, line), _owner(owner), _update(noop) {
-    if (acquire) {
-      lock();
-    }
-  }
-
-  ~RecursiveWriteLocker() { unlock(); }
-
-  bool isLocked() { return _locked; }
-
-  void lock() {
-    // recursive locking of the same instance is not yet supported (create a new
-    // instance instead)
-    TRI_ASSERT(_update != owned);
-
-    if (std::this_thread::get_id() != _owner.load()) {  // not recursive
-      _locker.lock();
-      _owner.store(std::this_thread::get_id());
-      _update = owned;
-    }
-
-    _locked = true;
-  }
-
-  void unlock() {
-    _update(*this);
-    _locked = false;
-  }
-
- private:
-  bool _locked;  // track locked state separately for recursive lock aquisition
-  arangodb::basics::WriteLocker<T> _locker;
-  std::atomic<std::thread::id>& _owner;
-  void (*_update)(RecursiveWriteLocker& locker);
-
-  static void noop(RecursiveWriteLocker&) {}
-  static void owned(RecursiveWriteLocker& locker) {
-    static std::thread::id unowned;
-    locker._owner.store(unowned);
-    locker._locker.unlock();
-    locker._update = noop;
-  }
-};
-
-#define NAME__(name, line) name##line
-#define NAME_EXPANDER__(name, line) NAME__(name, line)
-#define NAME(name) NAME_EXPANDER__(name, __LINE__)
-#define RECURSIVE_READ_LOCKER(lock, owner)                             \
-  RecursiveReadLocker<typename std::decay<decltype(lock)>::type> NAME( \
-      RecursiveLocker)(lock, owner, __FILE__, __LINE__)
-#define RECURSIVE_WRITE_LOCKER_NAMED(name, lock, owner, acquire)        \
-  RecursiveWriteLocker<typename std::decay<decltype(lock)>::type> name( \
-      lock, owner, arangodb::basics::LockerType::BLOCKING, acquire, __FILE__, __LINE__)
-#define RECURSIVE_WRITE_LOCKER(lock, owner) \
-  RECURSIVE_WRITE_LOCKER_NAMED(NAME(RecursiveLocker), lock, owner, true)
-
-}  // namespace
-
 using namespace arangodb;
 using namespace arangodb::basics;
 

lib/Basics/RecursiveLocker.h

@@ -0,0 +1,180 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Andrey Abramov
+/// @author Vasiliy Nabatchikov
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef LIB_BASICS_RECURSIVE_LOCKER_H
+#define LIB_BASICS_RECURSIVE_LOCKER_H 1
+
+#include "WriteLocker.h"
+
+namespace arangodb {
+
+// identical code to RecursiveWriteLocker except for type
+template <typename T>
+class RecursiveMutexLocker {
+ public:
+  RecursiveMutexLocker( // recursive locker
+      T& mutex, // mutex
+      std::atomic<std::thread::id>& owner, // owner
+      arangodb::basics::LockerType type, // locker type
+      bool acquire, // aquire flag
+      char const* file, // file
+      int line // line
+): _locker(&mutex, type, false, file, line), _owner(owner), _update(noop) {
+    if (acquire) {
+      lock();
+    }
+  }
+
+  ~RecursiveMutexLocker() { unlock(); }
+
+  bool isLocked() { return _locker.isLocked(); }
+
+  void lock() {
+    // recursive locking of the same instance is not yet supported (create a new instance instead)
+    TRI_ASSERT(_update != owned);
+
+    if (std::this_thread::get_id() != _owner.load()) {  // not recursive
+      _locker.lock();
+      _owner.store(std::this_thread::get_id());
+      _update = owned;
+    }
+  }
+
+  void unlock() { _update(*this); }
+
+ private:
+  arangodb::basics::MutexLocker<T> _locker;
+  std::atomic<std::thread::id>& _owner;
+  void (*_update)(RecursiveMutexLocker& locker);
+
+  static void noop(RecursiveMutexLocker&) {}
+  static void owned(RecursiveMutexLocker& locker) {
+    static std::thread::id unowned;
+    locker._owner.store(unowned);
+    locker._locker.unlock();
+    locker._update = noop;
+  }
+};
+
+#define NAME__(name, line) name##line
+#define NAME_EXPANDER__(name, line) NAME__(name, line)
+#define NAME(name) NAME_EXPANDER__(name, __LINE__)
+#define RECURSIVE_MUTEX_LOCKER_NAMED(name, lock, owner, acquire)                     \
+  RecursiveMutexLocker<typename std::decay<decltype(lock)>::type> name(              \
+    lock, owner, arangodb::basics::LockerType::BLOCKING, acquire, __FILE__, __LINE__ \
+  )
+#define RECURSIVE_MUTEX_LOCKER(lock, owner) \
+  RECURSIVE_MUTEX_LOCKER_NAMED(NAME(RecursiveLocker), lock, owner, true)
+
+template <typename T>
+class RecursiveReadLocker {
+ public:
+  RecursiveReadLocker( // recursive locker
+      T& mutex, // mutex
+      std::atomic<std::thread::id>& owner, // owner
+      char const* file, // file
+      int line // line
+): _locker(&mutex, arangodb::basics::LockerType::TRY, true, file, line) {
+    if (!_locker.isLocked() && owner.load() != std::this_thread::get_id()) {
+      _locker.lock();
+    }
+  }
+
+ private:
+  arangodb::basics::ReadLocker<T> _locker;
+};
+
+// identical code to RecursiveMutexLocker except for type
+template <typename T>
+class RecursiveWriteLocker {
+ public:
+  RecursiveWriteLocker( // recursive locker
+      T& mutex, // mutex
+      std::atomic<std::thread::id>& owner, // owner
+      arangodb::basics::LockerType type, // locker type
+      bool acquire, // aquire flag
+      char const* file, // file
+      int line // line
+): _locked(false), // locked
+   _locker(&mutex, type, false, file, line), // locker
+   _owner(owner), // owner
+   _update(noop) {
+    if (acquire) {
+      lock();
+    }
+  }
+
+  ~RecursiveWriteLocker() { unlock(); }
+
+  bool isLocked() { return _locked; }
+
+  void lock() {
+    // recursive locking of the same instance is not yet supported (create a new instance instead)
+    TRI_ASSERT(_update != owned);
+
+    if (std::this_thread::get_id() != _owner.load()) {  // not recursive
+      _locker.lock();
+      _owner.store(std::this_thread::get_id());
+      _update = owned;
+    }
+
+    _locked = true;
+  }
+
+  void unlock() {
+    _update(*this);
+    _locked = false;
+  }
+
+ private:
+  bool _locked;  // track locked state separately for recursive lock aquisition
+  arangodb::basics::WriteLocker<T> _locker;
+  std::atomic<std::thread::id>& _owner;
+  void (*_update)(RecursiveWriteLocker& locker);
+
+  static void noop(RecursiveWriteLocker&) {}
+  static void owned(RecursiveWriteLocker& locker) {
+    static std::thread::id unowned;
+    locker._owner.store(unowned);
+    locker._locker.unlock();
+    locker._update = noop;
+  }
+};
+
+#define NAME__(name, line) name##line
+#define NAME_EXPANDER__(name, line) NAME__(name, line)
+#define NAME(name) NAME_EXPANDER__(name, __LINE__)
+#define RECURSIVE_READ_LOCKER(lock, owner)                                             \
+  RecursiveReadLocker<typename std::decay<decltype(lock)>::type> NAME(RecursiveLocker)(\
+    lock, owner, __FILE__, __LINE__                                                    \
+  )
+#define RECURSIVE_WRITE_LOCKER_NAMED(name, lock, owner, acquire)                       \
+  RecursiveWriteLocker<typename std::decay<decltype(lock)>::type> name(                \
+      lock, owner, arangodb::basics::LockerType::BLOCKING, acquire, __FILE__, __LINE__ \
+  )
+#define RECURSIVE_WRITE_LOCKER(lock, owner) \
+  RECURSIVE_WRITE_LOCKER_NAMED(NAME(RecursiveLocker), lock, owner, true)
+
+} // arangodb
+
+#endif

tests/Cluster/ClusterInfo-test.cpp

@@ -141,7 +141,8 @@ struct ClusterInfoSetup {
     arangodb::EngineSelectorFeature::ENGINE = &engine;
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress INFO {cluster} Starting up with role SINGLE
     arangodb::LogTopic::setLogLevel(arangodb::Logger::CLUSTER.name(), arangodb::LogLevel::FATAL);

tests/IResearch/ClusterCommMock.cpp

@@ -61,13 +61,29 @@ arangodb::OperationID ClusterCommMock::asyncRequest(
    bool singleRequest,
    arangodb::ClusterCommTimeout initTimeout
 ) {
-  auto entry = _requests.emplace(
+  // execute before insertion to avoid consuming an '_operationId'
-    std::piecewise_construct,
+  if (arangodb::rest::RequestType::PUT == reqtype
-    std::forward_as_tuple(++_operationId), // non-zero value
+      && std::string::npos != path.find("/_api/aql/shutdown/")) {
-    std::forward_as_tuple(coordTransactionID, destination, reqtype, path, body, headerFields, callback, singleRequest)
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); // terminate query 'shutdown' infinite loops with exception
-  );
+  }
 
-  return entry.first->first;
+  if (_responses.empty()) {
+    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "request not expected since result not provided");
+  }
+
+  auto operationID = _responses.front().operationID; // set expected responce ID
+
+  _requests.emplace_back(coordTransactionID, destination, reqtype, path, body, headerFields, callback, singleRequest);
+
+  if (!callback) {
+    return operationID;
+  }
+
+  auto result = wait(coordTransactionID, 0, "", timeout); // OperationID == 0 same as ClusterComm::performRequests(...)
+
+  (*callback)(&result);
+
+  return operationID;
 }
 
 void ClusterCommMock::drop(
@@ -75,15 +91,7 @@ void ClusterCommMock::drop(
    arangodb::OperationID const operationID, // 0 == any opId
    arangodb::ShardID const& shardID // "" = any shardId
 ) {
-  auto itr = _responses.find(operationID);
+  _responses.pop_front();
-
-  TRI_ASSERT(itr != _responses.end());
-  TRI_ASSERT(!itr->second.empty());
-  itr->second.pop_front();
-
-  if (itr->second.empty()) {
-    _responses.erase(itr);
-  }
 }
 
 /*static*/ std::shared_ptr<ClusterCommMock> ClusterCommMock::setInstance(
@@ -102,25 +110,48 @@ void ClusterCommMock::drop(
   );
 }
 
+std::unique_ptr<arangodb::ClusterCommResult> ClusterCommMock::syncRequest(
+  arangodb::CoordTransactionID const coordTransactionID,
+  std::string const& destination,
+  arangodb::rest::RequestType reqtype,
+  std::string const& path,
+  std::string const& body,
+  std::unordered_map<std::string, std::string> const& headerFields,
+  arangodb::ClusterCommTimeout timeout
+) {
+  asyncRequest(
+    coordTransactionID,
+    destination,
+    reqtype,
+    path,
+    std::shared_ptr<std::string const>(&body, [](std::string const*)->void {}),
+    headerFields,
+    nullptr,
+    timeout,
+    true,
+    timeout
+  );
+
+  auto result = std::make_unique<arangodb::ClusterCommResult>(
+    wait(coordTransactionID, 0, "", timeout) // OperationID == 0 same as ClusterComm::performRequests(...)
+  );
+
+  return result;
+}
+
 arangodb::ClusterCommResult const ClusterCommMock::wait(
    arangodb::CoordTransactionID const coordTransactionID, // 0 == any trxId
    arangodb::OperationID const operationID, // 0 == any opId
    arangodb::ShardID const& shardID, // "" = any shardId
    arangodb::ClusterCommTimeout timeout
 ) {
-  auto itr = _responses.find(operationID);
+  if (_responses.empty()) {
-
-  if (itr == _responses.end() || itr->second.empty()) {
     return arangodb::ClusterCommResult();
   }
 
-  auto result = std::move(itr->second.front());
+  auto result = std::move(_responses.front());
 
-  itr->second.pop_front();
+  _responses.pop_front();
-
-  if (itr->second.empty()) {
-    _responses.erase(itr);
-  }
 
   return result;
 }

tests/IResearch/ClusterCommMock.h

@@ -54,8 +54,8 @@ class ClusterCommMock: public arangodb::ClusterComm {
     );
   };
 
-  std::unordered_map<arangodb::OperationID, Request> _requests;
+  std::vector<Request> _requests;
-  std::unordered_map<arangodb::OperationID, std::deque<arangodb::ClusterCommResult>> _responses;
+  std::deque<arangodb::ClusterCommResult> _responses;
 
   ClusterCommMock();
 
@@ -78,21 +78,26 @@ class ClusterCommMock: public arangodb::ClusterComm {
      arangodb::ShardID const& shardID // "" = any shardId
   ) override;
 
-  arangodb::OperationID nextOperationId() const { return _operationId + 1; }
-
   static std::shared_ptr<ClusterCommMock> setInstance(
     ClusterCommMock& instance
   );
 
+  std::unique_ptr<arangodb::ClusterCommResult> syncRequest(
+    arangodb::CoordTransactionID const coordTransactionID,
+    std::string const& destination,
+    arangodb::rest::RequestType reqtype,
+    std::string const& path,
+    std::string const& body,
+    std::unordered_map<std::string, std::string> const& headerFields,
+    arangodb::ClusterCommTimeout timeout
+  ) override;
+
   arangodb::ClusterCommResult const wait(
      arangodb::CoordTransactionID const coordTransactionID, // 0 == any trxId
      arangodb::OperationID const operationID, // 0 == any opId
      arangodb::ShardID const& shardID, // "" = any shardId
      arangodb::ClusterCommTimeout timeout
   ) override;
-
- private:
-  arangodb::OperationID _operationId{};
 };
 
 #endif

tests/IResearch/ExecutionBlockMock-test.cpp

@@ -87,8 +87,8 @@ struct IResearchBlockMockSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(),
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
-                                    arangodb::LogLevel::WARN);
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR);  // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/ExpressionFilter-test.cpp

@@ -265,7 +265,8 @@ struct IResearchExpressionFilterSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchDocument-test.cpp

@@ -138,7 +138,8 @@ struct IResearchDocumentSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // setup required application features
     features.emplace_back(new arangodb::AuthenticationFeature(server), true);

tests/IResearch/IResearchFeature-test.cpp

@@ -93,7 +93,8 @@ struct IResearchFeatureSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AGENCY.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchFilter-test.cpp

@@ -145,7 +145,8 @@ struct IResearchFilterSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchFilterBoolean-test.cpp

@@ -85,7 +85,8 @@ struct IResearchFilterBooleanSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchFilterCompare-test.cpp

@@ -86,7 +86,8 @@ struct IResearchFilterCompareSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchFilterFunction-test.cpp

@@ -87,7 +87,8 @@ struct IResearchFilterFunctionSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchFilterIn-test.cpp

@@ -85,7 +85,8 @@ struct IResearchFilterInSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchIndex-test.cpp

@@ -149,7 +149,8 @@ struct IResearchIndexSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchLink-test.cpp

@@ -85,7 +85,8 @@ struct IResearchLinkSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::ENGINES.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchLinkCoordinator-test.cpp

@@ -122,8 +122,8 @@ struct IResearchLinkCoordinatorSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
-
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // pretend we're on coordinator
     serverRoleBeforeSetup = arangodb::ServerState::instance()->getRole();

tests/IResearch/IResearchLinkHelper-test.cpp

@@ -66,9 +66,11 @@ struct IResearchLinkHelperSetup {
     arangodb::EngineSelectorFeature::ENGINE = &engine;
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AGENCYCOMM.name(), arangodb::LogLevel::FATAL);
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);
 
     features.emplace_back(new arangodb::AqlFeature(server), true); // required for UserManager::loadFromDB()
@@ -135,6 +137,7 @@ struct IResearchLinkHelperSetup {
     }
 
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::DEFAULT);
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AGENCYCOMM.name(), arangodb::LogLevel::DEFAULT);
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::DEFAULT);
     arangodb::EngineSelectorFeature::ENGINE = nullptr;
   }

tests/IResearch/IResearchLinkMeta-test.cpp

@@ -108,7 +108,8 @@ struct IResearchLinkMetaSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // setup required application features
     features.emplace_back(new arangodb::AuthenticationFeature(server), true);
@@ -156,7 +157,6 @@ struct IResearchLinkMetaSetup {
     >();
     arangodb::iresearch::IResearchAnalyzerFeature::EmplaceResult result;
 
-    analyzers->emplace(result, "empty", "empty", "en", irs::flags{ irs::frequency::type() }); // cache the 'empty' analyzer
     analyzers->emplace(result, "testVocbase::empty", "empty", "de", irs::flags{ irs::frequency::type() }); // cache the 'empty' analyzer for 'testVocbase'
 
     // suppress log messages since tests check error conditions
@@ -226,7 +226,7 @@ SECTION("test_inheritDefaults") {
   defaults._trackListPositions = true;
   defaults._storeValues = arangodb::iresearch::ValueStorage::FULL;
   defaults._analyzers.clear();
-  defaults._analyzers.emplace_back(analyzers.get(arangodb::StaticStrings::SystemDatabase + "::empty"));
+  defaults._analyzers.emplace_back(analyzers.get("testVocbase::empty"));
   defaults._fields["abc"]->_fields["xyz"] = arangodb::iresearch::IResearchLinkMeta();
 
   auto json = arangodb::velocypack::Parser::fromJson("{}");
@@ -263,8 +263,8 @@ SECTION("test_inheritDefaults") {
 
   CHECK(1U == meta._analyzers.size());
   CHECK((*(meta._analyzers.begin())));
-  CHECK((arangodb::StaticStrings::SystemDatabase + "::empty" == (*(meta._analyzers.begin()))->name()));
+  CHECK(("testVocbase::empty" == (*(meta._analyzers.begin()))->name()));
-  CHECK((irs::flags() == (*(meta._analyzers.begin()))->features()));
+  CHECK((irs::flags({irs::frequency::type()}) == (*(meta._analyzers.begin()))->features()));
   CHECK(false == !meta._analyzers.begin()->get());
 }
 
@@ -327,94 +327,9 @@ SECTION("test_readCustomizedValues") {
 
   // without active vocbase
   {
-    std::unordered_set<std::string> expectedFields = { "a", "b", "c" };
-    std::unordered_set<std::string> expectedOverrides = { "default", "all", "some", "none" };
-    std::unordered_set<std::string> expectedAnalyzers = { "empty", "identity" };
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string tmpString;
-    CHECK(true == meta.init(json->slice(), tmpString));
+    CHECK(false == meta.init(json->slice(), tmpString));
-    CHECK(3U == meta._fields.size());
-
-    for (auto& field: meta._fields) {
-      CHECK(1U == expectedFields.erase(field.key()));
-
-      for (auto& fieldOverride: field.value()->_fields) {
-        auto& actual = *(fieldOverride.value());
-
-        CHECK(1U == expectedOverrides.erase(fieldOverride.key()));
-
-        if ("default" == fieldOverride.key()) {
-          CHECK(true == actual._fields.empty());
-          CHECK(false == actual._includeAllFields);
-          CHECK(false == actual._trackListPositions);
-          CHECK((arangodb::iresearch::ValueStorage::NONE == actual._storeValues));
-          CHECK(1U == actual._analyzers.size());
-          CHECK((*(actual._analyzers.begin())));
-          CHECK(("identity" == (*(actual._analyzers.begin()))->name()));
-          CHECK((irs::flags({irs::norm::type(), irs::frequency::type()}) == (*(actual._analyzers.begin()))->features()));
-          CHECK(false == !actual._analyzers.begin()->get());
-        } else if ("all" == fieldOverride.key()) {
-          CHECK(2U == actual._fields.size());
-          CHECK(true == (actual._fields.find("d") != actual._fields.end()));
-          CHECK(true == (actual._fields.find("e") != actual._fields.end()));
-          CHECK(true == actual._includeAllFields);
-          CHECK(true == actual._trackListPositions);
-          CHECK((arangodb::iresearch::ValueStorage::FULL == actual._storeValues));
-          CHECK(1U == actual._analyzers.size());
-          CHECK((*(actual._analyzers.begin())));
-          CHECK(("empty" == (*(actual._analyzers.begin()))->name()));
-          CHECK((irs::flags({irs::frequency::type()}) == (*(actual._analyzers.begin()))->features()));
-          CHECK(false == !actual._analyzers.begin()->get());
-        } else if ("some" == fieldOverride.key()) {
-          CHECK(true == actual._fields.empty()); // not inherited
-          CHECK(true == actual._includeAllFields); // inherited
-          CHECK(true == actual._trackListPositions);
-          CHECK((arangodb::iresearch::ValueStorage::ID == actual._storeValues));
-          CHECK(2U == actual._analyzers.size());
-          auto itr = actual._analyzers.begin();
-          CHECK((*itr));
-          CHECK(("empty" == (*itr)->name()));
-          CHECK((irs::flags({irs::frequency::type()}) == (*itr)->features()));
-          CHECK(false == !itr->get());
-          ++itr;
-          CHECK((*itr));
-          CHECK(("identity" == (*itr)->name()));
-          CHECK((irs::flags({irs::norm::type(), irs::frequency::type()}) == (*itr)->features()));
-          CHECK(false == !itr->get());
-        } else if ("none" == fieldOverride.key()) {
-          CHECK(true == actual._fields.empty()); // not inherited
-          CHECK(true == actual._includeAllFields); // inherited
-          CHECK(true == actual._trackListPositions); // inherited
-          CHECK((arangodb::iresearch::ValueStorage::FULL == actual._storeValues));
-          auto itr = actual._analyzers.begin();
-          CHECK((*itr));
-          CHECK(("empty" == (*itr)->name()));
-          CHECK((irs::flags({irs::frequency::type()}) == (*itr)->features()));
-          CHECK(false == !itr->get());
-          ++itr;
-          CHECK((*itr));
-          CHECK(("identity" == (*itr)->name()));
-          CHECK((irs::flags({irs::norm::type(), irs::frequency::type()}) == (*itr)->features()));
-          CHECK(false == !itr->get());
-        }
-      }
-    }
-
-    CHECK(true == expectedOverrides.empty());
-    CHECK(true == expectedFields.empty());
-    CHECK(true == meta._includeAllFields);
-    CHECK(true == meta._trackListPositions);
-    CHECK((arangodb::iresearch::ValueStorage::FULL == meta._storeValues));
-    auto itr = meta._analyzers.begin();
-    CHECK((*itr));
-    CHECK(("empty" == (*itr)->name()));
-    CHECK((irs::flags({irs::frequency::type()}) == (*itr)->features()));
-    CHECK(false == !itr->get());
-    ++itr;
-    CHECK((*itr));
-    CHECK(("identity" == (*itr)->name()));
-    CHECK((irs::flags({irs::norm::type(), irs::frequency::type()}) == (*itr)->features()));
-    CHECK(false == !itr->get());
   }
 
   // with active vocbase
@@ -571,7 +486,7 @@ SECTION("test_writeDefaults") {
       && tmpSlice.at(0).isObject()
       && tmpSlice.at(0).get("name").isString() && std::string("identity") == tmpSlice.at(0).get("name").copyString()
       && tmpSlice.at(0).get("type").isString() && std::string("identity") == tmpSlice.at(0).get("type").copyString()
-      && tmpSlice.at(0).get("properties").isString() && std::string("") == tmpSlice.at(0).get("properties").copyString()
+      && tmpSlice.at(0).get("properties").isNull()
       && tmpSlice.at(0).get("features").isArray() && 2 == tmpSlice.at(0).get("features").length() // frequency+norm
     ));
   }
@@ -637,7 +552,7 @@ SECTION("test_writeDefaults") {
       && tmpSlice.at(0).isObject()
       && tmpSlice.at(0).get("name").isString() && std::string("identity") == tmpSlice.at(0).get("name").copyString()
       && tmpSlice.at(0).get("type").isString() && std::string("identity") == tmpSlice.at(0).get("type").copyString()
-      && tmpSlice.at(0).get("properties").isString() && std::string("") == tmpSlice.at(0).get("properties").copyString()
+      && tmpSlice.at(0).get("properties").isNull()
       && tmpSlice.at(0).get("features").isArray() && 2 == tmpSlice.at(0).get("features").length() // frequency+norm
     ));
   }
@@ -844,7 +759,7 @@ SECTION("test_writeCustomizedValues") {
             && tmpSlice.at(0).isObject()
             && tmpSlice.at(0).get("name").isString() && std::string("identity") == tmpSlice.at(0).get("name").copyString()
             && tmpSlice.at(0).get("type").isString() && std::string("identity") == tmpSlice.at(0).get("type").copyString()
-            && tmpSlice.at(0).get("properties").isString() && std::string("") == tmpSlice.at(0).get("properties").copyString()
+            && tmpSlice.at(0).get("properties").isNull()
             && tmpSlice.at(0).get("features").isArray() && 2 == tmpSlice.at(0).get("features").length() // frequency+norm
           ));
         } else if ("all" == fieldOverride.copyString()) {
@@ -902,9 +817,9 @@ SECTION("test_writeCustomizedValues") {
         true == value.isObject()
         && value.hasKey("name") && value.get("name").isString()
         && value.hasKey("type") && value.get("type").isString()
-        && value.hasKey("properties") && value.get("properties").isString()
+        && value.hasKey("properties") && (value.get("properties").isString() || value.get("properties").isNull())
         && value.hasKey("features") && value.get("features").isArray() && (1 == value.get("features").length() || 2 == value.get("features").length()) // empty/identity 1/2
-        && 1 == expectedAnalyzers.erase(std::make_pair(value.get("name").copyString(), value.get("properties").copyString()))
+        && 1 == expectedAnalyzers.erase(std::make_pair(value.get("name").copyString(), value.get("properties").isNull() ? "" : value.get("properties").copyString()))
      ));
     }
 
@@ -1073,7 +988,7 @@ SECTION("test_writeCustomizedValues") {
             && tmpSlice.at(0).isObject()
             && tmpSlice.at(0).get("name").isString() && std::string("identity") == tmpSlice.at(0).get("name").copyString()
             && tmpSlice.at(0).get("type").isString() && std::string("identity") == tmpSlice.at(0).get("type").copyString()
-            && tmpSlice.at(0).get("properties").isString() && std::string("") == tmpSlice.at(0).get("properties").copyString()
+            && tmpSlice.at(0).get("properties").isNull()
             && tmpSlice.at(0).get("features").isArray() && 2 == tmpSlice.at(0).get("features").length()  // frequency+norm
           ));
         } else if ("all" == fieldOverride.copyString()) {
@@ -1131,9 +1046,9 @@ SECTION("test_writeCustomizedValues") {
         true == value.isObject()
         && value.hasKey("name") && value.get("name").isString()
         && value.hasKey("type") && value.get("type").isString()
-        && value.hasKey("properties") && value.get("properties").isString()
+        && value.hasKey("properties") && (value.get("properties").isString() || value.get("properties").isNull())
         && value.hasKey("features") && value.get("features").isArray() && (1 == value.get("features").length() || 2 == value.get("features").length()) // empty/identity 1/2
-        && 1 == expectedAnalyzers.erase(std::make_pair(value.get("name").copyString(), value.get("properties").copyString()))
+        && 1 == expectedAnalyzers.erase(std::make_pair(value.get("name").copyString(), value.get("properties").isNull() ? "" : value.get("properties").copyString()))
      ));
     }
 
@@ -1250,6 +1165,8 @@ SECTION("test_writeMaskNone") {
 }
 
 SECTION("test_readAnalyzerDefinitions") {
+  TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
+
   // missing analyzer (name only)
   {
     auto json = arangodb::velocypack::Parser::fromJson("{ \
@@ -1257,7 +1174,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>empty1") == errorField));
   }
 
@@ -1271,7 +1188,7 @@ SECTION("test_readAnalyzerDefinitions") {
     auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>empty1") == errorField));
   }
 
@@ -1282,7 +1199,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]=>name") == errorField));
   }
 
@@ -1293,7 +1210,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]=>type") == errorField));
   }
 
@@ -1304,7 +1221,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]") == errorField));
   }
 
@@ -1315,7 +1232,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]") == errorField));
   }
 
@@ -1330,7 +1247,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]") == errorField));
   }
 
@@ -1345,9 +1262,9 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((true == meta.init(json->slice(), errorField)));
+    CHECK((true == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((1 == meta._analyzers.size()));
-    CHECK((std::string("missing2") == meta._analyzers[0]->name()));
+    CHECK((std::string("testVocbase::missing2") == meta._analyzers[0]->name()));
     CHECK((std::string("empty") == meta._analyzers[0]->type()));
     CHECK((std::string("ru") == meta._analyzers[0]->properties()));
     CHECK((1 == meta._analyzers[0]->features().size()));
@@ -1364,7 +1281,7 @@ SECTION("test_readAnalyzerDefinitions") {
     auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]") == errorField));
   }
 
@@ -1375,11 +1292,11 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((true == meta.init(json->slice(), errorField)));
+    CHECK((true == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((1 == meta._analyzers.size()));
-    CHECK((std::string("empty") == meta._analyzers[0]->name()));
+    CHECK((std::string("testVocbase::empty") == meta._analyzers[0]->name()));
     CHECK((std::string("empty") == meta._analyzers[0]->type()));
-    CHECK((std::string("en") == meta._analyzers[0]->properties()));
+    CHECK((std::string("de") == meta._analyzers[0]->properties()));
     CHECK((1 == meta._analyzers[0]->features().size()));
     CHECK((true == meta._analyzers[0]->features().check(irs::frequency::type())));
   }
@@ -1394,27 +1311,27 @@ SECTION("test_readAnalyzerDefinitions") {
     auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((true == meta.init(json->slice(), errorField)));
+    CHECK((true == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((1 == meta._analyzers.size()));
-    CHECK((std::string("empty") == meta._analyzers[0]->name()));
+    CHECK((std::string("testVocbase::empty") == meta._analyzers[0]->name()));
     CHECK((std::string("empty") == meta._analyzers[0]->type()));
-    CHECK((std::string("en") == meta._analyzers[0]->properties()));
+    CHECK((std::string("de") == meta._analyzers[0]->properties()));
     CHECK((1 == meta._analyzers[0]->features().size()));
     CHECK((true == meta._analyzers[0]->features().check(irs::frequency::type())));
   }
 
-  // existing analyzer (full) analyzer creation not allowed (passs)
+  // existing analyzer (full) analyzer creation not allowed (pass)
   {
     auto json = arangodb::velocypack::Parser::fromJson("{ \
-      \"analyzers\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": \"en\", \"features\": [ \"frequency\" ] } ] \
+      \"analyzers\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": \"de\", \"features\": [ \"frequency\" ] } ] \
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((true == meta.init(json->slice(), errorField)));
+    CHECK((true == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((1 == meta._analyzers.size()));
-    CHECK((std::string("empty") == meta._analyzers[0]->name()));
+    CHECK((std::string("testVocbase::empty") == meta._analyzers[0]->name()));
     CHECK((std::string("empty") == meta._analyzers[0]->type()));
-    CHECK((std::string("en") == meta._analyzers[0]->properties()));
+    CHECK((std::string("de") == meta._analyzers[0]->properties()));
     CHECK((1 == meta._analyzers[0]->features().size()));
     CHECK((true == meta._analyzers[0]->features().check(irs::frequency::type())));
   }
@@ -1422,15 +1339,15 @@ SECTION("test_readAnalyzerDefinitions") {
   // existing analyzer (full)
   {
     auto json = arangodb::velocypack::Parser::fromJson("{ \
-      \"analyzers\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": \"en\", \"features\": [ \"frequency\" ] } ] \
+      \"analyzers\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": \"de\", \"features\": [ \"frequency\" ] } ] \
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((true == meta.init(json->slice(), errorField)));
+    CHECK((true == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((1 == meta._analyzers.size()));
-    CHECK((std::string("empty") == meta._analyzers[0]->name()));
+    CHECK((std::string("testVocbase::empty") == meta._analyzers[0]->name()));
     CHECK((std::string("empty") == meta._analyzers[0]->type()));
-    CHECK((std::string("en") == meta._analyzers[0]->properties()));
+    CHECK((std::string("de") == meta._analyzers[0]->properties()));
     CHECK((1 == meta._analyzers[0]->features().size()));
     CHECK((true == meta._analyzers[0]->features().check(irs::frequency::type())));
   }
@@ -1438,18 +1355,18 @@ SECTION("test_readAnalyzerDefinitions") {
   // existing analyzer (full) inRecovery
   {
     auto json = arangodb::velocypack::Parser::fromJson("{ \
-      \"analyzers\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": \"en\", \"features\": [ \"frequency\" ] } ] \
+      \"analyzers\": [ { \"name\": \"empty\", \"type\": \"empty\", \"properties\": \"de\", \"features\": [ \"frequency\" ] } ] \
     }");
     auto before = StorageEngineMock::inRecoveryResult;
     StorageEngineMock::inRecoveryResult = true;
     auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((true == meta.init(json->slice(), errorField)));
+    CHECK((true == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((1 == meta._analyzers.size()));
-    CHECK((std::string("empty") == meta._analyzers[0]->name()));
+    CHECK((std::string("testVocbase::empty") == meta._analyzers[0]->name()));
     CHECK((std::string("empty") == meta._analyzers[0]->type()));
-    CHECK((std::string("en") == meta._analyzers[0]->properties()));
+    CHECK((std::string("de") == meta._analyzers[0]->properties()));
     CHECK((1 == meta._analyzers[0]->features().size()));
     CHECK((true == meta._analyzers[0]->features().check(irs::frequency::type())));
   }
@@ -1461,7 +1378,7 @@ SECTION("test_readAnalyzerDefinitions") {
     }");
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]") == errorField));
   }
 
@@ -1475,7 +1392,7 @@ SECTION("test_readAnalyzerDefinitions") {
     auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; });
     arangodb::iresearch::IResearchLinkMeta meta;
     std::string errorField;
-    CHECK((false == meta.init(json->slice(), errorField)));
+    CHECK((false == meta.init(json->slice(), errorField, &vocbase)));
     CHECK((std::string("analyzers=>[0]") == errorField));
   }
 }

tests/IResearch/IResearchQuery-test.cpp

@@ -152,7 +152,8 @@ struct IResearchQuerySetup {
     IcuInitializer::setup(ARGV0); // initialize ICU, required for Utf8Helper which is using by optimizer
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);

tests/IResearch/IResearchQueryAggregate-test.cpp

@@ -87,7 +87,8 @@ struct IResearchQueryAggregateSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQueryAnd-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryAndSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryBooleanTerm-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryBooleanTermSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryComplexBoolean-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryComplexBooleanSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryExists-test.cpp

@@ -90,7 +90,8 @@ struct IResearchQueryExistsSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryIn-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryInSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryJoin-test.cpp

@@ -96,7 +96,8 @@ struct IResearchQueryJoinSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryNullTerm-test.cpp

@@ -87,7 +87,8 @@ struct IResearchQueryNullTermSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryNumericTerm-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryNumericTermSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryOptions-test.cpp

@@ -91,7 +91,8 @@ struct IResearchQueryOptionsSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQueryOr-test.cpp

@@ -89,7 +89,8 @@ struct IResearchQueryOrSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryPhrase-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryPhraseSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryScorer-test.cpp

@@ -96,7 +96,8 @@ struct IResearchQueryScorerSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQuerySelectAll-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQuerySelectAllSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQueryStartsWith-test.cpp

@@ -88,7 +88,8 @@ struct IResearchQueryStartsWithSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQueryStringTerm-test.cpp

@@ -91,7 +91,8 @@ struct IResearchQueryStringTermSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQueryTokens-test.cpp

@@ -96,7 +96,8 @@ struct IResearchQueryTokensSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchQueryTraversal-test.cpp

@@ -87,7 +87,8 @@ struct IResearchQueryTraversalSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::AQL.name(), arangodb::LogLevel::ERR); // suppress WARNING {aql} Suboptimal AqlItemMatrix index lookup:

tests/IResearch/IResearchQueryValue-test.cpp

@@ -87,7 +87,8 @@ struct IResearchQueryValueSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR); // suppress WARNING DefaultCustomTypeHandler called

tests/IResearch/IResearchView-test.cpp

@@ -152,7 +152,8 @@ struct IResearchViewSetup {
     arangodb::tests::init();
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::FATAL); // suppress ERROR recovery failure due to error from callback

tests/IResearch/IResearchViewDBServer-test.cpp

@@ -92,7 +92,8 @@ struct IResearchViewDBServerSetup {
     arangodb::EngineSelectorFeature::ENGINE = &engine;
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress INFO {cluster} Starting up with role PRIMARY
     arangodb::LogTopic::setLogLevel(arangodb::Logger::CLUSTER.name(), arangodb::LogLevel::WARN);

tests/IResearch/IResearchViewNode-test.cpp

@@ -90,8 +90,8 @@ struct IResearchViewNodeSetup {
     arangodb::tests::init(true);
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(),
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
-                                    arangodb::LogLevel::WARN);
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // suppress log messages since tests check error conditions
     arangodb::LogTopic::setLogLevel(arangodb::Logger::FIXME.name(), arangodb::LogLevel::ERR);  // suppress WARNING DefaultCustomTypeHandler called

tests/RestHandler/RestUsersHandler-test.cpp

@@ -107,7 +107,8 @@ struct RestUsersHandlerSetup {
     arangodb::EngineSelectorFeature::ENGINE = &engine;
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     features.emplace_back(new arangodb::AuthenticationFeature(server), false); // required for VocbaseContext
     features.emplace_back(new arangodb::DatabaseFeature(server), false); // required for UserManager::updateUser(...)

tests/RestHandler/RestViewHandler-test.cpp

@@ -96,7 +96,8 @@ struct RestViewHandlerSetup {
     arangodb::EngineSelectorFeature::ENGINE = &engine;
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // setup required application features
     features.emplace_back(new arangodb::AuthenticationFeature(server), false); // required for VocbaseContext

tests/V8Server/v8-users-test.cpp

@@ -128,7 +128,8 @@ struct V8UsersSetup {
     arangodb::tests::v8Init(); // on-time initialize V8
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     features.emplace_back(new arangodb::AuthenticationFeature(server), false); // required for VocbaseContext
     features.emplace_back(new arangodb::DatabaseFeature(server), false); // required for UserManager::updateUser(...)

tests/V8Server/v8-views-test.cpp

@@ -125,7 +125,8 @@ struct V8ViewsSetup {
     arangodb::tests::v8Init(); // on-time initialize V8
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     // setup required application features
     features.emplace_back(new arangodb::AuthenticationFeature(server), false); // required for VocbaseContext

tests/VocBase/LogicalView-test.cpp

@@ -98,7 +98,8 @@ struct LogicalViewSetup {
     arangodb::EngineSelectorFeature::ENGINE = &engine;
 
     // suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
-    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
+    // suppress WARNING {authentication} --server.jwt-secret is insecure. Use --server.jwt-secret-keyfile instead
+    arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::ERR);
 
     features.emplace_back(new arangodb::AuthenticationFeature(server), false); // required for ExecContext
     features.emplace_back(new arangodb::QueryRegistryFeature(server), false); // required for TRI_vocbase_t