use JobGuard when querying users from DB in cluster (#8057)

* use JobGuard when querying users from DB in cluster

* fix test crashes

arangod/Aql/ClusterBlocks.cpp

@@ -40,7 +40,6 @@
 #include "Cluster/ClusterComm.h"
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/ServerState.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/Methods.h"
 #include "Transaction/StandaloneContext.h"

arangod/Aql/RestAqlHandler.cpp

@@ -45,7 +45,6 @@
 #include "Logger/Logger.h"
 #include "Rest/HttpRequest.h"
 #include "Rest/HttpResponse.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/Methods.h"

arangod/Auth/UserManager.cpp

@@ -41,6 +41,8 @@
 #include "RestServer/DatabaseFeature.h"
 #include "RestServer/InitDatabaseFeature.h"
 #include "RestServer/SystemDatabaseFeature.h"
+#include "Scheduler/JobGuard.h"
+#include "Scheduler/SchedulerFeature.h"
 #include "Ssl/SslInterface.h"
 #include "Transaction/StandaloneContext.h"
 #include "Utils/ExecContext.h"
@@ -192,11 +194,19 @@ void auth::UserManager::loadFromDB() {
   if (_internalVersion.load(std::memory_order_acquire) == globalVersion()) {
     return;
   }
+                     
   MUTEX_LOCKER(guard, _loadFromDBLock);
   uint64_t tmp = globalVersion();
   if (_internalVersion.load(std::memory_order_acquire) == tmp) {
     return;
   }
+  
+  JobGuard jobGuard(SchedulerFeature::SCHEDULER);
+  if (ServerState::instance()->isCoordinator()) {
+    // mark the current thread as being busy, because we will very soon be
+    // doing blocking cluster-internal communication
+    jobGuard.work();
+  }
 
   try {
     std::shared_ptr<VPackBuilder> builder = QueryAllUsers(_queryRegistry);

arangod/Cluster/ClusterComm.cpp

@@ -32,7 +32,6 @@
 #include "Cluster/ServerState.h"
 #include "GeneralServer/AuthenticationFeature.h"
 #include "Logger/Logger.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "SimpleHttpClient/SimpleHttpCommunicatorResult.h"
 #include "Transaction/Methods.h"

arangod/Cluster/HeartbeatThread.cpp

@@ -46,7 +46,6 @@
 #include "Replication/GlobalReplicationApplier.h"
 #include "Replication/ReplicationFeature.h"
 #include "RestServer/DatabaseFeature.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"

arangod/GeneralServer/GeneralCommTask.cpp

@@ -41,7 +41,6 @@
 #include "Replication/ReplicationFeature.h"
 #include "RestServer/DatabaseFeature.h"
 #include "RestServer/VocbaseContext.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Scheduler/Socket.h"

arangod/Scheduler/JobGuard.h

@@ -47,7 +47,10 @@ class JobGuard : public SameThreadAsserter {
     TRI_ASSERT(!_isWorkingFlag);
 
     if (0 == _isWorking++) {
-      _scheduler->incWorking();
+      if (_scheduler) {
+        // scheduler can be a nullptr during tests
+        _scheduler->incWorking();
+      }
     }
 
     _isWorkingFlag = true;
@@ -62,12 +65,16 @@ class JobGuard : public SameThreadAsserter {
       if (0 == --_isWorking) {
         // if this is the last JobGuard we inform the
         // scheduler that the thread is back to idle
-        _scheduler->decWorking();
+        if (_scheduler) {
+          // scheduler can be a nullptr during tests
+          _scheduler->decWorking();
+        }
       }
     }
   }
 
  private:
+  // note that the scheduler can be a nullptr during testing
   rest::Scheduler* _scheduler;
 
   bool _isWorkingFlag = false;

arangod/Scheduler/Socket.h

@@ -28,7 +28,7 @@
 #include "Basics/StringBuffer.h"
 #include "Basics/asio_ns.h"
 #include "Logger/Logger.h"
-#include "Scheduler/JobGuard.h"
+#include "Scheduler/Scheduler.h"
 
 namespace arangodb {
 namespace rest {

arangod/V8Server/V8DealerFeature.cpp

@@ -44,7 +44,6 @@
 #include "Rest/Version.h"
 #include "RestServer/DatabasePathFeature.h"
 #include "RestServer/SystemDatabaseFeature.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/V8Context.h"
 #include "V8/v8-buffer.h"

arangod/V8Server/v8-dispatcher.cpp

@@ -31,7 +31,6 @@
 #include "Basics/tri-strings.h"
 #include "Cluster/ServerState.h"
 #include "Logger/Logger.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/Hints.h"

arangod/VocBase/Methods/Tasks.cpp

@@ -33,7 +33,6 @@
 #include "Basics/tri-strings.h"
 #include "Cluster/ServerState.h"
 #include "Logger/Logger.h"
-#include "Scheduler/JobGuard.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/Hints.h"