Fix heartbeat thread hanging during shutdown (#7709)

arangod/Cluster/DBServerAgencySync.cpp

@@ -164,6 +164,14 @@ DBServerAgencySyncResult DBServerAgencySync::execute() {
   auto plan = clusterInfo->getPlan();
   auto serverId = arangodb::ServerState::instance()->getId();
 
+  if (plan == nullptr) {
+    // TODO increase log level, except during shutdown?
+    LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
+        << "DBServerAgencySync::execute no plan";
+    result.errorMessage = "DBServerAgencySync::execute no plan";
+    return result;
+  }
+
   VPackBuilder local;
   Result glc = getLocalCollections(local);
   if (!glc.ok()) {
@@ -200,6 +208,13 @@ DBServerAgencySyncResult DBServerAgencySync::execute() {
     }
 
     auto current = clusterInfo->getCurrent();
+    if (current == nullptr) {
+      // TODO increase log level, except during shutdown?
+      LOG_TOPIC(DEBUG, Logger::MAINTENANCE)
+          << "DBServerAgencySync::execute no current";
+      result.errorMessage = "DBServerAgencySync::execute no current";
+      return result;
+    }
     LOG_TOPIC(TRACE, Logger::MAINTENANCE) << "DBServerAgencySync::phaseTwo - current state: " << current->toJson();
 
     tmp = arangodb::maintenance::phaseTwo(

arangod/Cluster/HeartbeatThread.cpp

@@ -319,20 +319,6 @@ void HeartbeatThread::runDBServer() {
     return true;
   };
 
-  auto planAgencyCallback = std::make_shared<AgencyCallback>(
-      _agency, "Plan/Version", updatePlan, true);
-
-  bool registered = false;
-  while (!registered) {
-    registered =
-      _agencyCallbackRegistry->registerCallback(planAgencyCallback);
-    if (!registered) {
-      LOG_TOPIC(ERR, Logger::HEARTBEAT)
-          << "Couldn't register plan change in agency!";
-      std::this_thread::sleep_for(std::chrono::seconds(1));
-    }
-  }
-
   std::function<bool(VPackSlice const& result)> updateCurrent =
     [=](VPackSlice const& result) {
 
@@ -362,11 +348,25 @@ void HeartbeatThread::runDBServer() {
     return true;
   };
 
+  auto planAgencyCallback = std::make_shared<AgencyCallback>(
+      _agency, "Plan/Version", updatePlan, true);
+
   auto currentAgencyCallback = std::make_shared<AgencyCallback>(
       _agency, "Current/Version", updateCurrent, true);
 
+  bool registered = false;
+  while (!registered && !isStopping()) {
+    registered =
+      _agencyCallbackRegistry->registerCallback(planAgencyCallback);
+    if (!registered) {
+      LOG_TOPIC(ERR, Logger::HEARTBEAT)
+          << "Couldn't register plan change in agency!";
+      std::this_thread::sleep_for(std::chrono::seconds(1));
+    }
+  }
+
   registered = false;
-  while (!registered) {
+  while (!registered && !isStopping()) {
     registered =
       _agencyCallbackRegistry->registerCallback(currentAgencyCallback);
     if (!registered) {
@@ -515,6 +515,7 @@ void HeartbeatThread::runDBServer() {
     }
   }
 
+  // TODO should these be defered?
   _agencyCallbackRegistry->unregisterCallback(currentAgencyCallback);
   _agencyCallbackRegistry->unregisterCallback(planAgencyCallback);
 }