Try to fix drop database bug with Collector.

Wait for a phase in the Collector with no queued operations before
actually destroying the LogicalCollections. This is to make sure that
no more DOCUMENT ditches are around.

arangod/RestServer/DatabaseFeature.cpp

@@ -164,7 +164,20 @@ void DatabaseManagerThread::run() {
             }
           }
 
-          database->shutdown();
+          // To shutdown the database (which destroys all LogicalCollection
+          // objects of all collections) we need to make sure that the
+          // Collector does not interfere. Therefore we execute the shutdown
+          // in a phase in which the collector thread does not have any
+          // queued operations, a service which it offers:
+          auto callback = [&database]() {
+            database->shutdown();
+            usleep(10000);
+          };
+          while (!arangodb::wal::LogfileManager::instance()
+                  ->executeWhileNothingQueued(callback)) {
+            LOG(INFO) << "Trying to shutdown dropped database, waiting for phase in which the collector thread does not have queued operations.";
+          }
+
           engine->dropDatabase(database);
         }
 

arangod/StorageEngine/MMFilesCollectorThread.cpp

@@ -356,6 +356,18 @@ bool MMFilesCollectorThread::hasQueuedOperations(TRI_voc_cid_t cid) {
   return (_operationsQueue.find(cid) != _operationsQueue.end());
 }
 
+// execute a callback during a phase in which the collector has nothing
+// queued. This is used in the DatabaseManagerThread when dropping
+// a database to avoid existence of ditches of type DOCUMENT.
+bool MMFilesCollectorThread::executeWhileNothingQueued(std::function<void()> const& cb) {
+  MUTEX_LOCKER(mutexLocker, _operationsQueueLock);
+  if (!_operationsQueue.empty()) {
+    return false;
+  }
+  cb();
+  return true;
+}
+
 /// @brief step 1: perform collection of a logfile (if any)
 int MMFilesCollectorThread::collectLogfiles(bool& worked) {
   // always init result variable

arangod/StorageEngine/MMFilesCollectorThread.h

@@ -68,6 +68,11 @@ class MMFilesCollectorThread final : public Thread {
   /// collection
   bool hasQueuedOperations(TRI_voc_cid_t);
 
+  // execute a callback during a phase in which the collector has nothing
+  // queued. This is used in the DatabaseManagerThread when dropping
+  // a database to avoid existence of ditches of type DOCUMENT.
+  bool executeWhileNothingQueued(std::function<void()> const& cb);
+
  protected:
   /// @brief main loop
   void run() override;

arangod/Wal/LogfileManager.cpp

@@ -1684,6 +1684,16 @@ void LogfileManager::waitForCollector() {
   }
 }
 
+// execute a callback during a phase in which the collector has nothing
+// queued. This is used in the DatabaseManagerThread when dropping
+// a database to avoid existence of ditches of type DOCUMENT.
+bool LogfileManager::executeWhileNothingQueued(std::function<void()> const& cb) {
+  if (_collectorThread == nullptr) {
+    return true;
+  }
+  return _collectorThread->executeWhileNothingQueued(cb);
+}
+
 // wait until a specific logfile has been collected
 int LogfileManager::waitForCollector(Logfile::IdType logfileId,
                                      double maxWaitTime) {

arangod/Wal/LogfileManager.h

@@ -372,6 +372,11 @@ class LogfileManager final : public application_features::ApplicationFeature {
   
   void waitForCollector();
 
+  // execute a callback during a phase in which the collector has nothing
+  // queued. This is used in the DatabaseManagerThread when dropping
+  // a database to avoid existence of ditches of type DOCUMENT.
+  bool executeWhileNothingQueued(std::function<void()> const& cb);
+
  private:
   // hashes the transaction id into a bucket
   size_t getBucket(TRI_voc_tid_t id) const { return std::hash<TRI_voc_cid_t>()(id) % numBuckets; }