Add RocksDB background error listener. (#8535)

arangod/RocksDBEngine/CMakeLists.txt

@@ -39,6 +39,7 @@ endif()
 
 # add sources for rocksdb engine
 set(ROCKSDB_SOURCES
+  RocksDBEngine/RocksDBBackgroundErrorListener.cpp
   RocksDBEngine/RocksDBBackgroundThread.cpp
   RocksDBEngine/RocksDBBuilderIndex.cpp
   RocksDBEngine/RocksDBCollection.cpp

arangod/RocksDBEngine/RocksDBBackgroundErrorListener.cpp

@@ -0,0 +1,63 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Dan Larkin-York
+////////////////////////////////////////////////////////////////////////////////
+
+#include "RocksDBBackgroundErrorListener.h"
+
+#include "Logger/Logger.h"
+
+namespace arangodb {
+
+RocksDBBackgroundErrorListener::~RocksDBBackgroundErrorListener() {}
+
+void RocksDBBackgroundErrorListener::OnBackgroundError(rocksdb::BackgroundErrorReason reason,
+                                                       rocksdb::Status*) {
+  if (!_called) {
+    _called = true;
+
+    std::string operation = "unknown";
+    switch (reason) {
+      case rocksdb::BackgroundErrorReason::kFlush: {
+        operation = "flush";
+        break;
+      }
+      case rocksdb::BackgroundErrorReason::kCompaction: {
+        operation = "compaction";
+        break;
+      }
+      case rocksdb::BackgroundErrorReason::kWriteCallback: {
+        operation = "write callback";
+        break;
+      }
+      case rocksdb::BackgroundErrorReason::kMemTable: {
+        operation = "memtable";
+        break;
+      }
+    }
+
+    LOG_TOPIC(ERR, Logger::ROCKSDB)
+        << "RocksDB encountered a background error during a " << operation
+        << " operation; The database will be put in read-only "
+           "mode, and subsequent write errors are likely";
+  }
+}
+
+}  // namespace arangodb

arangod/RocksDBEngine/RocksDBBackgroundErrorListener.h

@@ -0,0 +1,44 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Dan Larkin-York
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGO_ROCKSDB_ROCKSDB_BACKGROUND_ERROR_LISTENER_H
+#define ARANGO_ROCKSDB_ROCKSDB_BACKGROUND_ERROR_LISTENER_H 1
+
+// public rocksdb headers
+#include <rocksdb/db.h>
+#include <rocksdb/listener.h>
+
+namespace arangodb {
+
+class RocksDBBackgroundErrorListener : public rocksdb::EventListener {
+ public:
+  virtual ~RocksDBBackgroundErrorListener();
+
+  void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* error) override;
+
+ private:
+  bool _called = false;
+};  // class RocksDBThrottle
+
+}  // namespace arangodb
+
+#endif

arangod/RocksDBEngine/RocksDBEngine.cpp

@@ -22,6 +22,7 @@
 /// @author Jan Christoph Uhde
 ////////////////////////////////////////////////////////////////////////////////
 
+#include "RocksDBEngine.h"
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "ApplicationFeatures/RocksDBOptionFeature.h"
 #include "Basics/Exceptions.h"
@@ -45,7 +46,7 @@
 #include "RestHandler/RestHandlerCreator.h"
 #include "RestServer/DatabasePathFeature.h"
 #include "RestServer/ServerIdFeature.h"
-#include "RocksDBEngine.h"
+#include "RocksDBEngine/RocksDBBackgroundErrorListener.h"
 #include "RocksDBEngine/RocksDBBackgroundThread.h"
 #include "RocksDBEngine/RocksDBCollection.h"
 #include "RocksDBEngine/RocksDBColumnFamily.h"
@@ -551,6 +552,8 @@ void RocksDBEngine::start() {
     _options.listeners.push_back(_listener);
   }
 
+  _options.listeners.push_back(std::make_shared<RocksDBBackgroundErrorListener>());
+
   if (opts->_totalWriteBufferSize > 0) {
     _options.db_write_buffer_size = opts->_totalWriteBufferSize;
   }