Work on RocksDBEngine and enable engine selection

arangod/RocksDBEngine/RocksDBCollection.cpp

@@ -52,10 +52,6 @@ TRI_voc_rid_t RocksDBCollection::revision() const {
 void RocksDBCollection::updateCount(int64_t) {
   throw std::logic_error("not implemented");
 }
-size_t RocksDBCollection::journalSize() const {
-  throw std::logic_error("not implemented");
-  return 0;
-}
 
 void RocksDBCollection::getPropertiesVPack(velocypack::Builder&) const {
   throw std::logic_error("not implemented");
@@ -112,16 +108,6 @@ bool RocksDBCollection::isFullyCollected() const {
   return false;
 }
 
-bool RocksDBCollection::doCompact() const {
-  throw std::logic_error("not implemented");
-  return false;
-}
-
-uint32_t RocksDBCollection::indexBuckets() const {
-  throw std::logic_error("not implemented");
-  return 0;
-}
-
 void RocksDBCollection::prepareIndexes(
     arangodb::velocypack::Slice indexesSlice) {
   throw std::logic_error("not implemented");

arangod/RocksDBEngine/RocksDBCollection.h

@@ -79,7 +79,6 @@ class RocksDBCollection final : public PhysicalCollection {
 
   int64_t initialCount() const override;
   void updateCount(int64_t) override;
-  size_t journalSize() const override;
 
   void getPropertiesVPack(velocypack::Builder&) const override;
 
@@ -109,14 +108,10 @@ class RocksDBCollection final : public PhysicalCollection {
 
   bool isFullyCollected() const override;
 
-  bool doCompact() const override;  // { return _doCompact; }
-
   ////////////////////////////////////
   // -- SECTION Indexes --
   ///////////////////////////////////
 
-  uint32_t indexBuckets() const override;
-
   void prepareIndexes(arangodb::velocypack::Slice indexesSlice) override;
 
   /// @brief Find index by definition

arangod/RocksDBEngine/RocksDBEngine.cpp

@@ -1,6 +1,26 @@
+#include "Basics/Exceptions.h"
+#include "Basics/FileUtils.h"
+#include "Basics/tri-strings.h"
+#include "Logger/Logger.h"
+#include "ProgramOptions/ProgramOptions.h"
+#include "ProgramOptions/Section.h"
+#include "RestServer/DatabasePathFeature.h"
 #include "RocksDBEngine.h"
 #include <Basics/Result.h>
 #include <stdexcept>
+#include <rocksdb/db.h>
+#include <rocksdb/convenience.h>
+#include <rocksdb/env.h>
+#include <rocksdb/filter_policy.h>
+#include <rocksdb/iterator.h>
+#include <rocksdb/options.h>
+#include <rocksdb/slice_transform.h>
+#include <rocksdb/table.h>
+#include <rocksdb/write_batch.h>
+
+using namespace arangodb;
+using namespace arangodb::application_features;
+using namespace arangodb::options;
 
 namespace arangodb {
 
@@ -11,7 +31,7 @@ std::string const RocksDBEngine::FeatureName("RocksDBEngine");
 RocksDBEngine::RocksDBEngine(application_features::ApplicationServer* server)
     : StorageEngine(server, EngineName, FeatureName, nullptr /*new
                                                               MMFilesIndexFactory()*/) {
-  startsAfter("MMFilesEngine");
+      //inherits order from StorageEngine
 }
 
 RocksDBEngine::~RocksDBEngine() { throw std::runtime_error("not implemented"); }
@@ -28,7 +48,30 @@ void RocksDBEngine::validateOptions(std::shared_ptr<options::ProgramOptions>) {
   throw std::runtime_error("not implemented");
 }
 
-void RocksDBEngine::start() { throw std::runtime_error("not implemented"); }
+void RocksDBEngine::start() {
+  //it is already decided that rocksdb is used
+  if (!isEnabled()) {
+    return;
+  }
+
+  // set the database sub-directory for RocksDB
+  auto database = ApplicationServer::getFeature<DatabasePathFeature>("DatabasePath");
+  _path = database->subdirectoryName("engine-rocksdb");
+
+  LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "initializing rocksdb, path: " << _path;
+
+  rocksdb::TransactionDBOptions transactionOptions;
+
+  _options.create_if_missing = true;
+  _options.max_open_files = -1;
+
+  rocksdb::Status status = rocksdb::TransactionDB::Open(_options, transactionOptions, _path, &_db);
+
+  if (! status.ok()) {
+    LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "unable to initialize RocksDB engine: " << status.ToString();
+    FATAL_ERROR_EXIT();
+  }
+}
 void RocksDBEngine::stop() { throw std::runtime_error("not implemented"); }
 // preparation phase for storage engine. can be used for internal setup.
 // the storage engine must not start any threads here or write any files

arangod/RocksDBEngine/RocksDBEngine.h

@@ -29,6 +29,9 @@
 #include <string>
 #include <vector>
 
+#include <rocksdb/db.h>
+#include <rocksdb/utilities/transaction_db.h>
+
 #include "Basics/Common.h"
 #include "Basics/Mutex.h"
 #include "StorageEngine/StorageEngine.h"
@@ -54,7 +57,6 @@ class RocksDBEngine final : public StorageEngine {
  public:
   // create the storage engine
   explicit RocksDBEngine(application_features::ApplicationServer*);
-
   ~RocksDBEngine();
 
   // inherited from ApplicationFeature
@@ -71,6 +73,7 @@ class RocksDBEngine final : public StorageEngine {
   // the storage engine must not start any threads here or write any files
   void prepare() override;
 
+
   transaction::ContextData* createTransactionContextData() override;
   TransactionState* createTransactionState(TRI_vocbase_t*) override;
   TransactionCollection* createTransactionCollection(
@@ -223,9 +226,14 @@ class RocksDBEngine final : public StorageEngine {
   /// @brief Add engine-specific REST handlers
   void addRestHandlers(rest::RestHandlerFactory*) override;
 
- public:
+public:
   static std::string const EngineName;
   static std::string const FeatureName;
+
+private:
+  rocksdb::TransactionDB* _db;
+  rocksdb::Options _options;
+  std::string _path;
 };
 }
 #endif

arangod/StorageEngine/EngineSelectorFeature.cpp

@@ -26,6 +26,7 @@
 #include "ProgramOptions/ProgramOptions.h"
 #include "ProgramOptions/Section.h"
 #include "MMFiles/MMFilesEngine.h"
+#include "RocksDBEngine/RocksDBEngine.h"
 #include "StorageEngine/StorageEngine.h"
 
 using namespace arangodb;
@@ -35,7 +36,7 @@ StorageEngine* EngineSelectorFeature::ENGINE = nullptr;
 
 EngineSelectorFeature::EngineSelectorFeature(
     application_features::ApplicationServer* server)
-    : ApplicationFeature(server, "EngineSelector"), _engine(MMFilesEngine::EngineName) {
+    : ApplicationFeature(server, "EngineSelector"), _engine("auto") {
   setOptional(false);
   requiresElevatedPrivileges(false);
   startsAfter("Logger");
@@ -44,9 +45,15 @@ EngineSelectorFeature::EngineSelectorFeature(
 void EngineSelectorFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
   options->addSection("server", "Server features");
 
-  options->addHiddenOption("--server.storage-engine", 
-                           "storage engine type",
-                           new DiscreteValuesParameter<StringParameter>(&_engine, availableEngineNames()));
+  options->addOption("--server.storage-engine", 
+                     "storage engine type",
+                     new DiscreteValuesParameter<StringParameter>(&_engine, availableEngineNames()));
+}
+
+void EngineSelectorFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
+  if(_engine == "auto"){
+    _engine = MMFilesEngine::EngineName;
+  }
 }
 
 void EngineSelectorFeature::prepare() {
@@ -83,12 +90,14 @@ std::unordered_set<std::string> EngineSelectorFeature::availableEngineNames() {
   for (auto const& it : availableEngines()) {
     result.emplace(it.first);
   }
-  return result; 
+  result.emplace("auto");
+  return result;
 }
 
 // return all available storage engines
 std::unordered_map<std::string, std::string> EngineSelectorFeature::availableEngines() { 
   return std::unordered_map<std::string, std::string>{
-    {MMFilesEngine::EngineName, MMFilesEngine::FeatureName}
+    {MMFilesEngine::EngineName, MMFilesEngine::FeatureName},
+    {RocksDBEngine::EngineName, RocksDBEngine::FeatureName}
   };
 }

arangod/StorageEngine/EngineSelectorFeature.h

@@ -35,6 +35,7 @@ class EngineSelectorFeature final : public application_features::ApplicationFeat
 
  public:
   void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;
+  void validateOptions(std::shared_ptr<options::ProgramOptions>) override final;
   void prepare() override final;
   void unprepare() override final;