From 82e784dcab634a96a644e8dd4f2e4501615b50a7 Mon Sep 17 00:00:00 2001
From: Andreas Streichardt <andreas.streichardt@gmail.com>
Date: Tue, 25 Oct 2016 08:47:11 +0200
Subject: [PATCH 01/27] WINDOWS_LEAN_AND_MEAN so the min max stuff is finally
 fixed

---
 lib/Basics/Common.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Basics/Common.h b/lib/Basics/Common.h
index f5ff23a335..fb85ac90b9 100644
--- a/lib/Basics/Common.h
+++ b/lib/Basics/Common.h
@@ -25,7 +25,7 @@
 #define ARANGODB_BASICS_COMMON_H 1
 
 #ifdef _WIN32
-
+#define WIN32_LEAN_AND_MEAN 1
 // debug malloc for Windows (only used when DEBUG is set)
 #define _CRTDBG_MAP_ALLOC
 #include <stdlib.h>

From a8cc4c802df4c6f84d837992d2781128351809ec Mon Sep 17 00:00:00 2001
From: Jan Christoph Uhde <Jan@UhdeJC.com>
Date: Mon, 8 May 2017 12:04:31 +0200
Subject: [PATCH 02/27] move logger state creation into engines

---
 arangod/MMFiles/MMFilesEngine.cpp     | 47 ++++++++++++++++++++++++++-
 arangod/MMFiles/MMFilesEngine.h       |  2 ++
 arangod/RocksDBEngine/RocksDBEngine.h |  2 +-
 arangod/StorageEngine/StorageEngine.h |  1 +
 arangod/V8Server/v8-replication.cpp   | 40 ++++-------------------
 5 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp
index 04dc494c8d..b7c6cc3a79 100644
--- a/arangod/MMFiles/MMFilesEngine.cpp
+++ b/arangod/MMFiles/MMFilesEngine.cpp
@@ -22,15 +22,16 @@
 /// @author Jan Christoph Uhde
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "MMFilesEngine.h"
 #include "Basics/FileUtils.h"
 #include "Basics/MutexLocker.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
+#include "Basics/build.h"
 #include "Basics/encoding.h"
 #include "Basics/files.h"
+#include "MMFilesEngine.h"
 #include "MMFiles/MMFilesAqlFunctions.h"
 #include "MMFiles/MMFilesCleanupThread.h"
 #include "MMFiles/MMFilesCollection.h"
@@ -54,6 +55,7 @@
 #include "Random/RandomGenerator.h"
 #include "RestServer/DatabaseFeature.h"
 #include "RestServer/DatabasePathFeature.h"
+#include "RestServer/ServerIdFeature.h"
 #include "RestServer/ViewTypesFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "VocBase/LogicalCollection.h"
@@ -3356,3 +3358,46 @@ int MMFilesEngine::handleSyncKeys(arangodb::InitialSyncer& syncer,
                           std::string& errorMsg) {
   return handleSyncKeysMMFiles(syncer, col, keysId, cid, collectionName,maxTick, errorMsg);
 }
+  
+Result MMFilesEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder){
+  MMFilesLogfileManagerState const s = MMFilesLogfileManager::instance()->state();
+
+  // "state" part
+  builder.add("state", VPackValue(VPackValueType::Object));  // open
+  builder.add("lastLogTick", VPackValue(std::to_string(s.lastCommittedTick)));
+  builder.add("lastUncommittedLogTick", VPackValue(std::to_string(s.lastAssignedTick)));
+  builder.add("totalEvents", VPackValue(static_cast<double>(s.numEvents + s.numEventsSync)));  // s.numEvents + s.numEventsSync
+  builder.add("time", VPackValue(s.timeString));
+  builder.close();
+
+  // "server" part
+  builder.add("server", VPackValue(VPackValueType::Object));  // open
+  builder.add("version", VPackValue(ARANGODB_VERSION));
+  builder.add("serverId", VPackValue(std::to_string(ServerIdFeature::getId())));
+  builder.close();
+
+  // "clients" part
+  builder.add("clients", VPackValue(VPackValueType::Array));  // open
+  if (vocbase != nullptr) {                                   // add clients
+    auto allClients = vocbase->getReplicationClients();
+    for (auto& it : allClients) {
+      // One client
+      builder.add(VPackValue(VPackValueType::Object));
+      builder.add("serverId", VPackValue(std::to_string(std::get<0>(it))));
+
+      char buffer[21];
+      TRI_GetTimeStampReplication(std::get<1>(it), &buffer[0], sizeof(buffer));
+      builder.add("time", VPackValue(buffer));
+
+      builder.add("lastServedTick",
+                  VPackValue(std::to_string(std::get<2>(it))));
+
+      builder.close();
+    }
+  }
+  builder.close();  // clients
+
+  builder.close();  // base
+
+  return Result();
+}
diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h
index 3cc8c08ac0..032cd28eb7 100644
--- a/arangod/MMFiles/MMFilesEngine.h
+++ b/arangod/MMFiles/MMFilesEngine.h
@@ -99,6 +99,8 @@ class MMFilesEngine final : public StorageEngine {
                      std::string const& collectionName, TRI_voc_tick_t maxTick,
                      std::string& errorMsg) override;
 
+  Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
+
   TransactionManager* createTransactionManager() override;
   transaction::ContextData* createTransactionContextData() override;
   TransactionState* createTransactionState(TRI_vocbase_t*) override;
diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h
index 38cb4fb106..7ae83d76e2 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.h
+++ b/arangod/RocksDBEngine/RocksDBEngine.h
@@ -259,7 +259,7 @@ class RocksDBEngine final : public StorageEngine {
   void addCollectionMapping(uint64_t, TRI_voc_tick_t, TRI_voc_cid_t);
   std::pair<TRI_voc_tick_t, TRI_voc_cid_t> mapObjectToCollection(uint64_t);
 
-  Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder);
+  Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
 
   void determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep);
   void pruneWalFiles();
diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h
index 22cf0eb04d..d82b1263f2 100644
--- a/arangod/StorageEngine/StorageEngine.h
+++ b/arangod/StorageEngine/StorageEngine.h
@@ -418,6 +418,7 @@ class StorageEngine : public application_features::ApplicationFeature {
                           std::string const& collectionName,
                           TRI_voc_tick_t maxTick,
                           std::string& errorMsg) = 0;
+  virtual Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) = 0;
   
   void getCapabilities(VPackBuilder& builder) const {
     builder.openObject();
diff --git a/arangod/V8Server/v8-replication.cpp b/arangod/V8Server/v8-replication.cpp
index e6a45cc170..2c0ef5cbe3 100644
--- a/arangod/V8Server/v8-replication.cpp
+++ b/arangod/V8Server/v8-replication.cpp
@@ -62,42 +62,16 @@ static void JS_StateLoggerReplication(
   TRI_V8_TRY_CATCH_BEGIN(isolate);
   v8::HandleScope scope(isolate);
 
-  std::string engineName = EngineSelectorFeature::ENGINE->typeName();
+  StorageEngine* engine = EngineSelectorFeature::ENGINE;
   v8::Handle<v8::Object> result = v8::Object::New(isolate);
 
-  if(engineName == "mmfiles"){
-    v8::Handle<v8::Object> state = v8::Object::New(isolate);
-    MMFilesLogfileManagerState const s = MMFilesLogfileManager::instance()->state();
-    state->Set(TRI_V8_ASCII_STRING("running"), v8::True(isolate));
-    state->Set(TRI_V8_ASCII_STRING("lastLogTick"),
-               TRI_V8UInt64String<TRI_voc_tick_t>(isolate, s.lastCommittedTick));
-    state->Set(TRI_V8_ASCII_STRING("lastUncommittedLogTick"),
-               TRI_V8UInt64String<TRI_voc_tick_t>(isolate, s.lastAssignedTick));
-    state->Set(TRI_V8_ASCII_STRING("totalEvents"),
-             v8::Number::New(isolate, static_cast<double>(s.numEvents + s.numEventsSync)));
-    state->Set(TRI_V8_ASCII_STRING("time"), TRI_V8_STD_STRING(s.timeString));
-    result->Set(TRI_V8_ASCII_STRING("state"), state);
-
-    v8::Handle<v8::Object> server = v8::Object::New(isolate);
-    server->Set(TRI_V8_ASCII_STRING("version"),
-                TRI_V8_ASCII_STRING(ARANGODB_VERSION));
-    server->Set(TRI_V8_ASCII_STRING("serverId"),
-                TRI_V8_STD_STRING(StringUtils::itoa(ServerIdFeature::getId())));
-    result->Set(TRI_V8_ASCII_STRING("server"), server);
-
-    v8::Handle<v8::Object> clients = v8::Object::New(isolate);
-    result->Set(TRI_V8_ASCII_STRING("clients"), clients);
-  } else if (engineName == "rocksdb") {
-    VPackBuilder builder;
-    auto res = rocksutils::globalRocksEngine()->createLoggerState(nullptr,builder);
-    if(res.fail()){
-      TRI_V8_THROW_EXCEPTION(res);
-    }
-    v8::Handle<v8::Value>resultValue = TRI_VPackToV8(isolate, builder.slice());
-    result = v8::Handle<v8::Object>::Cast(resultValue);
-  } else {
-    TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine");
+  VPackBuilder builder;
+  auto res = engine->createLoggerState(nullptr,builder);
+  if(res.fail()){
+    TRI_V8_THROW_EXCEPTION(res);
   }
+  v8::Handle<v8::Value>resultValue = TRI_VPackToV8(isolate, builder.slice());
+  result = v8::Handle<v8::Object>::Cast(resultValue);
 
   TRI_V8_RETURN(result);
   TRI_V8_TRY_CATCH_END

From cf333f3b0a31a440b739f0ede1b379d5030d754f Mon Sep 17 00:00:00 2001
From: Jan Christoph Uhde <Jan@UhdeJC.com>
Date: Mon, 8 May 2017 13:58:57 +0200
Subject: [PATCH 03/27] move creation of tickranges and first ticks into
 storage engines

---
 arangod/MMFiles/MMFilesEngine.cpp       |  29 +++++++
 arangod/MMFiles/MMFilesEngine.h         |   4 +-
 arangod/RocksDBEngine/RocksDBEngine.cpp |  52 ++++++++++++
 arangod/RocksDBEngine/RocksDBEngine.h   |   5 +-
 arangod/StorageEngine/StorageEngine.h   |   2 +
 arangod/V8Server/v8-replication.cpp     | 100 ++++--------------------
 6 files changed, 104 insertions(+), 88 deletions(-)

diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp
index b7c6cc3a79..a7547efe18 100644
--- a/arangod/MMFiles/MMFilesEngine.cpp
+++ b/arangod/MMFiles/MMFilesEngine.cpp
@@ -3401,3 +3401,32 @@ Result MMFilesEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& bu
 
   return Result();
 }
+
+Result MMFilesEngine::createTickRanges(VPackBuilder& builder){
+    auto const& ranges = MMFilesLogfileManager::instance()->ranges();
+    builder.isOpenArray();
+    for (auto& it : ranges) {
+      builder.openObject();
+      //filename and state are already of type string
+      builder.add("datafile", VPackValue(it.filename));
+      builder.add("state", VPackValue(it.state));
+      builder.add("tickMin", VPackValue(std::to_string(it.tickMin)));
+      builder.add("tickMax", VPackValue(std::to_string(it.tickMax)));
+      builder.close();
+    }
+    builder.close();
+    return Result{};
+}
+
+Result MMFilesEngine::firstTick(uint64_t& tick){
+  auto const& ranges = MMFilesLogfileManager::instance()->ranges();
+  for (auto& it : ranges) {
+    if (it.tickMin == 0) {
+      continue;
+    }
+    if (it.tickMin < tick) {
+      tick = it.tickMin;
+    }
+  }
+  return Result{};
+};
diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h
index 032cd28eb7..23c26acf81 100644
--- a/arangod/MMFiles/MMFilesEngine.h
+++ b/arangod/MMFiles/MMFilesEngine.h
@@ -100,7 +100,9 @@ class MMFilesEngine final : public StorageEngine {
                      std::string& errorMsg) override;
 
   Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
-
+  Result createTickRanges(VPackBuilder& builder) override;
+  Result firstTick(uint64_t& tick) override;
+  
   TransactionManager* createTransactionManager() override;
   transaction::ContextData* createTransactionContextData() override;
   TransactionState* createTransactionState(TRI_vocbase_t*) override;
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index 4d0edc31bf..b1cbe3b8b7 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -1335,4 +1335,56 @@ int RocksDBEngine::handleSyncKeys(arangodb::InitialSyncer& syncer,
   return handleSyncKeysRocksDB(syncer, col, keysId, cid, collectionName,
                                maxTick, errorMsg);
 }
+Result RocksDBEngine::createTickRanges(VPackBuilder& builder){
+    rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB();
+    rocksdb::VectorLogPtr walFiles;
+    rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
+    Result res = rocksutils::convertStatus(s);
+    if (res.fail()){
+      return res;
+    }
+
+    builder.isOpenArray();
+    for (auto lfile = walFiles.begin(); lfile != walFiles.end(); ++lfile) {
+      auto& logfile = *lfile;
+      builder.openObject();
+      //filename and state are already of type string
+      builder.add("datafile", VPackValue(logfile->PathName()));
+      if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) {
+        builder.add("state", VPackValue("open"));
+      } else if (logfile->Type() == rocksdb::WalFileType::kArchivedLogFile) {
+        builder.add("state", VPackValue("collected"));
+      }
+      rocksdb::SequenceNumber min = logfile->StartSequence();
+      builder.add("tickMin", VPackValue(std::to_string(min)));
+      rocksdb::SequenceNumber max;
+      if (std::next(lfile) != walFiles.end()) {
+        max = (*std::next(lfile))->StartSequence();
+      } else {
+        max = tdb->GetLatestSequenceNumber();
+      }
+      builder.add("tickMax", VPackValue(std::to_string(max)));
+      builder.close();
+    }
+    builder.close();
+    return Result{};
+}
+
+Result RocksDBEngine::firstTick(uint64_t& tick){
+  Result res{};
+  rocksdb::TransactionDB *tdb = rocksutils::globalRocksDB();
+  rocksdb::VectorLogPtr walFiles;
+  rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
+
+  if (!s.ok()) {
+    res = rocksutils::convertStatus(s);
+    return res;
+  }
+  // read minium possible tick
+  if (!walFiles.empty()) {
+    tick = walFiles[0]->StartSequence();
+  }
+  return res;
+}
+
 }  // namespace arangodb
diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h
index 7ae83d76e2..29a78cef5c 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.h
+++ b/arangod/RocksDBEngine/RocksDBEngine.h
@@ -125,6 +125,9 @@ class RocksDBEngine final : public StorageEngine {
                      std::string const& keysId, std::string const& cid,
                      std::string const& collectionName, TRI_voc_tick_t maxTick,
                      std::string& errorMsg) override;
+  Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
+  Result createTickRanges(VPackBuilder& builder) override;
+  Result firstTick(uint64_t& tick) override;
   // database, collection and index management
   // -----------------------------------------
 
@@ -259,8 +262,6 @@ class RocksDBEngine final : public StorageEngine {
   void addCollectionMapping(uint64_t, TRI_voc_tick_t, TRI_voc_cid_t);
   std::pair<TRI_voc_tick_t, TRI_voc_cid_t> mapObjectToCollection(uint64_t);
 
-  Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
-
   void determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep);
   void pruneWalFiles();
 
diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h
index d82b1263f2..3b66d9e512 100644
--- a/arangod/StorageEngine/StorageEngine.h
+++ b/arangod/StorageEngine/StorageEngine.h
@@ -419,6 +419,8 @@ class StorageEngine : public application_features::ApplicationFeature {
                           TRI_voc_tick_t maxTick,
                           std::string& errorMsg) = 0;
   virtual Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) = 0;
+  virtual Result createTickRanges(VPackBuilder& builder) = 0;
+  virtual Result firstTick(uint64_t& tick) = 0;
   
   void getCapabilities(VPackBuilder& builder) const {
     builder.openObject();
diff --git a/arangod/V8Server/v8-replication.cpp b/arangod/V8Server/v8-replication.cpp
index 2c0ef5cbe3..4d3a648023 100644
--- a/arangod/V8Server/v8-replication.cpp
+++ b/arangod/V8Server/v8-replication.cpp
@@ -36,6 +36,7 @@
 #include "V8/v8-vpack.h"
 #include "V8Server/v8-vocbaseprivate.h"
 #include "v8-replication.h"
+#include "StorageEngine/StorageEngine.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "RocksDBEngine/RocksDBEngine.h"
 #include "RocksDBEngine/RocksDBCommon.h"
@@ -86,63 +87,16 @@ static void JS_TickRangesLoggerReplication(
   TRI_V8_TRY_CATCH_BEGIN(isolate);
   v8::HandleScope scope(isolate);
   v8::Handle<v8::Array> result;
-  
-  std::string engineName = EngineSelectorFeature::ENGINE->typeName();
-  if (engineName == "mmfiles") {
-    auto const& ranges = MMFilesLogfileManager::instance()->ranges();
-    result = v8::Array::New(isolate, (int)ranges.size());
-    
-    uint32_t i = 0;
-    
-    for (auto& it : ranges) {
-      v8::Handle<v8::Object> df = v8::Object::New(isolate);
-      
-      df->ForceSet(TRI_V8_ASCII_STRING("datafile"), TRI_V8_STD_STRING(it.filename));
-      df->ForceSet(TRI_V8_ASCII_STRING("state"), TRI_V8_STD_STRING(it.state));
-      df->ForceSet(TRI_V8_ASCII_STRING("tickMin"), TRI_V8UInt64String<TRI_voc_tick_t>(isolate, it.tickMin));
-      df->ForceSet(TRI_V8_ASCII_STRING("tickMax"), TRI_V8UInt64String<TRI_voc_tick_t>(isolate, it.tickMax));
-      
-      result->Set(i++, df);
-    }
-  } else if (engineName == "rocksdb") {
-    rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB();
-    rocksdb::VectorLogPtr walFiles;
-    rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
-    if (!s.ok()) {
-      Result r = rocksutils::convertStatus(s);
-      TRI_V8_THROW_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage());
-    }
-    
-    result = v8::Array::New(isolate, (int)walFiles.size());
-    for(uint32_t i = 0; i < walFiles.size(); i++) {
-      std::unique_ptr<rocksdb::LogFile>& logfile = walFiles[i];
-      
-      v8::Handle<v8::Object> df = v8::Object::New(isolate);
-      df->ForceSet(TRI_V8_ASCII_STRING("datafile"), TRI_V8_STD_STRING(logfile->PathName()));
-      // setting state of each file
-      if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) {
-        df->ForceSet(TRI_V8_ASCII_STRING("state"), TRI_V8_STRING("open"));
-      } else if (logfile->Type() == rocksdb::WalFileType::kArchivedLogFile) {
-        df->ForceSet(TRI_V8_ASCII_STRING("state"), TRI_V8_STRING("collected"));
-      }
-      rocksdb::SequenceNumber min = logfile->StartSequence();
-      df->ForceSet(TRI_V8_ASCII_STRING("tickMin"),
-                   TRI_V8UInt64String<TRI_voc_tick_t>(isolate, min));
-      
-      rocksdb::SequenceNumber max;
-      if (i+1 < walFiles.size()) {
-        max = walFiles[i+1]->StartSequence();
-      } else {
-        max = tdb->GetLatestSequenceNumber();
-      }
-      df->ForceSet(TRI_V8_ASCII_STRING("tickMax"),
-                   TRI_V8UInt64String<rocksdb::SequenceNumber>(isolate, max));
-      
-      result->Set(i, df);
-    }
-  } else {
-    TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine");
-  }
+
+  VPackBuilder builder;
+  Result res = EngineSelectorFeature::ENGINE->createTickRanges(builder);
+  if(res.fail()){
+    TRI_V8_THROW_EXCEPTION(res);
+   }
+
+  v8::Handle<v8::Value>resultValue = TRI_VPackToV8(isolate, builder.slice());
+  result = v8::Handle<v8::Array>::Cast(resultValue);
+
   TRI_V8_RETURN(result);
   TRI_V8_TRY_CATCH_END
 }
@@ -157,35 +111,11 @@ static void JS_FirstTickLoggerReplication(
   v8::HandleScope scope(isolate);
 
   TRI_voc_tick_t tick = UINT64_MAX;
-  std::string engineName = EngineSelectorFeature::ENGINE->typeName();
-  if (engineName == "mmfiles") {
-    auto const& ranges = MMFilesLogfileManager::instance()->ranges();
-    
-    
-    for (auto& it : ranges) {
-      if (it.tickMin == 0) {
-        continue;
-      }
-      
-      if (it.tickMin < tick) {
-        tick = it.tickMin;
-      }
-    }
-  } else if (engineName == "rocksdb") {
-    rocksdb::TransactionDB *tdb = rocksutils::globalRocksDB();
-    rocksdb::VectorLogPtr walFiles;
-    rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
-    if (!s.ok()) {
-      Result r = rocksutils::convertStatus(s);
-      TRI_V8_THROW_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage());
-    }
-    // read minium possible tick
-    if (!walFiles.empty()) {
-      tick = walFiles[0]->StartSequence();
-    }
-  } else {
-    TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine");
+  Result res  = EngineSelectorFeature::ENGINE->firstTick(tick);
+  if(res.fail()){
+    TRI_V8_THROW_EXCEPTION(res);
   }
+
   if (tick == UINT64_MAX) {
     TRI_V8_RETURN(v8::Null(isolate));
   }

From 8ee0406c64d378ff61d7386935e648ea8221148b Mon Sep 17 00:00:00 2001
From: Jan Christoph Uhde <Jan@UhdeJC.com>
Date: Mon, 8 May 2017 15:18:19 +0200
Subject: [PATCH 04/27] move last logger creation to engines

---
 arangod/MMFiles/MMFilesEngine.cpp       | 20 ++++++-
 arangod/MMFiles/MMFilesEngine.h         |  1 +
 arangod/RocksDBEngine/RocksDBEngine.cpp | 18 ++++++
 arangod/RocksDBEngine/RocksDBEngine.h   |  1 +
 arangod/StorageEngine/StorageEngine.h   |  4 +-
 arangod/V8Server/v8-replication.cpp     | 79 +++++++------------------
 6 files changed, 63 insertions(+), 60 deletions(-)

diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp
index a7547efe18..706f5e8afd 100644
--- a/arangod/MMFiles/MMFilesEngine.cpp
+++ b/arangod/MMFiles/MMFilesEngine.cpp
@@ -31,13 +31,13 @@
 #include "Basics/build.h"
 #include "Basics/encoding.h"
 #include "Basics/files.h"
-#include "MMFilesEngine.h"
 #include "MMFiles/MMFilesAqlFunctions.h"
 #include "MMFiles/MMFilesCleanupThread.h"
 #include "MMFiles/MMFilesCollection.h"
 #include "MMFiles/MMFilesCompactorThread.h"
 #include "MMFiles/MMFilesDatafile.h"
 #include "MMFiles/MMFilesDatafileHelper.h"
+#include "MMFiles/MMFilesEngine.h"
 #include "MMFiles/MMFilesIndexFactory.h"
 #include "MMFiles/MMFilesInitialSync.h"
 #include "MMFiles/MMFilesLogfileManager.h"
@@ -52,6 +52,7 @@
 #include "MMFiles/MMFilesV8Functions.h"
 #include "MMFiles/MMFilesView.h"
 #include "MMFiles/MMFilesWalRecoveryFeature.h"
+#include "MMFiles/mmfiles-replication-dump.h"
 #include "Random/RandomGenerator.h"
 #include "RestServer/DatabaseFeature.h"
 #include "RestServer/DatabasePathFeature.h"
@@ -3430,3 +3431,20 @@ Result MMFilesEngine::firstTick(uint64_t& tick){
   }
   return Result{};
 };
+
+Result MMFilesEngine::lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) {
+  Result res{};
+  auto transactionContext = std::make_shared<transaction::StandaloneContext>(vocbase);
+  MMFilesReplicationDumpContext dump(transactionContext, 0, true, 0);
+  int r = MMFilesDumpLogReplication(&dump, std::unordered_set<TRI_voc_tid_t>(),
+                                      0, tickStart, tickEnd, true);
+  if (r != TRI_ERROR_NO_ERROR) {
+    res.reset(r);
+    return res;
+  }
+  // parsing JSON
+  VPackParser parser;
+  parser.parse(dump._buffer->_buffer);
+  builderSPtr = parser.steal();
+  return res;
+}
diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h
index 23c26acf81..877a998446 100644
--- a/arangod/MMFiles/MMFilesEngine.h
+++ b/arangod/MMFiles/MMFilesEngine.h
@@ -102,6 +102,7 @@ class MMFilesEngine final : public StorageEngine {
   Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
   Result createTickRanges(VPackBuilder& builder) override;
   Result firstTick(uint64_t& tick) override;
+  Result lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) override;
   
   TransactionManager* createTransactionManager() override;
   transaction::ContextData* createTransactionContextData() override;
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index b1cbe3b8b7..e52a23422e 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -60,6 +60,7 @@
 #include "RocksDBEngine/RocksDBV8Functions.h"
 #include "RocksDBEngine/RocksDBValue.h"
 #include "RocksDBEngine/RocksDBView.h"
+#include "RocksDBEngine/RocksDBReplicationTailing.h"
 #include "VocBase/replication-applier.h"
 #include "VocBase/ticks.h"
 
@@ -1387,4 +1388,21 @@ Result RocksDBEngine::firstTick(uint64_t& tick){
   return res;
 }
 
+Result RocksDBEngine::lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr){
+  bool includeSystem = true;
+  size_t chunkSize = 32 * 1024 * 1024; // TODO: determine good default value?
+
+  // construct vocbase with proper handler
+  std::shared_ptr<transaction::Context> transactionContext =
+  transaction::StandaloneContext::Create(vocbase);
+  VPackBuilder builder(transactionContext->getVPackOptions());
+
+  builder.openArray();
+  RocksDBReplicationResult rep = rocksutils::tailWal(vocbase, tickStart,
+                                                     tickEnd, chunkSize,
+                                                     includeSystem, 0, builder);
+  builder.close();
+  return rep;
+}
+
 }  // namespace arangodb
diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h
index 29a78cef5c..c50cab1102 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.h
+++ b/arangod/RocksDBEngine/RocksDBEngine.h
@@ -128,6 +128,7 @@ class RocksDBEngine final : public StorageEngine {
   Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
   Result createTickRanges(VPackBuilder& builder) override;
   Result firstTick(uint64_t& tick) override;
+  Result lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) override;
   // database, collection and index management
   // -----------------------------------------
 
diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h
index 3b66d9e512..19a4a34897 100644
--- a/arangod/StorageEngine/StorageEngine.h
+++ b/arangod/StorageEngine/StorageEngine.h
@@ -60,6 +60,7 @@ class ContextData;
 }
 
 class StorageEngine : public application_features::ApplicationFeature {
+
  public:
 
   // create the storage engine
@@ -421,7 +422,8 @@ class StorageEngine : public application_features::ApplicationFeature {
   virtual Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) = 0;
   virtual Result createTickRanges(VPackBuilder& builder) = 0;
   virtual Result firstTick(uint64_t& tick) = 0;
-  
+  virtual Result lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) = 0;
+
   void getCapabilities(VPackBuilder& builder) const {
     builder.openObject();
     builder.add("name", VPackValue(typeName()));
diff --git a/arangod/V8Server/v8-replication.cpp b/arangod/V8Server/v8-replication.cpp
index 4d3a648023..ec780d687b 100644
--- a/arangod/V8Server/v8-replication.cpp
+++ b/arangod/V8Server/v8-replication.cpp
@@ -24,23 +24,17 @@
 #include "Basics/ReadLocker.h"
 #include "Cluster/ClusterComm.h"
 #include "Cluster/ClusterFeature.h"
-// FIXME to be removed (should be storage engine independent - get it working now)
-#include "MMFiles/MMFilesLogfileManager.h"
-#include "MMFiles/mmfiles-replication-dump.h"
 #include "Replication/InitialSyncer.h"
 #include "Rest/Version.h"
 #include "RestServer/ServerIdFeature.h"
+#include "StorageEngine/EngineSelectorFeature.h"
+#include "StorageEngine/StorageEngine.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-globals.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
 #include "V8Server/v8-vocbaseprivate.h"
 #include "v8-replication.h"
-#include "StorageEngine/StorageEngine.h"
-#include "StorageEngine/EngineSelectorFeature.h"
-#include "RocksDBEngine/RocksDBEngine.h"
-#include "RocksDBEngine/RocksDBCommon.h"
-#include "RocksDBEngine/RocksDBReplicationTailing.h"
 
 #include <velocypack/Builder.h>
 #include <velocypack/Parser.h>
@@ -132,69 +126,38 @@ static void JS_LastLoggerReplication(
     v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_BEGIN(isolate);
   v8::HandleScope scope(isolate);
-  
+
   TRI_vocbase_t* vocbase = GetContextVocBase(isolate);
-  
+
   if (vocbase == nullptr) {
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
   }
-  
+
   if (args.Length() != 2) {
-    TRI_V8_THROW_EXCEPTION_USAGE(
-                                 "REPLICATION_LOGGER_LAST(<fromTick>, <toTick>)");
+    TRI_V8_THROW_EXCEPTION_USAGE("REPLICATION_LOGGER_LAST(<fromTick>, <toTick>)");
   }
+
   TRI_voc_tick_t tickStart = TRI_ObjectToUInt64(args[0], true);
   TRI_voc_tick_t tickEnd = TRI_ObjectToUInt64(args[1], true);
   if (tickEnd <= tickStart) {
-    TRI_V8_THROW_EXCEPTION_USAGE(
-                                 "tickStart < tickEnd");
+    TRI_V8_THROW_EXCEPTION_USAGE("tickStart < tickEnd");
   }
-  
-  
-  v8::Handle<v8::Value> result;
-  std::string engineName = EngineSelectorFeature::ENGINE->typeName();
-  if(engineName == "mmfiles"){
-    auto transactionContext = std::make_shared<transaction::StandaloneContext>(vocbase);
-    
-    MMFilesReplicationDumpContext dump(transactionContext, 0, true, 0);
-    
-    
-    int res = MMFilesDumpLogReplication(&dump, std::unordered_set<TRI_voc_tid_t>(),
-                                        0, tickStart, tickEnd, true);
-    
-    if (res != TRI_ERROR_NO_ERROR) {
-      TRI_V8_THROW_EXCEPTION(res);
-    }
-    // parsing JSON
-    VPackParser parser;
-    parser.parse(dump._buffer->_buffer);
-    result = TRI_VPackToV8(isolate, VPackSlice(parser.start()));
 
-  } else if (engineName == "rocksdb") {
-    bool includeSystem = true;
-    size_t chunkSize = 32 * 1024 * 1024; // TODO: determine good default value?
-    
-    // construct vocbase with proper handler
-    std::shared_ptr<transaction::Context> transactionContext =
-    transaction::StandaloneContext::Create(vocbase);
-    VPackBuilder builder(transactionContext->getVPackOptions());
-    
-    builder.openArray();
-    RocksDBReplicationResult rep = rocksutils::tailWal(vocbase, tickStart,
-                                                       tickEnd, chunkSize,
-                                                       includeSystem, 0, builder);
-    builder.close();
-    
-    if (rep.ok()) {
-      result = TRI_VPackToV8(isolate, builder.slice(),
-                             transactionContext->getVPackOptions());
-    } else {
-      result = v8::Null(isolate);
-    }
-  } else {
-    TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid storage engine");
+  auto builderSPtr = std::make_shared<VPackBuilder>();
+  Result res = EngineSelectorFeature::ENGINE->lastLogger(
+    vocbase, tickStart, tickEnd, builderSPtr);
+  v8::Handle<v8::Value> result;
+
+  if(res.fail()){
+    result = v8::Null(isolate);
+    TRI_V8_THROW_EXCEPTION(res);
   }
 
+  // do we need the options?
+  //result = TRI_VPackToV8(isolate, builderSPtr->slice(),
+  //                       transactionContext->getVPackOptions());
+  result = TRI_VPackToV8(isolate, VPackSlice(builderSPtr->slice()));
+
   TRI_V8_RETURN(result);
   TRI_V8_TRY_CATCH_END
 }

From acee2700756d831c27e79cfc924b482e84c2dae1 Mon Sep 17 00:00:00 2001
From: Jan Christoph Uhde <Jan@UhdeJC.com>
Date: Mon, 8 May 2017 15:43:09 +0200
Subject: [PATCH 05/27] fix small bugs revealed by tests

---
 arangod/MMFiles/MMFilesEngine.cpp       | 5 +++--
 arangod/RocksDBEngine/RocksDBEngine.cpp | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp
index 706f5e8afd..1f00cfd4bd 100644
--- a/arangod/MMFiles/MMFilesEngine.cpp
+++ b/arangod/MMFiles/MMFilesEngine.cpp
@@ -3362,9 +3362,10 @@ int MMFilesEngine::handleSyncKeys(arangodb::InitialSyncer& syncer,
   
 Result MMFilesEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder){
   MMFilesLogfileManagerState const s = MMFilesLogfileManager::instance()->state();
-
+  builder.openObject();  // Base
   // "state" part
   builder.add("state", VPackValue(VPackValueType::Object));  // open
+  builder.add("running", VPackValue(true));
   builder.add("lastLogTick", VPackValue(std::to_string(s.lastCommittedTick)));
   builder.add("lastUncommittedLogTick", VPackValue(std::to_string(s.lastAssignedTick)));
   builder.add("totalEvents", VPackValue(static_cast<double>(s.numEvents + s.numEventsSync)));  // s.numEvents + s.numEventsSync
@@ -3405,7 +3406,7 @@ Result MMFilesEngine::createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& bu
 
 Result MMFilesEngine::createTickRanges(VPackBuilder& builder){
     auto const& ranges = MMFilesLogfileManager::instance()->ranges();
-    builder.isOpenArray();
+    builder.openArray();
     for (auto& it : ranges) {
       builder.openObject();
       //filename and state are already of type string
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index e52a23422e..34b88e915c 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -996,7 +996,7 @@ Result RocksDBEngine::createLoggerState(TRI_vocbase_t* vocbase,
                                         VPackBuilder& builder) {
   syncWal();
 
-  builder.add(VPackValue(VPackValueType::Object));  // Base
+  builder.openObject();  // Base
   rocksdb::SequenceNumber lastTick = _db->GetLatestSequenceNumber();
 
   // "state" part
@@ -1345,7 +1345,7 @@ Result RocksDBEngine::createTickRanges(VPackBuilder& builder){
       return res;
     }
 
-    builder.isOpenArray();
+    builder.openArray();
     for (auto lfile = walFiles.begin(); lfile != walFiles.end(); ++lfile) {
       auto& logfile = *lfile;
       builder.openObject();

From a4a04350a3cb9b0f97b7c8edef680273ce39900f Mon Sep 17 00:00:00 2001
From: Jan Christoph Uhde <Jan@UhdeJC.com>
Date: Mon, 8 May 2017 17:43:36 +0200
Subject: [PATCH 06/27] fix replication

---
 arangod/MMFiles/MMFilesEngine.cpp       |  8 ++-
 arangod/MMFiles/MMFilesEngine.h         |  4 +-
 arangod/RocksDBEngine/RocksDBEngine.cpp | 74 ++++++++++++-------------
 arangod/RocksDBEngine/RocksDBEngine.h   |  3 +-
 arangod/StorageEngine/StorageEngine.h   |  6 +-
 arangod/V8Server/v8-replication.cpp     | 14 ++---
 6 files changed, 57 insertions(+), 52 deletions(-)

diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp
index 1f00cfd4bd..5d806f1660 100644
--- a/arangod/MMFiles/MMFilesEngine.cpp
+++ b/arangod/MMFiles/MMFilesEngine.cpp
@@ -3433,10 +3433,12 @@ Result MMFilesEngine::firstTick(uint64_t& tick){
   return Result{};
 };
 
-Result MMFilesEngine::lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) {
+Result MMFilesEngine::lastLogger(TRI_vocbase_t* /*vocbase*/, std::shared_ptr<transaction::Context> transactionContext, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) {
   Result res{};
-  auto transactionContext = std::make_shared<transaction::StandaloneContext>(vocbase);
-  MMFilesReplicationDumpContext dump(transactionContext, 0, true, 0);
+  std::shared_ptr<transaction::StandaloneContext> scontext =
+    std::dynamic_pointer_cast<transaction::StandaloneContext>(transactionContext);
+  TRI_ASSERT(scontext);
+  MMFilesReplicationDumpContext dump(scontext, 0, true, 0);
   int r = MMFilesDumpLogReplication(&dump, std::unordered_set<TRI_voc_tid_t>(),
                                       0, tickStart, tickEnd, true);
   if (r != TRI_ERROR_NO_ERROR) {
diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h
index 877a998446..7a8a60711c 100644
--- a/arangod/MMFiles/MMFilesEngine.h
+++ b/arangod/MMFiles/MMFilesEngine.h
@@ -102,8 +102,8 @@ class MMFilesEngine final : public StorageEngine {
   Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
   Result createTickRanges(VPackBuilder& builder) override;
   Result firstTick(uint64_t& tick) override;
-  Result lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) override;
-  
+  Result lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr<transaction::Context>, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) override;
+
   TransactionManager* createTransactionManager() override;
   transaction::ContextData* createTransactionContextData() override;
   TransactionState* createTransactionState(TRI_vocbase_t*) override;
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index 34b88e915c..44742df003 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -1038,7 +1038,7 @@ Result RocksDBEngine::createLoggerState(TRI_vocbase_t* vocbase,
 
   builder.close();  // base
 
-  return Result();
+  return Result{};
 }
 
 void RocksDBEngine::determinePrunableWalFiles(TRI_voc_tick_t minTickToKeep) {
@@ -1337,38 +1337,38 @@ int RocksDBEngine::handleSyncKeys(arangodb::InitialSyncer& syncer,
                                maxTick, errorMsg);
 }
 Result RocksDBEngine::createTickRanges(VPackBuilder& builder){
-    rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB();
-    rocksdb::VectorLogPtr walFiles;
-    rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
-    Result res = rocksutils::convertStatus(s);
-    if (res.fail()){
-      return res;
-    }
+  rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB();
+  rocksdb::VectorLogPtr walFiles;
+  rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
+  Result res = rocksutils::convertStatus(s);
+  if (res.fail()){
+    return res;
+  }
 
-    builder.openArray();
-    for (auto lfile = walFiles.begin(); lfile != walFiles.end(); ++lfile) {
-      auto& logfile = *lfile;
-      builder.openObject();
-      //filename and state are already of type string
-      builder.add("datafile", VPackValue(logfile->PathName()));
-      if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) {
-        builder.add("state", VPackValue("open"));
-      } else if (logfile->Type() == rocksdb::WalFileType::kArchivedLogFile) {
-        builder.add("state", VPackValue("collected"));
-      }
-      rocksdb::SequenceNumber min = logfile->StartSequence();
-      builder.add("tickMin", VPackValue(std::to_string(min)));
-      rocksdb::SequenceNumber max;
-      if (std::next(lfile) != walFiles.end()) {
-        max = (*std::next(lfile))->StartSequence();
-      } else {
-        max = tdb->GetLatestSequenceNumber();
-      }
-      builder.add("tickMax", VPackValue(std::to_string(max)));
-      builder.close();
+  builder.openArray();
+  for (auto lfile = walFiles.begin(); lfile != walFiles.end(); ++lfile) {
+    auto& logfile = *lfile;
+    builder.openObject();
+    //filename and state are already of type string
+    builder.add("datafile", VPackValue(logfile->PathName()));
+    if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) {
+      builder.add("state", VPackValue("open"));
+    } else if (logfile->Type() == rocksdb::WalFileType::kArchivedLogFile) {
+      builder.add("state", VPackValue("collected"));
     }
+    rocksdb::SequenceNumber min = logfile->StartSequence();
+    builder.add("tickMin", VPackValue(std::to_string(min)));
+    rocksdb::SequenceNumber max;
+    if (std::next(lfile) != walFiles.end()) {
+      max = (*std::next(lfile))->StartSequence();
+    } else {
+      max = tdb->GetLatestSequenceNumber();
+    }
+    builder.add("tickMax", VPackValue(std::to_string(max)));
     builder.close();
-    return Result{};
+  }
+  builder.close();
+  return Result{};
 }
 
 Result RocksDBEngine::firstTick(uint64_t& tick){
@@ -1388,20 +1388,20 @@ Result RocksDBEngine::firstTick(uint64_t& tick){
   return res;
 }
 
-Result RocksDBEngine::lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr){
+Result RocksDBEngine::lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr<transaction::Context> transactionContext, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr){
   bool includeSystem = true;
   size_t chunkSize = 32 * 1024 * 1024; // TODO: determine good default value?
 
   // construct vocbase with proper handler
-  std::shared_ptr<transaction::Context> transactionContext =
-  transaction::StandaloneContext::Create(vocbase);
-  VPackBuilder builder(transactionContext->getVPackOptions());
+  auto builder = std::make_unique<VPackBuilder>(transactionContext->getVPackOptions());
 
-  builder.openArray();
+  builder->openArray();
   RocksDBReplicationResult rep = rocksutils::tailWal(vocbase, tickStart,
                                                      tickEnd, chunkSize,
-                                                     includeSystem, 0, builder);
-  builder.close();
+                                                     includeSystem, 0, *builder);
+  builder->close();
+  builderSPtr = std::move(builder);
+
   return rep;
 }
 
diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h
index c50cab1102..e19b4eab81 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.h
+++ b/arangod/RocksDBEngine/RocksDBEngine.h
@@ -128,7 +128,8 @@ class RocksDBEngine final : public StorageEngine {
   Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) override;
   Result createTickRanges(VPackBuilder& builder) override;
   Result firstTick(uint64_t& tick) override;
-  Result lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) override;
+  Result lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr<transaction::Context>
+                   ,uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) override;
   // database, collection and index management
   // -----------------------------------------
 
diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h
index 19a4a34897..7d0f9377d2 100644
--- a/arangod/StorageEngine/StorageEngine.h
+++ b/arangod/StorageEngine/StorageEngine.h
@@ -56,6 +56,7 @@ class RestHandlerFactory;
 }
 
 namespace transaction {
+class Context;
 class ContextData;
 }
 
@@ -422,7 +423,10 @@ class StorageEngine : public application_features::ApplicationFeature {
   virtual Result createLoggerState(TRI_vocbase_t* vocbase, VPackBuilder& builder) = 0;
   virtual Result createTickRanges(VPackBuilder& builder) = 0;
   virtual Result firstTick(uint64_t& tick) = 0;
-  virtual Result lastLogger(TRI_vocbase_t* vocbase, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr) = 0;
+  virtual Result lastLogger(TRI_vocbase_t* vocbase
+                           ,std::shared_ptr<transaction::Context>
+                           ,uint64_t tickStart, uint64_t tickEnd
+                           ,std::shared_ptr<VPackBuilder>& builderSPtr) = 0;
 
   void getCapabilities(VPackBuilder& builder) const {
     builder.openObject();
diff --git a/arangod/V8Server/v8-replication.cpp b/arangod/V8Server/v8-replication.cpp
index ec780d687b..8c298bc44d 100644
--- a/arangod/V8Server/v8-replication.cpp
+++ b/arangod/V8Server/v8-replication.cpp
@@ -122,8 +122,7 @@ static void JS_FirstTickLoggerReplication(
 /// @brief get the last WAL entries
 ////////////////////////////////////////////////////////////////////////////////
 
-static void JS_LastLoggerReplication(
-    v8::FunctionCallbackInfo<v8::Value> const& args) {
+static void JS_LastLoggerReplication( v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_BEGIN(isolate);
   v8::HandleScope scope(isolate);
 
@@ -143,20 +142,19 @@ static void JS_LastLoggerReplication(
     TRI_V8_THROW_EXCEPTION_USAGE("tickStart < tickEnd");
   }
 
+  auto transactionContext = transaction::StandaloneContext::Create(vocbase);
   auto builderSPtr = std::make_shared<VPackBuilder>();
   Result res = EngineSelectorFeature::ENGINE->lastLogger(
-    vocbase, tickStart, tickEnd, builderSPtr);
-  v8::Handle<v8::Value> result;
+    vocbase, transactionContext, tickStart, tickEnd, builderSPtr);
 
+  v8::Handle<v8::Value> result;
   if(res.fail()){
     result = v8::Null(isolate);
     TRI_V8_THROW_EXCEPTION(res);
   }
 
-  // do we need the options?
-  //result = TRI_VPackToV8(isolate, builderSPtr->slice(),
-  //                       transactionContext->getVPackOptions());
-  result = TRI_VPackToV8(isolate, VPackSlice(builderSPtr->slice()));
+  result = TRI_VPackToV8(isolate, builderSPtr->slice(),
+                         transactionContext->getVPackOptions());
 
   TRI_V8_RETURN(result);
   TRI_V8_TRY_CATCH_END

From 788fc07427ef84ba0a46d38d6f1ff56af0656ffa Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 17:57:42 +0200
Subject: [PATCH 07/27] fix race

---
 arangod/Scheduler/ListenTask.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp
index bb3c61d771..edb9c4fd15 100644
--- a/arangod/Scheduler/ListenTask.cpp
+++ b/arangod/Scheduler/ListenTask.cpp
@@ -58,9 +58,13 @@ void ListenTask::start() {
   } catch (std::exception const& err) {
     LOG_TOPIC(WARN, arangodb::Logger::COMMUNICATION) << "failed to open endpoint '" << _endpoint->specification()
               << "' with error: " << err.what();
+    return;
   }
 
-  _handler = [this](boost::system::error_code const& ec) {
+  TRI_ASSERT(_bound);
+
+  auto self = shared_from_this();
+  _handler = [this, self](boost::system::error_code const& ec) {
     // copy the shared_ptr so nobody can delete the Acceptor while the
     // callback is running
     std::shared_ptr<Acceptor> acceptorCopy(_acceptor);

From 905f1efb9cc95bbdd38f3062b3c48cd3e6f79950 Mon Sep 17 00:00:00 2001
From: Wilfried Goesgens <willi@arangodb.com>
Date: Mon, 8 May 2017 18:02:28 +0200
Subject: [PATCH 08/27] clean out the working directory for snap - just in
 case.

---
 cmake/packages/snap.cmake | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cmake/packages/snap.cmake b/cmake/packages/snap.cmake
index 5898f246a5..edba6f2e3c 100644
--- a/cmake/packages/snap.cmake
+++ b/cmake/packages/snap.cmake
@@ -32,8 +32,13 @@ if(SNAPCRAFT_FOUND)
     DESTINATION "${SNAPCRAFT_SOURCE_DIR}/"
     )
 
+snapcraft clean arangodb3 -s pull
+
+arangodb3
+  
   add_custom_target(snap
     COMMENT "create snap-package"
+    COMMAND ${SNAP_EXE} clean ${CPACK_PACKAGE_NAME}
     COMMAND ${SNAP_EXE} snap
     COMMAND ${CMAKE_COMMAND} -E copy ${SNAPCRAFT_SOURCE_DIR}/${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}*_${ARANGODB_PACKAGE_ARCHITECTURE}.snap ${PROJECT_BINARY_DIR}
     DEPENDS TGZ_package

From 284bb6fb513f0164b0b6a4e8da51a770b408c0b4 Mon Sep 17 00:00:00 2001
From: Dan Larkin <danielhlarkin@gmail.com>
Date: Mon, 8 May 2017 12:08:41 -0400
Subject: [PATCH 09/27] Fixed some issues with RocksDB passthrough options.

---
 arangod/RocksDBEngine/RocksDBEngine.cpp          | 14 ++++++++++++++
 lib/ApplicationFeatures/RocksDBOptionFeature.cpp |  6 ++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index 4d0edc31bf..902a836b85 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -216,6 +216,20 @@ void RocksDBEngine::start() {
   _options.env->SetBackgroundThreads(opts->_numThreadsLow,
                                      rocksdb::Env::Priority::LOW);
 
+  if (opts->_blockCacheSize > 0) {
+    auto cache =
+        rocksdb::NewLRUCache(opts->_blockCacheSize, opts->_blockCacheShardBits);
+    rocksdb::BlockBasedTableOptions table_options;
+    table_options.block_cache = cache;
+    _options.table_factory.reset(
+        rocksdb::NewBlockBasedTableFactory(table_options));
+  } else {
+    rocksdb::BlockBasedTableOptions table_options;
+    table_options.no_block_cache = true;
+    _options.table_factory.reset(
+        rocksdb::NewBlockBasedTableFactory(table_options));
+  }
+
   _options.create_if_missing = true;
   _options.max_open_files = -1;
   _options.comparator = _cmp.get();
diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
index 30f771fdf1..1fda95e374 100644
--- a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
+++ b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
@@ -113,11 +113,13 @@ void RocksDBOptionFeature::collectOptions(
                      new UInt64Parameter(&_numLevels));
 
   options->addHiddenOption("--rocksdb.max-bytes-for-level-base",
-                           "control maximum total data size for a level",
+                           "control maximum total data size for level-1",
                            new UInt64Parameter(&_maxBytesForLevelBase));
 
   options->addOption("--rocksdb.max-bytes-for-level-multiplier",
-                     "control maximum total data size for a level",
+                     "maximum number of bytes for level L can be calculated as "
+                     "max-bytes-for-level-base * "
+                     "(max-bytes-for-level-multiplier ^ (L-1))",
                      new DoubleParameter(&_maxBytesForLevelMultiplier));
 
   options->addHiddenOption(

From 7577fed6fb5efedd109182652477b71d1a7c0d85 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 18:24:40 +0200
Subject: [PATCH 10/27] added logger

---
 arangod/RocksDBEngine/RocksDBEngine.cpp       |  7 ++
 .../RocksDBOptionFeature.cpp                  | 24 +++++
 .../RocksDBOptionFeature.h                    |  4 +-
 lib/Basics/RocksDBLogger.h                    | 98 +++++++++++++++++++
 4 files changed, 130 insertions(+), 3 deletions(-)
 create mode 100644 lib/Basics/RocksDBLogger.h

diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index 4d0edc31bf..0f1e173fc1 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -27,6 +27,7 @@
 #include "Basics/Exceptions.h"
 #include "Basics/FileUtils.h"
 #include "Basics/Result.h"
+#include "Basics/RocksDBLogger.h"
 #include "Basics/StaticStrings.h"
 #include "Basics/Thread.h"
 #include "Basics/VelocyPackHelper.h"
@@ -216,6 +217,11 @@ void RocksDBEngine::start() {
   _options.env->SetBackgroundThreads(opts->_numThreadsLow,
                                      rocksdb::Env::Priority::LOW);
 
+  _options.info_log_level = rocksdb::InfoLogLevel::ERROR_LEVEL;
+  // intentionally do not start the logger (yet)
+  // as it will produce a lot of log spam
+  // _options.info_log = std::make_shared<RocksDBLogger>(rocksdb::InfoLogLevel::ERROR_LEVEL);
+
   _options.create_if_missing = true;
   _options.max_open_files = -1;
   _options.comparator = _cmp.get();
@@ -236,6 +242,7 @@ void RocksDBEngine::start() {
         << "unable to initialize RocksDB engine: " << status.ToString();
     FATAL_ERROR_EXIT();
   }
+  
 
   TRI_ASSERT(_db != nullptr);
   _counterManager.reset(new RocksDBCounterManager(_db));
diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
index 30f771fdf1..e8d862b6a6 100644
--- a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
+++ b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
@@ -261,3 +261,27 @@ void RocksDBOptionFeature::validateOptions(
     FATAL_ERROR_EXIT();
   }
 }
+
+void RocksDBOptionFeature::start() {
+  LOG_TOPIC(TRACE, Logger::FIXME) << "using RocksDB options:"
+                                  << " write_buffer_size: " << _writeBufferSize
+                                  << " max_write_buffer_number: " << _maxWriteBufferNumber
+                                  << " delayed_write_rate: " << _delayedWriteRate
+                                  << " min_write_buffer_number_to_merge: " << _minWriteBufferNumberToMerge
+                                  << " num_levels: " << _numLevels
+                                  << " max_bytes_for_level_base: " << _maxBytesForLevelBase
+                                  << " max_bytes_for_level_multiplier: " << _maxBytesForLevelMultiplier
+                                  << " base_background_compactions: " << _baseBackgroundCompactions
+                                  << " max_background_compactions: " << _maxBackgroundCompactions
+                                  << " max_flushes: " << _maxFlushes
+                                  << " num_threads_high: " << _numThreadsHigh
+                                  << " num_threads_low: " << _numThreadsLow
+                                  << " block_cache_size: " << _blockCacheSize
+                                  << " block_cache_shard_bits: " << _blockCacheShardBits
+                                  << " compaction_read_ahead_size: " << _compactionReadaheadSize
+                                  << " verify_checksums_in_compaction: " << std::boolalpha << _verifyChecksumsInCompaction
+                                  << " optimize_filters_for_hits: " << std::boolalpha << _optimizeFiltersForHits
+                                  << " use_direct_reads: " << std::boolalpha << _useDirectReads
+                                  << " use_direct_writes: " << std::boolalpha << _useDirectWrites
+                                  << " use_fsync: " << std::boolalpha << _useFSync;
+}
diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.h b/lib/ApplicationFeatures/RocksDBOptionFeature.h
index 33a08cbb33..5e7e823508 100644
--- a/lib/ApplicationFeatures/RocksDBOptionFeature.h
+++ b/lib/ApplicationFeatures/RocksDBOptionFeature.h
@@ -45,9 +45,7 @@ class RocksDBOptionFeature final
 
   void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;
   void validateOptions(std::shared_ptr<options::ProgramOptions>) override final;
-  void prepare() override final{};
-  void start() override final {}
-  void unprepare() override final {}
+  void start() override final;
 
   uint64_t _writeBufferSize;
   uint64_t _maxWriteBufferNumber;
diff --git a/lib/Basics/RocksDBLogger.h b/lib/Basics/RocksDBLogger.h
new file mode 100644
index 0000000000..80330657f1
--- /dev/null
+++ b/lib/Basics/RocksDBLogger.h
@@ -0,0 +1,98 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGODB_BASICS_ROCKSDB_LOGGER_H
+#define ARANGODB_BASICS_ROCKSDB_LOGGER_H 1
+
+#include "Basics/Common.h"
+#include "Basics/StringRef.h"
+#include "Logger/Logger.h"
+
+#include <rocksdb/env.h>
+
+namespace arangodb {
+
+class RocksDBLogger final : public rocksdb::Logger {
+ public:
+  explicit RocksDBLogger(rocksdb::InfoLogLevel level) : rocksdb::Logger(level) {}
+  ~RocksDBLogger() {}
+  
+  // intentionally do not log header information here
+  // as this does not seem to honor the loglevel correctly
+  void LogHeader(const char* format, va_list ap) override {}
+
+  void Logv(char const* format, va_list ap) override {
+    static constexpr size_t prefixSize = 9; // strlen("rocksdb: ");
+    char buffer[2048];
+    memcpy(&buffer[0], "rocksdb: \0", prefixSize); // add trailing \0 byte already for safety
+
+    va_list backup;
+    va_copy(backup, ap);
+    int length = vsnprintf(&buffer[0] + prefixSize, sizeof(buffer) - prefixSize - 1, format, backup);
+    va_end(backup);
+    buffer[sizeof(buffer) - 1] = '\0';  // Windows
+
+    if (length == 0) {
+      return;
+    }
+
+    size_t l = static_cast<size_t>(length) + prefixSize;
+    if (l >= sizeof(buffer)) {
+      // truncation!
+      l = sizeof(buffer) - 1;
+    }
+
+    TRI_ASSERT(l > 0 && l < sizeof(buffer));
+    if (buffer[l - 1] == '\n' || buffer[l - 1] == '\0') {
+      // strip tailing \n or \0 in log message
+      --l;
+    }
+
+    switch (GetInfoLogLevel()) {
+      case rocksdb::InfoLogLevel::DEBUG_LEVEL:
+        LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << StringRef(buffer, l);
+        break;
+      case rocksdb::InfoLogLevel::INFO_LEVEL:
+        LOG_TOPIC(INFO, arangodb::Logger::FIXME) << StringRef(buffer, l);
+        break;
+      case rocksdb::InfoLogLevel::WARN_LEVEL:
+        LOG_TOPIC(WARN, arangodb::Logger::FIXME) << StringRef(buffer, l);
+        break;
+      case rocksdb::InfoLogLevel::ERROR_LEVEL:
+      case rocksdb::InfoLogLevel::FATAL_LEVEL:
+        LOG_TOPIC(ERR, arangodb::Logger::FIXME) << StringRef(buffer, l);
+        break;
+      default: {
+        // ignore these levels
+      }
+    }
+  }
+
+  // nothing to do here, as ArangoDB logger infrastructure takes care of flushing itself
+  void Flush() override {}
+ 
+};
+
+}  // namespace arangodb
+
+#endif

From 4e70775ec970ebace09e311516190def682c47f9 Mon Sep 17 00:00:00 2001
From: Dan Larkin <danielhlarkin@gmail.com>
Date: Mon, 8 May 2017 12:34:51 -0400
Subject: [PATCH 11/27] Removed some hard-coded default options for RocksDB.

---
 lib/ApplicationFeatures/RocksDBOptionFeature.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
index 1fda95e374..ae8386b323 100644
--- a/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
+++ b/lib/ApplicationFeatures/RocksDBOptionFeature.cpp
@@ -55,8 +55,8 @@ RocksDBOptionFeature::RocksDBOptionFeature(
       _baseBackgroundCompactions(rocksDBDefaults.base_background_compactions),
       _maxBackgroundCompactions(rocksDBDefaults.max_background_compactions),
       _maxFlushes(rocksDBDefaults.max_background_flushes),
-      _numThreadsHigh(1),
-      _numThreadsLow(1),
+      _numThreadsHigh(rocksDBDefaults.max_background_flushes),
+      _numThreadsLow(rocksDBDefaults.max_background_compactions),
       _blockCacheSize(8 * 1024 * 1024),
       _blockCacheShardBits(4),
       _maxLogFileSize(rocksDBDefaults.max_log_file_size),

From 129abe71b5041d195126f2a789f99d18a7d95eee Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 18:39:47 +0200
Subject: [PATCH 12/27] mmh. did the previous change break the cluster?

---
 arangod/Scheduler/ListenTask.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp
index edb9c4fd15..38e3471bd6 100644
--- a/arangod/Scheduler/ListenTask.cpp
+++ b/arangod/Scheduler/ListenTask.cpp
@@ -63,8 +63,8 @@ void ListenTask::start() {
 
   TRI_ASSERT(_bound);
 
-  auto self = shared_from_this();
-  _handler = [this, self](boost::system::error_code const& ec) {
+//  auto self = shared_from_this();
+  _handler = [this](boost::system::error_code const& ec) {
     // copy the shared_ptr so nobody can delete the Acceptor while the
     // callback is running
     std::shared_ptr<Acceptor> acceptorCopy(_acceptor);

From 1a9d8e0298cd06b45e7bdf287ff0e88411b4f56d Mon Sep 17 00:00:00 2001
From: Andreas Streichardt <andreas.streichardt@gmail.com>
Date: Mon, 8 May 2017 20:40:44 +0200
Subject: [PATCH 13/27] Skip ssl stuff when not required

---
 cmake/ArangoDBInstall.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ArangoDBInstall.cmake b/cmake/ArangoDBInstall.cmake
index 352d4f2ca8..ee6ecac902 100644
--- a/cmake/ArangoDBInstall.cmake
+++ b/cmake/ArangoDBInstall.cmake
@@ -203,7 +203,7 @@ install(FILES ${ICU_DT}
   DESTINATION "${INSTALL_ICU_DT_DEST}"
   RENAME ${ICU_DT_DEST})
 
-if (MSVC)
+if (MSVC AND NOT(SKIP_PACKAGING))
   # so we don't need to ship dll's twice, make it one directory:
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/InstallMacros.cmake)
   set(CMAKE_INSTALL_FULL_SBINDIR     "${CMAKE_INSTALL_FULL_BINDIR}")

From 6d80a770055e1a7ad8c616a1fbf5afdad157628c Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 22:05:40 +0200
Subject: [PATCH 14/27] dont copy words

---
 arangod/MMFiles/mmfiles-fulltext-index.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arangod/MMFiles/mmfiles-fulltext-index.cpp b/arangod/MMFiles/mmfiles-fulltext-index.cpp
index 3397365d14..5948ed79ff 100644
--- a/arangod/MMFiles/mmfiles-fulltext-index.cpp
+++ b/arangod/MMFiles/mmfiles-fulltext-index.cpp
@@ -1280,7 +1280,7 @@ bool TRI_InsertWordsMMFilesFulltextIndex(TRI_fts_index_t* const ftx,
     // LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << "checking word " << wordlist->_words[w];
 
     if (w > 0) {
-      std::string tmp = wordlist[w];
+      std::string const& tmp = wordlist[w];
       // check if current word has a shared/common prefix with the previous word
       // inserted
       // in case this is true, we can use an optimisation and do not need to

From fc0170f2a85e8f175a422bf4be54cbd4eb3471a2 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 22:06:03 +0200
Subject: [PATCH 15/27] re-enable shared_from_this

---
 arangod/GeneralServer/GeneralServer.cpp | 11 +++--------
 arangod/GeneralServer/GeneralServer.h   |  2 +-
 arangod/Scheduler/ListenTask.cpp        |  7 ++++---
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/arangod/GeneralServer/GeneralServer.cpp b/arangod/GeneralServer/GeneralServer.cpp
index 751ed1b192..8d4777f280 100644
--- a/arangod/GeneralServer/GeneralServer.cpp
+++ b/arangod/GeneralServer/GeneralServer.cpp
@@ -46,11 +46,7 @@ using namespace arangodb::rest;
 // --SECTION--                                      constructors and destructors
 // -----------------------------------------------------------------------------
 
-GeneralServer::~GeneralServer() {
-  for (auto& task : _listenTasks) {
-    delete task;
-  }
-}
+GeneralServer::~GeneralServer() {}
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                    public methods
@@ -106,7 +102,7 @@ bool GeneralServer::openEndpoint(Endpoint* endpoint) {
     }
   }
 
-  std::unique_ptr<ListenTask> task(new GeneralListenTask(
+  std::shared_ptr<ListenTask> task(new GeneralListenTask(
       SchedulerFeature::SCHEDULER->eventLoop(), this, endpoint, protocolType));
   task->start();
 
@@ -114,7 +110,6 @@ bool GeneralServer::openEndpoint(Endpoint* endpoint) {
     return false;
   }
 
-  _listenTasks.emplace_back(task.get());
-  task.release();
+  _listenTasks.emplace_back(std::move(task));
   return true;
 }
diff --git a/arangod/GeneralServer/GeneralServer.h b/arangod/GeneralServer/GeneralServer.h
index b725019e4b..bfd00b0445 100644
--- a/arangod/GeneralServer/GeneralServer.h
+++ b/arangod/GeneralServer/GeneralServer.h
@@ -56,7 +56,7 @@ class GeneralServer {
   bool openEndpoint(Endpoint* endpoint);
 
  private:
-  std::vector<ListenTask*> _listenTasks;
+  std::vector<std::shared_ptr<ListenTask>> _listenTasks;
   EndpointList const* _endpointList = nullptr;
 };
 }
diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp
index 38e3471bd6..071899bb3d 100644
--- a/arangod/Scheduler/ListenTask.cpp
+++ b/arangod/Scheduler/ListenTask.cpp
@@ -63,8 +63,9 @@ void ListenTask::start() {
 
   TRI_ASSERT(_bound);
 
-//  auto self = shared_from_this();
-  _handler = [this](boost::system::error_code const& ec) {
+  auto self = shared_from_this();
+
+  _handler = [this, self](boost::system::error_code const& ec) {
     // copy the shared_ptr so nobody can delete the Acceptor while the
     // callback is running
     std::shared_ptr<Acceptor> acceptorCopy(_acceptor);
@@ -111,7 +112,7 @@ void ListenTask::start() {
       acceptorCopy->asyncAccept(_handler);
     }
   };
-
+  
   _acceptor->asyncAccept(_handler);
 }
 

From f40760c3b212496dc954c66380ada5fab2d5c737 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 22:21:05 +0200
Subject: [PATCH 16/27] updated release notes

---
 .../Books/Manual/ReleaseNotes/NewFeatures32.md | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/Documentation/Books/Manual/ReleaseNotes/NewFeatures32.md b/Documentation/Books/Manual/ReleaseNotes/NewFeatures32.md
index 01dd923357..ed6a29338a 100644
--- a/Documentation/Books/Manual/ReleaseNotes/NewFeatures32.md
+++ b/Documentation/Books/Manual/ReleaseNotes/NewFeatures32.md
@@ -94,7 +94,7 @@ These will be resolved in the following releases:
 * index selectivity estimates are missing. All indexes will report their selectivity
   estimate as `0.2`. This may lead to non-optimal indexes being used in a query.
 
-* geo and fulltext indexes are not yet implemented
+* the geo index is not yet implemented
 
 * the number of documents reported for collections (`db.<collection>.count()`) may be
   slightly wrong during transactions
@@ -112,14 +112,18 @@ These will be resolved in the following releases:
 The existing indexes in the RocksDB engine are all persistent. The following indexes are
 supported there:
 
-* primary: automatically created, indexing `_id` / `_key`
+* primary: this type of index is automatically created. It indexes `_id` / `_key`
 
-* edge: automatically created for edge collections, indexing `_from` and `_to`
+* edge: this index is automatically created for edge collections. It indexes 
+  `_from` and `_to`
 
-* hash, skiplist, persistent: user-defined index, technically it is neither a hash
-  nor a skiplist index. All these index types map to the same RocksDB-based
-  sorted index implementation. The names "hash", "skiplist" and "persistent" are
-  only used for compatibility with the MMFiles engine.
+* hash, skiplist, persistent: these are user-defined indexes, Despite their names, they are 
+  neither hash nor skiplist indexes. These index types map to the same RocksDB-based
+  sorted index implementation. The same is true for the "persistent" index. The names 
+  "hash", "skiplist" and "persistent" are only used for compatibility with the MMFiles 
+  engine where these indexes existed in previous and the current version of ArangoDB.
+
+* fulltext: user-defined sorted reverted index on words occurring in documents
 
 
 Memory management

From e601e1efd9ce47a04b204d825235143d93a32989 Mon Sep 17 00:00:00 2001
From: Dan Larkin <danielhlarkin@gmail.com>
Date: Mon, 8 May 2017 16:47:36 -0400
Subject: [PATCH 17/27] Added prefix extractor.

---
 arangod/RocksDBEngine/CMakeLists.txt          |  1 +
 arangod/RocksDBEngine/RocksDBComparator.cpp   | 21 ++++-
 arangod/RocksDBEngine/RocksDBEngine.cpp       | 49 +++++++-----
 .../RocksDBEngine/RocksDBPrefixExtractor.cpp  | 78 +++++++++++++++++++
 .../RocksDBEngine/RocksDBPrefixExtractor.h    | 58 ++++++++++++++
 arangod/RocksDBEngine/RocksDBTypes.h          |  6 +-
 6 files changed, 187 insertions(+), 26 deletions(-)
 create mode 100644 arangod/RocksDBEngine/RocksDBPrefixExtractor.cpp
 create mode 100644 arangod/RocksDBEngine/RocksDBPrefixExtractor.h

diff --git a/arangod/RocksDBEngine/CMakeLists.txt b/arangod/RocksDBEngine/CMakeLists.txt
index 43bbe86e02..54d1be5ee3 100644
--- a/arangod/RocksDBEngine/CMakeLists.txt
+++ b/arangod/RocksDBEngine/CMakeLists.txt
@@ -19,6 +19,7 @@ set(ROCKSDB_SOURCES
   RocksDBEngine/RocksDBKey.cpp
   RocksDBEngine/RocksDBKeyBounds.cpp
   RocksDBEngine/RocksDBLogValue.cpp
+  RocksDBEngine/RocksDBPrefixExtractor.cpp
   RocksDBEngine/RocksDBPrimaryIndex.cpp
   RocksDBEngine/RocksDBReplicationCommon.cpp
   RocksDBEngine/RocksDBReplicationContext.cpp
diff --git a/arangod/RocksDBEngine/RocksDBComparator.cpp b/arangod/RocksDBEngine/RocksDBComparator.cpp
index e22acd4e7c..b3fbec996a 100644
--- a/arangod/RocksDBEngine/RocksDBComparator.cpp
+++ b/arangod/RocksDBEngine/RocksDBComparator.cpp
@@ -25,6 +25,7 @@
 #include "RocksDBEngine/RocksDBComparator.h"
 #include "Basics/VelocyPackHelper.h"
 #include "RocksDBEngine/RocksDBKey.h"
+#include "RocksDBEngine/RocksDBPrefixExtractor.h"
 #include "RocksDBEngine/RocksDBTypes.h"
 
 using namespace arangodb;
@@ -77,9 +78,6 @@ int RocksDBComparator::compareLexicographic(rocksdb::Slice const& lhs,
 
 int RocksDBComparator::compareIndexValues(rocksdb::Slice const& lhs,
                                           rocksdb::Slice const& rhs) const {
-  TRI_ASSERT(lhs.size() > sizeof(char) + sizeof(uint64_t));
-  TRI_ASSERT(rhs.size() > sizeof(char) + sizeof(uint64_t));
-
   size_t offset = sizeof(char);
   int result =
       memcmp((lhs.data() + offset), (rhs.data() + offset), sizeof(uint64_t));
@@ -87,6 +85,19 @@ int RocksDBComparator::compareIndexValues(rocksdb::Slice const& lhs,
     return result;
   }
 
+  size_t prefixLength = RocksDBPrefixExtractor::getPrefixLength(
+      static_cast<RocksDBEntryType>(lhs[0]));
+  if (lhs.size() == prefixLength || rhs.size() == prefixLength) {
+    if (lhs.size() == rhs.size()) {
+      return 0;
+    }
+
+    return ((lhs.size() < rhs.size()) ? -1 : 1);
+  }
+
+  TRI_ASSERT(lhs.size() > sizeof(char) + sizeof(uint64_t));
+  TRI_ASSERT(rhs.size() > sizeof(char) + sizeof(uint64_t));
+
   VPackSlice const lSlice = RocksDBKey::indexedVPack(lhs);
   VPackSlice const rSlice = RocksDBKey::indexedVPack(rhs);
 
@@ -127,7 +138,9 @@ int RocksDBComparator::compareIndexedValues(VPackSlice const& lhs,
   size_t const rLength = rhs.length();
   size_t const n = lLength < rLength ? rLength : lLength;
 
-  // LOG_TOPIC(ERR, Logger::FIXME) << "COMPARING INDEX VALUES: " << lhs.toJson() << "; " << rhs.toJson() << "; LLENGTH: " << lLength << ", RLENGTH: " << rLength << ", N: " << n;
+  // LOG_TOPIC(ERR, Logger::FIXME) << "COMPARING INDEX VALUES: " << lhs.toJson()
+  // << "; " << rhs.toJson() << "; LLENGTH: " << lLength << ", RLENGTH: " <<
+  // rLength << ", N: " << n;
 
   for (size_t i = 0; i < n; ++i) {
     int res = arangodb::basics::VelocyPackHelper::compare(
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index 17c749ae29..45ed74e5f8 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -51,7 +51,9 @@
 #include "RocksDBEngine/RocksDBInitialSync.h"
 #include "RocksDBEngine/RocksDBKey.h"
 #include "RocksDBEngine/RocksDBLogValue.h"
+#include "RocksDBEngine/RocksDBPrefixExtractor.h"
 #include "RocksDBEngine/RocksDBReplicationManager.h"
+#include "RocksDBEngine/RocksDBReplicationTailing.h"
 #include "RocksDBEngine/RocksDBRestHandlers.h"
 #include "RocksDBEngine/RocksDBTransactionCollection.h"
 #include "RocksDBEngine/RocksDBTransactionContextData.h"
@@ -61,7 +63,6 @@
 #include "RocksDBEngine/RocksDBV8Functions.h"
 #include "RocksDBEngine/RocksDBValue.h"
 #include "RocksDBEngine/RocksDBView.h"
-#include "RocksDBEngine/RocksDBReplicationTailing.h"
 #include "VocBase/replication-applier.h"
 #include "VocBase/ticks.h"
 
@@ -221,21 +222,20 @@ void RocksDBEngine::start() {
   _options.info_log_level = rocksdb::InfoLogLevel::ERROR_LEVEL;
   // intentionally do not start the logger (yet)
   // as it will produce a lot of log spam
-  // _options.info_log = std::make_shared<RocksDBLogger>(rocksdb::InfoLogLevel::ERROR_LEVEL);
+  // _options.info_log =
+  // std::make_shared<RocksDBLogger>(rocksdb::InfoLogLevel::ERROR_LEVEL);
 
+  rocksdb::BlockBasedTableOptions table_options;
   if (opts->_blockCacheSize > 0) {
     auto cache =
         rocksdb::NewLRUCache(opts->_blockCacheSize, opts->_blockCacheShardBits);
-    rocksdb::BlockBasedTableOptions table_options;
     table_options.block_cache = cache;
-    _options.table_factory.reset(
-        rocksdb::NewBlockBasedTableFactory(table_options));
   } else {
-    rocksdb::BlockBasedTableOptions table_options;
     table_options.no_block_cache = true;
-    _options.table_factory.reset(
-        rocksdb::NewBlockBasedTableFactory(table_options));
   }
+  table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
+  _options.table_factory.reset(
+      rocksdb::NewBlockBasedTableFactory(table_options));
 
   _options.create_if_missing = true;
   _options.max_open_files = -1;
@@ -247,7 +247,11 @@ void RocksDBEngine::start() {
                                                  // garbage collect them
   _options.WAL_size_limit_MB = 0;
   double counter_sync_seconds = 2.5;
-  // TODO: prefix_extractior +  memtable_insert_with_hint_prefix
+
+  _options.prefix_extractor.reset(new RocksDBPrefixExtractor());
+  _options.memtable_prefix_bloom_size_ratio = 0.1;  // TODO: pick better value?
+  // TODO: enable memtable_insert_with_hint_prefix_extractor?
+  _options.bloom_locality = 1;
 
   rocksdb::Status status =
       rocksdb::TransactionDB::Open(_options, transactionOptions, _path, &_db);
@@ -257,7 +261,6 @@ void RocksDBEngine::start() {
         << "unable to initialize RocksDB engine: " << status.ToString();
     FATAL_ERROR_EXIT();
   }
-  
 
   TRI_ASSERT(_db != nullptr);
   _counterManager.reset(new RocksDBCounterManager(_db));
@@ -1357,12 +1360,12 @@ int RocksDBEngine::handleSyncKeys(arangodb::InitialSyncer& syncer,
   return handleSyncKeysRocksDB(syncer, col, keysId, cid, collectionName,
                                maxTick, errorMsg);
 }
-Result RocksDBEngine::createTickRanges(VPackBuilder& builder){
+Result RocksDBEngine::createTickRanges(VPackBuilder& builder) {
   rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB();
   rocksdb::VectorLogPtr walFiles;
   rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
   Result res = rocksutils::convertStatus(s);
-  if (res.fail()){
+  if (res.fail()) {
     return res;
   }
 
@@ -1370,7 +1373,7 @@ Result RocksDBEngine::createTickRanges(VPackBuilder& builder){
   for (auto lfile = walFiles.begin(); lfile != walFiles.end(); ++lfile) {
     auto& logfile = *lfile;
     builder.openObject();
-    //filename and state are already of type string
+    // filename and state are already of type string
     builder.add("datafile", VPackValue(logfile->PathName()));
     if (logfile->Type() == rocksdb::WalFileType::kAliveLogFile) {
       builder.add("state", VPackValue("open"));
@@ -1392,9 +1395,9 @@ Result RocksDBEngine::createTickRanges(VPackBuilder& builder){
   return Result{};
 }
 
-Result RocksDBEngine::firstTick(uint64_t& tick){
+Result RocksDBEngine::firstTick(uint64_t& tick) {
   Result res{};
-  rocksdb::TransactionDB *tdb = rocksutils::globalRocksDB();
+  rocksdb::TransactionDB* tdb = rocksutils::globalRocksDB();
   rocksdb::VectorLogPtr walFiles;
   rocksdb::Status s = tdb->GetSortedWalFiles(walFiles);
 
@@ -1409,17 +1412,21 @@ Result RocksDBEngine::firstTick(uint64_t& tick){
   return res;
 }
 
-Result RocksDBEngine::lastLogger(TRI_vocbase_t* vocbase, std::shared_ptr<transaction::Context> transactionContext, uint64_t tickStart, uint64_t tickEnd,  std::shared_ptr<VPackBuilder>& builderSPtr){
+Result RocksDBEngine::lastLogger(
+    TRI_vocbase_t* vocbase,
+    std::shared_ptr<transaction::Context> transactionContext,
+    uint64_t tickStart, uint64_t tickEnd,
+    std::shared_ptr<VPackBuilder>& builderSPtr) {
   bool includeSystem = true;
-  size_t chunkSize = 32 * 1024 * 1024; // TODO: determine good default value?
+  size_t chunkSize = 32 * 1024 * 1024;  // TODO: determine good default value?
 
   // construct vocbase with proper handler
-  auto builder = std::make_unique<VPackBuilder>(transactionContext->getVPackOptions());
+  auto builder =
+      std::make_unique<VPackBuilder>(transactionContext->getVPackOptions());
 
   builder->openArray();
-  RocksDBReplicationResult rep = rocksutils::tailWal(vocbase, tickStart,
-                                                     tickEnd, chunkSize,
-                                                     includeSystem, 0, *builder);
+  RocksDBReplicationResult rep = rocksutils::tailWal(
+      vocbase, tickStart, tickEnd, chunkSize, includeSystem, 0, *builder);
   builder->close();
   builderSPtr = std::move(builder);
 
diff --git a/arangod/RocksDBEngine/RocksDBPrefixExtractor.cpp b/arangod/RocksDBEngine/RocksDBPrefixExtractor.cpp
new file mode 100644
index 0000000000..7f2114b14a
--- /dev/null
+++ b/arangod/RocksDBEngine/RocksDBPrefixExtractor.cpp
@@ -0,0 +1,78 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2017 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Daniel H. Larkin
+////////////////////////////////////////////////////////////////////////////////
+
+#include "RocksDBEngine/RocksDBPrefixExtractor.h"
+#include "Basics/VelocyPackHelper.h"
+#include "RocksDBEngine/RocksDBKey.h"
+#include "RocksDBEngine/RocksDBTypes.h"
+
+using namespace arangodb;
+using namespace arangodb::velocypack;
+
+RocksDBPrefixExtractor::RocksDBPrefixExtractor()
+    : _name("ArangoRocksDBPrefixExtractor") {}
+
+RocksDBPrefixExtractor::~RocksDBPrefixExtractor() {}
+
+const char* RocksDBPrefixExtractor::Name() const { return _name.data(); };
+
+rocksdb::Slice RocksDBPrefixExtractor::Transform(
+    rocksdb::Slice const& key) const {
+  size_t length = _prefixLength[static_cast<uint8_t>(key[0])];
+  return rocksdb::Slice(key.data(), length);
+}
+
+bool RocksDBPrefixExtractor::InDomain(rocksdb::Slice const& key) const {
+  return ((key.size() > 0) &&
+          (_prefixLength[static_cast<uint8_t>(key[0])] > 0) &&
+          (_prefixLength[static_cast<uint8_t>(key[0])] <= key.size()));
+}
+
+bool RocksDBPrefixExtractor::InRange(rocksdb::Slice const& dst) const {
+  return ((dst.size() > 0) &&
+          (dst.size() == _prefixLength[static_cast<uint8_t>(dst[0])]));
+}
+
+size_t RocksDBPrefixExtractor::getPrefixLength(RocksDBEntryType type) {
+  return _prefixLength[static_cast<uint8_t>(type)];
+}
+
+const size_t RocksDBPrefixExtractor::_prefixLength[] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x00 - 0x0f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x10 - 0x1f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x20 - 0x2f
+    1, 1, 1, 9, 9, 9, 9, 9, 1, 1, 1, 1, 0, 0, 0, 0,  // 0x30 - 0x3f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x40 - 0x4f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x50 - 0x5f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x60 - 0x6f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x70 - 0x7f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x80 - 0x8f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0x90 - 0x9f
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0xa0 - 0xaf
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0xb0 - 0xbf
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0xc0 - 0xcf
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0xd0 - 0xdf
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 0xe0 - 0xef
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0   // 0xf0 - 0xff
+};
diff --git a/arangod/RocksDBEngine/RocksDBPrefixExtractor.h b/arangod/RocksDBEngine/RocksDBPrefixExtractor.h
new file mode 100644
index 0000000000..c94f4955c5
--- /dev/null
+++ b/arangod/RocksDBEngine/RocksDBPrefixExtractor.h
@@ -0,0 +1,58 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2017 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Daniel H. Larkin
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGO_ROCKSDB_ROCKSDB_PREFIX_EXTRACTOR_H
+#define ARANGO_ROCKSDB_ROCKSDB_PREFIX_EXTRACTOR_H 1
+
+#include "Basics/Common.h"
+#include "RocksDBEngine/RocksDBTypes.h"
+
+#include <rocksdb/slice.h>
+#include <rocksdb/slice_transform.h>
+
+#include <velocypack/Slice.h>
+#include <velocypack/velocypack-aliases.h>
+
+namespace arangodb {
+
+class RocksDBPrefixExtractor final : public rocksdb::SliceTransform {
+ public:
+  RocksDBPrefixExtractor();
+  ~RocksDBPrefixExtractor();
+
+  const char* Name() const;
+  rocksdb::Slice Transform(rocksdb::Slice const& key) const;
+  bool InDomain(rocksdb::Slice const& key) const;
+  bool InRange(rocksdb::Slice const& dst) const;
+
+  static size_t getPrefixLength(RocksDBEntryType type);
+
+ private:
+  const std::string _name;
+  static const size_t _prefixLength[];
+};
+
+}  // namespace arangodb
+
+#endif
diff --git a/arangod/RocksDBEngine/RocksDBTypes.h b/arangod/RocksDBEngine/RocksDBTypes.h
index 6a6a706499..d633fc2d2c 100644
--- a/arangod/RocksDBEngine/RocksDBTypes.h
+++ b/arangod/RocksDBEngine/RocksDBTypes.h
@@ -31,6 +31,10 @@
 
 namespace arangodb {
 
+////////////////////////////////////////////////////////////////////////////////
+/// If these values change, make sure to reflect the changes in
+/// RocksDBPrefixExtractor as well.
+////////////////////////////////////////////////////////////////////////////////
 enum class RocksDBEntryType : char {
   Database = '0',
   Collection = '1',
@@ -67,6 +71,6 @@ enum class RocksDBLogType : char {
 };
 
 rocksdb::Slice const& rocksDBSlice(RocksDBEntryType const& type);
-}
+}  // namespace arangodb
 
 #endif

From 7db04133dd9026b99fc3c9c76b6ed8eae397d7c8 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Mon, 8 May 2017 23:53:04 +0200
Subject: [PATCH 18/27] move RocksDBLogger code into cpp file

---
 lib/Basics/RocksDBLogger.cpp | 78 ++++++++++++++++++++++++++++++++++++
 lib/Basics/RocksDBLogger.h   | 54 ++-----------------------
 lib/CMakeLists.txt           |  1 +
 3 files changed, 82 insertions(+), 51 deletions(-)
 create mode 100644 lib/Basics/RocksDBLogger.cpp

diff --git a/lib/Basics/RocksDBLogger.cpp b/lib/Basics/RocksDBLogger.cpp
new file mode 100644
index 0000000000..1d185d0f52
--- /dev/null
+++ b/lib/Basics/RocksDBLogger.cpp
@@ -0,0 +1,78 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#include "RocksDBLogger.h"
+#include "Basics/StringRef.h"
+#include "Logger/Logger.h"
+
+using namespace arangodb;
+
+RocksDBLogger::RocksDBLogger(rocksdb::InfoLogLevel level) : rocksdb::Logger(level) {}
+RocksDBLogger::~RocksDBLogger() {}
+  
+void RocksDBLogger::Logv(char const* format, va_list ap) {
+  static constexpr size_t prefixSize = 9; // strlen("rocksdb: ");
+  char buffer[2048];
+  memcpy(&buffer[0], "rocksdb: \0", prefixSize); // add trailing \0 byte already for safety
+
+  va_list backup;
+  va_copy(backup, ap);
+  int length = vsnprintf(&buffer[0] + prefixSize, sizeof(buffer) - prefixSize - 1, format, backup);
+  va_end(backup);
+  buffer[sizeof(buffer) - 1] = '\0';  // Windows
+
+  if (length == 0) {
+    return;
+  }
+
+  size_t l = static_cast<size_t>(length) + prefixSize;
+  if (l >= sizeof(buffer)) {
+    // truncation!
+    l = sizeof(buffer) - 1;
+  }
+
+  TRI_ASSERT(l > 0 && l < sizeof(buffer));
+  if (buffer[l - 1] == '\n' || buffer[l - 1] == '\0') {
+    // strip tailing \n or \0 in log message
+    --l;
+  }
+
+  switch (GetInfoLogLevel()) {
+    case rocksdb::InfoLogLevel::DEBUG_LEVEL:
+      LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << StringRef(buffer, l);
+      break;
+    case rocksdb::InfoLogLevel::INFO_LEVEL:
+      LOG_TOPIC(INFO, arangodb::Logger::FIXME) << StringRef(buffer, l);
+      break;
+    case rocksdb::InfoLogLevel::WARN_LEVEL:
+      LOG_TOPIC(WARN, arangodb::Logger::FIXME) << StringRef(buffer, l);
+      break;
+    case rocksdb::InfoLogLevel::ERROR_LEVEL:
+    case rocksdb::InfoLogLevel::FATAL_LEVEL:
+      LOG_TOPIC(ERR, arangodb::Logger::FIXME) << StringRef(buffer, l);
+      break;
+    default: { 
+      // ignore other levels 
+    }
+  }
+}
diff --git a/lib/Basics/RocksDBLogger.h b/lib/Basics/RocksDBLogger.h
index 80330657f1..2965dddb86 100644
--- a/lib/Basics/RocksDBLogger.h
+++ b/lib/Basics/RocksDBLogger.h
@@ -25,8 +25,6 @@
 #define ARANGODB_BASICS_ROCKSDB_LOGGER_H 1
 
 #include "Basics/Common.h"
-#include "Basics/StringRef.h"
-#include "Logger/Logger.h"
 
 #include <rocksdb/env.h>
 
@@ -34,63 +32,17 @@ namespace arangodb {
 
 class RocksDBLogger final : public rocksdb::Logger {
  public:
-  explicit RocksDBLogger(rocksdb::InfoLogLevel level) : rocksdb::Logger(level) {}
-  ~RocksDBLogger() {}
+  explicit RocksDBLogger(rocksdb::InfoLogLevel level); 
+  ~RocksDBLogger();
   
   // intentionally do not log header information here
   // as this does not seem to honor the loglevel correctly
   void LogHeader(const char* format, va_list ap) override {}
 
-  void Logv(char const* format, va_list ap) override {
-    static constexpr size_t prefixSize = 9; // strlen("rocksdb: ");
-    char buffer[2048];
-    memcpy(&buffer[0], "rocksdb: \0", prefixSize); // add trailing \0 byte already for safety
-
-    va_list backup;
-    va_copy(backup, ap);
-    int length = vsnprintf(&buffer[0] + prefixSize, sizeof(buffer) - prefixSize - 1, format, backup);
-    va_end(backup);
-    buffer[sizeof(buffer) - 1] = '\0';  // Windows
-
-    if (length == 0) {
-      return;
-    }
-
-    size_t l = static_cast<size_t>(length) + prefixSize;
-    if (l >= sizeof(buffer)) {
-      // truncation!
-      l = sizeof(buffer) - 1;
-    }
-
-    TRI_ASSERT(l > 0 && l < sizeof(buffer));
-    if (buffer[l - 1] == '\n' || buffer[l - 1] == '\0') {
-      // strip tailing \n or \0 in log message
-      --l;
-    }
-
-    switch (GetInfoLogLevel()) {
-      case rocksdb::InfoLogLevel::DEBUG_LEVEL:
-        LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << StringRef(buffer, l);
-        break;
-      case rocksdb::InfoLogLevel::INFO_LEVEL:
-        LOG_TOPIC(INFO, arangodb::Logger::FIXME) << StringRef(buffer, l);
-        break;
-      case rocksdb::InfoLogLevel::WARN_LEVEL:
-        LOG_TOPIC(WARN, arangodb::Logger::FIXME) << StringRef(buffer, l);
-        break;
-      case rocksdb::InfoLogLevel::ERROR_LEVEL:
-      case rocksdb::InfoLogLevel::FATAL_LEVEL:
-        LOG_TOPIC(ERR, arangodb::Logger::FIXME) << StringRef(buffer, l);
-        break;
-      default: {
-        // ignore these levels
-      }
-    }
-  }
+  void Logv(char const* format, va_list ap) override;
 
   // nothing to do here, as ArangoDB logger infrastructure takes care of flushing itself
   void Flush() override {}
- 
 };
 
 }  // namespace arangodb
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index d24183d48e..58e1ad2f07 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -135,6 +135,7 @@ add_library(${LIB_ARANGO} STATIC
   Basics/Nonce.cpp
   Basics/OpenFilesTracker.cpp
   Basics/ReadWriteLockCPP11.cpp
+  Basics/RocksDBLogger.cpp
   Basics/StaticStrings.cpp
   Basics/StringBuffer.cpp
   Basics/StringHeap.cpp

From 05e0c08824e1be60b4bf85adffd5b85579dfe452 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 02:03:59 +0200
Subject: [PATCH 19/27] there still seems to be a race between

- WorkMonitorThread asynchronously deleting handlers, and handlers accessing the io service
- the io service being deleted by the scheduler
---
 arangod/GeneralServer/GeneralListenTask.cpp |  1 +
 arangod/GeneralServer/GeneralServer.cpp     | 16 +++----
 arangod/GeneralServer/GeneralServer.h       |  9 ++--
 arangod/Scheduler/ListenTask.cpp            | 46 ++++++++++-----------
 arangod/Scheduler/ListenTask.h              | 14 +++----
 5 files changed, 40 insertions(+), 46 deletions(-)

diff --git a/arangod/GeneralServer/GeneralListenTask.cpp b/arangod/GeneralServer/GeneralListenTask.cpp
index 467a4776f9..ece7305137 100644
--- a/arangod/GeneralServer/GeneralListenTask.cpp
+++ b/arangod/GeneralServer/GeneralListenTask.cpp
@@ -26,6 +26,7 @@
 
 #include "GeneralServer/GeneralServer.h"
 #include "GeneralServer/GeneralServerFeature.h"
+#include "GeneralServer/HttpCommTask.h"
 #include "GeneralServer/VppCommTask.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
diff --git a/arangod/GeneralServer/GeneralServer.cpp b/arangod/GeneralServer/GeneralServer.cpp
index 8d4777f280..c44ca9c874 100644
--- a/arangod/GeneralServer/GeneralServer.cpp
+++ b/arangod/GeneralServer/GeneralServer.cpp
@@ -27,13 +27,9 @@
 #include "Basics/ConditionLocker.h"
 #include "Basics/MutexLocker.h"
 #include "Endpoint/EndpointList.h"
-#include "GeneralServer/AsyncJobManager.h"
+#include "GeneralServer/GeneralDefinitions.h"
 #include "GeneralServer/GeneralListenTask.h"
-#include "GeneralServer/RestHandler.h"
 #include "Logger/Logger.h"
-#include "Rest/CommonDefines.h"
-#include "Rest/GeneralResponse.h"
-#include "Scheduler/ListenTask.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Scheduler/Task.h"
@@ -46,7 +42,9 @@ using namespace arangodb::rest;
 // --SECTION--                                      constructors and destructors
 // -----------------------------------------------------------------------------
 
-GeneralServer::~GeneralServer() {}
+GeneralServer::~GeneralServer() {
+  _listenTasks.clear();
+}
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                    public methods
@@ -102,11 +100,9 @@ bool GeneralServer::openEndpoint(Endpoint* endpoint) {
     }
   }
 
-  std::shared_ptr<ListenTask> task(new GeneralListenTask(
+  std::unique_ptr<ListenTask> task(new GeneralListenTask(
       SchedulerFeature::SCHEDULER->eventLoop(), this, endpoint, protocolType));
-  task->start();
-
-  if (!task->isBound()) {
+  if (!task->start()) {
     return false;
   }
 
diff --git a/arangod/GeneralServer/GeneralServer.h b/arangod/GeneralServer/GeneralServer.h
index bfd00b0445..81e303bf88 100644
--- a/arangod/GeneralServer/GeneralServer.h
+++ b/arangod/GeneralServer/GeneralServer.h
@@ -28,11 +28,8 @@
 
 #include "Basics/Common.h"
 
-#include "Basics/ConditionVariable.h"
-#include "Endpoint/ConnectionInfo.h"
-#include "GeneralServer/GeneralDefinitions.h"
 #include "GeneralServer/HttpCommTask.h"
-#include "GeneralServer/RestHandler.h"
+#include "Scheduler/ListenTask.h"
 
 namespace arangodb {
 class EndpointList;
@@ -48,7 +45,7 @@ class GeneralServer {
   virtual ~GeneralServer();
 
  public:
-  void setEndpointList(const EndpointList* list);
+  void setEndpointList(EndpointList const* list);
   void startListening();
   void stopListening();
 
@@ -56,7 +53,7 @@ class GeneralServer {
   bool openEndpoint(Endpoint* endpoint);
 
  private:
-  std::vector<std::shared_ptr<ListenTask>> _listenTasks;
+  std::vector<std::unique_ptr<ListenTask>> _listenTasks;
   EndpointList const* _endpointList = nullptr;
 };
 }
diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp
index 071899bb3d..825b335f7a 100644
--- a/arangod/Scheduler/ListenTask.cpp
+++ b/arangod/Scheduler/ListenTask.cpp
@@ -24,10 +24,10 @@
 
 #include "ListenTask.h"
 
+#include "Basics/MutexLocker.h"
 #include "GeneralServer/GeneralServerFeature.h"
 #include "Logger/Logger.h"
 #include "Scheduler/Acceptor.h"
-#include "Ssl/SslServerFeature.h"
 
 using namespace arangodb;
 using namespace arangodb::rest;
@@ -40,43 +40,39 @@ ListenTask::ListenTask(EventLoop loop, Endpoint* endpoint)
     : Task(loop, "ListenTask"),
       _endpoint(endpoint),
       _bound(false),
-      _ioService(loop._ioService),
       _acceptor(Acceptor::factory(*loop._ioService, endpoint)) {}
 
+ListenTask::~ListenTask() {}
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                    public methods
 // -----------------------------------------------------------------------------
 
-void ListenTask::start() {
+bool ListenTask::start() {
+  MUTEX_LOCKER(mutex, _shutdownMutex);
+
   try {
     _acceptor->open();
-    _bound = true;
   } catch (boost::system::system_error const& err) {
     LOG_TOPIC(WARN, arangodb::Logger::COMMUNICATION) << "failed to open endpoint '" << _endpoint->specification()
               << "' with error: " << err.what();
-    return;
+    return false;
   } catch (std::exception const& err) {
     LOG_TOPIC(WARN, arangodb::Logger::COMMUNICATION) << "failed to open endpoint '" << _endpoint->specification()
               << "' with error: " << err.what();
-    return;
+    return true;
   }
 
-  TRI_ASSERT(_bound);
+  _handler = [this](boost::system::error_code const& ec) {
+    MUTEX_LOCKER(mutex, _shutdownMutex);
 
-  auto self = shared_from_this();
-
-  _handler = [this, self](boost::system::error_code const& ec) {
-    // copy the shared_ptr so nobody can delete the Acceptor while the
-    // callback is running
-    std::shared_ptr<Acceptor> acceptorCopy(_acceptor);
-
-    if (acceptorCopy == nullptr) {
-      // ListenTask already stopped
+    if (!_bound) {
+      _handler = nullptr;
       return;
     }
 
-    // now it is safe to use acceptorCopy
-    TRI_ASSERT(acceptorCopy != nullptr);
+    TRI_ASSERT(_handler != nullptr);
+    TRI_ASSERT(_acceptor != nullptr);
 
     if (ec) {
       if (ec == boost::asio::error::operation_aborted) {
@@ -95,7 +91,7 @@ void ListenTask::start() {
 
     ConnectionInfo info;
 
-    auto peer = acceptorCopy->movePeer();
+    auto peer = _acceptor->movePeer();
 
     // set the endpoint
     info.endpoint = _endpoint->specification();
@@ -108,20 +104,24 @@ void ListenTask::start() {
 
     handleConnected(std::move(peer), std::move(info));
 
-    if (_bound) {
-      acceptorCopy->asyncAccept(_handler);
-    }
+    _acceptor->asyncAccept(_handler);
   };
-  
+
+  _bound = true;
   _acceptor->asyncAccept(_handler);
+  return true;
 }
 
 void ListenTask::stop() {
+  MUTEX_LOCKER(mutex, _shutdownMutex);
+
   if (!_bound) {
     return;
   }
 
   _bound = false;
+  _handler = nullptr;
+
   _acceptor->close();
   _acceptor.reset();
 }
diff --git a/arangod/Scheduler/ListenTask.h b/arangod/Scheduler/ListenTask.h
index 66a5ef1783..64d26f62de 100644
--- a/arangod/Scheduler/ListenTask.h
+++ b/arangod/Scheduler/ListenTask.h
@@ -27,6 +27,7 @@
 
 #include "Scheduler/Task.h"
 
+#include "Basics/Mutex.h"
 #include "Endpoint/ConnectionInfo.h"
 #include "Endpoint/Endpoint.h"
 #include "Scheduler/Acceptor.h"
@@ -39,15 +40,15 @@ class ListenTask : virtual public rest::Task {
 
  public:
   ListenTask(EventLoop, Endpoint*);
+  ~ListenTask();
 
  public:
   virtual void handleConnected(std::unique_ptr<Socket>, ConnectionInfo&&) = 0;
 
  public:
-  bool isBound() const { return _bound.load(); }
   Endpoint* endpoint() const { return _endpoint; }
 
-  void start();
+  bool start();
   void stop();
 
  private:
@@ -56,12 +57,11 @@ class ListenTask : virtual public rest::Task {
  private:
   Endpoint* _endpoint;
   size_t _acceptFailures = 0;
-  std::atomic<bool> _bound;
-
-  boost::asio::io_service* _ioService;
-
-  std::shared_ptr<Acceptor> _acceptor;
+  
+  Mutex _shutdownMutex; 
+  bool _bound;
 
+  std::unique_ptr<Acceptor> _acceptor;
   std::function<void(boost::system::error_code const&)> _handler;
 };
 }

From 587418fd1b52d8402d63f93a957125e5665011d9 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 02:13:51 +0200
Subject: [PATCH 20/27] constified

---
 arangod/RocksDBEngine/RocksDBEngine.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index 45ed74e5f8..f22fddf586 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -175,7 +175,7 @@ void RocksDBEngine::start() {
   // transactionOptions.num_stripes = TRI_numberProcessors();
 
   // options imported set by RocksDBOptionFeature
-  auto* opts = ApplicationServer::getFeature<arangodb::RocksDBOptionFeature>(
+  auto const* opts = ApplicationServer::getFeature<arangodb::RocksDBOptionFeature>(
       "RocksDBOption");
 
   _options.write_buffer_size = static_cast<size_t>(opts->_writeBufferSize);

From 2067f5567517b7bd0f4c19f59d6d44cee70f6164 Mon Sep 17 00:00:00 2001
From: Wilfried Goesgens <willi@arangodb.com>
Date: Tue, 9 May 2017 09:54:13 +0200
Subject: [PATCH 21/27] remove cut'n'paste leftovers accidently forgotten

---
 cmake/packages/snap.cmake | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cmake/packages/snap.cmake b/cmake/packages/snap.cmake
index edba6f2e3c..8ece3be8a7 100644
--- a/cmake/packages/snap.cmake
+++ b/cmake/packages/snap.cmake
@@ -31,10 +31,6 @@ if(SNAPCRAFT_FOUND)
     COPY        "${SNAPCRAFT_TEMPLATE_DIR}/arangodb.png"
     DESTINATION "${SNAPCRAFT_SOURCE_DIR}/"
     )
-
-snapcraft clean arangodb3 -s pull
-
-arangodb3
   
   add_custom_target(snap
     COMMENT "create snap-package"

From fe0d987642498e3e9127fa583c6e4aea1ff2996e Mon Sep 17 00:00:00 2001
From: Wilfried Goesgens <willi@arangodb.com>
Date: Tue, 9 May 2017 10:17:14 +0200
Subject: [PATCH 22/27] fix condition to force full rebuild on windows

---
 Installation/Jenkins/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh
index 94dbc55260..91abcf232b 100755
--- a/Installation/Jenkins/build.sh
+++ b/Installation/Jenkins/build.sh
@@ -653,7 +653,7 @@ fi
 PARTIAL_STATE=$?
 set -e
 
-if test "${isCygwin}" == 1 -a "${PARTIAL_STATE}" == 0; then
+if test "${isCygwin}" == 0 -a "${PARTIAL_STATE}" == 0; then
     # windows fails to partialy re-configure - so do a complete configure run.
     if test -f CMakeFiles/generate.stamp -a CMakeFiles/generate.stamp -ot "${SOURCE_DIR}/CMakeList.txt"; then
         echo "CMakeList older - Forcing complete configure run!"

From 217d41f6f5fa97f6fa24704ed0d1aaaf42166216 Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 10:24:40 +0200
Subject: [PATCH 23/27] fix shutdown races

---
 arangod/Scheduler/Scheduler.cpp      | 13 ++++++++++
 arangod/Utils/WorkMonitorArangod.cpp | 39 +++++++++++++++-------------
 lib/Basics/WorkMonitor.h             |  1 +
 lib/Basics/WorkMonitorDummy.cpp      |  5 ++++
 4 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/arangod/Scheduler/Scheduler.cpp b/arangod/Scheduler/Scheduler.cpp
index 4b091145d7..1841f15a50 100644
--- a/arangod/Scheduler/Scheduler.cpp
+++ b/arangod/Scheduler/Scheduler.cpp
@@ -34,6 +34,7 @@
 #include "Basics/MutexLocker.h"
 #include "Basics/StringUtils.h"
 #include "Basics/Thread.h"
+#include "Basics/WorkMonitor.h"
 #include "GeneralServer/RestHandler.h"
 #include "Logger/Logger.h"
 #include "Rest/GeneralResponse.h"
@@ -440,6 +441,18 @@ void Scheduler::shutdown() {
 
   deleteOldThreads();
 
+  // remove all queued work descriptions in the work monitor first
+  // before freeing the io service a few lines later
+  // this is required because the work descriptions may have captured
+  // HttpCommTasks etc. which have references to the io service and
+  // access it in their destructors
+  // so the proper shutdown order is:
+  // - stop accepting further requests (already done by GeneralServerFeature::stop)
+  // - cancel all running scheduler tasks
+  // - free all work descriptions in work monitor
+  // - delete io service
+  WorkMonitor::clearWorkDescriptions();
+
   _managerService.reset();
   _ioService.reset();
 }
diff --git a/arangod/Utils/WorkMonitorArangod.cpp b/arangod/Utils/WorkMonitorArangod.cpp
index a86581081b..40d0d28e38 100644
--- a/arangod/Utils/WorkMonitorArangod.cpp
+++ b/arangod/Utils/WorkMonitorArangod.cpp
@@ -43,6 +43,24 @@ using namespace arangodb::rest;
 // -----------------------------------------------------------------------------
 // --SECTION--                                                       WorkMonitor
 // -----------------------------------------------------------------------------
+      
+bool WorkMonitor::clearWorkDescriptions() {
+  bool found = false;
+  WorkDescription* desc;
+
+  // handle freeable work descriptions
+  while (_freeableWorkDescription.pop(desc)) {
+    found = true;
+
+    if (desc != nullptr) {
+      deleteWorkDescription(desc, false);
+      found = true;
+      desc = nullptr;
+    }
+  }
+
+  return found;
+}
 
 void WorkMonitor::run() {
   CONDITION_LOCKER(guard, _waiter);
@@ -54,17 +72,7 @@ void WorkMonitor::run() {
   // clean old entries and create summary if requested
   while (!isStopping()) {
     try {
-      bool found = false;
-      WorkDescription* desc;
-
-      // handle freeable work descriptions
-      while (_freeableWorkDescription.pop(desc)) {
-        found = true;
-
-        if (desc != nullptr) {
-          deleteWorkDescription(desc, false);
-        }
-      }
+      bool found = clearWorkDescriptions();
 
       if (found) {
         s = minSleep;
@@ -129,14 +137,9 @@ void WorkMonitor::run() {
   _stopped.store(true);
 
   // cleanup old entries
+  clearWorkDescriptions();
+
   WorkDescription* desc;
-
-  while (_freeableWorkDescription.pop(desc)) {
-    if (desc != nullptr) {
-      deleteWorkDescription(desc, false);
-    }
-  }
-
   while (_emptyWorkDescription.pop(desc)) {
     if (desc != nullptr) {
       delete desc;
diff --git a/lib/Basics/WorkMonitor.h b/lib/Basics/WorkMonitor.h
index 2225b99cc0..7047c6f164 100644
--- a/lib/Basics/WorkMonitor.h
+++ b/lib/Basics/WorkMonitor.h
@@ -67,6 +67,7 @@ class WorkMonitor : public Thread {
   static void initialize();
   static void shutdown();
   static void clearHandlers();
+  static bool clearWorkDescriptions();
 
  private:
   static WorkDescription* createWorkDescription(WorkType);
diff --git a/lib/Basics/WorkMonitorDummy.cpp b/lib/Basics/WorkMonitorDummy.cpp
index b5943005b4..a3cd80f4ad 100644
--- a/lib/Basics/WorkMonitorDummy.cpp
+++ b/lib/Basics/WorkMonitorDummy.cpp
@@ -48,3 +48,8 @@ void WorkMonitor::addWorkOverview(std::shared_ptr<rest::RestHandler>,
 void WorkMonitor::clearAllHandlers() {
   TRI_ASSERT(false);
 }
+
+bool WorkMonitor::clearWorkDescriptions() {
+  TRI_ASSERT(false);
+  return false;
+}

From d0ae7673b9cc5030a5fd41fd30b8110138e228ff Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 10:26:59 +0200
Subject: [PATCH 24/27] do not print errors on shutdown

---
 js/server/modules/@arangodb/statistics.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/js/server/modules/@arangodb/statistics.js b/js/server/modules/@arangodb/statistics.js
index ed20a48e9f..ae3e33acb5 100644
--- a/js/server/modules/@arangodb/statistics.js
+++ b/js/server/modules/@arangodb/statistics.js
@@ -486,7 +486,10 @@ exports.historian = function () {
       }
     }
   } catch (err) {
-    require('console').warn('catch error in historian: %s', err.stack);
+    // errors on shutdown are expected. do not log them in case they occur
+    if (err.errorNum !== internal.errors.ERROR_SHUTTING_DOWN.code) {
+      require('console').warn('catch error in historian: %s', err.stack);
+    }
   }
 };
 

From e2039d587b595b9c3adfffc6b36a9355af420c3c Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 11:24:41 +0200
Subject: [PATCH 25/27] added debug output in case tests fail

---
 js/common/tests/replication/replication.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/js/common/tests/replication/replication.js b/js/common/tests/replication/replication.js
index ae7f7721d8..fff3087178 100644
--- a/js/common/tests/replication/replication.js
+++ b/js/common/tests/replication/replication.js
@@ -928,7 +928,7 @@ function ReplicationLoggerSuite () {
       }
 
       var entry = getLogEntries(tick, 2300);
-      assertEqual(100, entry.length);
+      assertEqual(100, entry.length, JSON.stringify(entry));
     },
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1562,7 +1562,7 @@ function ReplicationLoggerSuite () {
       });
 
       var entry = getLogEntries(tick, [ 2200, 2201, 2202, 2300 ]);
-      assertEqual(4, entry.length);
+      assertEqual(4, entry.length, JSON.stringify(entry));
 
       assertEqual(2200, entry[0].type);
       assertEqual(2300, entry[1].type);

From 1bfc52d8002ca9fa7bc5367929fc94a2234c4b2f Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 12:40:45 +0200
Subject: [PATCH 26/27] suppress non-error warnings on shutdown

---
 lib/Basics/Thread.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/Basics/Thread.cpp b/lib/Basics/Thread.cpp
index b7b9c3863b..5c78571417 100644
--- a/lib/Basics/Thread.cpp
+++ b/lib/Basics/Thread.cpp
@@ -244,7 +244,9 @@ void Thread::shutdown() {
   if (_state.load() == ThreadState::STARTED) {
     beginShutdown();
 
-    if (!isSilent() && _state.load() != ThreadState::STOPPING) {
+    if (!isSilent() && 
+        _state.load() != ThreadState::STOPPING && 
+        _state.load() != ThreadState::STOPPED) {
       LOG_TOPIC(WARN, Logger::THREADS) << "forcefully shutting down thread '"
                                        << _name << "' in state "
                                        << stringify(_state.load());

From 488e4be48598d5bc03c6163c94b4953f02f5c02b Mon Sep 17 00:00:00 2001
From: jsteemann <jan@arangodb.com>
Date: Tue, 9 May 2017 12:42:25 +0200
Subject: [PATCH 27/27] preparation for RocksDB logger

---
 arangod/GeneralServer/GeneralCommTask.cpp         |  4 +---
 arangod/RocksDBEngine/RocksDBEngine.cpp           |  6 ++++--
 arangod/RocksDBEngine/RocksDBTransactionState.cpp |  6 ++++++
 arangod/Utils/WorkMonitorArangod.cpp              |  1 -
 lib/Basics/RocksDBLogger.cpp                      | 13 +++++++++++--
 lib/Basics/RocksDBLogger.h                        |  1 +
 6 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/arangod/GeneralServer/GeneralCommTask.cpp b/arangod/GeneralServer/GeneralCommTask.cpp
index 92bef6fe1e..6b5c005f91 100644
--- a/arangod/GeneralServer/GeneralCommTask.cpp
+++ b/arangod/GeneralServer/GeneralCommTask.cpp
@@ -61,15 +61,13 @@ GeneralCommTask::GeneralCommTask(EventLoop loop, GeneralServer* server,
       _server(server) {}
 
 GeneralCommTask::~GeneralCommTask() {
-  for (auto&& statistics : _statisticsMap) {
+  for (auto& statistics : _statisticsMap) {
     auto stat = statistics.second;
 
     if (stat != nullptr) {
       stat->release();
     }
   }
-
-  _statisticsMap.clear();
 }
 
 // -----------------------------------------------------------------------------
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
index f22fddf586..6cb0a96010 100644
--- a/arangod/RocksDBEngine/RocksDBEngine.cpp
+++ b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -222,8 +222,10 @@ void RocksDBEngine::start() {
   _options.info_log_level = rocksdb::InfoLogLevel::ERROR_LEVEL;
   // intentionally do not start the logger (yet)
   // as it will produce a lot of log spam
-  // _options.info_log =
-  // std::make_shared<RocksDBLogger>(rocksdb::InfoLogLevel::ERROR_LEVEL);
+  // _options.info_log = std::make_shared<RocksDBLogger>(_options.info_log_level);
+
+  // _options.statistics = rocksdb::CreateDBStatistics();
+  // _options.stats_dump_period_sec = 1;
 
   rocksdb::BlockBasedTableOptions table_options;
   if (opts->_blockCacheSize > 0) {
diff --git a/arangod/RocksDBEngine/RocksDBTransactionState.cpp b/arangod/RocksDBEngine/RocksDBTransactionState.cpp
index 935a888c17..774ab15d3b 100644
--- a/arangod/RocksDBEngine/RocksDBTransactionState.cpp
+++ b/arangod/RocksDBEngine/RocksDBTransactionState.cpp
@@ -203,7 +203,13 @@ Result RocksDBTransactionState::commitTransaction(
 
         // TODO wait for response on github issue to see how we can use the
         // sequence number
+        double t1 = TRI_microtime();
         result = rocksutils::convertStatus(_rocksTransaction->Commit());
+
+        double t2 = TRI_microtime();
+        if (t2 - t1 > 0.25) {
+          LOG_TOPIC(ERR, Logger::FIXME) << "COMMIT TOOK: " << (t2 - t1) << " S. NUMINSERTS: " << _numInserts << ", NUMUPDATES: " << _numUpdates << ", NUMREMOVES: " << _numRemoves << ", TRANSACTIONSIZE: " << _transactionSize;
+        }
         rocksdb::SequenceNumber latestSeq =
             rocksutils::globalRocksDB()->GetLatestSequenceNumber();
         if (!result.ok()) {
diff --git a/arangod/Utils/WorkMonitorArangod.cpp b/arangod/Utils/WorkMonitorArangod.cpp
index 40d0d28e38..5666b05adf 100644
--- a/arangod/Utils/WorkMonitorArangod.cpp
+++ b/arangod/Utils/WorkMonitorArangod.cpp
@@ -34,7 +34,6 @@
 #include "Rest/HttpRequest.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
-#include "Statistics/RequestStatistics.h"
 #include "VocBase/vocbase.h"
 
 using namespace arangodb;
diff --git a/lib/Basics/RocksDBLogger.cpp b/lib/Basics/RocksDBLogger.cpp
index 1d185d0f52..0aa4773528 100644
--- a/lib/Basics/RocksDBLogger.cpp
+++ b/lib/Basics/RocksDBLogger.cpp
@@ -30,7 +30,11 @@ using namespace arangodb;
 RocksDBLogger::RocksDBLogger(rocksdb::InfoLogLevel level) : rocksdb::Logger(level) {}
 RocksDBLogger::~RocksDBLogger() {}
   
-void RocksDBLogger::Logv(char const* format, va_list ap) {
+void RocksDBLogger::Logv(const rocksdb::InfoLogLevel logLevel, char const* format, va_list ap) {
+  if (logLevel < GetInfoLogLevel()) {
+    return;
+  }
+  
   static constexpr size_t prefixSize = 9; // strlen("rocksdb: ");
   char buffer[2048];
   memcpy(&buffer[0], "rocksdb: \0", prefixSize); // add trailing \0 byte already for safety
@@ -57,7 +61,7 @@ void RocksDBLogger::Logv(char const* format, va_list ap) {
     --l;
   }
 
-  switch (GetInfoLogLevel()) {
+  switch (logLevel) {
     case rocksdb::InfoLogLevel::DEBUG_LEVEL:
       LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << StringRef(buffer, l);
       break;
@@ -76,3 +80,8 @@ void RocksDBLogger::Logv(char const* format, va_list ap) {
     }
   }
 }
+
+void RocksDBLogger::Logv(char const* format, va_list ap) {
+  // forward to the level-aware method
+  Logv(rocksdb::InfoLogLevel::INFO_LEVEL, format, ap);
+}
diff --git a/lib/Basics/RocksDBLogger.h b/lib/Basics/RocksDBLogger.h
index 2965dddb86..dae4445f70 100644
--- a/lib/Basics/RocksDBLogger.h
+++ b/lib/Basics/RocksDBLogger.h
@@ -40,6 +40,7 @@ class RocksDBLogger final : public rocksdb::Logger {
   void LogHeader(const char* format, va_list ap) override {}
 
   void Logv(char const* format, va_list ap) override;
+  void Logv(const rocksdb::InfoLogLevel, char const* format, va_list ap) override;
 
   // nothing to do here, as ArangoDB logger infrastructure takes care of flushing itself
   void Flush() override {}