Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

CMakeLists.txt

@@ -39,6 +39,7 @@ else ()
   project(arangodb3 CXX C ASM)
 endif ()
 
+# required for clang completion in editors
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # where to find CMAKE modules
@@ -949,7 +950,11 @@ if (NOT USE_PRECOMPILED_V8)
   add_dependencies(arangosh      v8_build)
 endif ()
 
-# copy compile commands to source dir
+# This copies the compile commands to the source dir.
+# There they can be used to repeat parts of compile
+# steps generating an AST. This AST can in turn be used
+# for meaningful auto-completion in editors without
+# heuristics, dice rolling and other guessing.
 if( EXISTS "${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json" )
   message(STATUS "copy compile_commands.json")
 

arangod/Agency/Constituent.cpp

@@ -40,7 +40,7 @@
 #include "Utils/OperationOptions.h"
 #include "Utils/OperationResult.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "VocBase/ticks.h"
 #include "VocBase/vocbase.h"
 
@@ -127,7 +127,7 @@ void Constituent::termNoLock(term_t t) {
 
     TRI_ASSERT(_vocbase != nullptr);
     auto transactionContext =
-        std::make_shared<StandaloneTransactionContext>(_vocbase);
+        std::make_shared<transaction::StandaloneContext>(_vocbase);
     SingleCollectionTransaction trx(transactionContext, "election",
                                     AccessMode::Type::WRITE);
 

arangod/Agency/State.cpp

@@ -45,7 +45,7 @@
 #include "Utils/OperationOptions.h"
 #include "Utils/OperationResult.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/vocbase.h"
 
@@ -101,7 +101,7 @@ bool State::persist(arangodb::consensus::index_t index, term_t term,
   
   TRI_ASSERT(_vocbase != nullptr);
   auto transactionContext =
-    std::make_shared<StandaloneTransactionContext>(_vocbase);
+    std::make_shared<transaction::StandaloneContext>(_vocbase);
   SingleCollectionTransaction trx(
     transactionContext, "log", AccessMode::Type::WRITE);
   
@@ -620,7 +620,7 @@ bool State::loadOrPersistConfiguration() {
     _agent->id(to_string(boost::uuids::random_generator()()));
 
     auto transactionContext =
-        std::make_shared<StandaloneTransactionContext>(_vocbase);
+        std::make_shared<transaction::StandaloneContext>(_vocbase);
     SingleCollectionTransaction trx(transactionContext, "configuration",
                                     AccessMode::Type::WRITE);
 
@@ -819,7 +819,7 @@ bool State::persistReadDB(arangodb::consensus::index_t cind) {
 
     TRI_ASSERT(_vocbase != nullptr);
     auto transactionContext =
-        std::make_shared<StandaloneTransactionContext>(_vocbase);
+        std::make_shared<transaction::StandaloneContext>(_vocbase);
     SingleCollectionTransaction trx(transactionContext, "compact",
                                     AccessMode::Type::WRITE);
 

arangod/Aql/AqlTransaction.h

@@ -26,7 +26,7 @@
 
 #include "Basics/Common.h"
 #include "Aql/Collection.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Methods.h"
 #include "VocBase/vocbase.h"
 
@@ -38,7 +38,7 @@ class AqlTransaction final : public transaction::Methods {
   /// @brief create the transaction and add all collections from the query
   /// context
   AqlTransaction(
-      std::shared_ptr<TransactionContext> transactionContext, 
+      std::shared_ptr<transaction::Context> transactionContext, 
       std::map<std::string, aql::Collection*> const* collections,
       bool isMainTransaction)
       : transaction::Methods(transactionContext),
@@ -88,7 +88,7 @@ class AqlTransaction final : public transaction::Methods {
   /// distributed
   /// AQL query running on the coordinator
   transaction::Methods* clone() const override {
-    return new AqlTransaction(StandaloneTransactionContext::Create(vocbase()),
+    return new AqlTransaction(transaction::StandaloneContext::Create(vocbase()),
         &_collections, false);
   }
 

arangod/Aql/AqlValue.cpp

@@ -26,7 +26,7 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Transaction/Helpers.h"
 #include "Transaction/Methods.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-vpack.h"
 

arangod/Aql/EnumerateCollectionBlock.cpp

@@ -185,7 +185,7 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t,  // atLeast,
     std::function<void(DocumentIdentifierToken const& tkn)> cb;
     if (_mustStoreResult) {
       cb = [&] (DocumentIdentifierToken const& tkn) {
-        if (c->readDocument(_trx, *_mmdr, tkn)) {
+        if (c->readDocument(_trx, tkn, *_mmdr)) {
           // The result is in the first variable of this depth,
           // we do not need to do a lookup in getPlanNode()->_registerPlan->varInfo,
           // but can just take cur->getNrRegs() as registerId:

arangod/Aql/ExecutionEngine.cpp

@@ -45,6 +45,7 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Cluster/ClusterComm.h"
 #include "Cluster/ClusterInfo.h"
+#include "Cluster/CollectionLockState.h"
 #include "Cluster/TraverserEngineRegistry.h"
 #include "Logger/Logger.h"
 #include "Transaction/Methods.h"
@@ -1139,8 +1140,8 @@ int ExecutionEngine::shutdown(int errorCode) {
   if (_root != nullptr && !_wasShutdown) {
     // Take care of locking prevention measures in the cluster:
     if (_lockedShards != nullptr) {
-      if (transaction::Methods::_makeNolockHeaders == _lockedShards) {
+      if (CollectionLockState::_noLockHeaders == _lockedShards) {
-        transaction::Methods::_makeNolockHeaders = _previouslyLockedShards;
+        CollectionLockState::_noLockHeaders = _previouslyLockedShards;
       }
       delete _lockedShards;
       _lockedShards = nullptr;
@@ -1190,10 +1191,10 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan(
         engine = inst.get()->buildEngines();
         root = engine->root();
         // Now find all shards that take part:
-        if (transaction::Methods::_makeNolockHeaders != nullptr) {
+        if (CollectionLockState::_noLockHeaders != nullptr) {
           engine->_lockedShards = new std::unordered_set<std::string>(
-              *transaction::Methods::_makeNolockHeaders);
+              *CollectionLockState::_noLockHeaders);
-          engine->_previouslyLockedShards = transaction::Methods::_makeNolockHeaders;
+          engine->_previouslyLockedShards = CollectionLockState::_noLockHeaders;
         } else {
           engine->_lockedShards = new std::unordered_set<std::string>();
           engine->_previouslyLockedShards = nullptr;
@@ -1291,7 +1292,7 @@ ExecutionEngine* ExecutionEngine::instantiateFromPlan(
                 TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, message);
           }
         }
-        transaction::Methods::_makeNolockHeaders = engine->_lockedShards;
+        CollectionLockState::_noLockHeaders = engine->_lockedShards;
       } catch (...) {
         // We need to destroy all queries that we have built and stuffed
         // into the QueryRegistry as well as those that we have pushed to

arangod/Aql/Functions.cpp

@@ -44,7 +44,7 @@
 #include "Utils/CollectionNameResolver.h"
 #include "Transaction/Helpers.h"
 #include "Transaction/Methods.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ManagedDocumentResult.h"
 

arangod/Aql/IndexBlock.cpp

@@ -439,7 +439,7 @@ bool IndexBlock::readIndex(size_t atMost) {
 
     if (hasMultipleIndexes) {
       for (auto const& element : _result) {
-        if (collection->readDocument(_trx, *_mmdr, element)) {
+        if (collection->readDocument(_trx, element, *_mmdr)) {
           uint8_t const* vpack = _mmdr->vpack(); //back();
           // uniqueness checks
           if (!isLastIndex) {
@@ -457,7 +457,7 @@ bool IndexBlock::readIndex(size_t atMost) {
       }
     } else {
       for (auto const& element : _result) {
-        if (collection->readDocument(_trx, *_mmdr, element)) {
+        if (collection->readDocument(_trx, element, *_mmdr)) {
           uint8_t const* vpack = _mmdr->vpack(); //back();
           _documents.emplace_back(vpack);
         }

arangod/Aql/ModificationBlocks.cpp

@@ -48,7 +48,7 @@ ModificationBlock::ModificationBlock(ExecutionEngine* engine,
       _isDBServer(false),
       _usesDefaultSharding(true) {
 
-  _trx->orderDitch(_collection->cid());
+  _trx->pinData(_collection->cid());
 
   auto const& registerPlan = ep->getRegisterPlan()->varInfo;
 

arangod/Aql/Query.cpp

@@ -43,8 +43,8 @@
 #include "RestServer/AqlFeature.h"
 #include "StorageEngine/TransactionState.h"
 #include "Transaction/Methods.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-vpack.h"
 #include "V8Server/V8DealerFeature.h"
@@ -285,7 +285,7 @@ Query::~Query() {
     ISOLATE;
     TRI_GET_GLOBALS();
     auto ctx =
-        static_cast<arangodb::V8TransactionContext*>(v8g->_transactionContext);
+        static_cast<arangodb::transaction::V8Context*>(v8g->_transactionContext);
     if (ctx != nullptr) {
       ctx->unregisterTransaction();
     }
@@ -669,7 +669,7 @@ QueryResult Query::execute(QueryRegistry* registry) {
         QueryResult res;
         // we don't have yet a transaction when we're here, so let's create
         // a mimimal context to build the result
-        res.context = std::make_shared<StandaloneTransactionContext>(_vocbase);
+        res.context = std::make_shared<transaction::StandaloneContext>(_vocbase);
 
         res.warnings = warningsToVelocyPack();
         TRI_ASSERT(cacheEntry->_queryResult != nullptr);
@@ -853,7 +853,7 @@ QueryResultV8 Query::executeV8(v8::Isolate* isolate, QueryRegistry* registry) {
         QueryResultV8 result;
         // we don't have yet a transaction when we're here, so let's create
         // a mimimal context to build the result
-        result.context = std::make_shared<StandaloneTransactionContext>(_vocbase);
+        result.context = std::make_shared<transaction::StandaloneContext>(_vocbase);
 
         v8::Handle<v8::Value> values =
             TRI_VPackToV8(isolate, cacheEntry->_queryResult->slice(),
@@ -1168,7 +1168,7 @@ void Query::enterContext() {
 
       ISOLATE;
       TRI_GET_GLOBALS();
-      auto ctx = static_cast<arangodb::V8TransactionContext*>(
+      auto ctx = static_cast<arangodb::transaction::V8Context*>(
           v8g->_transactionContext);
       if (ctx != nullptr) {
         ctx->registerTransaction(_trx->state());
@@ -1186,7 +1186,7 @@ void Query::exitContext() {
       // unregister transaction and resolver in context
       ISOLATE;
       TRI_GET_GLOBALS();
-      auto ctx = static_cast<arangodb::V8TransactionContext*>(
+      auto ctx = static_cast<arangodb::transaction::V8Context*>(
           v8g->_transactionContext);
       if (ctx != nullptr) {
         ctx->unregisterTransaction();
@@ -1474,14 +1474,14 @@ void Query::cleanupPlanAndEngine(int errorCode, VPackBuilder* statsBuilder) {
   _plan.reset();
 }
 
-/// @brief create a TransactionContext
+/// @brief create a transaction::Context
-std::shared_ptr<TransactionContext> Query::createTransactionContext() {
+std::shared_ptr<transaction::Context> Query::createTransactionContext() {
   if (_contextOwnedByExterior) {
     // we can use v8
-    return arangodb::V8TransactionContext::Create(_vocbase, true);
+    return arangodb::transaction::V8Context::Create(_vocbase, true);
   }
 
-  return arangodb::StandaloneTransactionContext::Create(_vocbase);
+  return arangodb::transaction::StandaloneContext::Create(_vocbase);
 }
 
 /// @brief look up a graph either from our cache list or from the _graphs

arangod/Aql/Query.h

@@ -42,9 +42,9 @@
 struct TRI_vocbase_t;
 
 namespace arangodb {
-class TransactionContext;
 
 namespace transaction {
+class Context;
 class Methods;
 }
 
@@ -383,8 +383,8 @@ class Query {
   /// @brief cleanup plan and engine for current query
   void cleanupPlanAndEngine(int, VPackBuilder* statsBuilder = nullptr);
 
-  /// @brief create a TransactionContext
+  /// @brief create a transaction::Context
-  std::shared_ptr<TransactionContext> createTransactionContext();
+  std::shared_ptr<transaction::Context> createTransactionContext();
 
   /// @brief returns the next query id
   static TRI_voc_tick_t NextId();

arangod/Aql/QueryRegistry.cpp

@@ -26,9 +26,11 @@
 #include "Aql/Query.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/WriteLocker.h"
+#include "Cluster/CollectionLockState.h"
 #include "Logger/Logger.h"
 #include "Transaction/Methods.h"
 
+using namespace arangodb;
 using namespace arangodb::aql;
 
 QueryRegistry::~QueryRegistry() {
@@ -93,10 +95,10 @@ void QueryRegistry::insert(QueryId id, Query* query, double ttl) {
     TRI_ASSERT(_queries.find(vocbase->name())->second.find(id) !=
                          _queries.find(vocbase->name())->second.end());
 
-    // If we have set _makeNolockHeaders, we need to unset it:
+    // If we have set _noLockHeaders, we need to unset it:
-    if (transaction::Methods::_makeNolockHeaders != nullptr) {
+    if (CollectionLockState::_noLockHeaders != nullptr) {
-      if (transaction::Methods::_makeNolockHeaders == query->engine()->lockedShards()) {
+      if (CollectionLockState::_noLockHeaders == query->engine()->lockedShards()) {
-        transaction::Methods::_makeNolockHeaders = nullptr;
+        CollectionLockState::_noLockHeaders = nullptr;
       }
       // else {
       // We have not set it, just leave it alone. This happens in particular
@@ -130,11 +132,11 @@ Query* QueryRegistry::open(TRI_vocbase_t* vocbase, QueryId id) {
   }
   qi->_isOpen = true;
 
-  // If we had set _makeNolockHeaders, we need to reset it:
+  // If we had set _noLockHeaders, we need to reset it:
   if (qi->_query->engine()->lockedShards() != nullptr) {
-    if (transaction::Methods::_makeNolockHeaders == nullptr) {
+    if (CollectionLockState::_noLockHeaders == nullptr) {
-      // std::cout << "Setting _makeNolockHeaders\n";
+      // std::cout << "Setting _noLockHeaders\n";
-      transaction::Methods::_makeNolockHeaders = qi->_query->engine()->lockedShards();
+      CollectionLockState::_noLockHeaders = qi->_query->engine()->lockedShards();
     } else {
       LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "Found strange lockedShards in thread, not overwriting!";
     }
@@ -164,17 +166,17 @@ void QueryRegistry::close(TRI_vocbase_t* vocbase, QueryId id, double ttl) {
         TRI_ERROR_INTERNAL, "query with given vocbase and id is not open");
   }
 
-  // If we have set _makeNolockHeaders, we need to unset it:
+  // If we have set _noLockHeaders, we need to unset it:
-  if (transaction::Methods::_makeNolockHeaders != nullptr) {
+  if (CollectionLockState::_noLockHeaders != nullptr) {
-    if (transaction::Methods::_makeNolockHeaders ==
+    if (CollectionLockState::_noLockHeaders ==
         qi->_query->engine()->lockedShards()) {
-      // std::cout << "Resetting _makeNolockHeaders to nullptr\n";
+      // std::cout << "Resetting _noLockHeaders to nullptr\n";
-      transaction::Methods::_makeNolockHeaders = nullptr;
+      CollectionLockState::_noLockHeaders = nullptr;
     } else {
-      if (transaction::Methods::_makeNolockHeaders != nullptr) {
+      if (CollectionLockState::_noLockHeaders != nullptr) {
-        if (transaction::Methods::_makeNolockHeaders ==
+        if (CollectionLockState::_noLockHeaders ==
             qi->_query->engine()->lockedShards()) {
-          transaction::Methods::_makeNolockHeaders = nullptr;
+          CollectionLockState::_noLockHeaders = nullptr;
         }
         // else {
         // We have not set it, just leave it alone. This happens in particular
@@ -209,10 +211,10 @@ void QueryRegistry::destroy(std::string const& vocbase, QueryId id,
   // to register the transaction with the current context and adjust
   // the debugging counters for transactions:
   if (!qi->_isOpen) {
-    // If we had set _makeNolockHeaders, we need to reset it:
+    // If we had set _noLockHeaders, we need to reset it:
     if (qi->_query->engine()->lockedShards() != nullptr) {
-      if (transaction::Methods::_makeNolockHeaders == nullptr) {
+      if (CollectionLockState::_noLockHeaders == nullptr) {
-        transaction::Methods::_makeNolockHeaders = qi->_query->engine()->lockedShards();
+        CollectionLockState::_noLockHeaders = qi->_query->engine()->lockedShards();
       } else {
         LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "Found strange lockedShards in thread, not overwriting!";
       }

arangod/Aql/QueryResult.h

@@ -31,7 +31,9 @@ namespace velocypack {
 class Builder;
 }
 
-class TransactionContext;
+namespace transaction {
+class Context;
+}
 
 namespace aql {
 
@@ -63,7 +65,7 @@ struct QueryResult {
   std::shared_ptr<arangodb::velocypack::Builder> result;
   std::shared_ptr<arangodb::velocypack::Builder> stats;
   std::shared_ptr<arangodb::velocypack::Builder> profile;
-  std::shared_ptr<TransactionContext> context;
+  std::shared_ptr<transaction::Context> context;
 };
 }
 }

arangod/Aql/RestAqlHandler.cpp

@@ -39,7 +39,7 @@
 #include "Scheduler/JobQueue.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/Methods.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/ticks.h"
 
 #include <velocypack/Dumper.h>
@@ -902,7 +902,7 @@ std::shared_ptr<VPackBuilder> RestAqlHandler::parseVelocyPackBody() {
 // Send slice as result with the given response type.
 void RestAqlHandler::sendResponse(
     rest::ResponseCode code, VPackSlice const slice,
-    TransactionContext* transactionContext) {
+    transaction::Context* transactionContext) {
   resetResponse(code);
   writeResult(slice, *(transactionContext->getVPackOptionsForDump()));
 }

arangod/Aql/RestAqlHandler.h

@@ -102,7 +102,7 @@ class RestAqlHandler : public RestVocbaseBaseHandler {
  private:
   // Send slice as result with the given response type.
   void sendResponse(rest::ResponseCode,
-                    arangodb::velocypack::Slice const, TransactionContext*);
+                    arangodb::velocypack::Slice const, transaction::Context*);
 
   // handle for useQuery
   void handleUseQuery(std::string const&, Query*,

arangod/Aql/ShortestPathBlock.cpp

@@ -84,7 +84,7 @@ struct ConstDistanceExpanderLocal {
     
       LogicalCollection* collection = edgeCursor->collection();
       auto cb = [&] (DocumentIdentifierToken const& element) {
-        if (collection->readDocument(_block->transaction(), *mmdr, element)) {
+        if (collection->readDocument(_block->transaction(), element, *mmdr)) {
           VPackSlice edge(mmdr->vpack());
           VPackSlice from =
               transaction::helpers::extractFromFromDocument(edge);
@@ -216,7 +216,7 @@ struct EdgeWeightExpanderLocal {
 
       LogicalCollection* collection = edgeCursor->collection();
       auto cb = [&] (DocumentIdentifierToken const& element) {
-        if (collection->readDocument(_block->transaction(), *mmdr, element)) {
+        if (collection->readDocument(_block->transaction(), element, *mmdr)) {
           VPackSlice edge(mmdr->vpack());
           VPackSlice from =
               transaction::helpers::extractFromFromDocument(edge);

arangod/CMakeLists.txt

@@ -191,6 +191,7 @@ SET(ARANGOD_SOURCES
   Cluster/ClusterInfo.cpp
   Cluster/ClusterMethods.cpp
   Cluster/ClusterTraverser.cpp
+  Cluster/CollectionLockState.cpp
   Cluster/FollowerInfo.cpp
   Cluster/DBServerAgencySync.cpp
   Cluster/HeartbeatThread.cpp
@@ -229,6 +230,7 @@ SET(ARANGOD_SOURCES
   MMFiles/MMFilesCompactorThread.cpp
   MMFiles/MMFilesDatafile.cpp
   MMFiles/MMFilesDatafileStatistics.cpp
+  MMFiles/MMFilesDitch.cpp
   MMFiles/MMFilesDocumentOperation.cpp
   MMFiles/MMFilesEdgeIndex.cpp
   MMFiles/MMFilesEngine.cpp
@@ -248,6 +250,7 @@ SET(ARANGOD_SOURCES
   MMFiles/MMFilesSkiplistIndex.cpp
   MMFiles/MMFilesSynchronizerThread.cpp
   MMFiles/MMFilesTransactionCollection.cpp
+  MMFiles/MMFilesTransactionContextData.cpp
   MMFiles/MMFilesTransactionState.cpp
   MMFiles/MMFilesWalLogfile.cpp
   MMFiles/MMFilesWalRecoverState.cpp
@@ -326,8 +329,11 @@ SET(ARANGOD_SOURCES
   StorageEngine/EngineSelectorFeature.cpp
   StorageEngine/TransactionCollection.cpp
   StorageEngine/TransactionState.cpp
+  Transaction/Context.cpp
   Transaction/Helpers.cpp
   Transaction/Methods.cpp
+  Transaction/StandaloneContext.cpp
+  Transaction/V8Context.cpp
   Utils/CollectionExport.cpp
   Utils/CollectionKeys.cpp
   Utils/CollectionKeysRepository.cpp
@@ -336,9 +342,6 @@ SET(ARANGOD_SOURCES
   Utils/CursorRepository.cpp
   Utils/OperationCursor.cpp
   Utils/SingleCollectionTransaction.cpp
-  Utils/StandaloneTransactionContext.cpp
-  Utils/TransactionContext.cpp
-  Utils/V8TransactionContext.cpp
   Utils/WorkMonitorArangod.cpp
   V8Server/FoxxQueuesFeature.cpp
   V8Server/V8Context.cpp
@@ -358,7 +361,6 @@ SET(ARANGOD_SOURCES
   V8Server/v8-vocindex.cpp
   VocBase/AuthInfo.cpp
   VocBase/DatafileStatisticsContainer.cpp
-  VocBase/Ditch.cpp
   VocBase/EdgeCollectionInfo.cpp
   VocBase/Graphs.cpp
   VocBase/KeyGenerator.cpp

arangod/Cluster/ClusterComm.cpp

@@ -26,6 +26,7 @@
 #include "Basics/HybridLogicalClock.h"
 #include "Basics/StringUtils.h"
 #include "Cluster/ClusterInfo.h"
+#include "Cluster/CollectionLockState.h"
 #include "Cluster/ServerState.h"
 #include "GeneralServer/AuthenticationFeature.h"
 #include "Logger/Logger.h"
@@ -1079,11 +1080,11 @@ std::pair<ClusterCommResult*, HttpRequest*> ClusterComm::prepareRequest(std::str
 
   std::unordered_map<std::string, std::string> headersCopy(headerFields);
   if (destination.substr(0, 6) == "shard:") {
-    if (transaction::Methods::_makeNolockHeaders != nullptr) {
+    if (CollectionLockState::_noLockHeaders != nullptr) {
       // LOCKING-DEBUG
       // std::cout << "Found Nolock header\n";
-      auto it = transaction::Methods::_makeNolockHeaders->find(result->shardID);
+      auto it = CollectionLockState::_noLockHeaders->find(result->shardID);
-      if (it != transaction::Methods::_makeNolockHeaders->end()) {
+      if (it != CollectionLockState::_noLockHeaders->end()) {
         // LOCKING-DEBUG
         // std::cout << "Found our shard\n";
         headersCopy["X-Arango-Nolock"] = result->shardID;

arangod/Cluster/ClusterInfo.cpp

@@ -42,6 +42,7 @@
 #include "RestServer/DatabaseFeature.h"
 #include "Utils/Events.h"
 #include "VocBase/LogicalCollection.h"
+#include "VocBase/PhysicalCollection.h"
 
 #ifdef USE_ENTERPRISE
 #include "Enterprise/VocBase/SmartVertexCollection.h"
@@ -1144,7 +1145,7 @@ int ClusterInfo::createCollectionCoordinator(std::string const& databaseName,
 
   AgencyOperation createCollection(
       "Plan/Collections/" + databaseName + "/" + collectionID,
-      AgencyValueOperationType::SET, builder.slice());
+      AgencyValueOperationType::SET, builder.slice()); 
   AgencyOperation increaseVersion("Plan/Version",
                                   AgencySimpleOperationType::INCREMENT_OP);
 
@@ -1393,8 +1394,8 @@ int ClusterInfo::setCollectionPropertiesCoordinator(
         copy.add(key, entry.value);
       }
     }
-    copy.add("doCompact", VPackValue(info->doCompact()));
+    copy.add("doCompact", VPackValue(info->getPhysical()->doCompact()));
-    copy.add("journalSize", VPackValue(info->journalSize()));
+    copy.add("journalSize", VPackValue(info->getPhysical()->journalSize()));
     copy.add("waitForSync", VPackValue(info->waitForSync()));
     copy.add("indexBuckets", VPackValue(info->indexBuckets()));
   } catch (...) {
@@ -1548,7 +1549,7 @@ int ClusterInfo::ensureIndexCoordinator(
   auto numberOfShardsMutex = std::make_shared<Mutex>();
   auto numberOfShards = std::make_shared<int>(0);
   auto resBuilder = std::make_shared<VPackBuilder>();
-  auto collectionBuilder = std::make_shared<VPackBuilder>();
+  VPackBuilder collectionBuilder;
 
   {
     std::shared_ptr<LogicalCollection> c =
@@ -1606,9 +1607,14 @@ int ClusterInfo::ensureIndexCoordinator(
     }
 
     // now create a new index
-    c->toVelocyPackForAgency(*collectionBuilder);
+    std::unordered_set<std::string> const ignoreKeys{
+        "allowUserKeys", "cid", /* cid really ignore?*/
+        "count",         "planId", "version",
+    };
+    c->setStatus(TRI_VOC_COL_STATUS_LOADED);
+    collectionBuilder = c->toVelocyPackIgnore(ignoreKeys, false);
   }
-  VPackSlice const collectionSlice = collectionBuilder->slice();
+  VPackSlice const collectionSlice = collectionBuilder.slice();
 
   auto newBuilder = std::make_shared<VPackBuilder>();
   if (!collectionSlice.isObject()) {

arangod/Cluster/ClusterMethods.cpp

@@ -2242,8 +2242,12 @@ ClusterMethods::persistCollectionInAgency(LogicalCollection* col) {
   }
   col->setShardMap(shards);
 
-  VPackBuilder velocy;
+  std::unordered_set<std::string> const ignoreKeys{
-  col->toVelocyPackForAgency(velocy);
+      "allowUserKeys", "cid", /* cid really ignore?*/
+      "count",         "planId", "version",
+  };
+  col->setStatus(TRI_VOC_COL_STATUS_LOADED);
+  VPackBuilder velocy = col->toVelocyPackIgnore(ignoreKeys, false);
 
   std::string errorMsg;
   int myerrno = ci->createCollectionCoordinator(

arangod/Cluster/CollectionLockState.cpp

@@ -0,0 +1,33 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+////////////////////////////////////////////////////////////////////////////////
+
+#include "CollectionLockState.h"
+
+using namespace arangodb;
+
+/// @brief if this pointer is set to an actual set, then for each request
+/// sent to a shardId using the ClusterComm library, an X-Arango-Nolock
+/// header is generated.
+thread_local std::unordered_set<std::string>* CollectionLockState::_noLockHeaders =
+    nullptr;
+  

arangod/Cluster/CollectionLockState.h

@@ -0,0 +1,38 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGOD_CLUSTER_COLLECTION_LOCK_STATE_H
+#define ARANGOD_CLUSTER_COLLECTION_LOCK_STATE_H 1
+
+#include "Basics/Common.h"
+
+namespace arangodb {
+
+struct CollectionLockState {
+  /// @brief collections for which there does not need to be an extra lock
+  static thread_local std::unordered_set<std::string>* _noLockHeaders;
+};
+
+}
+
+#endif

arangod/Cluster/TraverserEngine.cpp

@@ -27,7 +27,7 @@
 #include "Aql/Ast.h"
 #include "Aql/Query.h"
 #include "Utils/CollectionNameResolver.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/ManagedDocumentResult.h"
 #include "VocBase/TraverserOptions.h"
 
@@ -95,7 +95,7 @@ BaseTraverserEngine::BaseTraverserEngine(TRI_vocbase_t* vocbase,
   auto opts = std::make_shared<VPackBuilder>();
 
   _trx = new aql::AqlTransaction(
-      arangodb::StandaloneTransactionContext::Create(vocbase),
+      arangodb::transaction::StandaloneContext::Create(vocbase),
       _collections.collections(), false);
   _query = new aql::Query(true, vocbase, "", 0, params, opts, aql::PART_DEPENDENT);
   _query->injectTransaction(_trx);
@@ -294,7 +294,7 @@ bool BaseTraverserEngine::lockCollection(std::string const& shard) {
   if (cid == 0) {
     return false;
   }
-  _trx->orderDitch(cid); // will throw when it fails 
+  _trx->pinData(cid); // will throw when it fails 
   int res = _trx->lock(_trx->trxCollection(cid), AccessMode::Type::READ);
   if (res != TRI_ERROR_NO_ERROR) {
     LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "Logging Shard " << shard << " lead to exception '"
@@ -304,7 +304,7 @@ bool BaseTraverserEngine::lockCollection(std::string const& shard) {
   return true;
 }
 
-std::shared_ptr<TransactionContext> BaseTraverserEngine::context() const {
+std::shared_ptr<transaction::Context> BaseTraverserEngine::context() const {
   return _trx->transactionContext();
 }
 

arangod/Cluster/TraverserEngine.h

@@ -35,8 +35,10 @@ namespace arangodb {
 namespace transaction {
 class Methods;
 }
-;
+
-class TransactionContext;
+namespace transaction {
+class Context;
+}
 
 namespace aql {
 class Collections;
@@ -83,7 +85,7 @@ class BaseTraverserEngine {
 
    bool lockCollection(std::string const&);
 
-   std::shared_ptr<TransactionContext> context() const;
+   std::shared_ptr<transaction::Context> context() const;
 
   protected:
     std::unique_ptr<TraverserOptions> _opts;

arangod/Cluster/TraverserEngineRegistry.cpp

@@ -29,6 +29,9 @@
 #include "Cluster/TraverserEngine.h"
 #include "VocBase/ticks.h"
 
+#include <velocypack/Slice.h>
+#include <velocypack/velocypack-aliases.h>
+
 using namespace arangodb::traverser;
 
 #ifndef USE_ENTERPRISE

arangod/Cluster/v8-cluster.cpp

@@ -31,12 +31,14 @@
 #include "Cluster/ServerState.h"
 #include "Cluster/ClusterComm.h"
 #include "GeneralServer/AuthenticationFeature.h"
+#include "MMFiles/MMFilesCollection.h"
 #include "V8/v8-buffer.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-globals.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
 #include "VocBase/LogicalCollection.h"
+#include "VocBase/PhysicalCollection.h"
 #include "VocBase/ticks.h"
 #include "V8Server/v8-vocbaseprivate.h"
 
@@ -598,15 +600,18 @@ static void JS_GetCollectionInfoClusterInfo(
   result->Set(TRI_V8_ASCII_STRING("deleted"),
               v8::Boolean::New(isolate, ci->deleted()));
   result->Set(TRI_V8_ASCII_STRING("doCompact"),
-              v8::Boolean::New(isolate, ci->doCompact()));
+              v8::Boolean::New(isolate, ci->getPhysical()->doCompact()));
   result->Set(TRI_V8_ASCII_STRING("isSystem"),
               v8::Boolean::New(isolate, ci->isSystem()));
-  result->Set(TRI_V8_ASCII_STRING("isVolatile"),
+  result->Set(
-              v8::Boolean::New(isolate, ci->isVolatile()));
+      TRI_V8_ASCII_STRING("isVolatile"),
+      v8::Boolean::New(
+          isolate,
+          static_cast<MMFilesCollection*>(ci->getPhysical())->isVolatile()));
   result->Set(TRI_V8_ASCII_STRING("waitForSync"),
               v8::Boolean::New(isolate, ci->waitForSync()));
   result->Set(TRI_V8_ASCII_STRING("journalSize"),
-              v8::Number::New(isolate, static_cast<double>(ci->journalSize())));
+              v8::Number::New(isolate, static_cast<double>(ci->getPhysical()->journalSize())));
   result->Set(TRI_V8_ASCII_STRING("replicationFactor"),
               v8::Number::New(isolate, ci->replicationFactor()));
   result->Set(TRI_V8_ASCII_STRING("isSmart"),

arangod/Indexes/IndexIterator.h

@@ -88,8 +88,6 @@ class IndexIterator {
   virtual bool next(TokenCallback const& callback, size_t limit) = 0;
   virtual bool nextExtra(ExtraCallback const& callback, size_t limit);
 
-//  virtual DocumentIdentifierToken next();
-
   virtual void reset();
 
   virtual void skip(uint64_t count, uint64_t& skipped);

arangod/Indexes/IndexLookupContext.h

@@ -48,7 +48,7 @@ class IndexLookupContext {
 
   uint8_t const* lookup(DocumentIdentifierToken token) {
     try {
-      if (_collection->readDocument(_trx, *_result, token)) {
+      if (_collection->readDocument(_trx, token, *_result)) {
         return _result->vpack();
       } 
     } catch (...) {

arangod/MMFiles/MMFilesAqlFunctions.cpp

@@ -106,7 +106,7 @@ static AqlValue buildGeoResult(transaction::Methods* trx,
       for (auto& it : distances) {
         VPackObjectBuilder docGuard(builder.get());
         builder->add(attributeName, VPackValue(it._distance));
-        if (collection->readDocument(trx, mmdr, it._token)) {
+        if (collection->readDocument(trx, it._token, mmdr)) {
           VPackSlice doc(mmdr.vpack());
           for (auto const& entry : VPackObjectIterator(doc)) {
             std::string key = entry.key.copyString();
@@ -119,7 +119,7 @@ static AqlValue buildGeoResult(transaction::Methods* trx,
 
     } else {
       for (auto& it : distances) {
-        if (collection->readDocument(trx, mmdr, it._token)) {
+        if (collection->readDocument(trx, it._token, mmdr)) {
           builder->addExternal(mmdr.vpack());
         }
       }
@@ -165,7 +165,7 @@ static arangodb::MMFilesGeoIndex* getGeoIndex(
                                   collectionName.c_str());
   }
 
-  trx->orderDitch(cid);
+  trx->pinData(cid);
 
   return index;
 }
@@ -260,7 +260,7 @@ AqlValue MMFilesAqlFunctions::Fulltext(
                                   collectionName.c_str());
   }
 
-  trx->orderDitch(cid);
+  trx->pinData(cid);
 
   TRI_fulltext_query_t* ft =
       TRI_CreateQueryMMFilesFulltextIndex(TRI_FULLTEXT_SEARCH_MAX_WORDS, maxResults);
@@ -286,7 +286,7 @@ AqlValue MMFilesAqlFunctions::Fulltext(
     THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
   }
   
-  TRI_ASSERT(trx->hasDitch(cid));
+  TRI_ASSERT(trx->isPinned(cid));
 
   transaction::BuilderLeaser builder(trx);
   try {
@@ -295,7 +295,7 @@ AqlValue MMFilesAqlFunctions::Fulltext(
     ManagedDocumentResult mmdr;
     size_t const numResults = queryResult->_numDocuments;
     for (size_t i = 0; i < numResults; ++i) {
-      if (collection->readDocument(trx, mmdr, queryResult->_documents[i])) {
+      if (collection->readDocument(trx, queryResult->_documents[i], mmdr)) {
         builder->addExternal(mmdr.vpack());
       }
     }
@@ -364,7 +364,7 @@ AqlValue MMFilesAqlFunctions::Near(arangodb::aql::Query* query,
   arangodb::MMFilesGeoIndex* index = getGeoIndex(trx, cid, collectionName);
 
   TRI_ASSERT(index != nullptr);
-  TRI_ASSERT(trx->hasDitch(cid));
+  TRI_ASSERT(trx->isPinned(cid));
 
   GeoCoordinates* cors = index->nearQuery(
       trx, latitude.toDouble(trx), longitude.toDouble(trx), static_cast<size_t>(limitValue));
@@ -415,7 +415,7 @@ AqlValue MMFilesAqlFunctions::Within(
   arangodb::MMFilesGeoIndex* index = getGeoIndex(trx, cid, collectionName);
 
   TRI_ASSERT(index != nullptr);
-  TRI_ASSERT(trx->hasDitch(cid));
+  TRI_ASSERT(trx->isPinned(cid));
 
   GeoCoordinates* cors = index->withinQuery(
       trx, latitudeValue.toDouble(trx), longitudeValue.toDouble(trx), radiusValue.toDouble(trx));

arangod/MMFiles/MMFilesCleanupThread.cpp

@@ -30,10 +30,10 @@
 #include "Basics/files.h"
 #include "Logger/Logger.h"
 #include "MMFiles/MMFilesCollection.h"
+#include "MMFiles/MMFilesDitch.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "Utils/CursorRepository.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
 #include "MMFiles/MMFilesLogfileManager.h"
 
@@ -164,13 +164,14 @@ void MMFilesCleanupThread::cleanupCollection(arangodb::LogicalCollection* collec
   // loop until done
 
   while (true) {
-    auto ditches = collection->ditches();
+    auto mmfiles = arangodb::MMFilesCollection::toMMFilesCollection(collection);
+    auto ditches = mmfiles->ditches();
 
     TRI_ASSERT(ditches != nullptr);
 
     // check and remove all callback elements at the beginning of the list
-    auto callback = [&](arangodb::Ditch const* ditch) -> bool {
+    auto callback = [&](arangodb::MMFilesDitch const* ditch) -> bool {
-      if (ditch->type() == arangodb::Ditch::TRI_DITCH_COLLECTION_UNLOAD) {
+      if (ditch->type() == arangodb::MMFilesDitch::TRI_DITCH_COLLECTION_UNLOAD) {
         // check if we can really unload, this is only the case if the
         // collection's WAL markers
         // were fully collected
@@ -214,9 +215,9 @@ void MMFilesCleanupThread::cleanupCollection(arangodb::LogicalCollection* collec
       if (gotStatus && !isUnloading) {
         popped = false;
         auto unloader =
-            ditches->process(popped, [](arangodb::Ditch const* ditch) -> bool {
+            ditches->process(popped, [](arangodb::MMFilesDitch const* ditch) -> bool {
               return (ditch->type() ==
-                      arangodb::Ditch::TRI_DITCH_COLLECTION_UNLOAD);
+                      arangodb::MMFilesDitch::TRI_DITCH_COLLECTION_UNLOAD);
             });
         if (popped) {
           // we've changed the list. try with current state in next turn
@@ -226,7 +227,7 @@ void MMFilesCleanupThread::cleanupCollection(arangodb::LogicalCollection* collec
         }
       }
   
-      if (!collection->isFullyCollected()) {
+      if (!collection->getPhysical()->isFullyCollected()) {
         bool isDeleted = false;
 
         // if there is still some garbage collection to perform,
@@ -263,17 +264,17 @@ void MMFilesCleanupThread::cleanupCollection(arangodb::LogicalCollection* collec
     // execute callback, some of the callbacks might delete or unload our collection
     auto const type = ditch->type();
   
-    if (type == arangodb::Ditch::TRI_DITCH_DATAFILE_DROP) {
+    if (type == arangodb::MMFilesDitch::TRI_DITCH_DATAFILE_DROP) {
-      dynamic_cast<arangodb::DropDatafileDitch*>(ditch)->executeCallback();
+      dynamic_cast<arangodb::MMFilesDropDatafileDitch*>(ditch)->executeCallback();
       delete ditch;
       // next iteration
-    } else if (type == arangodb::Ditch::TRI_DITCH_DATAFILE_RENAME) {
+    } else if (type == arangodb::MMFilesDitch::TRI_DITCH_DATAFILE_RENAME) {
-      dynamic_cast<arangodb::RenameDatafileDitch*>(ditch)->executeCallback();
+      dynamic_cast<arangodb::MMFilesRenameDatafileDitch*>(ditch)->executeCallback();
       delete ditch;
       // next iteration
-    } else if (type == arangodb::Ditch::TRI_DITCH_COLLECTION_UNLOAD) {
+    } else if (type == arangodb::MMFilesDitch::TRI_DITCH_COLLECTION_UNLOAD) {
       // collection will be unloaded
-      bool hasUnloaded = dynamic_cast<arangodb::UnloadCollectionDitch*>(ditch)
+      bool hasUnloaded = dynamic_cast<arangodb::MMFilesUnloadCollectionDitch*>(ditch)
                              ->executeCallback();
       delete ditch;
 
@@ -281,9 +282,9 @@ void MMFilesCleanupThread::cleanupCollection(arangodb::LogicalCollection* collec
         // this has unloaded and freed the collection
         return;
       }
-    } else if (type == arangodb::Ditch::TRI_DITCH_COLLECTION_DROP) {
+    } else if (type == arangodb::MMFilesDitch::TRI_DITCH_COLLECTION_DROP) {
       // collection will be dropped
-      bool hasDropped = dynamic_cast<arangodb::DropCollectionDitch*>(ditch)
+      bool hasDropped = dynamic_cast<arangodb::MMFilesDropCollectionDitch*>(ditch)
                             ->executeCallback();
       delete ditch;
 

arangod/MMFiles/MMFilesCollection.h

@@ -28,8 +28,8 @@
 #include "Basics/ReadWriteLock.h"
 #include "Indexes/IndexLookupContext.h"
 #include "MMFiles/MMFilesDatafileStatistics.h"
+#include "MMFiles/MMFilesDitch.h"
 #include "MMFiles/MMFilesRevisionsCache.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/KeyGenerator.h"
 #include "VocBase/ManagedDocumentResult.h"
 #include "VocBase/PhysicalCollection.h"
@@ -40,13 +40,29 @@ struct TRI_df_marker_t;
 namespace arangodb {
 class LogicalCollection;
 class ManagedDocumentResult;
+struct MMFilesDocumentOperation;
 class MMFilesPrimaryIndex;
+class MMFilesWalMarker;
 
 class MMFilesCollection final : public PhysicalCollection {
- friend class MMFilesCompactorThread;
+  friend class MMFilesCompactorThread;
- friend class MMFilesEngine;
+  friend class MMFilesEngine;
 
  public:
+  static inline MMFilesCollection* toMMFilesCollection(
+      PhysicalCollection* physical) {
+    auto rv = static_cast<MMFilesCollection*>(physical);
+    TRI_ASSERT(rv != nullptr);
+    return rv;
+  }
+
+  static inline MMFilesCollection* toMMFilesCollection(
+      LogicalCollection* logical) {
+    auto phys = logical->getPhysical();
+    TRI_ASSERT(phys != nullptr);
+    return toMMFilesCollection(phys);
+  }
+
   /// @brief state during opening of a collection
   struct OpenIteratorState {
     LogicalCollection* _collection;
@@ -64,9 +80,10 @@ class MMFilesCollection final : public PhysicalCollection {
     int64_t _initialCount;
     bool const _trackKeys;
 
-    OpenIteratorState(LogicalCollection* collection, transaction::Methods* trx) 
+    OpenIteratorState(LogicalCollection* collection, transaction::Methods* trx)
         : _collection(collection),
-          _primaryIndex(collection->primaryIndex()),
+          _primaryIndex(static_cast<MMFilesCollection*>(collection->getPhysical())
+                            ->primaryIndex()),
           _tid(0),
           _fid(0),
           _stats(),
@@ -78,7 +95,7 @@ class MMFilesCollection final : public PhysicalCollection {
           _documents(0),
           _operations(0),
           _initialCount(-1),
-          _trackKeys(collection->keyGenerator()->trackKeys()) {
+          _trackKeys(collection->getPhysical()->keyGenerator()->trackKeys()) {
       TRI_ASSERT(collection != nullptr);
       TRI_ASSERT(trx != nullptr);
     }
@@ -99,17 +116,24 @@ class MMFilesCollection final : public PhysicalCollection {
   };
 
  public:
-  explicit MMFilesCollection(LogicalCollection*);
+  explicit MMFilesCollection(LogicalCollection*, VPackSlice const& info);
+  explicit MMFilesCollection(LogicalCollection*, PhysicalCollection*); //use in cluster only!!!!!
+
   ~MMFilesCollection();
-  
+
   std::string const& path() const override {
     return _path;
   };
-  
+
   void setPath(std::string const& path) override {
     _path = path;
   };
 
+  CollectionResult updateProperties(VPackSlice const& slice, bool doSync) override;
+  virtual int persistProperties() noexcept override;
+
+  virtual PhysicalCollection* clone(LogicalCollection*, PhysicalCollection*) override;
+
   TRI_voc_rid_t revision() const override;
 
   void setRevision(TRI_voc_rid_t revision, bool force);
@@ -118,10 +142,14 @@ class MMFilesCollection final : public PhysicalCollection {
 
   int64_t initialCount() const override;
   void updateCount(int64_t) override;
-  
+  size_t journalSize() const override;
-  /// @brief return engine-specific figures
+  bool isVolatile() const;
-  void figures(std::shared_ptr<arangodb::velocypack::Builder>&) override;
+ 
-  
+  TRI_voc_tick_t maxTick() const { return _maxTick; }
+  void maxTick(TRI_voc_tick_t value) { _maxTick = value; }
+
+  void getPropertiesVPack(velocypack::Builder&) const override;
+
   // datafile management
   bool applyForTickRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax,
                          std::function<bool(TRI_voc_tick_t foundTick, TRI_df_marker_t const* marker)> const& callback) override;
@@ -155,20 +183,17 @@ class MMFilesCollection final : public PhysicalCollection {
   int sealDatafile(MMFilesDatafile* datafile, bool isCompactor);
 
   /// @brief increase dead stats for a datafile, if it exists
-  void updateStats(TRI_voc_fid_t fid, DatafileStatisticsContainer const& values) override {
+  void updateStats(TRI_voc_fid_t fid, DatafileStatisticsContainer const& values) {
     _datafileStatistics.update(fid, values);
   }
    
+  uint64_t numberDocuments() const override;
+
+  void sizeHint(transaction::Methods* trx, int64_t hint) override;
+
   /// @brief report extra memory used by indexes etc.
   size_t memory() const override;
 
-  //void preventCompaction() override;
-  //bool tryPreventCompaction() override;
-  //void allowCompaction() override;
-  //void lockForCompaction() override;
-  //bool tryLockForCompaction() override;
-  //void finishCompaction() override;
-
   void preventCompaction();
   bool tryPreventCompaction();
   void allowCompaction();
@@ -194,8 +219,7 @@ class MMFilesCollection final : public PhysicalCollection {
   double lastCompactionStamp() const { return _lastCompactionStamp; }
   void lastCompactionStamp(double value) { _lastCompactionStamp = value; }
 
-  
+  MMFilesDitches* ditches() const { return &_ditches; }
-  Ditches* ditches() const override { return &_ditches; }
   
   void open(bool ignoreErrors) override;
 
@@ -203,6 +227,9 @@ class MMFilesCollection final : public PhysicalCollection {
   int iterateMarkersOnLoad(arangodb::transaction::Methods* trx) override;
   
   bool isFullyCollected() const override;
+
+  bool doCompact() const override { return _doCompact; }
+
   
   int64_t uncollectedLogfileEntries() const {
     return _uncollectedLogfileEntries.load();
@@ -219,6 +246,33 @@ class MMFilesCollection final : public PhysicalCollection {
     }
   }
 
+  ////////////////////////////////////
+  // -- SECTION Indexes --
+  ///////////////////////////////////
+
+  // WARNING: Make sure that this Collection Instance
+  // is somehow protected. If it goes out of all scopes
+  // or it's indexes are freed the pointer returned will get invalidated.
+  MMFilesPrimaryIndex* primaryIndex() const;
+ 
+  inline bool useSecondaryIndexes() const { return _useSecondaryIndexes; }
+
+  void useSecondaryIndexes(bool value) { _useSecondaryIndexes = value; }
+
+  int fillAllIndexes(transaction::Methods*);
+
+  int saveIndex(transaction::Methods* trx, std::shared_ptr<arangodb::Index> idx) override;
+  
+  std::unique_ptr<IndexIterator> getAllIterator(transaction::Methods* trx, ManagedDocumentResult* mdr, bool reverse) override;
+  std::unique_ptr<IndexIterator> getAnyIterator(transaction::Methods* trx, ManagedDocumentResult* mdr)  override;
+  void invokeOnAllElements(std::function<bool(DocumentIdentifierToken const&)> callback) override;
+
+  /// @brief Restores an index from VelocyPack.
+  int restoreIndex(transaction::Methods*, velocypack::Slice const&,
+                   std::shared_ptr<Index>&) override;
+
+  /// @brief Drop an index with the given iid.
+  bool dropIndex(TRI_idx_iid_t iid) override;
 
   int cleanupIndexes();
 
@@ -243,6 +297,15 @@ class MMFilesCollection final : public PhysicalCollection {
   int read(transaction::Methods*, arangodb::velocypack::Slice const key,
            ManagedDocumentResult& result, bool) override;
 
+  bool readDocument(transaction::Methods* trx,
+                    DocumentIdentifierToken const& token,
+                    ManagedDocumentResult& result) override;
+
+  bool readDocumentConditional(transaction::Methods* trx,
+                               DocumentIdentifierToken const& token,
+                               TRI_voc_tick_t maxTick,
+                               ManagedDocumentResult& result) override;
+
   int insert(arangodb::transaction::Methods* trx,
              arangodb::velocypack::Slice const newSlice,
              arangodb::ManagedDocumentResult& result,
@@ -270,8 +333,8 @@ class MMFilesCollection final : public PhysicalCollection {
              arangodb::velocypack::Slice const slice,
              arangodb::ManagedDocumentResult& previous,
              OperationOptions& options, TRI_voc_tick_t& resultMarkerTick,
-             bool lock, TRI_voc_rid_t const& revisionId, TRI_voc_rid_t& prevRev,
+             bool lock, TRI_voc_rid_t const& revisionId,
-             arangodb::velocypack::Slice const toRemove) override;
+             TRI_voc_rid_t& prevRev) override;
 
   int rollbackOperation(transaction::Methods*, TRI_voc_document_operation_e,
                         TRI_voc_rid_t oldRevisionId,
@@ -294,6 +357,20 @@ class MMFilesCollection final : public PhysicalCollection {
 
  private:
 
+  bool openIndex(VPackSlice const& description, transaction::Methods* trx);
+
+  /// @brief initializes an index with all existing documents
+  void fillIndex(basics::LocalTaskQueue*, transaction::Methods*,
+                 Index*,
+                 std::vector<std::pair<TRI_voc_rid_t, VPackSlice>> const&,
+                 bool);
+
+
+  /// @brief Fill indexes used in recovery
+  int fillIndexes(transaction::Methods*,
+                  std::vector<std::shared_ptr<Index>> const&,
+                  bool skipPersistent = true);
+ 
   int openWorker(bool ignoreErrors);
 
   int removeFastPath(arangodb::transaction::Methods* trx,
@@ -355,8 +432,15 @@ class MMFilesCollection final : public PhysicalCollection {
                        MMFilesWalMarker const* marker, bool& waitForSync);
 
    private:
+
+    /// @brief return engine-specific figures
+    void figuresSpecific(std::shared_ptr<arangodb::velocypack::Builder>&) override;
+  
     // SECTION: Index storage
 
+    /// @brief Detect all indexes form file
+    int detectIndexes(transaction::Methods* trx);
+
     int insertIndexes(transaction::Methods * trx, TRI_voc_rid_t revisionId,
                       velocypack::Slice const& doc);
 
@@ -383,7 +467,7 @@ class MMFilesCollection final : public PhysicalCollection {
                        bool& waitForSync);
 
    private:
-    mutable arangodb::Ditches _ditches;
+    mutable arangodb::MMFilesDitches _ditches;
 
     // lock protecting the indexes
     mutable basics::ReadWriteLock _idxLock;
@@ -412,6 +496,13 @@ class MMFilesCollection final : public PhysicalCollection {
     char const* _lastCompactionStatus;
     double _lastCompactionStamp;
     std::string _path;
+    TRI_voc_size_t _journalSize;
+    bool const _isVolatile;
+
+    // whether or not secondary indexes should be filled
+    bool _useSecondaryIndexes;
+    bool _doCompact;
+    TRI_voc_tick_t _maxTick;
 };
 
 }

arangod/MMFiles/MMFilesCollectorCache.h

@@ -25,8 +25,8 @@
 #define ARANGOD_MMFILES_COLLECTOR_CACHE_H 1
 
 #include "Basics/Common.h"
+#include "MMFiles/MMFilesDitch.h"
 #include "VocBase/DatafileStatisticsContainer.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/voc-types.h"
 
 struct MMFilesDatafile;
@@ -76,7 +76,7 @@ struct MMFilesCollectorCache {
 
   ~MMFilesCollectorCache() {
     delete operations;
-    freeDitches();
+    freeMMFilesDitches();
   }
 
   /// @brief return a reference to an existing datafile statistics struct
@@ -99,15 +99,15 @@ struct MMFilesCollectorCache {
   }
 
   /// @brief add a ditch
-  void addDitch(arangodb::DocumentDitch* ditch) {
+  void addDitch(arangodb::MMFilesDocumentDitch* ditch) {
     TRI_ASSERT(ditch != nullptr);
     ditches.emplace_back(ditch);
   }
 
   /// @brief free all ditches
-  void freeDitches() {
+  void freeMMFilesDitches() {
     for (auto& it : ditches) {
-      it->ditches()->freeDocumentDitch(it, false);
+      it->ditches()->freeMMFilesDocumentDitch(it, false);
     }
 
     ditches.clear();
@@ -129,7 +129,7 @@ struct MMFilesCollectorCache {
   std::vector<MMFilesCollectorOperation>* operations;
 
   /// @brief ditches held by the operations
-  std::vector<arangodb::DocumentDitch*> ditches;
+  std::vector<arangodb::MMFilesDocumentDitch*> ditches;
 
   /// @brief datafile info cache, updated when the collector transfers markers
   std::unordered_map<TRI_voc_fid_t, DatafileStatisticsContainer> dfi;

arangod/MMFiles/MMFilesCollectorThread.cpp

@@ -32,9 +32,11 @@
 #include "Basics/memory-map.h"
 #include "Logger/Logger.h"
 #include "MMFiles/MMFilesCollection.h"
+#include "MMFiles/MMFilesCompactionLocker.h"
 #include "MMFiles/MMFilesDatafileHelper.h"
-#include "MMFiles/MMFilesLogfileManager.h"
+#include "MMFiles/MMFilesEngine.h"
 #include "MMFiles/MMFilesIndexElement.h"
+#include "MMFiles/MMFilesLogfileManager.h"
 #include "MMFiles/MMFilesPersistentIndex.h"
 #include "MMFiles/MMFilesPrimaryIndex.h"
 #include "MMFiles/MMFilesWalLogfile.h"
@@ -45,9 +47,8 @@
 #include "Utils/CollectionGuard.h"
 #include "Utils/DatabaseGuard.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Hints.h"
-#include "VocBase/CompactionLocker.h"
 #include "VocBase/LogicalCollection.h"
 
 using namespace arangodb;
@@ -562,6 +563,7 @@ void MMFilesCollectorThread::processCollectionMarker(
     LogicalCollection* collection, MMFilesCollectorCache* cache,
     MMFilesCollectorOperation const& operation) {
   auto physical = static_cast<MMFilesCollection*>(collection->getPhysical());
+  TRI_ASSERT(physical != nullptr);
   auto const* walMarker = reinterpret_cast<TRI_df_marker_t const*>(operation.walPosition);
   TRI_ASSERT(walMarker != nullptr);
   TRI_ASSERT(reinterpret_cast<TRI_df_marker_t const*>(operation.datafilePosition));
@@ -582,7 +584,7 @@ void MMFilesCollectorThread::processCollectionMarker(
     transaction::helpers::extractKeyAndRevFromDocument(slice, keySlice, revisionId);
   
     bool wasAdjusted = false;
-    MMFilesSimpleIndexElement element = collection->primaryIndex()->lookupKey(&trx, keySlice);
+    MMFilesSimpleIndexElement element = physical->primaryIndex()->lookupKey(&trx, keySlice);
 
     if (element &&
         element.revisionId() == revisionId) { 
@@ -613,7 +615,7 @@ void MMFilesCollectorThread::processCollectionMarker(
     TRI_voc_rid_t revisionId = 0;
     transaction::helpers::extractKeyAndRevFromDocument(slice, keySlice, revisionId);
 
-    MMFilesSimpleIndexElement found = collection->primaryIndex()->lookupKey(&trx, keySlice);
+    MMFilesSimpleIndexElement found = physical->primaryIndex()->lookupKey(&trx, keySlice);
 
     if (found && 
         found.revisionId() > revisionId) {
@@ -648,7 +650,7 @@ int MMFilesCollectorThread::processCollectionOperations(MMFilesCollectorCache* c
   }
 
   arangodb::SingleCollectionTransaction trx(
-      arangodb::StandaloneTransactionContext::Create(collection->vocbase()),
+      arangodb::transaction::StandaloneContext::Create(collection->vocbase()),
       collection->cid(), AccessMode::Type::WRITE);
   trx.addHint(transaction::Hints::Hint::NO_USAGE_LOCK);  // already locked by guard above
   trx.addHint(transaction::Hints::Hint::NO_COMPACTION_LOCK);  // already locked above
@@ -899,7 +901,8 @@ int MMFilesCollectorThread::transferMarkers(MMFilesWalLogfile* logfile,
   int res = TRI_ERROR_INTERNAL;
 
   try {
-    res = engine->transferMarkers(collection, cache.get(), operations);
+    auto en = static_cast<MMFilesEngine*>(engine);
+    res = en->transferMarkers(collection, cache.get(), operations);
     
     if (res == TRI_ERROR_NO_ERROR && !cache->operations->empty()) {
       queueOperations(logfile, cache);
@@ -979,7 +982,9 @@ int MMFilesCollectorThread::updateDatafileStatistics(
   // iterate over all datafile infos and update the collection's datafile stats
   for (auto it = cache->dfi.begin(); it != cache->dfi.end();
        /* no hoisting */) {
-    collection->updateStats((*it).first, (*it).second);
+    MMFilesCollection* mmfiles = static_cast<MMFilesCollection*>(collection->getPhysical());
+    TRI_ASSERT(mmfiles);
+    mmfiles->updateStats((*it).first, (*it).second);
 
     // flush the local datafile info so we don't update the statistics twice
     // with the same values
@@ -989,7 +994,7 @@ int MMFilesCollectorThread::updateDatafileStatistics(
 
   return TRI_ERROR_NO_ERROR;
 }
-      
+
 void MMFilesCollectorThread::broadcastCollectorResult(int res) { 
   CONDITION_LOCKER(guard, _collectorResultCondition);
   _collectorResult = res;

arangod/MMFiles/MMFilesCollectorThread.h

@@ -30,7 +30,7 @@
 #include "Basics/Thread.h"
 #include "MMFiles/MMFilesCollectorCache.h"
 #include "MMFiles/MMFilesDatafile.h"
-#include "VocBase/Ditch.h"
+#include "MMFiles/MMFilesDitch.h"
 #include "VocBase/voc-types.h"
 
 namespace arangodb {

arangod/MMFiles/MMFilesCompactionLocker.h

@@ -21,8 +21,8 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#ifndef ARANGOD_VOCBASE_COMPACTION_LOCKER_H
+#ifndef ARANGOD_MMFILES_MMFILES_COMPACTION_LOCKER_H
-#define ARANGOD_VOCBASE_COMPACTION_LOCKER_H 1
+#define ARANGOD_MMFILES_MMFILES_COMPACTION_LOCKER_H 1
 
 #include "Basics/Common.h"
 #include "MMFiles/MMFilesCollection.h"

arangod/MMFiles/MMFilesCompactorThread.cpp

@@ -31,6 +31,7 @@
 #include "Basics/memory-map.h"
 #include "Logger/Logger.h"
 #include "MMFiles/MMFilesCollection.h"
+#include "MMFiles/MMFilesCompactionLocker.h"
 #include "MMFiles/MMFilesDatafileHelper.h"
 #include "MMFiles/MMFilesDocumentPosition.h"
 #include "MMFiles/MMFilesIndexElement.h"
@@ -38,10 +39,9 @@
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Helpers.h"
 #include "Transaction/Hints.h"
-#include "VocBase/CompactionLocker.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/vocbase.h"
 
@@ -272,6 +272,9 @@ MMFilesCompactorThread::CompactionInitialContext MMFilesCompactorThread::getComp
     /// @brief datafile iterator, calculates necessary total size
     auto calculateSize = [&context](TRI_df_marker_t const* marker, MMFilesDatafile* datafile) -> bool {
       LogicalCollection* collection = context._collection;
+      TRI_ASSERT(collection != nullptr);
+      auto physical = static_cast<MMFilesCollection*>(collection->getPhysical());
+      TRI_ASSERT(physical != nullptr);
       TRI_df_marker_type_t const type = marker->getType();
 
       // new or updated document
@@ -282,12 +285,15 @@ MMFilesCompactorThread::CompactionInitialContext MMFilesCompactorThread::getComp
         VPackSlice keySlice = transaction::helpers::extractKeyFromDocument(slice);
 
         // check if the document is still active
-        auto primaryIndex = collection->primaryIndex();
+        auto primaryIndex = physical->primaryIndex();
         TRI_df_marker_t const* markerPtr = nullptr;
         MMFilesSimpleIndexElement element = primaryIndex->lookupKey(context._trx, keySlice);
         if (element) {
-          MMFilesDocumentPosition const old = static_cast<MMFilesCollection*>(collection->getPhysical())->lookupRevision(element.revisionId());
+          MMFilesDocumentPosition const old =
-          markerPtr = reinterpret_cast<TRI_df_marker_t const*>(static_cast<uint8_t const*>(old.dataptr()) - MMFilesDatafileHelper::VPackOffset(TRI_DF_MARKER_VPACK_DOCUMENT));
+              physical->lookupRevision(element.revisionId());
+          markerPtr = reinterpret_cast<TRI_df_marker_t const*>(
+              static_cast<uint8_t const*>(old.dataptr()) -
+              MMFilesDatafileHelper::VPackOffset(TRI_DF_MARKER_VPACK_DOCUMENT));
         }
 
         bool deleted = (markerPtr == nullptr || marker != markerPtr);
@@ -361,7 +367,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
   /// file.
   /// IMPORTANT: if the logic inside this function is adjusted, the total size
   /// calculated by function CalculateSize might need adjustment, too!!
-  auto compactifier = [&context, &collection, &physical, this](TRI_df_marker_t const* marker, MMFilesDatafile* datafile) -> bool {
+  auto compactifier = [&context, &physical, this](TRI_df_marker_t const* marker, MMFilesDatafile* datafile) -> bool {
     TRI_voc_fid_t const targetFid = context->_compactor->fid();
 
     TRI_df_marker_type_t const type = marker->getType();
@@ -374,7 +380,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
       VPackSlice keySlice = transaction::helpers::extractKeyFromDocument(slice);
 
       // check if the document is still active
-      auto primaryIndex = collection->primaryIndex();
+      auto primaryIndex = physical->primaryIndex();
       TRI_df_marker_t const* markerPtr = nullptr;
       MMFilesSimpleIndexElement element = primaryIndex->lookupKey(context->_trx, keySlice);
       if (element) {
@@ -432,7 +438,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
     return true;
   };
 
-  arangodb::SingleCollectionTransaction trx(arangodb::StandaloneTransactionContext::Create(collection->vocbase()), 
+  arangodb::SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(collection->vocbase()), 
       collection->cid(), AccessMode::Type::WRITE);
   trx.addHint(transaction::Hints::Hint::NO_BEGIN_MARKER);
   trx.addHint(transaction::Hints::Hint::NO_ABORT_MARKER);
@@ -547,10 +553,12 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
       removeDatafile(collection, compaction._datafile);
 
       // add a deletion ditch to the collection
-      auto b = collection->ditches()->createDropDatafileDitch(
+      auto b = arangodb::MMFilesCollection::toMMFilesCollection(collection)
-          compaction._datafile, collection, DropDatafileCallback, __FILE__,
+                   ->ditches()
-          __LINE__);
+                   ->createMMFilesDropDatafileDitch(compaction._datafile, collection,
-      
+                                             DropDatafileCallback, __FILE__,
+                                             __LINE__);
+
       if (b == nullptr) {
         LOG_TOPIC(ERR, Logger::COMPACTOR) << "out of memory when creating datafile-drop ditch";
       }
@@ -576,9 +584,12 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
 
       if (i == 0) {
         // add a rename marker
-        auto b = collection->ditches()->createRenameDatafileDitch(
+        auto b = arangodb::MMFilesCollection::toMMFilesCollection(collection)
-            compaction._datafile, context->_compactor, context->_collection, RenameDatafileCallback, __FILE__,
+                     ->ditches()
-            __LINE__);
+                     ->createMMFilesRenameDatafileDitch(
+                         compaction._datafile, context->_compactor,
+                         context->_collection, RenameDatafileCallback, __FILE__,
+                         __LINE__);
 
         if (b == nullptr) {
           LOG_TOPIC(ERR, Logger::COMPACTOR) << "out of memory when creating datafile-rename ditch";
@@ -591,9 +602,11 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
         removeDatafile(collection, compaction._datafile);
 
         // add a drop datafile marker
-        auto b = collection->ditches()->createDropDatafileDitch(
+        auto b = arangodb::MMFilesCollection::toMMFilesCollection(collection)
-            compaction._datafile, collection, DropDatafileCallback, __FILE__,
+                     ->ditches()
-            __LINE__);
+                     ->createMMFilesDropDatafileDitch(compaction._datafile, collection,
+                                               DropDatafileCallback, __FILE__,
+                                               __LINE__);
 
         if (b == nullptr) {
           LOG_TOPIC(ERR, Logger::COMPACTOR) << "out of memory when creating datafile-drop ditch";
@@ -660,7 +673,7 @@ bool MMFilesCompactorThread::compactCollection(LogicalCollection* collection, bo
   uint64_t const numDocuments = getNumberOfDocuments(collection);
 
   // get maximum size of result file
-  uint64_t maxSize = maxSizeFactor() * (uint64_t)collection->journalSize();
+  uint64_t maxSize = maxSizeFactor() * (uint64_t)collection->getPhysical()->journalSize();
   if (maxSize < 8 * 1024 * 1024) {
     maxSize = 8 * 1024 * 1024;
   }
@@ -871,7 +884,7 @@ void MMFilesCompactorThread::run() {
               return;
             }
 
-            bool doCompact = collection->doCompact();
+            bool doCompact = collection->getPhysical()->doCompact();
 
             // for document collection, compactify datafiles
             if (collection->status() == TRI_VOC_COL_STATUS_LOADED && doCompact) {
@@ -891,8 +904,10 @@ void MMFilesCompactorThread::run() {
               try {
                 double const now = TRI_microtime();
                 if (physical->lastCompactionStamp() + compactionCollectionInterval() <= now) {
-                  auto ce = collection->ditches()->createCompactionDitch(__FILE__,
+                  auto ce = arangodb::MMFilesCollection::toMMFilesCollection(
-                                                                        __LINE__);
+                                collection)
+                                ->ditches()
+                                ->createMMFilesCompactionDitch(__FILE__, __LINE__);
 
                   if (ce == nullptr) {
                     // out of memory
@@ -913,7 +928,9 @@ void MMFilesCompactorThread::run() {
                       // in case an error occurs, we must still free this ditch
                     }
 
-                    collection->ditches()->freeDitch(ce);
+                    arangodb::MMFilesCollection::toMMFilesCollection(collection)
+                        ->ditches()
+                        ->freeDitch(ce);
                   }
                 }
               } catch (...) {
@@ -964,7 +981,7 @@ void MMFilesCompactorThread::run() {
 /// @brief determine the number of documents in the collection
 uint64_t MMFilesCompactorThread::getNumberOfDocuments(LogicalCollection* collection) {
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), collection->cid(),
+      transaction::StandaloneContext::Create(_vocbase), collection->cid(),
       AccessMode::Type::READ);
   // only try to acquire the lock here
   // if lock acquisition fails, we go on and report an (arbitrary) positive number

arangod/MMFiles/MMFilesDitch.cpp

@@ -21,103 +21,103 @@
 /// @author Dr. Frank Celler
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "Ditch.h"
+#include "MMFilesDitch.h"
 #include "Basics/MutexLocker.h"
 #include "Logger/Logger.h"
 #include "MMFiles/MMFilesDatafile.h"
 
 using namespace arangodb;
 
-Ditch::Ditch(Ditches* ditches, char const* filename, int line)
+MMFilesDitch::MMFilesDitch(MMFilesDitches* ditches, char const* filename, int line)
     : _ditches(ditches),
       _prev(nullptr),
       _next(nullptr),
       _filename(filename),
       _line(line) {}
 
-Ditch::~Ditch() {}
+MMFilesDitch::~MMFilesDitch() {}
 
 /// @brief return the associated collection
-LogicalCollection* Ditch::collection() const {
+LogicalCollection* MMFilesDitch::collection() const {
   return _ditches->collection();
 }
 
-DocumentDitch::DocumentDitch(Ditches* ditches, bool usedByTransaction,
+MMFilesDocumentDitch::MMFilesDocumentDitch(MMFilesDitches* ditches, bool usedByTransaction,
                              char const* filename, int line)
-    : Ditch(ditches, filename, line),
+    : MMFilesDitch(ditches, filename, line),
       _usedByTransaction(usedByTransaction) {}
 
-DocumentDitch::~DocumentDitch() {}
+MMFilesDocumentDitch::~MMFilesDocumentDitch() {}
 
-ReplicationDitch::ReplicationDitch(Ditches* ditches, char const* filename,
+MMFilesReplicationDitch::MMFilesReplicationDitch(MMFilesDitches* ditches, char const* filename,
                                    int line)
-    : Ditch(ditches, filename, line) {}
+    : MMFilesDitch(ditches, filename, line) {}
 
-ReplicationDitch::~ReplicationDitch() {}
+MMFilesReplicationDitch::~MMFilesReplicationDitch() {}
 
-CompactionDitch::CompactionDitch(Ditches* ditches, char const* filename,
+MMFilesCompactionDitch::MMFilesCompactionDitch(MMFilesDitches* ditches, char const* filename,
                                  int line)
-    : Ditch(ditches, filename, line) {}
+    : MMFilesDitch(ditches, filename, line) {}
 
-CompactionDitch::~CompactionDitch() {}
+MMFilesCompactionDitch::~MMFilesCompactionDitch() {}
 
-DropDatafileDitch::DropDatafileDitch(
+MMFilesDropDatafileDitch::MMFilesDropDatafileDitch(
-    Ditches* ditches, MMFilesDatafile* datafile, LogicalCollection* collection,
+    MMFilesDitches* ditches, MMFilesDatafile* datafile, LogicalCollection* collection,
     std::function<void(MMFilesDatafile*, LogicalCollection*)> const& callback, char const* filename,
     int line)
-    : Ditch(ditches, filename, line),
+    : MMFilesDitch(ditches, filename, line),
       _datafile(datafile),
       _collection(collection),
       _callback(callback) {}
 
-DropDatafileDitch::~DropDatafileDitch() { delete _datafile; }
+MMFilesDropDatafileDitch::~MMFilesDropDatafileDitch() { delete _datafile; }
 
-RenameDatafileDitch::RenameDatafileDitch(
+MMFilesRenameDatafileDitch::MMFilesRenameDatafileDitch(
-    Ditches* ditches, MMFilesDatafile* datafile, MMFilesDatafile* compactor,
+    MMFilesDitches* ditches, MMFilesDatafile* datafile, MMFilesDatafile* compactor,
     LogicalCollection* collection,
     std::function<void(MMFilesDatafile*, MMFilesDatafile*, LogicalCollection*)> const& callback, char const* filename,
     int line)
-    : Ditch(ditches, filename, line),
+    : MMFilesDitch(ditches, filename, line),
       _datafile(datafile),
       _compactor(compactor),
       _collection(collection),
       _callback(callback) {}
 
-RenameDatafileDitch::~RenameDatafileDitch() {}
+MMFilesRenameDatafileDitch::~MMFilesRenameDatafileDitch() {}
 
-UnloadCollectionDitch::UnloadCollectionDitch(
+MMFilesUnloadCollectionDitch::MMFilesUnloadCollectionDitch(
-    Ditches* ditches, LogicalCollection* collection,
+    MMFilesDitches* ditches, LogicalCollection* collection,
     std::function<bool(LogicalCollection*)> const& callback,
     char const* filename, int line)
-    : Ditch(ditches, filename, line),
+    : MMFilesDitch(ditches, filename, line),
       _collection(collection),
       _callback(callback) {}
 
-UnloadCollectionDitch::~UnloadCollectionDitch() {}
+MMFilesUnloadCollectionDitch::~MMFilesUnloadCollectionDitch() {}
 
-DropCollectionDitch::DropCollectionDitch(
+MMFilesDropCollectionDitch::MMFilesDropCollectionDitch(
-    Ditches* ditches, arangodb::LogicalCollection* collection,
+    MMFilesDitches* ditches, arangodb::LogicalCollection* collection,
     std::function<bool(arangodb::LogicalCollection*)> callback,
     char const* filename, int line)
-    : Ditch(ditches, filename, line),
+    : MMFilesDitch(ditches, filename, line),
       _collection(collection),
       _callback(callback) {}
 
-DropCollectionDitch::~DropCollectionDitch() {}
+MMFilesDropCollectionDitch::~MMFilesDropCollectionDitch() {}
 
-Ditches::Ditches(LogicalCollection* collection)
+MMFilesDitches::MMFilesDitches(LogicalCollection* collection)
     : _collection(collection),
       _lock(),
       _begin(nullptr),
       _end(nullptr),
-      _numDocumentDitches(0) {
+      _numMMFilesDocumentMMFilesDitches(0) {
   TRI_ASSERT(_collection != nullptr);
 }
 
-Ditches::~Ditches() { destroy(); }
+MMFilesDitches::~MMFilesDitches() { destroy(); }
 
 /// @brief destroy the ditches - to be called on shutdown only
-void Ditches::destroy() {
+void MMFilesDitches::destroy() {
   MUTEX_LOCKER(mutexLocker, _lock);
   auto* ptr = _begin;
 
@@ -125,14 +125,14 @@ void Ditches::destroy() {
     auto* next = ptr->next();
     auto const type = ptr->type();
 
-    if (type == Ditch::TRI_DITCH_COLLECTION_UNLOAD ||
+    if (type == MMFilesDitch::TRI_DITCH_COLLECTION_UNLOAD ||
-        type == Ditch::TRI_DITCH_COLLECTION_DROP ||
+        type == MMFilesDitch::TRI_DITCH_COLLECTION_DROP ||
-        type == Ditch::TRI_DITCH_DATAFILE_DROP ||
+        type == MMFilesDitch::TRI_DITCH_DATAFILE_DROP ||
-        type == Ditch::TRI_DITCH_DATAFILE_RENAME ||
+        type == MMFilesDitch::TRI_DITCH_DATAFILE_RENAME ||
-        type == Ditch::TRI_DITCH_REPLICATION ||
+        type == MMFilesDitch::TRI_DITCH_REPLICATION ||
-        type == Ditch::TRI_DITCH_COMPACTION) {
+        type == MMFilesDitch::TRI_DITCH_COMPACTION) {
       delete ptr;
-    } else if (type == Ditch::TRI_DITCH_DOCUMENT) {
+    } else if (type == MMFilesDitch::TRI_DITCH_DOCUMENT) {
       LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "logic error. shouldn't have document ditches on unload";
       TRI_ASSERT(false);
     } else {
@@ -144,20 +144,20 @@ void Ditches::destroy() {
 }
 
 /// @brief return the associated collection
-LogicalCollection* Ditches::collection() const { return _collection; }
+LogicalCollection* MMFilesDitches::collection() const { return _collection; }
 
 /// @brief run a user-defined function under the lock
-void Ditches::executeProtected(std::function<void()> callback) {
+void MMFilesDitches::executeProtected(std::function<void()> callback) {
   MUTEX_LOCKER(mutexLocker, _lock);
   callback();
 }
 
 /// @brief process the first element from the list
 /// the list will remain unchanged if the first element is either a
-/// DocumentDitch, a ReplicationDitch or a CompactionDitch, or if the list
+/// MMFilesDocumentDitch, a MMFilesReplicationDitch or a MMFilesCompactionDitch, or if the list
-/// contains any DocumentDitches.
+/// contains any MMFilesDocumentMMFilesDitches.
-Ditch* Ditches::process(bool& popped,
+MMFilesDitch* MMFilesDitches::process(bool& popped,
-                        std::function<bool(Ditch const*)> callback) {
+                        std::function<bool(MMFilesDitch const*)> callback) {
   popped = false;
 
   MUTEX_LOCKER(mutexLocker, _lock);
@@ -173,11 +173,11 @@ Ditch* Ditches::process(bool& popped,
 
   auto const type = ditch->type();
 
-  // if it is a DocumentDitch, it means that there is still a reference held
+  // if it is a MMFilesDocumentDitch, it means that there is still a reference held
   // to document data in a datafile. We must then not unload or remove a file
-  if (type == Ditch::TRI_DITCH_DOCUMENT ||
+  if (type == MMFilesDitch::TRI_DITCH_DOCUMENT ||
-      type == Ditch::TRI_DITCH_REPLICATION ||
+      type == MMFilesDitch::TRI_DITCH_REPLICATION ||
-      type == Ditch::TRI_DITCH_COMPACTION || _numDocumentDitches > 0) {
+      type == MMFilesDitch::TRI_DITCH_COMPACTION || _numMMFilesDocumentMMFilesDitches > 0) {
     // did not find anything at the head of the barrier list or found an element
     // marker
     // this means we must exit and cannot throw away datafiles and can unload
@@ -185,11 +185,11 @@ Ditch* Ditches::process(bool& popped,
     return nullptr;
   }
 
-  // no DocumentDitch at the head of the ditches list. This means that there is
+  // no MMFilesDocumentDitch at the head of the ditches list. This means that there is
   // some other action we can perform (i.e. unloading a datafile or a
   // collection)
 
-  // note that there is no need to check the entire list for a DocumentDitch as
+  // note that there is no need to check the entire list for a MMFilesDocumentDitch as
   // the list is filled up in chronological order. New ditches are always added
   // to the
   // tail of the list, and if we have the following list
@@ -218,7 +218,7 @@ Ditch* Ditches::process(bool& popped,
 }
 
 /// @brief return the type name of the ditch at the head of the active ditches
-char const* Ditches::head() {
+char const* MMFilesDitches::head() {
   MUTEX_LOCKER(mutexLocker, _lock);
 
   auto ditch = _begin;
@@ -230,19 +230,19 @@ char const* Ditches::head() {
 }
 
 /// @brief return the number of document ditches active
-uint64_t Ditches::numDocumentDitches() {
+uint64_t MMFilesDitches::numMMFilesDocumentMMFilesDitches() {
   MUTEX_LOCKER(mutexLocker, _lock);
 
-  return _numDocumentDitches;
+  return _numMMFilesDocumentMMFilesDitches;
 }
 
 /// @brief check whether the ditches contain a ditch of a certain type
-bool Ditches::contains(Ditch::DitchType type) {
+bool MMFilesDitches::contains(MMFilesDitch::DitchType type) {
   MUTEX_LOCKER(mutexLocker, _lock);
 
-  if (type == Ditch::TRI_DITCH_DOCUMENT) {
+  if (type == MMFilesDitch::TRI_DITCH_DOCUMENT) {
     // shortcut
-    return (_numDocumentDitches > 0);
+    return (_numMMFilesDocumentMMFilesDitches > 0);
   }
 
   auto const* ptr = _begin;
@@ -259,19 +259,19 @@ bool Ditches::contains(Ditch::DitchType type) {
 }
 
 /// @brief removes and frees a ditch
-void Ditches::freeDitch(Ditch* ditch) {
+void MMFilesDitches::freeDitch(MMFilesDitch* ditch) {
   TRI_ASSERT(ditch != nullptr);
 
-  bool const isDocumentDitch = (ditch->type() == Ditch::TRI_DITCH_DOCUMENT);
+  bool const isMMFilesDocumentDitch = (ditch->type() == MMFilesDitch::TRI_DITCH_DOCUMENT);
 
   {
     MUTEX_LOCKER(mutexLocker, _lock);
 
     unlink(ditch);
 
-    if (isDocumentDitch) {
+    if (isMMFilesDocumentDitch) {
       // decrease counter
-      --_numDocumentDitches;
+      --_numMMFilesDocumentMMFilesDitches;
     }
   }
 
@@ -281,7 +281,7 @@ void Ditches::freeDitch(Ditch* ditch) {
 /// @brief removes and frees a ditch
 /// this is used for ditches used by transactions or by externals to protect
 /// the flags by the lock
-void Ditches::freeDocumentDitch(DocumentDitch* ditch, bool fromTransaction) {
+void MMFilesDitches::freeMMFilesDocumentDitch(MMFilesDocumentDitch* ditch, bool fromTransaction) {
   TRI_ASSERT(ditch != nullptr);
     
   // First see who might still be using the ditch:
@@ -295,17 +295,17 @@ void Ditches::freeDocumentDitch(DocumentDitch* ditch, bool fromTransaction) {
     unlink(ditch);
 
     // decrease counter
-    --_numDocumentDitches;
+    --_numMMFilesDocumentMMFilesDitches;
   }
 
   delete ditch;
 }
 
 /// @brief creates a new document ditch and links it
-DocumentDitch* Ditches::createDocumentDitch(bool usedByTransaction,
+MMFilesDocumentDitch* MMFilesDitches::createMMFilesDocumentDitch(bool usedByTransaction,
                                             char const* filename, int line) {
   try {
-    auto ditch = new DocumentDitch(this, usedByTransaction, filename, line);
+    auto ditch = new MMFilesDocumentDitch(this, usedByTransaction, filename, line);
     link(ditch);
 
     return ditch;
@@ -315,10 +315,10 @@ DocumentDitch* Ditches::createDocumentDitch(bool usedByTransaction,
 }
 
 /// @brief creates a new replication ditch and links it
-ReplicationDitch* Ditches::createReplicationDitch(char const* filename,
+MMFilesReplicationDitch* MMFilesDitches::createMMFilesReplicationDitch(char const* filename,
                                                   int line) {
   try {
-    auto ditch = new ReplicationDitch(this, filename, line);
+    auto ditch = new MMFilesReplicationDitch(this, filename, line);
     link(ditch);
 
     return ditch;
@@ -328,10 +328,10 @@ ReplicationDitch* Ditches::createReplicationDitch(char const* filename,
 }
 
 /// @brief creates a new compaction ditch and links it
-CompactionDitch* Ditches::createCompactionDitch(char const* filename,
+MMFilesCompactionDitch* MMFilesDitches::createMMFilesCompactionDitch(char const* filename,
                                                 int line) {
   try {
-    auto ditch = new CompactionDitch(this, filename, line);
+    auto ditch = new MMFilesCompactionDitch(this, filename, line);
     link(ditch);
 
     return ditch;
@@ -341,13 +341,13 @@ CompactionDitch* Ditches::createCompactionDitch(char const* filename,
 }
 
 /// @brief creates a new datafile deletion ditch
-DropDatafileDitch* Ditches::createDropDatafileDitch(
+MMFilesDropDatafileDitch* MMFilesDitches::createMMFilesDropDatafileDitch(
     MMFilesDatafile* datafile, LogicalCollection* collection, 
     std::function<void(MMFilesDatafile*, LogicalCollection*)> const& callback,
     char const* filename, int line) {
   try {
     auto ditch =
-        new DropDatafileDitch(this, datafile, collection, callback, filename, line);
+        new MMFilesDropDatafileDitch(this, datafile, collection, callback, filename, line);
     link(ditch);
       
     return ditch;
@@ -357,13 +357,13 @@ DropDatafileDitch* Ditches::createDropDatafileDitch(
 }
 
 /// @brief creates a new datafile rename ditch
-RenameDatafileDitch* Ditches::createRenameDatafileDitch(
+MMFilesRenameDatafileDitch* MMFilesDitches::createMMFilesRenameDatafileDitch(
     MMFilesDatafile* datafile, MMFilesDatafile* compactor, LogicalCollection* collection,
     std::function<void(MMFilesDatafile*, MMFilesDatafile*, LogicalCollection*)> const& callback,
     char const* filename, int line) {
   try {
     auto ditch =
-        new RenameDatafileDitch(this, datafile, compactor, collection, callback, filename, line);
+        new MMFilesRenameDatafileDitch(this, datafile, compactor, collection, callback, filename, line);
     link(ditch);
 
     return ditch;
@@ -373,12 +373,12 @@ RenameDatafileDitch* Ditches::createRenameDatafileDitch(
 }
 
 /// @brief creates a new collection unload ditch
-UnloadCollectionDitch* Ditches::createUnloadCollectionDitch(
+MMFilesUnloadCollectionDitch* MMFilesDitches::createMMFilesUnloadCollectionDitch(
     LogicalCollection* collection, 
     std::function<bool(LogicalCollection*)> const& callback,
     char const* filename, int line) {
   try {
-    auto ditch = new UnloadCollectionDitch(this, collection, callback,
+    auto ditch = new MMFilesUnloadCollectionDitch(this, collection, callback,
                                            filename, line);
     link(ditch);
 
@@ -389,12 +389,12 @@ UnloadCollectionDitch* Ditches::createUnloadCollectionDitch(
 }
 
 /// @brief creates a new datafile drop ditch
-DropCollectionDitch* Ditches::createDropCollectionDitch(
+MMFilesDropCollectionDitch* MMFilesDitches::createMMFilesDropCollectionDitch(
     arangodb::LogicalCollection* collection,
     std::function<bool(arangodb::LogicalCollection*)> callback,
     char const* filename, int line) {
   try {
-    auto ditch = new DropCollectionDitch(this, collection, callback,
+    auto ditch = new MMFilesDropCollectionDitch(this, collection, callback,
                                          filename, line);
     link(ditch);
 
@@ -405,13 +405,13 @@ DropCollectionDitch* Ditches::createDropCollectionDitch(
 }
 
 /// @brief inserts the ditch into the linked list of ditches
-void Ditches::link(Ditch* ditch) {
+void MMFilesDitches::link(MMFilesDitch* ditch) {
   TRI_ASSERT(ditch != nullptr);
     
   ditch->_next = nullptr;
   ditch->_prev = nullptr;
   
-  bool const isDocumentDitch = (ditch->type() == Ditch::TRI_DITCH_DOCUMENT);
+  bool const isMMFilesDocumentDitch = (ditch->type() == MMFilesDitch::TRI_DITCH_DOCUMENT);
 
   MUTEX_LOCKER(mutexLocker, _lock);  // FIX_MUTEX
 
@@ -428,14 +428,14 @@ void Ditches::link(Ditch* ditch) {
     
   _end = ditch;
 
-  if (isDocumentDitch) {
+  if (isMMFilesDocumentDitch) {
     // increase counter
-    ++_numDocumentDitches;
+    ++_numMMFilesDocumentMMFilesDitches;
   }
 }
 
 /// @brief unlinks the ditch from the linked list of ditches
-void Ditches::unlink(Ditch* ditch) {
+void MMFilesDitches::unlink(MMFilesDitch* ditch) {
   // ditch is at the beginning of the chain
   if (ditch->_prev == nullptr) {
     _begin = ditch->_next;

arangod/MMFiles/MMFilesDitch.h

@@ -21,8 +21,8 @@
 /// @author Dr. Frank Celler
 ////////////////////////////////////////////////////////////////////////////////
 
-#ifndef ARANGOD_VOC_BASE_DITCH_H
+#ifndef ARANGOD_MMFILES_MMFILES_DITCH_H
-#define ARANGOD_VOC_BASE_DITCH_H 1
+#define ARANGOD_MMFILES_MMFILES_DITCH_H 1
 
 #include "Basics/Common.h"
 #include "Basics/Mutex.h"
@@ -32,19 +32,19 @@ struct MMFilesDatafile;
 namespace arangodb {
 class LogicalCollection;
 
-class Ditches;
+class MMFilesDitches;
 
-class Ditch {
+class MMFilesDitch {
-  friend class Ditches;
+  friend class MMFilesDitches;
 
  protected:
-  Ditch(Ditch const&) = delete;
+  MMFilesDitch(MMFilesDitch const&) = delete;
-  Ditch& operator=(Ditch const&) = delete;
+  MMFilesDitch& operator=(MMFilesDitch const&) = delete;
 
-  Ditch(Ditches*, char const*, int);
+  MMFilesDitch(MMFilesDitches*, char const*, int);
 
  public:
-  virtual ~Ditch();
+  virtual ~MMFilesDitch();
 
  public:
   /// @brief ditch type
@@ -72,33 +72,33 @@ class Ditch {
   int line() const { return _line; }
 
   /// @brief return the next ditch in the linked list
-  inline Ditch* next() const { return _next; }
+  inline MMFilesDitch* next() const { return _next; }
 
   /// @brief return the link to all ditches
-  Ditches* ditches() { return _ditches; }
+  MMFilesDitches* ditches() { return _ditches; }
 
   /// @brief return the associated collection
   LogicalCollection* collection() const;
 
  protected:
-  Ditches* _ditches;
+  MMFilesDitches* _ditches;
 
  private:
-  Ditch* _prev;
+  MMFilesDitch* _prev;
-  Ditch* _next;
+  MMFilesDitch* _next;
   char const* _filename;
   int _line;
 };
 
 /// @brief document ditch
-class DocumentDitch final : public Ditch {
+class MMFilesDocumentDitch final : public MMFilesDitch {
-  friend class Ditches;
+  friend class MMFilesDitches;
 
  public:
-  DocumentDitch(Ditches* ditches, bool usedByTransaction, char const* filename,
+  MMFilesDocumentDitch(MMFilesDitches* ditches, bool usedByTransaction, char const* filename,
                 int line);
 
-  ~DocumentDitch();
+  ~MMFilesDocumentDitch();
 
  public:
   DitchType type() const override final { return TRI_DITCH_DOCUMENT; }
@@ -112,11 +112,11 @@ class DocumentDitch final : public Ditch {
 };
 
 /// @brief replication ditch
-class ReplicationDitch final : public Ditch {
+class MMFilesReplicationDitch final : public MMFilesDitch {
  public:
-  ReplicationDitch(Ditches* ditches, char const* filename, int line);
+  MMFilesReplicationDitch(MMFilesDitches* ditches, char const* filename, int line);
 
-  ~ReplicationDitch();
+  ~MMFilesReplicationDitch();
 
  public:
   DitchType type() const override final { return TRI_DITCH_REPLICATION; }
@@ -125,11 +125,11 @@ class ReplicationDitch final : public Ditch {
 };
 
 /// @brief compaction ditch
-class CompactionDitch final : public Ditch {
+class MMFilesCompactionDitch final : public MMFilesDitch {
  public:
-  CompactionDitch(Ditches* ditches, char const* filename, int line);
+  MMFilesCompactionDitch(MMFilesDitches* ditches, char const* filename, int line);
 
-  ~CompactionDitch();
+  ~MMFilesCompactionDitch();
 
  public:
   DitchType type() const override final { return TRI_DITCH_COMPACTION; }
@@ -138,14 +138,14 @@ class CompactionDitch final : public Ditch {
 };
 
 /// @brief datafile removal ditch
-class DropDatafileDitch final : public Ditch {
+class MMFilesDropDatafileDitch final : public MMFilesDitch {
  public:
-  DropDatafileDitch(Ditches* ditches, MMFilesDatafile* datafile,
+  MMFilesDropDatafileDitch(MMFilesDitches* ditches, MMFilesDatafile* datafile,
                     LogicalCollection* collection,
                     std::function<void(MMFilesDatafile*, LogicalCollection*)> const& callback,
                     char const* filename, int line);
 
-  ~DropDatafileDitch();
+  ~MMFilesDropDatafileDitch();
 
  public:
   DitchType type() const override final { return TRI_DITCH_DATAFILE_DROP; }
@@ -161,14 +161,14 @@ class DropDatafileDitch final : public Ditch {
 };
 
 /// @brief datafile rename ditch
-class RenameDatafileDitch final : public Ditch {
+class MMFilesRenameDatafileDitch final : public MMFilesDitch {
  public:
-  RenameDatafileDitch(Ditches* ditches, MMFilesDatafile* datafile,
+  MMFilesRenameDatafileDitch(MMFilesDitches* ditches, MMFilesDatafile* datafile,
                       MMFilesDatafile* compactor, LogicalCollection* collection,
                       std::function<void(MMFilesDatafile*, MMFilesDatafile*, LogicalCollection*)> const& callback,
                       char const* filename, int line);
 
-  ~RenameDatafileDitch();
+  ~MMFilesRenameDatafileDitch();
 
  public:
   DitchType type() const override final { return TRI_DITCH_DATAFILE_RENAME; }
@@ -185,14 +185,14 @@ class RenameDatafileDitch final : public Ditch {
 };
 
 /// @brief collection unload ditch
-class UnloadCollectionDitch final : public Ditch {
+class MMFilesUnloadCollectionDitch final : public MMFilesDitch {
  public:
-  UnloadCollectionDitch(
+  MMFilesUnloadCollectionDitch(
-      Ditches* ditches, LogicalCollection* collection,
+      MMFilesDitches* ditches, LogicalCollection* collection,
       std::function<bool(LogicalCollection*)> const& callback,
       char const* filename, int line);
 
-  ~UnloadCollectionDitch();
+  ~MMFilesUnloadCollectionDitch();
 
   DitchType type() const override final { return TRI_DITCH_COLLECTION_UNLOAD; }
 
@@ -206,14 +206,14 @@ class UnloadCollectionDitch final : public Ditch {
 };
 
 /// @brief collection drop ditch
-class DropCollectionDitch final : public Ditch {
+class MMFilesDropCollectionDitch final : public MMFilesDitch {
  public:
-  DropCollectionDitch(
+  MMFilesDropCollectionDitch(
-      arangodb::Ditches* ditches, arangodb::LogicalCollection* collection,
+      arangodb::MMFilesDitches* ditches, arangodb::LogicalCollection* collection,
       std::function<bool(arangodb::LogicalCollection*)> callback,
       char const* filename, int line);
 
-  ~DropCollectionDitch();
+  ~MMFilesDropCollectionDitch();
 
   DitchType type() const override final { return TRI_DITCH_COLLECTION_DROP; }
 
@@ -227,14 +227,14 @@ class DropCollectionDitch final : public Ditch {
 };
 
 /// @brief doubly linked list of ditches
-class Ditches {
+class MMFilesDitches {
  public:
-  Ditches(Ditches const&) = delete;
+  MMFilesDitches(MMFilesDitches const&) = delete;
-  Ditches& operator=(Ditches const&) = delete;
+  MMFilesDitches& operator=(MMFilesDitches const&) = delete;
-  Ditches() = delete;
+  MMFilesDitches() = delete;
 
-  explicit Ditches(LogicalCollection*);
+  explicit MMFilesDitches(LogicalCollection*);
-  ~Ditches();
+  ~MMFilesDitches();
 
  public:
   /// @brief destroy the ditches - to be called on shutdown only
@@ -248,75 +248,75 @@ class Ditches {
 
   /// @brief process the first element from the list
   /// the list will remain unchanged if the first element is either a
-  /// DocumentDitch, a ReplicationDitch or a CompactionDitch, or if the list
+  /// MMFilesDocumentDitch, a MMFilesReplicationDitch or a MMFilesCompactionDitch, or if the list
-  /// contains any DocumentDitches.
+  /// contains any MMFilesDocumentMMFilesDitches.
-  Ditch* process(bool&, std::function<bool(Ditch const*)>);
+  MMFilesDitch* process(bool&, std::function<bool(MMFilesDitch const*)>);
 
   /// @brief return the type name of the ditch at the head of the active ditches
   char const* head();
 
   /// @brief return the number of document ditches active
-  uint64_t numDocumentDitches();
+  uint64_t numMMFilesDocumentMMFilesDitches();
 
   /// @brief check whether the ditches contain a ditch of a certain type
-  bool contains(Ditch::DitchType);
+  bool contains(MMFilesDitch::DitchType);
 
   /// @brief unlinks and frees a ditch
-  void freeDitch(Ditch*);
+  void freeDitch(MMFilesDitch*);
 
   /// @brief unlinks and frees a document ditch
   /// this is used for ditches used by transactions or by externals to protect
   /// the flags by the lock
-  void freeDocumentDitch(DocumentDitch*, bool fromTransaction);
+  void freeMMFilesDocumentDitch(MMFilesDocumentDitch*, bool fromTransaction);
 
   /// @brief creates a new document ditch and links it
-  DocumentDitch* createDocumentDitch(bool usedByTransaction,
+  MMFilesDocumentDitch* createMMFilesDocumentDitch(bool usedByTransaction,
                                      char const* filename, int line);
 
   /// @brief creates a new replication ditch and links it
-  ReplicationDitch* createReplicationDitch(char const* filename, int line);
+  MMFilesReplicationDitch* createMMFilesReplicationDitch(char const* filename, int line);
 
   /// @brief creates a new compaction ditch and links it
-  CompactionDitch* createCompactionDitch(char const* filename, int line);
+  MMFilesCompactionDitch* createMMFilesCompactionDitch(char const* filename, int line);
 
   /// @brief creates a new datafile deletion ditch
-  DropDatafileDitch* createDropDatafileDitch(
+  MMFilesDropDatafileDitch* createMMFilesDropDatafileDitch(
       MMFilesDatafile* datafile, LogicalCollection* collection, 
       std::function<void(MMFilesDatafile*, LogicalCollection*)> const& callback,
       char const* filename, int line);
 
   /// @brief creates a new datafile rename ditch
-  RenameDatafileDitch* createRenameDatafileDitch(
+  MMFilesRenameDatafileDitch* createMMFilesRenameDatafileDitch(
       MMFilesDatafile* datafile, MMFilesDatafile* compactor, LogicalCollection* collection,
       std::function<void(MMFilesDatafile*, MMFilesDatafile*, LogicalCollection*)> const& callback,
       char const* filename, int line);
 
   /// @brief creates a new collection unload ditch
-  UnloadCollectionDitch* createUnloadCollectionDitch(
+  MMFilesUnloadCollectionDitch* createMMFilesUnloadCollectionDitch(
       LogicalCollection* collection,
       std::function<bool(LogicalCollection*)> const& callback,
       char const* filename, int line);
 
   /// @brief creates a new collection drop ditch
-  DropCollectionDitch* createDropCollectionDitch(
+  MMFilesDropCollectionDitch* createMMFilesDropCollectionDitch(
       arangodb::LogicalCollection* collection, 
       std::function<bool(arangodb::LogicalCollection*)> callback,
       char const* filename, int line);
 
  private:
   /// @brief inserts the ditch into the linked list of ditches
-  void link(Ditch*);
+  void link(MMFilesDitch*);
 
   /// @brief unlinks the ditch from the linked list of ditches
-  void unlink(Ditch*);
+  void unlink(MMFilesDitch*);
 
  private:
   LogicalCollection* _collection;
 
   arangodb::Mutex _lock;
-  Ditch* _begin;
+  MMFilesDitch* _begin;
-  Ditch* _end;
+  MMFilesDitch* _end;
-  uint64_t _numDocumentDitches;
+  uint64_t _numMMFilesDocumentMMFilesDitches;
 };
 }
 

arangod/MMFiles/MMFilesDocumentOperation.cpp

@@ -157,8 +157,9 @@ void MMFilesDocumentOperation::revert(transaction::Methods* trx) {
       }
     }
    
-    // let the primary index entry point to the correct document 
+    // let the primary index entry point to the correct document
-    MMFilesSimpleIndexElement* element = _collection->primaryIndex()->lookupKeyRef(trx, transaction::helpers::extractKeyFromDocument(newDoc));
+    MMFilesSimpleIndexElement* element = physical->primaryIndex()->lookupKeyRef(
+        trx, transaction::helpers::extractKeyFromDocument(newDoc));
     if (element != nullptr && element->revisionId() != 0) {
       VPackSlice keySlice(transaction::helpers::extractKeyFromDocument(oldDoc));
       element->updateRevisionId(oldRevisionId, static_cast<uint32_t>(keySlice.begin() - oldDoc.begin()));

arangod/MMFiles/MMFilesEdgeIndex.cpp

@@ -37,7 +37,7 @@
 #include "Transaction/Helpers.h"
 #include "Transaction/Methods.h"
 #include "Utils/CollectionNameResolver.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/LogicalCollection.h"
 
 #include <velocypack/Iterator.h>

arangod/MMFiles/MMFilesEngine.cpp

@@ -31,10 +31,6 @@
 #include "Basics/WriteLocker.h"
 #include "Basics/encoding.h"
 #include "Basics/files.h"
-#include "Random/RandomGenerator.h"
-#include "RestServer/DatabaseFeature.h"
-#include "RestServer/DatabasePathFeature.h"
-#include "StorageEngine/EngineSelectorFeature.h"
 #include "MMFiles/MMFilesAqlFunctions.h"
 #include "MMFiles/MMFilesCleanupThread.h"
 #include "MMFiles/MMFilesCompactorThread.h"
@@ -45,7 +41,12 @@
 #include "MMFiles/MMFilesPersistentIndex.h"
 #include "MMFiles/MMFilesPersistentIndexFeature.h"
 #include "MMFiles/MMFilesTransactionCollection.h"
+#include "MMFiles/MMFilesTransactionContextData.h"
 #include "MMFiles/MMFilesTransactionState.h"
+#include "Random/RandomGenerator.h"
+#include "RestServer/DatabaseFeature.h"
+#include "RestServer/DatabasePathFeature.h"
+#include "StorageEngine/EngineSelectorFeature.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ticks.h"
 #include "VocBase/vocbase.h"
@@ -223,7 +224,11 @@ void MMFilesEngine::stop() {
   logfileManager->flush(true, true, false);
   logfileManager->waitForCollector();
 }
-  
+ 
+transaction::ContextData* MMFilesEngine::createTransactionContextData() {
+  return new MMFilesTransactionContextData;
+}
+
 TransactionState* MMFilesEngine::createTransactionState(TRI_vocbase_t* vocbase) {
   return new MMFilesTransactionState(vocbase);
 }
@@ -233,9 +238,9 @@ TransactionCollection* MMFilesEngine::createTransactionCollection(TransactionSta
 }
 
 // create storage-engine specific collection
-PhysicalCollection* MMFilesEngine::createPhysicalCollection(LogicalCollection* collection) {
+PhysicalCollection* MMFilesEngine::createPhysicalCollection(LogicalCollection* collection, VPackSlice const& info) {
   TRI_ASSERT(EngineSelectorFeature::ENGINE == this);
-  return new MMFilesCollection(collection);
+  return new MMFilesCollection(collection, info);
 }
 
 void MMFilesEngine::recoveryDone(TRI_vocbase_t* vocbase) {    
@@ -495,15 +500,16 @@ int MMFilesEngine::getCollectionsAndIndexes(TRI_vocbase_t* vocbase,
     int res = TRI_ERROR_NO_ERROR;
 
     try {
-      std::unique_ptr<LogicalCollection> collection(loadCollectionInfo(vocbase, directory));
+      VPackBuilder builder = loadCollectionInfo(vocbase, directory);
-     
+      VPackSlice info = builder.slice();
-      if (collection->deleted()) {
+
-        _deleted.emplace_back(std::make_pair(collection->name(), directory));
+      if (VelocyPackHelper::readBooleanValue(info, "deleted", false)) {
+        std::string name = VelocyPackHelper::getStringValue(info, "name", "");
+        _deleted.emplace_back(std::make_pair(name, directory));
         continue;
       }
-
       // add collection info
-      collection->toVelocyPack(result, true);
+      result.add(info);
     } catch (arangodb::basics::Exception const& e) {
       std::string tmpfile = FileUtils::buildFilename(directory, ".tmp");
 
@@ -618,10 +624,10 @@ std::string MMFilesEngine::createCollection(TRI_vocbase_t* vocbase, TRI_voc_cid_
   std::string const path = databasePath(vocbase);
 
   // sanity check
-  if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) > parameters->journalSize()) {
+  if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) > parameters->getPhysical()->journalSize()) {
     LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot create datafile '" << parameters->name() << "' in '"
              << path << "', maximal size '"
-             << parameters->journalSize() << "' is too small";
+             << parameters->getPhysical()->journalSize() << "' is too small";
     THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_DATAFILE_FULL);
   }
 
@@ -841,29 +847,6 @@ void MMFilesEngine::createIndex(TRI_vocbase_t* vocbase, TRI_voc_cid_t collection
   }
 }
 
-void MMFilesEngine::createIndexWalMarker(TRI_vocbase_t* vocbase, TRI_voc_cid_t collectionId,
-                          arangodb::velocypack::Slice const& data, bool writeMarker, int& status){
-
-  status = TRI_ERROR_NO_ERROR;
-  if (!writeMarker) {
-    return;
-  }
-
-  try {
-    MMFilesCollectionMarker marker(TRI_DF_MARKER_VPACK_CREATE_INDEX,
-                                   vocbase->id(), collectionId, data);
-
-    MMFilesWalSlotInfoCopy slotInfo =
-        MMFilesLogfileManager::instance()->allocateAndWrite(marker, false);
-    status=slotInfo.errorCode;
-  } catch (arangodb::basics::Exception const& ex) {
-    status = ex.code();
-  } catch (...) {
-    status = TRI_ERROR_INTERNAL;
-  }
-
-};
-
 // asks the storage engine to drop the specified index and persist the deletion 
 // info. Note that physical deletion of the index must not be carried out by this call, 
 // as there may still be users of the index. It is recommended that this operation
@@ -1293,7 +1276,13 @@ TRI_vocbase_t* MMFilesEngine::openExistingDatabase(TRI_voc_tick_t id, std::strin
     for (auto const& it : VPackArrayIterator(slice)) {
       // we found a collection that is still active
       TRI_ASSERT(!it.get("id").isNone() || !it.get("cid").isNone());
-      arangodb::LogicalCollection* collection = StorageEngine::registerCollection(vocbase.get(), it);
+      auto uniqCol = std::make_unique<arangodb::LogicalCollection>(vocbase.get(), it, true);
+      auto collection = uniqCol.get();
+      TRI_ASSERT(collection != nullptr);
+      StorageEngine::registerCollection(vocbase.get(), uniqCol.get());
+      // The vocbase has taken over control
+      uniqCol.release();
+
 
       auto physical = static_cast<MMFilesCollection*>(collection->getPhysical());
       TRI_ASSERT(physical != nullptr);
@@ -1423,9 +1412,8 @@ void MMFilesEngine::saveCollectionInfo(TRI_vocbase_t* vocbase,
                                        bool forceSync) const {
   std::string const filename = collectionParametersFilename(vocbase->id(), id);
 
-  VPackBuilder builder;
+  VPackBuilder builder =
-  parameters->toVelocyPack(builder, false);
+      parameters->toVelocyPackIgnore({"path", "statusString"}, true);
-
   TRI_ASSERT(id != 0);
 
   bool ok = VelocyPackHelper::velocyPackToFile(filename,
@@ -1438,7 +1426,7 @@ void MMFilesEngine::saveCollectionInfo(TRI_vocbase_t* vocbase,
   }
 }
 
-LogicalCollection* MMFilesEngine::loadCollectionInfo(TRI_vocbase_t* vocbase, std::string const& path) {
+VPackBuilder MMFilesEngine::loadCollectionInfo(TRI_vocbase_t* vocbase, std::string const& path) {
   // find parameter file
   std::string filename =
       arangodb::basics::FileUtils::buildFilename(path, parametersFilename());
@@ -1555,10 +1543,7 @@ LogicalCollection* MMFilesEngine::loadCollectionInfo(TRI_vocbase_t* vocbase, std
   indexesPatch.close();
   indexesPatch.close();
 
-  VPackBuilder b3 = VPackCollection::merge(slice, indexesPatch.slice(), false);
+  return VPackCollection::merge(slice, indexesPatch.slice(), false);
-  slice = b3.slice();
-  
-  return new LogicalCollection(vocbase, slice, true);
 }
 
 /// @brief remove data of expired compaction blockers
@@ -2149,7 +2134,9 @@ int MMFilesEngine::transferMarkersWorker(
   // used only for crash / recovery tests
   int numMarkers = 0;
 
-  TRI_voc_tick_t const minTransferTick = collection->maxTick();
+  MMFilesCollection* mmfiles = static_cast<MMFilesCollection*>(collection->getPhysical());
+  TRI_ASSERT(mmfiles);
+  TRI_voc_tick_t const minTransferTick = mmfiles->maxTick();
   TRI_ASSERT(!operations.empty());
 
   for (auto it2 = operations.begin(); it2 != operations.end(); ++it2) {
@@ -2239,8 +2226,9 @@ char* MMFilesEngine::nextFreeMarkerPosition(
 
     // we only need the ditches when we are outside the recovery
     // the compactor will not run during recovery
-    auto ditch =
+    auto ditch = arangodb::MMFilesCollection::toMMFilesCollection(collection)
-        collection->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
+                     ->ditches()
+                     ->createMMFilesDocumentDitch(false, __FILE__, __LINE__);
 
     if (ditch == nullptr) {
       THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
@@ -2270,8 +2258,10 @@ void MMFilesEngine::finishMarker(char const* walPosition,
   // update ticks
   TRI_UpdateTicksDatafile(datafile, marker);
 
-  TRI_ASSERT(collection->maxTick() < tick);
+  MMFilesCollection* mmfiles = static_cast<MMFilesCollection*>(collection->getPhysical());
-  collection->maxTick(tick);
+  TRI_ASSERT(mmfiles);
+  TRI_ASSERT(mmfiles->maxTick() < tick);
+  mmfiles->maxTick(tick);
 
   cache->operations->emplace_back(MMFilesCollectorOperation(
       datafilePosition, marker->getSize(), walPosition, cache->lastFid));

arangod/MMFiles/MMFilesEngine.h

@@ -28,6 +28,7 @@
 #include "Basics/Common.h"
 #include "Basics/Mutex.h"
 #include "MMFiles/MMFilesDatafile.h"
+#include "MMFiles/MMFilesCollectorCache.h"
 #include "StorageEngine/StorageEngine.h"
 #include "VocBase/AccessMode.h"
 
@@ -39,6 +40,10 @@ class MMFilesCompactorThread;
 class TransactionCollection;
 class TransactionState;
 
+namespace transaction {
+class ContextData;
+}
+
 /// @brief collection file structure
 struct MMFilesEngineCollectionFiles {
   std::vector<std::string> journals;
@@ -74,11 +79,12 @@ class MMFilesEngine final : public StorageEngine {
   // flush wal wait for collector
   void stop() override;
 
+  transaction::ContextData* createTransactionContextData() override;
   TransactionState* createTransactionState(TRI_vocbase_t*) override;
   TransactionCollection* createTransactionCollection(TransactionState* state, TRI_voc_cid_t cid, AccessMode::Type accessType, int nestingLevel) override;
 
   // create storage-engine specific collection
-  PhysicalCollection* createPhysicalCollection(LogicalCollection*) override;
+  PhysicalCollection* createPhysicalCollection(LogicalCollection*, VPackSlice const&) override;
 
   // inventory functionality
   // -----------------------
@@ -180,9 +186,6 @@ public:
   void createIndex(TRI_vocbase_t* vocbase, TRI_voc_cid_t collectionId,
                    TRI_idx_iid_t id, arangodb::velocypack::Slice const& data) override;
   
-  virtual void createIndexWalMarker(TRI_vocbase_t* vocbase, TRI_voc_cid_t collectionId,
-                                    arangodb::velocypack::Slice const& data, bool useMarker, int&) override;
-
   // asks the storage engine to drop the specified index and persist the deletion 
   // info. Note that physical deletion of the index must not be carried out by this call, 
   // as there may still be users of the index. It is recommended that this operation
@@ -252,7 +255,7 @@ public:
   
   /// @brief transfer markers into a collection
   int transferMarkers(LogicalCollection* collection, MMFilesCollectorCache*,
-                      MMFilesOperationsType const&) override;
+                      MMFilesOperationsType const&);
 
   /// @brief Add engine specific AQL functions.
 
@@ -329,7 +332,7 @@ public:
                           arangodb::LogicalCollection const* parameters,
                           bool forceSync) const;
 
-  LogicalCollection* loadCollectionInfo(TRI_vocbase_t* vocbase, std::string const& path);
+  arangodb::velocypack::Builder loadCollectionInfo(TRI_vocbase_t* vocbase, std::string const& path);
   
   // start the cleanup thread for the database 
   int startCleanup(TRI_vocbase_t* vocbase);

arangod/MMFiles/MMFilesHashIndex.cpp

@@ -34,7 +34,7 @@
 #include "MMFiles/MMFilesToken.h"
 #include "StorageEngine/TransactionState.h"
 #include "Transaction/Helpers.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/LogicalCollection.h"
 
 #include <velocypack/Iterator.h>

arangod/MMFiles/MMFilesPersistentIndex.cpp

@@ -29,6 +29,7 @@
 #include "Basics/StaticStrings.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Indexes/IndexLookupContext.h"
+#include "MMFiles/MMFilesCollection.h"
 #include "MMFiles/MMFilesIndexElement.h"
 #include "MMFiles/MMFilesPrimaryIndex.h"
 #include "MMFiles/MMFilesPersistentIndexFeature.h"
@@ -359,7 +360,9 @@ int PersistentIndex::insert(transaction::Methods* trx, TRI_voc_rid_t revisionId,
       if (uniqueConstraintViolated) {
         // duplicate key
         res = TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED;
-        if (!_collection->useSecondaryIndexes()) {
+        auto physical = static_cast<MMFilesCollection*>(_collection->getPhysical());
+        TRI_ASSERT(physical != nullptr);
+        if (!physical->useSecondaryIndexes()) {
           // suppress the error during recovery
           res = TRI_ERROR_NO_ERROR;
         }
@@ -568,7 +571,8 @@ PersistentIndexIterator* PersistentIndex::lookup(transaction::Methods* trx,
   // Secured by trx. The shared_ptr index stays valid in
   // _collection at least as long as trx is running.
   // Same for the iterator
-  auto idx = _collection->primaryIndex();
+  auto physical = static_cast<MMFilesCollection*>(_collection->getPhysical());
+  auto idx = physical->primaryIndex();
   return new PersistentIndexIterator(_collection, trx, mmdr, this, idx, _db, reverse, leftBorder, rightBorder);
 }
 

arangod/MMFiles/MMFilesPrimaryIndex.cpp

@@ -34,7 +34,7 @@
 #include "StorageEngine/TransactionState.h"
 #include "Transaction/Helpers.h"
 #include "Transaction/Methods.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/LogicalCollection.h"
 
 #include <velocypack/Builder.h>

arangod/MMFiles/MMFilesTransactionCollection.cpp

@@ -23,6 +23,7 @@
 
 #include "MMFilesTransactionCollection.h"
 #include "Basics/Exceptions.h"
+#include "Cluster/CollectionLockState.h"
 #include "Logger/Logger.h"
 #include "MMFiles/MMFilesDocumentOperation.h"
 #include "MMFiles/MMFilesCollection.h"
@@ -142,7 +143,7 @@ void MMFilesTransactionCollection::freeOperations(transaction::Methods* activeTr
 
   if (mustRollback) {
     physical->setRevision(_originalRevision, true);
-  } else if (!_collection->isVolatile() && !isSingleOperationTransaction) {
+  } else if (!physical->isVolatile() && !isSingleOperationTransaction) {
     // only count logfileEntries if the collection is durable
     physical->increaseUncollectedLogfileEntries(_operations->size());
   }
@@ -319,10 +320,10 @@ int MMFilesTransactionCollection::doLock(AccessMode::Type type, int nestingLevel
 
   TRI_ASSERT(_collection != nullptr);
 
-  if (transaction::Methods::_makeNolockHeaders != nullptr) {
+  if (CollectionLockState::_noLockHeaders != nullptr) {
     std::string collName(_collection->name());
-    auto it = transaction::Methods::_makeNolockHeaders->find(collName);
+    auto it = CollectionLockState::_noLockHeaders->find(collName);
-    if (it != transaction::Methods::_makeNolockHeaders->end()) {
+    if (it != CollectionLockState::_noLockHeaders->end()) {
       // do not lock by command
       // LOCKING-DEBUG
       // std::cout << "LockCollection blocked: " << collName << std::endl;
@@ -371,10 +372,10 @@ int MMFilesTransactionCollection::doUnlock(AccessMode::Type type, int nestingLev
 
   TRI_ASSERT(_collection != nullptr);
 
-  if (transaction::Methods::_makeNolockHeaders != nullptr) {
+  if (CollectionLockState::_noLockHeaders != nullptr) {
     std::string collName(_collection->name());
-    auto it = transaction::Methods::_makeNolockHeaders->find(collName);
+    auto it = CollectionLockState::_noLockHeaders->find(collName);
-    if (it != transaction::Methods::_makeNolockHeaders->end()) {
+    if (it != CollectionLockState::_noLockHeaders->end()) {
       // do not lock by command
       // LOCKING-DEBUG
       // std::cout << "UnlockCollection blocked: " << collName << std::endl;

arangod/MMFiles/MMFilesTransactionContextData.cpp

@@ -0,0 +1,83 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#include "MMFilesTransactionContextData.h"
+#include "MMFiles/MMFilesCollection.h"
+#include "MMFiles/MMFilesDitch.h"
+#include "VocBase/LogicalCollection.h"
+
+using namespace arangodb;
+
+MMFilesTransactionContextData::MMFilesTransactionContextData()
+    : _lastPinnedCid(0) {}
+
+MMFilesTransactionContextData::~MMFilesTransactionContextData() {
+  for (auto& it : _ditches) {
+    // we're done with this ditch
+    auto& ditch = it.second;
+    ditch->ditches()->freeMMFilesDocumentDitch(ditch, true /* fromTransaction */);
+    // If some external entity is still using the ditch, it is kept!
+  }
+}
+
+/// @brief pin data for the collection
+void MMFilesTransactionContextData::pinData(LogicalCollection* collection) {
+  TRI_voc_cid_t const cid = collection->cid();
+
+  if (_lastPinnedCid == cid) {
+    // already pinned data for this collection
+    return;
+  }
+
+  auto it = _ditches.find(cid);
+
+  if (it != _ditches.end()) {
+    // tell everyone else this ditch is still in use,
+    // at least until the transaction is over
+    TRI_ASSERT((*it).second->usedByTransaction());
+    // ditch already exists
+    return;
+  }
+
+  // this method will not throw, but may return a nullptr
+  auto ditch = arangodb::MMFilesCollection::toMMFilesCollection(collection)->ditches()->createMMFilesDocumentDitch(true, __FILE__, __LINE__);
+
+  if (ditch == nullptr) {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
+  }
+
+  try {
+    _ditches.emplace(cid, ditch);
+  }
+  catch (...) {
+    ditch->ditches()->freeMMFilesDocumentDitch(ditch, true);
+    throw;
+  }
+
+  _lastPinnedCid = cid;
+}
+
+/// @brief whether or not the data for the collection is pinned
+bool MMFilesTransactionContextData::isPinned(TRI_voc_cid_t cid) const {
+  return (_ditches.find(cid) != _ditches.end());
+}

arangod/MMFiles/MMFilesTransactionContextData.h

@@ -0,0 +1,55 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGOD_MMFILES_TRANSACTION_CONTEXT_DATA_H
+#define ARANGOD_MMFILES_TRANSACTION_CONTEXT_DATA_H 1
+
+#include "Basics/Common.h"
+#include "Transaction/ContextData.h"
+#include "VocBase/voc-types.h"
+                                
+namespace arangodb {
+class LogicalCollection;
+class MMFilesDocumentDitch;
+
+/// @brief transaction type
+class MMFilesTransactionContextData final : public transaction::ContextData {
+ public:
+  MMFilesTransactionContextData();
+  ~MMFilesTransactionContextData();
+  
+  /// @brief pin data for the collection
+  void pinData(arangodb::LogicalCollection*) override;
+
+  /// @brief whether or not the data for the collection is pinned
+  bool isPinned(TRI_voc_cid_t) const override;
+  
+ private:
+  std::unordered_map<TRI_voc_cid_t, MMFilesDocumentDitch*> _ditches;
+
+  TRI_voc_cid_t _lastPinnedCid;
+};
+
+}
+
+#endif

arangod/MMFiles/MMFilesTransactionState.cpp

@@ -71,10 +71,10 @@ rocksdb::Transaction* MMFilesTransactionState::rocksTransaction() {
 }
   
 /// @brief start a transaction
-int MMFilesTransactionState::beginTransaction(transaction::Hints hints, int nestingLevel) {
+int MMFilesTransactionState::beginTransaction(transaction::Hints hints) {
-  LOG_TRX(this, nestingLevel) << "beginning " << AccessMode::typeString(_type) << " transaction";
+  LOG_TRX(this, _nestingLevel) << "beginning " << AccessMode::typeString(_type) << " transaction";
 
-  if (nestingLevel == 0) {
+  if (_nestingLevel == 0) {
     TRI_ASSERT(_status == transaction::Status::CREATED);
 
     auto logfileManager = MMFilesLogfileManager::instance();
@@ -114,43 +114,43 @@ int MMFilesTransactionState::beginTransaction(transaction::Hints hints, int nest
     TRI_ASSERT(_status == transaction::Status::RUNNING);
   }
 
-  int res = useCollections(nestingLevel);
+  int res = useCollections(_nestingLevel);
 
   if (res == TRI_ERROR_NO_ERROR) {
     // all valid
-    if (nestingLevel == 0) {
+    if (_nestingLevel == 0) {
       updateStatus(transaction::Status::RUNNING);
 
       // defer writing of the begin marker until necessary!
     }
   } else {
     // something is wrong
-    if (nestingLevel == 0) {
+    if (_nestingLevel == 0) {
       updateStatus(transaction::Status::ABORTED);
     }
 
     // free what we have got so far
-    unuseCollections(nestingLevel);
+    unuseCollections(_nestingLevel);
   }
 
   return res;
 }
 
 /// @brief commit a transaction
-int MMFilesTransactionState::commitTransaction(transaction::Methods* activeTrx, int nestingLevel) {
+int MMFilesTransactionState::commitTransaction(transaction::Methods* activeTrx) {
-  LOG_TRX(this, nestingLevel) << "committing " << AccessMode::typeString(_type) << " transaction";
+  LOG_TRX(this, _nestingLevel) << "committing " << AccessMode::typeString(_type) << " transaction";
 
   TRI_ASSERT(_status == transaction::Status::RUNNING);
 
   int res = TRI_ERROR_NO_ERROR;
 
-  if (nestingLevel == 0) {
+  if (_nestingLevel == 0) {
     if (_rocksTransaction != nullptr) {
       auto status = _rocksTransaction->Commit();
 
       if (!status.ok()) {
         res = TRI_ERROR_INTERNAL;
-        abortTransaction(activeTrx, nestingLevel);
+        abortTransaction(activeTrx);
         return res;
       }
     }
@@ -159,7 +159,7 @@ int MMFilesTransactionState::commitTransaction(transaction::Methods* activeTrx,
 
     if (res != TRI_ERROR_NO_ERROR) {
       // TODO: revert rocks transaction somehow
-      abortTransaction(activeTrx, nestingLevel);
+      abortTransaction(activeTrx);
 
       // return original error
       return res;
@@ -177,20 +177,20 @@ int MMFilesTransactionState::commitTransaction(transaction::Methods* activeTrx,
     freeOperations(activeTrx);
   }
 
-  unuseCollections(nestingLevel);
+  unuseCollections(_nestingLevel);
 
   return res;
 }
 
 /// @brief abort and rollback a transaction
-int MMFilesTransactionState::abortTransaction(transaction::Methods* activeTrx, int nestingLevel) {
+int MMFilesTransactionState::abortTransaction(transaction::Methods* activeTrx) {
-  LOG_TRX(this, nestingLevel) << "aborting " << AccessMode::typeString(_type) << " transaction";
+  LOG_TRX(this, _nestingLevel) << "aborting " << AccessMode::typeString(_type) << " transaction";
 
   TRI_ASSERT(_status == transaction::Status::RUNNING);
 
   int res = TRI_ERROR_NO_ERROR;
 
-  if (nestingLevel == 0) {
+  if (_nestingLevel == 0) {
     res = writeAbortMarker();
 
     updateStatus(transaction::Status::ABORTED);
@@ -204,7 +204,7 @@ int MMFilesTransactionState::abortTransaction(transaction::Methods* activeTrx, i
     freeOperations(activeTrx);
   }
 
-  unuseCollections(nestingLevel);
+  unuseCollections(_nestingLevel);
 
   return res;
 }

arangod/MMFiles/MMFilesTransactionState.h

@@ -45,7 +45,6 @@ class MMFilesWalMarker;
 namespace transaction {
 class Methods;
 }
-;
 class TransactionCollection;
 
 /// @brief transaction type
@@ -55,13 +54,13 @@ class MMFilesTransactionState final : public TransactionState {
   ~MMFilesTransactionState();
 
   /// @brief begin a transaction
-  int beginTransaction(transaction::Hints hints, int nestingLevel) override;
+  int beginTransaction(transaction::Hints hints) override;
 
   /// @brief commit a transaction
-  int commitTransaction(transaction::Methods* trx, int nestingLevel) override;
+  int commitTransaction(transaction::Methods* trx) override;
 
   /// @brief abort a transaction
-  int abortTransaction(transaction::Methods* trx, int nestingLevel) override;
+  int abortTransaction(transaction::Methods* trx) override;
 
   bool hasFailedOperations() const override {
     return (_hasOperations && _status == transaction::Status::ABORTED);

arangod/MMFiles/MMFilesWalRecoverState.cpp

@@ -40,7 +40,7 @@
 #include "Transaction/Hints.h"
 #include "Utils/OperationOptions.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "VocBase/LogicalCollection.h"
 
 #include <velocypack/Collection.h>
@@ -218,8 +218,10 @@ arangodb::LogicalCollection* MMFilesWalRecoverState::useCollection(
     return nullptr;
   }
 
+  auto physical = static_cast<MMFilesCollection*>(collection->getPhysical());
+  TRI_ASSERT(physical != nullptr);
   // disable secondary indexes for the moment
-  collection->useSecondaryIndexes(false);
+  physical->useSecondaryIndexes(false);
 
   openedCollections.emplace(collectionId, collection);
   res = TRI_ERROR_NO_ERROR;
@@ -277,7 +279,9 @@ int MMFilesWalRecoverState::executeSingleOperation(
     return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
   }
 
-  TRI_voc_tick_t maxTick = collection->maxTick();
+  auto mmfiles = static_cast<MMFilesCollection*>(collection->getPhysical());
+  TRI_ASSERT(mmfiles);
+  TRI_voc_tick_t maxTick = mmfiles->maxTick();
   if (marker->getTick() <= maxTick) {
     // already transferred this marker
     return TRI_ERROR_NO_ERROR;
@@ -286,7 +290,7 @@ int MMFilesWalRecoverState::executeSingleOperation(
   res = TRI_ERROR_INTERNAL;
 
   try {
-    SingleCollectionTransaction trx(arangodb::StandaloneTransactionContext::Create(vocbase), collectionId, AccessMode::Type::WRITE);
+    SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(vocbase), collectionId, AccessMode::Type::WRITE);
 
     trx.addHint(transaction::Hints::Hint::SINGLE_OPERATION);
     trx.addHint(transaction::Hints::Hint::NO_BEGIN_MARKER);
@@ -476,7 +480,9 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker,
             databaseId, collectionId, marker, datafile->fid(),
             [&](SingleCollectionTransaction* trx,
                 MMFilesMarkerEnvelope* envelope) -> int {
-              if (trx->documentCollection()->isVolatile()) {
+              if (arangodb::MMFilesCollection::toMMFilesCollection(
+                      trx->documentCollection())
+                      ->isVolatile()) {
                 return TRI_ERROR_NO_ERROR;
               }
 
@@ -550,7 +556,10 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker,
             databaseId, collectionId, marker, datafile->fid(),
             [&](SingleCollectionTransaction* trx,
                 MMFilesMarkerEnvelope* envelope) -> int {
-              if (trx->documentCollection()->isVolatile()) {
+
+              if (arangodb::MMFilesCollection::toMMFilesCollection(
+                      trx->documentCollection())
+                      ->isVolatile()) {
                 return TRI_ERROR_NO_ERROR;
               }
 
@@ -723,12 +732,11 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker,
         // be
         // dropped later
         bool const forceSync = state->willBeDropped(databaseId, collectionId);
-        int res = collection->updateProperties(payloadSlice, forceSync);
+        CollectionResult res = collection->updateProperties(payloadSlice, forceSync);
-
+        if (res.successful()) {
-        if (res != TRI_ERROR_NO_ERROR) {
           LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "cannot change collection properties for collection "
                     << collectionId << " in database " << databaseId << ": "
-                    << TRI_errno_string(res);
+                    << res.errorMessage;
           ++state->errorCount;
           return state->canContinue();
         }
@@ -800,10 +808,10 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker,
           return state->canContinue();
         } else {
           arangodb::SingleCollectionTransaction trx(
-              arangodb::StandaloneTransactionContext::Create(vocbase),
+              arangodb::transaction::StandaloneContext::Create(vocbase),
               collectionId, AccessMode::Type::WRITE);
           std::shared_ptr<arangodb::Index> unused;
-          int res = col->restoreIndex(&trx, payloadSlice, unused);
+          int res = physical->restoreIndex(&trx, payloadSlice, unused);
 
           if (res != TRI_ERROR_NO_ERROR) {
             LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "cannot create index " << indexId << ", collection "
@@ -1042,7 +1050,7 @@ bool MMFilesWalRecoverState::ReplayMarker(TRI_df_marker_t const* marker,
         // ignore any potential error returned by this call
         auto physical = static_cast<MMFilesCollection*>(col->getPhysical());
         TRI_ASSERT(physical != nullptr);
-        col->dropIndex(indexId, false);
+        col->dropIndex(indexId);
 
         PersistentIndexFeature::dropIndex(databaseId, collectionId, indexId);
 
@@ -1255,14 +1263,16 @@ int MMFilesWalRecoverState::fillIndexes() {
        ++it) {
     arangodb::LogicalCollection* collection = (*it).second;
 
+    auto physical = static_cast<MMFilesCollection*>(collection->getPhysical());
+    TRI_ASSERT(physical != nullptr);
     // activate secondary indexes
-    collection->useSecondaryIndexes(true);
+    physical->useSecondaryIndexes(true);
 
     arangodb::SingleCollectionTransaction trx(
-        arangodb::StandaloneTransactionContext::Create(collection->vocbase()),
+        arangodb::transaction::StandaloneContext::Create(collection->vocbase()),
         collection->cid(), AccessMode::Type::WRITE);
 
-    int res = collection->fillIndexes(&trx, *(collection->indexList()));
+    int res = physical->fillAllIndexes(&trx);
 
     if (res != TRI_ERROR_NO_ERROR) {
       return res;

arangod/MMFiles/README.md

@@ -0,0 +1,23 @@
+MMFilesEngine
+=============
+
+How operations are stored - Overview
+------------------------------------
+
+All operations like insert or remove are written as markers to a write ahead
+log (WAL). This WAL consists of files of a certain size and if such a file is
+full (or is manually flushed), all relevant markers are transferred
+(transferMarkers()) to the journals of the respective collections. During the
+transfer any obsolete markers will be thrown away: a sequence of insert, remove,
+insert on the same document will result in the last insert discarding the
+previous operations. When a journal file of size (journalSize()) is full, it
+will be sealed and renamed. By applying these operations it will become a
+datafile that is read-only. Datafiles will eventually be merged by a compactor
+thread that does about the same work as the transferMarkers function, reducing
+the size of the stored data.
+
+Ditches
+-------
+
+Ditches are used to pin objects in WAL or journal as long as they are used by
+other operations.

arangod/Replication/ContinuousSyncer.cpp

@@ -571,7 +571,7 @@ int ContinuousSyncer::processDocument(TRI_replication_operation_e type,
   else {
     // standalone operation
     // update the apply tick for all standalone operations
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase),
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase),
                                             cid, AccessMode::Type::WRITE);
     trx.addHint(transaction::Hints::Hint::SINGLE_OPERATION);
 
@@ -784,7 +784,7 @@ int ContinuousSyncer::changeCollection(VPackSlice const& slice) {
 
   arangodb::CollectionGuard guard(_vocbase, cid);
   bool doSync = application_features::ApplicationServer::getFeature<DatabaseFeature>("Database")->forceSyncProperties();
-  return guard.collection()->updateProperties(data, doSync);
+  return guard.collection()->updateProperties(data, doSync).code;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

arangod/Replication/InitialSyncer.cpp

@@ -33,14 +33,18 @@
 #include "RestServer/DatabaseFeature.h"
 #include "SimpleHttpClient/SimpleHttpClient.h"
 #include "SimpleHttpClient/SimpleHttpResult.h"
+#include "StorageEngine/EngineSelectorFeature.h"
+#include "StorageEngine/StorageEngine.h"
 #include "Utils/CollectionGuard.h"
+#include "MMFiles/MMFilesCollection.h" //TODO -- Remove -- ditches 
+#include "MMFiles/MMFilesDitch.h"
 #include "MMFiles/MMFilesDatafileHelper.h"
 #include "MMFiles/MMFilesIndexElement.h"
 #include "MMFiles/MMFilesPrimaryIndex.h"
 #include "Utils/OperationOptions.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ManagedDocumentResult.h"
+#include "VocBase/PhysicalCollection.h"
 #include "VocBase/vocbase.h"
 #include "VocBase/voc-types.h"
 
@@ -775,7 +779,7 @@ int InitialSyncer::handleCollectionDump(
     }
 
     if (res == TRI_ERROR_NO_ERROR) {
-      SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
+      SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
       
       res = trx.begin();
 
@@ -785,7 +789,7 @@ int InitialSyncer::handleCollectionDump(
         return res;
       }
      
-      trx.orderDitch(col->cid()); // will throw when it fails
+      trx.pinData(col->cid()); // will throw when it fails
 
       if (res == TRI_ERROR_NO_ERROR) {
         res = applyCollectionDump(trx, collectionName, response.get(), markersProcessed, errorMsg);
@@ -979,7 +983,7 @@ int InitialSyncer::handleCollectionSync(
 
   if (count.getNumber<size_t>() <= 0) {
     // remote collection has no documents. now truncate our local collection
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
 
     int res = trx.begin();
 
@@ -1034,12 +1038,12 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
   // fetch all local keys from primary index
   std::vector<uint8_t const*> markers;
     
-  DocumentDitch* ditch = nullptr;
+  MMFilesDocumentDitch* ditch = nullptr;
 
   // acquire a replication ditch so no datafiles are thrown away from now on
   // note: the ditch also protects against unloading the collection
   {    
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::READ);
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ);
   
     int res = trx.begin();
   
@@ -1047,8 +1051,10 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
       errorMsg = std::string("unable to start transaction: ") + TRI_errno_string(res);
       return res;
     }
-    
+
-    ditch = col->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
+    ditch = arangodb::MMFilesCollection::toMMFilesCollection(col)
+                ->ditches()
+                ->createMMFilesDocumentDitch(false, __FILE__, __LINE__);
 
     if (ditch == nullptr) {
       return TRI_ERROR_OUT_OF_MEMORY;
@@ -1057,10 +1063,12 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
 
   TRI_ASSERT(ditch != nullptr);
 
-  TRI_DEFER(col->ditches()->freeDitch(ditch));
+  TRI_DEFER(arangodb::MMFilesCollection::toMMFilesCollection(col)
+                ->ditches()
+                ->freeDitch(ditch));
 
   {
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::READ);
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ);
   
     int res = trx.begin();
   
@@ -1072,13 +1080,13 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
     // We do not take responsibility for the index.
     // The LogicalCollection is protected by trx.
     // Neither it nor it's indexes can be invalidated
-    auto idx = trx.documentCollection()->primaryIndex();
+
-    markers.reserve(idx->size());
+    markers.reserve(trx.documentCollection()->numberDocuments());
 
     uint64_t iterations = 0;
     ManagedDocumentResult mmdr;
-    trx.invokeOnAllElements(trx.name(), [this, &trx, &mmdr, &markers, &iterations, &idx](DocumentIdentifierToken const& token) {
+    trx.invokeOnAllElements(trx.name(), [this, &trx, &mmdr, &markers, &iterations](DocumentIdentifierToken const& token) {
-      if (idx->collection()->readDocument(&trx, mmdr, token)) {
+      if (trx.documentCollection()->readDocument(&trx, token, mmdr)) {
         markers.emplace_back(mmdr.vpack());
         
         if (++iterations % 10000 == 0) {
@@ -1200,7 +1208,7 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
   // remove all keys that are below first remote key or beyond last remote key
   if (n > 0) {
     // first chunk
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
   
     int res = trx.begin();
   
@@ -1271,7 +1279,7 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
       return TRI_ERROR_REPLICATION_APPLIER_STOPPED;
     }
     
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
   
     int res = trx.begin();
   
@@ -1280,12 +1288,16 @@ int InitialSyncer::handleSyncKeys(arangodb::LogicalCollection* col,
       return res;
     }
     
-    trx.orderDitch(col->cid()); // will throw when it fails
+    trx.pinData(col->cid()); // will throw when it fails
     
     // We do not take responsibility for the index.
     // The LogicalCollection is protected by trx.
     // Neither it nor it's indexes can be invalidated
-    auto idx = trx.documentCollection()->primaryIndex();
+
+    // TODO Move to MMFiles
+    auto physical = static_cast<MMFilesCollection*>(
+        trx.documentCollection()->getPhysical());
+    auto idx = physical->primaryIndex();
 
     size_t const currentChunkId = i;
     progress = "processing keys chunk " + std::to_string(currentChunkId) +
@@ -1639,7 +1651,7 @@ int InitialSyncer::changeCollection(arangodb::LogicalCollection* col,
           "Database")
           ->forceSyncProperties();
 
-  return guard.collection()->updateProperties(slice, doSync);
+  return guard.collection()->updateProperties(slice, doSync).code;
 }
  
 ////////////////////////////////////////////////////////////////////////////////
@@ -1647,7 +1659,7 @@ int InitialSyncer::changeCollection(arangodb::LogicalCollection* col,
 ////////////////////////////////////////////////////////////////////////////////
 
 int64_t InitialSyncer::getSize(arangodb::LogicalCollection* col) {
-  SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::READ);
+  SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::READ);
 
   int res = trx.begin();
 
@@ -1735,7 +1747,7 @@ int InitialSyncer::handleCollection(VPackSlice const& parameters,
           // system collection
           setProgress("truncating " + collectionMsg);
 
-          SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
+          SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
 
           int res = trx.begin();
 
@@ -1850,7 +1862,7 @@ int InitialSyncer::handleCollection(VPackSlice const& parameters,
           setProgress(progress);
 
           try {
-            SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
+            SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), col->cid(), AccessMode::Type::WRITE);
       
             res = trx.begin();
 
@@ -1859,10 +1871,12 @@ int InitialSyncer::handleCollection(VPackSlice const& parameters,
               return res;
             }
 
-            trx.orderDitch(col->cid()); // will throw when it fails
+            trx.pinData(col->cid()); // will throw when it fails
       
             LogicalCollection* document = trx.documentCollection();
             TRI_ASSERT(document != nullptr);
+            auto physical = document->getPhysical();
+            TRI_ASSERT(physical != nullptr);
 
             for (auto const& idxDef : VPackArrayIterator(indexes)) {
               std::shared_ptr<arangodb::Index> idx;
@@ -1877,22 +1891,12 @@ int InitialSyncer::handleCollection(VPackSlice const& parameters,
                 }
               }
 
-              res = document->restoreIndex(&trx, idxDef, idx);
+              res = physical->restoreIndex(&trx, idxDef, idx);
 
               if (res != TRI_ERROR_NO_ERROR) {
                 errorMsg = "could not create index: " +
                             std::string(TRI_errno_string(res));
                 break;
-              } else {
-                TRI_ASSERT(idx != nullptr);
-
-                res = document->saveIndex(idx.get(), true);
-
-                if (res != TRI_ERROR_NO_ERROR) {
-                  errorMsg = "could not save index: " +
-                              std::string(TRI_errno_string(res));
-                  break;
-                }
               }
             }
       

arangod/Replication/ReplicationTransaction.h

@@ -27,7 +27,7 @@
 #include "Basics/Common.h"
 #include "StorageEngine/TransactionState.h"
 #include "Utils/DatabaseGuard.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Methods.h"
 #include "VocBase/vocbase.h"
 
@@ -37,7 +37,7 @@ class ReplicationTransaction : public transaction::Methods {
  public:
   /// @brief create the transaction
   ReplicationTransaction(TRI_vocbase_t* vocbase)
-      : transaction::Methods(StandaloneTransactionContext::Create(vocbase)),
+      : transaction::Methods(transaction::StandaloneContext::Create(vocbase)),
         _guard(vocbase) {}
 
  public:

arangod/Replication/Syncer.cpp

@@ -29,12 +29,15 @@
 #include "SimpleHttpClient/GeneralClientConnection.h"
 #include "SimpleHttpClient/SimpleHttpClient.h"
 #include "SimpleHttpClient/SimpleHttpResult.h"
+#include "StorageEngine/EngineSelectorFeature.h"
+#include "StorageEngine/StorageEngine.h"
 #include "StorageEngine/TransactionState.h"
 #include "Utils/CollectionGuard.h"
 #include "Utils/OperationOptions.h"
 #include "Utils/OperationResult.h"
 #include "Utils/SingleCollectionTransaction.h"
 #include "VocBase/LogicalCollection.h"
+#include "VocBase/PhysicalCollection.h"
 #include "VocBase/vocbase.h"
 #include "VocBase/voc-types.h"
 
@@ -552,7 +555,7 @@ int Syncer::createIndex(VPackSlice const& slice) {
 
     LogicalCollection* collection = guard.collection();
 
-    SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_vocbase), guard.collection()->cid(), AccessMode::Type::WRITE);
+    SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_vocbase), guard.collection()->cid(), AccessMode::Type::WRITE);
 
     int res = trx.begin();
 
@@ -560,13 +563,10 @@ int Syncer::createIndex(VPackSlice const& slice) {
       return res;
     }
 
+    auto physical = collection->getPhysical();
+    TRI_ASSERT(physical != nullptr);
     std::shared_ptr<arangodb::Index> idx;
-    res = collection->restoreIndex(&trx, indexSlice, idx);
+    res = physical->restoreIndex(&trx, indexSlice, idx);
-
-    if (res == TRI_ERROR_NO_ERROR) {
-      res = collection->saveIndex(idx.get(), true);
-    }
-
     res = trx.finish(res);
 
     return res;
@@ -602,7 +602,7 @@ int Syncer::dropIndex(arangodb::velocypack::Slice const& slice) {
 
     LogicalCollection* collection = guard.collection();
 
-    bool result = collection->dropIndex(iid, true);
+    bool result = collection->dropIndex(iid);
 
     if (!result) {
       return TRI_ERROR_NO_ERROR;

arangod/RestHandler/RestBaseHandler.cpp

@@ -33,7 +33,7 @@
 #include "Meta/conversion.h"
 #include "Rest/HttpRequest.h"
 #include "Rest/HttpResponse.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 
 using namespace arangodb;
 using namespace arangodb::basics;
@@ -94,7 +94,7 @@ void RestBaseHandler::generateResult(rest::ResponseCode code,
 template<typename Payload>
 void RestBaseHandler::generateResult(
     rest::ResponseCode code, Payload&& payload,
-    std::shared_ptr<TransactionContext> context) {
+    std::shared_ptr<transaction::Context> context) {
   resetResponse(code);
   writeResult(std::forward<Payload>(payload), *(context->getVPackOptionsForDump()));
 }
@@ -185,9 +185,9 @@ template void RestBaseHandler::generateResult<VPackBuffer<uint8_t>>(rest::Respon
 template void RestBaseHandler::generateResult<VPackSlice>(rest::ResponseCode, VPackSlice&&, VPackOptions const*);
 template void RestBaseHandler::generateResult<VPackSlice&>(rest::ResponseCode, VPackSlice&, VPackOptions const*);
 
-template void RestBaseHandler::generateResult<VPackBuffer<uint8_t>>(rest::ResponseCode, VPackBuffer<uint8_t>&&, std::shared_ptr<TransactionContext>);
+template void RestBaseHandler::generateResult<VPackBuffer<uint8_t>>(rest::ResponseCode, VPackBuffer<uint8_t>&&, std::shared_ptr<transaction::Context>);
-template void RestBaseHandler::generateResult<VPackSlice>(rest::ResponseCode, VPackSlice&&, std::shared_ptr<TransactionContext>);
+template void RestBaseHandler::generateResult<VPackSlice>(rest::ResponseCode, VPackSlice&&, std::shared_ptr<transaction::Context>);
-template void RestBaseHandler::generateResult<VPackSlice&>(rest::ResponseCode, VPackSlice&, std::shared_ptr<TransactionContext>);
+template void RestBaseHandler::generateResult<VPackSlice&>(rest::ResponseCode, VPackSlice&, std::shared_ptr<transaction::Context>);
 
 template void RestBaseHandler::writeResult<VPackBuffer<uint8_t>>(VPackBuffer<uint8_t>&& payload, VPackOptions const&);
 template void RestBaseHandler::writeResult<VPackSlice>(VPackSlice&& payload, VPackOptions const&);

arangod/RestHandler/RestBaseHandler.h

@@ -29,7 +29,9 @@
 #include "Rest/GeneralResponse.h"
 
 namespace arangodb {
-class TransactionContext;
+namespace transaction {
+class Context;
+}
 
 namespace velocypack {
 class Builder;
@@ -55,7 +57,7 @@ class RestBaseHandler : public rest::RestHandler {
   // generates a result from VelocyPack
   template <typename Payload>
   void generateResult(rest::ResponseCode, Payload&&,
-                      std::shared_ptr<TransactionContext> context);
+                      std::shared_ptr<transaction::Context> context);
 
   // generates an error
   void generateError(rest::ResponseCode, int);

arangod/RestHandler/RestCursorHandler.cpp

@@ -31,7 +31,7 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Utils/Cursor.h"
 #include "Utils/CursorRepository.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 
 #include <velocypack/Iterator.h>
 #include <velocypack/Value.h>

arangod/RestHandler/RestDocumentHandler.cpp

@@ -29,7 +29,7 @@
 #include "Rest/HttpRequest.h"
 #include "Utils/OperationOptions.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Hints.h"
 #include "VocBase/vocbase.h"
 
@@ -120,7 +120,7 @@ bool RestDocumentHandler::createDocument() {
   opOptions.silent = extractBooleanParameter(StaticStrings::SilentString, false);
 
   // find and load collection given by name or identifier
-  auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
+  auto transactionContext(transaction::StandaloneContext::Create(_vocbase));
   SingleCollectionTransaction trx(transactionContext, collectionName,
                                   AccessMode::Type::WRITE);
   bool const isMultiple = body.isArray();
@@ -227,7 +227,7 @@ bool RestDocumentHandler::readSingleDocument(bool generateBody) {
   VPackSlice search = builder.slice();
 
   // find and load collection given by name or identifier
-  auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
+  auto transactionContext(transaction::StandaloneContext::Create(_vocbase));
   SingleCollectionTransaction trx(transactionContext, collection,
                                   AccessMode::Type::READ);
   trx.addHint(transaction::Hints::Hint::SINGLE_OPERATION);
@@ -414,7 +414,7 @@ bool RestDocumentHandler::modifyDocument(bool isPatch) {
   }
 
   // find and load collection given by name or identifier
-  auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
+  auto transactionContext(transaction::StandaloneContext::Create(_vocbase));
   SingleCollectionTransaction trx(transactionContext, collectionName,
                                   AccessMode::Type::WRITE);
   if (!isArrayCase) {
@@ -503,7 +503,7 @@ bool RestDocumentHandler::deleteDocument() {
   opOptions.waitForSync = extractBooleanParameter(StaticStrings::WaitForSyncString, false);
   opOptions.silent = extractBooleanParameter(StaticStrings::SilentString, false);
 
-  auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
+  auto transactionContext(transaction::StandaloneContext::Create(_vocbase));
 
   VPackBuilder builder;
   VPackSlice search;
@@ -593,7 +593,7 @@ bool RestDocumentHandler::readManyDocuments() {
   OperationOptions opOptions;
   opOptions.ignoreRevs = extractBooleanParameter(StaticStrings::IgnoreRevsString, true);
 
-  auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
+  auto transactionContext(transaction::StandaloneContext::Create(_vocbase));
   SingleCollectionTransaction trx(transactionContext, collectionName,
                                   AccessMode::Type::READ);
 

arangod/RestHandler/RestEdgesHandler.cpp

@@ -30,7 +30,7 @@
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/OperationCursor.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 
 #include <velocypack/Iterator.h>
 #include <velocypack/velocypack-aliases.h>
@@ -96,7 +96,7 @@ bool RestEdgesHandler::getEdgesForVertex(
     std::string const& id, std::string const& collectionName,
     TRI_edge_direction_e direction, SingleCollectionTransaction& trx,
     std::function<void(DocumentIdentifierToken const&)> cb) {
-  trx.orderDitch(trx.cid());  // will throw when it fails
+  trx.pinData(trx.cid());  // will throw when it fails
 
   // Create a conditionBuilder that manages the AstNodes for querying
   aql::EdgeConditionBuilderContainer condBuilder;
@@ -224,7 +224,7 @@ bool RestEdgesHandler::readEdges() {
 
   // find and load collection given by name or identifier
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), collectionName,
+      transaction::StandaloneContext::Create(_vocbase), collectionName,
       AccessMode::Type::READ);
 
   // .............................................................................
@@ -251,7 +251,7 @@ bool RestEdgesHandler::readEdges() {
   std::unordered_set<DocumentIdentifierToken> foundTokens;
   auto cb = [&] (DocumentIdentifierToken const& token) {
     if (foundTokens.find(token) == foundTokens.end()) {
-      if (collection->readDocument(&trx, mmdr, token)) {
+      if (collection->readDocument(&trx, token, mmdr)) {
         resultBuilder.add(VPackSlice(mmdr.vpack()));
       }
       scannedIndex++;
@@ -368,7 +368,7 @@ bool RestEdgesHandler::readEdgesForMultipleVertices() {
 
   // find and load collection given by name or identifier
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), collectionName,
+      transaction::StandaloneContext::Create(_vocbase), collectionName,
       AccessMode::Type::READ);
 
   // .............................................................................
@@ -396,7 +396,7 @@ bool RestEdgesHandler::readEdgesForMultipleVertices() {
   std::unordered_set<DocumentIdentifierToken> foundTokens;
   auto cb = [&] (DocumentIdentifierToken const& token) {
     if (foundTokens.find(token) == foundTokens.end()) {
-      if (collection->readDocument(&trx, mmdr, token)) {
+      if (collection->readDocument(&trx, token, mmdr)) {
         resultBuilder.add(VPackSlice(mmdr.vpack()));
       }
       scannedIndex++;

arangod/RestHandler/RestImportHandler.cpp

@@ -31,7 +31,7 @@
 #include "Transaction/Helpers.h"
 #include "Utils/OperationOptions.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "VocBase/vocbase.h"
 
 #include <velocypack/Collection.h>
@@ -357,7 +357,7 @@ bool RestImportHandler::createFromJson(std::string const& type) {
 
   // find and load collection given by name or identifier
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), collectionName,
+      transaction::StandaloneContext::Create(_vocbase), collectionName,
       AccessMode::Type::WRITE);
 
   // .............................................................................
@@ -563,7 +563,7 @@ bool RestImportHandler::createFromVPack(std::string const& type) {
 
   // find and load collection given by name or identifier
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), collectionName,
+      transaction::StandaloneContext::Create(_vocbase), collectionName,
       AccessMode::Type::WRITE);
 
   // .............................................................................
@@ -737,7 +737,7 @@ bool RestImportHandler::createFromKeyValueList() {
 
   // find and load collection given by name or identifier
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), collectionName,
+      transaction::StandaloneContext::Create(_vocbase), collectionName,
       AccessMode::Type::WRITE);
 
   // .............................................................................

arangod/RestHandler/RestReplicationHandler.cpp

@@ -47,10 +47,11 @@
 #include "Utils/CollectionKeysRepository.h"
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/OperationOptions.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "Transaction/Hints.h"
 #include "VocBase/LogicalCollection.h"
+#include "VocBase/PhysicalCollection.h"
 #include "VocBase/replication-applier.h"
 #include "VocBase/replication-dump.h"
 #include "VocBase/ticks.h"
@@ -962,7 +963,7 @@ void RestReplicationHandler::handleCommandLoggerFollow() {
   }
 
   auto transactionContext =
-      std::make_shared<StandaloneTransactionContext>(_vocbase);
+      std::make_shared<transaction::StandaloneContext>(_vocbase);
 
   // initialize the dump container
   TRI_replication_dump_t dump(transactionContext,
@@ -1066,7 +1067,7 @@ void RestReplicationHandler::handleCommandDetermineOpenTransactions() {
   }
 
   auto transactionContext =
-      std::make_shared<StandaloneTransactionContext>(_vocbase);
+      std::make_shared<transaction::StandaloneContext>(_vocbase);
 
   // initialize the dump container
   TRI_replication_dump_t dump(
@@ -1260,7 +1261,7 @@ int RestReplicationHandler::createCollection(VPackSlice slice,
   TRI_ASSERT(col != nullptr);
 
   /* Temporary ASSERTS to prove correctness of new constructor */
-  TRI_ASSERT(col->doCompact() ==
+  TRI_ASSERT(col->getPhysical()->doCompact() ==
              arangodb::basics::VelocyPackHelper::getBooleanValue(
                  slice, "doCompact", true));
   TRI_ASSERT(
@@ -1270,9 +1271,6 @@ int RestReplicationHandler::createCollection(VPackSlice slice,
           application_features::ApplicationServer::getFeature<DatabaseFeature>(
               "Database")
               ->waitForSync()));
-  TRI_ASSERT(col->isVolatile() ==
-             arangodb::basics::VelocyPackHelper::getBooleanValue(
-                 slice, "isVolatile", false));
   TRI_ASSERT(col->isSystem() == (name[0] == '_'));
   TRI_ASSERT(
       col->indexBuckets() ==
@@ -1496,7 +1494,7 @@ int RestReplicationHandler::processRestoreCollection(
 
         // instead, truncate them
         SingleCollectionTransaction trx(
-            StandaloneTransactionContext::Create(_vocbase), col->cid(),
+            transaction::StandaloneContext::Create(_vocbase), col->cid(),
             AccessMode::Type::WRITE);
         trx.addHint(transaction::Hints::Hint::RECOVERY); // to turn off waitForSync!
 
@@ -1750,7 +1748,7 @@ int RestReplicationHandler::processRestoreIndexes(VPackSlice const& collection,
     LogicalCollection* collection = guard.collection();
 
     SingleCollectionTransaction trx(
-        StandaloneTransactionContext::Create(_vocbase), collection->cid(),
+        transaction::StandaloneContext::Create(_vocbase), collection->cid(),
         AccessMode::Type::WRITE);
 
     int res = trx.begin();
@@ -1761,12 +1759,14 @@ int RestReplicationHandler::processRestoreIndexes(VPackSlice const& collection,
       THROW_ARANGO_EXCEPTION(res);
     }
 
+    auto physical = collection->getPhysical();
+    TRI_ASSERT(physical != nullptr);
     for (VPackSlice const& idxDef : VPackArrayIterator(indexes)) {
       std::shared_ptr<arangodb::Index> idx;
 
       // {"id":"229907440927234","type":"hash","unique":false,"fields":["x","Y"]}
 
-      res = collection->restoreIndex(&trx, idxDef, idx);
+      res = physical->restoreIndex(&trx, idxDef, idx);
 
       if (res == TRI_ERROR_NOT_IMPLEMENTED) {
         continue;
@@ -1776,17 +1776,8 @@ int RestReplicationHandler::processRestoreIndexes(VPackSlice const& collection,
         errorMsg =
             "could not create index: " + std::string(TRI_errno_string(res));
         break;
-      } else {
-        TRI_ASSERT(idx != nullptr);
-
-        res = collection->saveIndex(idx.get(), true);
-
-        if (res != TRI_ERROR_NO_ERROR) {
-          errorMsg =
-              "could not save index: " + std::string(TRI_errno_string(res));
-          break;
-        }
       }
+      TRI_ASSERT(idx != nullptr);
     }
   } catch (arangodb::basics::Exception const& ex) {
     errorMsg =
@@ -2256,7 +2247,7 @@ int RestReplicationHandler::processRestoreData(
     bool force, std::string& errorMsg) {
 
   SingleCollectionTransaction trx(
-      StandaloneTransactionContext::Create(_vocbase), colName,
+      transaction::StandaloneContext::Create(_vocbase), colName,
       AccessMode::Type::WRITE);
   trx.addHint(transaction::Hints::Hint::RECOVERY); // to turn off waitForSync!
 
@@ -2536,8 +2527,8 @@ void RestReplicationHandler::handleCommandFetchKeys() {
   }
 
   try {
-    std::shared_ptr<TransactionContext> transactionContext =
+    std::shared_ptr<transaction::Context> transactionContext =
-        StandaloneTransactionContext::Create(_vocbase);
+        transaction::StandaloneContext::Create(_vocbase);
 
     VPackBuilder resultBuilder(transactionContext->getVPackOptions());
     resultBuilder.openArray();
@@ -2712,7 +2703,7 @@ void RestReplicationHandler::handleCommandDump() {
   TRI_ASSERT(col != nullptr);
 
   auto transactionContext =
-      std::make_shared<StandaloneTransactionContext>(_vocbase);
+      std::make_shared<transaction::StandaloneContext>(_vocbase);
 
   // initialize the dump container
   TRI_replication_dump_t dump(transactionContext,
@@ -3444,7 +3435,7 @@ void RestReplicationHandler::handleCommandHoldReadLockCollection() {
     _holdReadLockJobs.emplace(id, false);
   }
 
-  auto trxContext = StandaloneTransactionContext::Create(_vocbase);
+  auto trxContext = transaction::StandaloneContext::Create(_vocbase);
   SingleCollectionTransaction trx(trxContext, col->cid(), AccessMode::Type::READ);
   trx.addHint(transaction::Hints::Hint::LOCK_ENTIRELY);
   int res = trx.begin();

arangod/RestHandler/RestSimpleHandler.cpp

@@ -32,8 +32,8 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/Traverser.h"
 

arangod/RestHandler/RestVocbaseBaseHandler.cpp

@@ -30,9 +30,10 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/VPackStringBufferAdapter.h"
 #include "Meta/conversion.h"
+#include "Cluster/CollectionLockState.h"
 #include "Cluster/ServerState.h"
 #include "Rest/HttpRequest.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Methods.h"
 
 #include <velocypack/Builder.h>
@@ -291,7 +292,7 @@ void RestVocbaseBaseHandler::generatePreconditionFailed(
     }
   }
 
-  auto transactionContext(StandaloneTransactionContext::Create(_vocbase));
+  auto transactionContext(transaction::StandaloneContext::Create(_vocbase));
   writeResult(builder.slice(), *(transactionContext->getVPackOptionsForDump()));
 }
 
@@ -633,7 +634,7 @@ void RestVocbaseBaseHandler::prepareExecute() {
   if (found) {
     _nolockHeaderSet =
         new std::unordered_set<std::string>{std::string(shardId)};
-    transaction::Methods::_makeNolockHeaders = _nolockHeaderSet;
+    CollectionLockState::_noLockHeaders = _nolockHeaderSet;
   }
 }
 
@@ -643,7 +644,7 @@ void RestVocbaseBaseHandler::prepareExecute() {
 
 void RestVocbaseBaseHandler::finalizeExecute() {
   if (_nolockHeaderSet != nullptr) {
-    transaction::Methods::_makeNolockHeaders = nullptr;
+    CollectionLockState::_noLockHeaders = nullptr;
     delete _nolockHeaderSet;
     _nolockHeaderSet = nullptr;
   }

arangod/RestServer/UpgradeFeature.cpp

@@ -23,15 +23,16 @@
 #include "UpgradeFeature.h"
 
 #include "Cluster/ClusterFeature.h"
+#include "Logger/Logger.h"
 #include "ProgramOptions/ProgramOptions.h"
 #include "ProgramOptions/Section.h"
 #include "RestServer/DatabaseFeature.h"
 #include "RestServer/InitDatabaseFeature.h"
-#include "MMFiles/MMFilesLogfileManager.h"
 #include "V8/v8-globals.h"
 #include "V8Server/V8Context.h"
 #include "V8Server/V8DealerFeature.h"
 #include "V8Server/v8-vocbase.h"
+#include "VocBase/vocbase.h"
 
 using namespace arangodb;
 using namespace arangodb::application_features;

arangod/Scheduler/JobGuard.h

@@ -25,6 +25,7 @@
 
 #include "Basics/Common.h"
 
+#include "Basics/SameThreadAsserter.h"
 #include "Scheduler/EventLoop.h"
 #include "Scheduler/Scheduler.h"
 
@@ -33,13 +34,13 @@ namespace rest {
 class Scheduler;
 }
 
-class JobGuard {
+class JobGuard : public SameThreadAsserter {
  public:
   JobGuard(JobGuard const&) = delete;
   JobGuard& operator=(JobGuard const&) = delete;
 
-  explicit JobGuard(EventLoop const& loop) : _scheduler(loop._scheduler) {}
+  explicit JobGuard(EventLoop const& loop) : SameThreadAsserter(), _scheduler(loop._scheduler) {}
-  explicit JobGuard(rest::Scheduler* scheduler) : _scheduler(scheduler) {}
+  explicit JobGuard(rest::Scheduler* scheduler) : SameThreadAsserter(), _scheduler(scheduler) {}
   ~JobGuard() { release(); }
 
  public:

arangod/StorageEngine/RocksDBEngine.cpp

@@ -37,10 +37,6 @@
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ticks.h"
 #include "VocBase/vocbase.h"
-#include "MMFiles/MMFilesLogfileManager.h"
-
-#include "MMFiles/MMFilesPersistentIndexFeature.h"
-#include "MMFiles/MMFilesPersistentIndex.h"
 
 #include <velocypack/Collection.h>
 #include <velocypack/Iterator.h>

arangod/StorageEngine/StorageEngine.h

@@ -28,7 +28,6 @@
 #include "Basics/Common.h"
 #include "ApplicationFeatures/ApplicationFeature.h"
 #include "Indexes/IndexFactory.h"
-#include "MMFiles/MMFilesCollectorCache.h"
 #include "VocBase/AccessMode.h"
 #include "VocBase/voc-types.h"
 #include "VocBase/vocbase.h"
@@ -46,6 +45,10 @@ class PhysicalCollection;
 class TransactionCollection;
 class TransactionState;
 
+namespace transaction {
+class ContextData;
+}
+
 class StorageEngine : public application_features::ApplicationFeature {
  public:
 
@@ -72,11 +75,12 @@ class StorageEngine : public application_features::ApplicationFeature {
   virtual void start() {}
   virtual void stop() {}
 
+  virtual transaction::ContextData* createTransactionContextData() = 0;
   virtual TransactionState* createTransactionState(TRI_vocbase_t*) = 0;
   virtual TransactionCollection* createTransactionCollection(TransactionState*, TRI_voc_cid_t, AccessMode::Type, int nestingLevel) = 0;
 
   // create storage-engine specific collection
-  virtual PhysicalCollection* createPhysicalCollection(LogicalCollection*) = 0;
+  virtual PhysicalCollection* createPhysicalCollection(LogicalCollection*, VPackSlice const&) = 0;
 
 
   // status functionality
@@ -236,9 +240,6 @@ class StorageEngine : public application_features::ApplicationFeature {
   virtual void createIndex(TRI_vocbase_t* vocbase, TRI_voc_cid_t collectionId,
                            TRI_idx_iid_t id, arangodb::velocypack::Slice const& data) = 0;
 
-  virtual void createIndexWalMarker(TRI_vocbase_t* vocbase, TRI_voc_cid_t collectionId,
-                                    arangodb::velocypack::Slice const& data, bool useMarker, int&) = 0;
-
   // asks the storage engine to drop the specified index and persist the deletion
   // info. Note that physical deletion of the index must not be carried out by this call,
   // as there may still be users of the index. It is recommended that this operation
@@ -312,10 +313,6 @@ class StorageEngine : public application_features::ApplicationFeature {
 
   virtual int openCollection(TRI_vocbase_t* vocbase, LogicalCollection* collection, bool ignoreErrors) = 0;
 
-  /// @brief transfer markers into a collection
-  virtual int transferMarkers(LogicalCollection* collection, MMFilesCollectorCache*,
-                              MMFilesOperationsType const&) = 0;
-
   // AQL functions
   // -------------
 
@@ -323,9 +320,9 @@ class StorageEngine : public application_features::ApplicationFeature {
   virtual void addAqlFunctions() const = 0;
 
  protected:
-  arangodb::LogicalCollection* registerCollection(
+  void registerCollection(TRI_vocbase_t* vocbase,
-      TRI_vocbase_t* vocbase, arangodb::velocypack::Slice params) {
+                          arangodb::LogicalCollection* collection) {
-    return vocbase->registerCollection(true, params);
+    vocbase->registerCollection(true, collection);
   }
 
  private:

arangod/StorageEngine/TransactionState.h

@@ -125,13 +125,13 @@ class TransactionState {
   void setHint(transaction::Hints::Hint hint) { _hints.set(hint); }
 
   /// @brief begin a transaction
-  virtual int beginTransaction(transaction::Hints hints, int nestingLevel) = 0;
+  virtual int beginTransaction(transaction::Hints hints) = 0;
 
   /// @brief commit a transaction
-  virtual int commitTransaction(transaction::Methods* trx, int nestingLevel) = 0;
+  virtual int commitTransaction(transaction::Methods* trx) = 0;
 
   /// @brief abort a transaction
-  virtual int abortTransaction(transaction::Methods* trx, int nestingLevel) = 0;
+  virtual int abortTransaction(transaction::Methods* trx) = 0;
 
   virtual bool hasFailedOperations() const = 0;
 

arangod/Transaction/Context.cpp

@@ -21,18 +21,18 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "TransactionContext.h"
+#include "Context.h"
-#include "Basics/MutexLocker.h"
 #include "Basics/StringBuffer.h"
 #include "RestServer/TransactionManagerFeature.h"
-#include "MMFiles/MMFilesDatafileHelper.h"
+#include "StorageEngine/EngineSelectorFeature.h"
-#include "MMFiles/MMFilesLogfileManager.h"
+#include "StorageEngine/StorageEngine.h"
+#include "Transaction/ContextData.h"
 #include "Transaction/Helpers.h"
 #include "Transaction/Methods.h"
 #include "Utils/CollectionNameResolver.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ManagedDocumentResult.h"
+#include "VocBase/TransactionManager.h"
 
 #include <velocypack/Builder.h>
 #include <velocypack/Dumper.h>
@@ -64,11 +64,10 @@ struct CustomTypeHandler final : public VPackCustomTypeHandler {
 };
 
 /// @brief create the context
-TransactionContext::TransactionContext(TRI_vocbase_t* vocbase) 
+transaction::Context::Context(TRI_vocbase_t* vocbase) 
     : _vocbase(vocbase), 
       _resolver(nullptr), 
       _customTypeHandler(),
-      _ditches(),
       _builders{_arena},
       _stringBuffer(),
       _options(arangodb::velocypack::Options::Defaults),
@@ -79,19 +78,12 @@ TransactionContext::TransactionContext(TRI_vocbase_t* vocbase)
 }
 
 /// @brief destroy the context
-TransactionContext::~TransactionContext() {
+transaction::Context::~Context() {
   // unregister the transaction from the logfile manager
   if (_transaction.id > 0) {
     TransactionManagerFeature::MANAGER->unregisterTransaction(_transaction.id, _transaction.hasFailedOperations);
   }
 
-  for (auto& it : _ditches) {
-    // we're done with this ditch
-    auto& ditch = it.second;
-    ditch->ditches()->freeDocumentDitch(ditch, true /* fromTransaction */);
-    // If some external entity is still using the ditch, it is kept!
-  }
-
   // free all VPackBuilders we handed out
   for (auto& it : _builders) {
     delete it;
@@ -104,57 +96,23 @@ TransactionContext::~TransactionContext() {
 }
 
 /// @brief factory to create a custom type handler, not managed
-VPackCustomTypeHandler* TransactionContext::createCustomTypeHandler(TRI_vocbase_t* vocbase, 
+VPackCustomTypeHandler* transaction::Context::createCustomTypeHandler(TRI_vocbase_t* vocbase, 
                                                                     CollectionNameResolver const* resolver) {
   return new CustomTypeHandler(vocbase, resolver);
 }
   
-/// @brief order a document ditch for the collection
+/// @brief pin data for the collection
-/// this will create one if none exists. if no ditch can be created, the
+void transaction::Context::pinData(LogicalCollection* collection) {
-/// function will return a nullptr!
+  contextData()->pinData(collection);
-DocumentDitch* TransactionContext::orderDitch(LogicalCollection* collection) {
-
-  TRI_voc_cid_t cid = collection->cid();
-
-  auto it = _ditches.find(cid);
-
-  if (it != _ditches.end()) {
-    // tell everyone else this ditch is still in use,
-    // at least until the transaction is over
-    TRI_ASSERT((*it).second->usedByTransaction());
-    // ditch already exists, return it
-    return (*it).second;
-  }
-
-  // this method will not throw, but may return a nullptr
-  auto ditch = collection->ditches()->createDocumentDitch(true, __FILE__, __LINE__);
-
-  if (ditch != nullptr) {
-    try {
-      _ditches.emplace(cid, ditch);
-    }
-    catch (...) {
-      ditch->ditches()->freeDocumentDitch(ditch, true);
-      ditch = nullptr; // return a nullptr
-    }
-  }
-
-  return ditch;
 }
-  
-/// @brief return the ditch for a collection
-/// this will return a nullptr if no ditch exists
-DocumentDitch* TransactionContext::ditch(TRI_voc_cid_t cid) const {
-  auto it = _ditches.find(cid);
 
-  if (it == _ditches.end()) {
+/// @brief whether or not the data for the collection is pinned
-    return nullptr;
+bool transaction::Context::isPinned(TRI_voc_cid_t cid) {
-  }
+  return contextData()->isPinned(cid);
-  return (*it).second;
 }
   
 /// @brief temporarily lease a StringBuffer object
-basics::StringBuffer* TransactionContext::leaseStringBuffer(size_t initialSize) {
+basics::StringBuffer* transaction::Context::leaseStringBuffer(size_t initialSize) {
   if (_stringBuffer == nullptr) {
     _stringBuffer.reset(new basics::StringBuffer(TRI_UNKNOWN_MEM_ZONE, initialSize, false));
   } else {
@@ -165,12 +123,12 @@ basics::StringBuffer* TransactionContext::leaseStringBuffer(size_t initialSize)
 }
 
 /// @brief return a temporary StringBuffer object
-void TransactionContext::returnStringBuffer(basics::StringBuffer* stringBuffer) {
+void transaction::Context::returnStringBuffer(basics::StringBuffer* stringBuffer) {
   _stringBuffer.reset(stringBuffer);
 }
 
 /// @brief temporarily lease a Builder object
-VPackBuilder* TransactionContext::leaseBuilder() {
+VPackBuilder* transaction::Context::leaseBuilder() {
   if (_builders.empty()) {
     // create a new builder and return it
     return new VPackBuilder();
@@ -185,7 +143,7 @@ VPackBuilder* TransactionContext::leaseBuilder() {
 }
   
 /// @brief return a temporary Builder object
-void TransactionContext::returnBuilder(VPackBuilder* builder) {
+void transaction::Context::returnBuilder(VPackBuilder* builder) {
   try {
     // put builder back into our vector of builders
     _builders.emplace_back(builder);
@@ -196,7 +154,7 @@ void TransactionContext::returnBuilder(VPackBuilder* builder) {
 }
   
 /// @brief get velocypack options with a custom type handler
-VPackOptions* TransactionContext::getVPackOptions() {
+VPackOptions* transaction::Context::getVPackOptions() {
   if (_customTypeHandler == nullptr) {
     // this modifies options!
     orderCustomTypeHandler();
@@ -205,7 +163,7 @@ VPackOptions* TransactionContext::getVPackOptions() {
 }
 
 /// @brief get velocypack options with a custom type handler for dumping
-VPackOptions* TransactionContext::getVPackOptionsForDump() {
+VPackOptions* transaction::Context::getVPackOptionsForDump() {
   if (_customTypeHandler == nullptr) {
     // this modifies options!
     orderCustomTypeHandler();
@@ -214,7 +172,7 @@ VPackOptions* TransactionContext::getVPackOptionsForDump() {
 }
 
 /// @brief create a resolver
-CollectionNameResolver const* TransactionContext::createResolver() {
+CollectionNameResolver const* transaction::Context::createResolver() {
   TRI_ASSERT(_resolver == nullptr);
   _resolver = new CollectionNameResolver(_vocbase);
   _ownsResolver = true;
@@ -223,10 +181,17 @@ CollectionNameResolver const* TransactionContext::createResolver() {
 
 /// @brief unregister the transaction
 /// this will save the transaction's id and status locally
-void TransactionContext::storeTransactionResult(TRI_voc_tid_t id, bool hasFailedOperations) noexcept {
+void transaction::Context::storeTransactionResult(TRI_voc_tid_t id, bool hasFailedOperations) noexcept {
   TRI_ASSERT(_transaction.id == 0);
 
   _transaction.id = id;
   _transaction.hasFailedOperations = hasFailedOperations;
 }
-
+  
+transaction::ContextData* transaction::Context::contextData() {
+  if (_contextData == nullptr) {
+    StorageEngine* engine = EngineSelectorFeature::ENGINE;
+    _contextData.reset(engine->createTransactionContextData());
+  }
+  return _contextData.get();
+}

arangod/Transaction/Context.h

@@ -21,11 +21,10 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#ifndef ARANGOD_UTILS_TRANSACTION_CONTEXT_H
+#ifndef ARANGOD_TRANSACTION_CONTEXT_H
-#define ARANGOD_UTILS_TRANSACTION_CONTEXT_H 1
+#define ARANGOD_TRANSACTION_CONTEXT_H 1
 
 #include "Basics/Common.h"
-#include "Basics/Mutex.h"
 #include "Basics/SmallVector.h"
 #include "VocBase/voc-types.h"
 
@@ -43,30 +42,28 @@ class Builder;
 struct CustomTypeHandler;
 }
 
-namespace transaction {
-class Methods;
-}
-
-
 class CollectionNameResolver;
-class DocumentDitch;
 class LogicalCollection;
 class TransactionState;
 
-class TransactionContext {
+namespace transaction {
+class ContextData;
+class Methods;
+
+class Context {
  public:
-  TransactionContext(TransactionContext const&) = delete;
+  Context(Context const&) = delete;
-  TransactionContext& operator=(TransactionContext const&) = delete;
+  Context& operator=(Context const&) = delete;
 
  protected:
 
   /// @brief create the context
-  explicit TransactionContext(TRI_vocbase_t* vocbase);
+  explicit Context(TRI_vocbase_t* vocbase);
 
  public:
 
   /// @brief destroy the context
-  virtual ~TransactionContext();
+  virtual ~Context();
 
   /// @brief factory to create a custom type handler, not managed
   static arangodb::velocypack::CustomTypeHandler* createCustomTypeHandler(
@@ -76,15 +73,12 @@ class TransactionContext {
   /// @brief return the vocbase
   TRI_vocbase_t* vocbase() const { return _vocbase; }
   
-  /// @brief order a document ditch for the collection
+  /// @brief pin data for the collection
-  /// this will create one if none exists. if no ditch can be created, the
+  void pinData(arangodb::LogicalCollection*);
-  /// function will return a nullptr!
-  DocumentDitch* orderDitch(arangodb::LogicalCollection*);
-  
-  /// @brief return the ditch for a collection
-  /// this will return a nullptr if no ditch exists
-  DocumentDitch* ditch(TRI_voc_cid_t) const;
 
+  /// @brief whether or not the data for the collection is pinned
+  bool isPinned(TRI_voc_cid_t);
+  
   /// @brief temporarily lease a StringBuffer object
   basics::StringBuffer* leaseStringBuffer(size_t initialSize);
 
@@ -108,8 +102,9 @@ class TransactionContext {
   void storeTransactionResult(TRI_voc_tid_t id, bool hasFailedOperations) noexcept;
   
   /// @brief get a custom type handler
-  virtual std::shared_ptr<VPackCustomTypeHandler> orderCustomTypeHandler() = 0;
+  virtual std::shared_ptr<arangodb::velocypack::CustomTypeHandler>
-  
+  orderCustomTypeHandler() = 0;
+
   /// @brief return the resolver
   virtual CollectionNameResolver const* getResolver() = 0;
 
@@ -129,6 +124,8 @@ class TransactionContext {
   
   /// @brief create a resolver
   CollectionNameResolver const* createResolver();
+
+  transaction::ContextData* contextData();
  
  protected:
   
@@ -138,8 +135,6 @@ class TransactionContext {
   
   std::shared_ptr<velocypack::CustomTypeHandler> _customTypeHandler;
   
-  std::unordered_map<TRI_voc_cid_t, DocumentDitch*> _ditches;
-
   SmallVector<arangodb::velocypack::Builder*, 32>::allocator_type::arena_type _arena;
   SmallVector<arangodb::velocypack::Builder*, 32> _builders;
   
@@ -147,6 +142,8 @@ class TransactionContext {
 
   arangodb::velocypack::Options _options;
   arangodb::velocypack::Options _dumpOptions;
+  
+  std::unique_ptr<transaction::ContextData> _contextData;
 
   struct {
     TRI_voc_tid_t id; 
@@ -155,6 +152,8 @@ class TransactionContext {
 
   bool _ownsResolver;
 };
+
+}
 }
 
 #endif

arangod/Transaction/ContextData.h

@@ -0,0 +1,59 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGOD_TRANSACTION_CONTEXT_DATA_H
+#define ARANGOD_TRANSACTION_CONTEXT_DATA_H 1
+
+#include "Basics/Common.h"
+#include "VocBase/voc-types.h"
+
+namespace arangodb {
+class LogicalCollection;
+
+namespace transaction {
+class ContextData {
+ public:
+  ContextData(ContextData const&) = delete;
+  ContextData& operator=(ContextData const&) = delete;
+
+ protected:
+
+  /// @brief create the context data
+  ContextData() = default;
+
+ public:
+
+  /// @brief destroy the context data
+  virtual ~ContextData() = default;
+  
+  /// @brief pin data for the collection
+  virtual void pinData(arangodb::LogicalCollection*) = 0;
+
+  /// @brief whether or not the data for the collection is pinned
+  virtual bool isPinned(TRI_voc_cid_t) const = 0;
+};
+
+}
+}
+
+#endif

arangod/Transaction/Helpers.cpp

@@ -28,7 +28,7 @@
 #include "Basics/encoding.h"
 #include "Transaction/Methods.h"
 #include "Utils/CollectionNameResolver.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 
 #include <velocypack/Builder.h>
 
@@ -388,7 +388,7 @@ transaction::StringBufferLeaser::StringBufferLeaser(transaction::Methods* trx)
 }
 
 /// @brief constructor, leases a StringBuffer
-transaction::StringBufferLeaser::StringBufferLeaser(TransactionContext* transactionContext) 
+transaction::StringBufferLeaser::StringBufferLeaser(transaction::Context* transactionContext) 
       : _transactionContext(transactionContext), 
         _stringBuffer(_transactionContext->leaseStringBuffer(32)) {
 }
@@ -406,7 +406,7 @@ transaction::BuilderLeaser::BuilderLeaser(transaction::Methods* trx)
 }
 
 /// @brief constructor, leases a builder
-transaction::BuilderLeaser::BuilderLeaser(TransactionContext* transactionContext) 
+transaction::BuilderLeaser::BuilderLeaser(transaction::Context* transactionContext) 
       : _transactionContext(transactionContext), 
         _builder(_transactionContext->leaseBuilder()) {
   TRI_ASSERT(_builder != nullptr);

arangod/Transaction/Helpers.h

@@ -34,7 +34,6 @@
 
 namespace arangodb {
 class CollectionNameResolver;
-class TransactionContext;
 
 namespace basics {
 class StringBuffer;
@@ -45,6 +44,7 @@ class Builder;
 }
 
 namespace transaction {
+class Context;
 class Methods;
 
 namespace helpers {
@@ -98,20 +98,20 @@ namespace helpers {
 class StringBufferLeaser {
  public:
   explicit StringBufferLeaser(Methods*); 
-  explicit StringBufferLeaser(TransactionContext*); 
+  explicit StringBufferLeaser(transaction::Context*); 
   ~StringBufferLeaser();
   arangodb::basics::StringBuffer* stringBuffer() const { return _stringBuffer; }
   arangodb::basics::StringBuffer* operator->() const { return _stringBuffer; }
   arangodb::basics::StringBuffer* get() const { return _stringBuffer; }
  private:
-  TransactionContext* _transactionContext;
+  transaction::Context* _transactionContext;
   arangodb::basics::StringBuffer* _stringBuffer;
 };
 
 class BuilderLeaser {
  public:
   explicit BuilderLeaser(transaction::Methods*); 
-  explicit BuilderLeaser(TransactionContext*); 
+  explicit BuilderLeaser(transaction::Context*); 
   ~BuilderLeaser();
   inline arangodb::velocypack::Builder* builder() const { return _builder; }
   inline arangodb::velocypack::Builder* operator->() const { return _builder; }
@@ -122,7 +122,7 @@ class BuilderLeaser {
     return res;
   }
  private:
-  TransactionContext* _transactionContext;
+  transaction::Context* _transactionContext;
   arangodb::velocypack::Builder* _builder;
 };
 

arangod/Transaction/Methods.cpp

@@ -39,9 +39,7 @@
 #include "Cluster/ServerState.h"
 #include "Indexes/Index.h"
 #include "Logger/Logger.h"
-#include "MMFiles/MMFilesLogfileManager.h"
+#include "MMFiles/MMFilesLogfileManager.h" //TODO -- remove -- waitForTick 
-#include "MMFiles/MMFilesPrimaryIndex.h"
-#include "MMFiles/MMFilesIndexElement.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "StorageEngine/TransactionCollection.h"
@@ -52,8 +50,7 @@
 #include "Utils/OperationCursor.h"
 #include "Utils/OperationOptions.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ManagedDocumentResult.h"
 #include "VocBase/ticks.h"
@@ -532,17 +529,8 @@ bool transaction::Methods::findIndexHandleForAndNode(
   return true;
 }
 
-
+transaction::Methods::Methods(std::shared_ptr<transaction::Context> transactionContext)
-/// @brief if this pointer is set to an actual set, then for each request
+    : _state(nullptr),
-/// sent to a shardId using the ClusterComm library, an X-Arango-Nolock
-/// header is generated.
-thread_local std::unordered_set<std::string>* transaction::Methods::_makeNolockHeaders =
-    nullptr;
-  
-      
-transaction::Methods::Methods(std::shared_ptr<TransactionContext> transactionContext)
-    : _hints(),
-      _state(nullptr),
       _transactionContext(transactionContext),
       _transactionContextPtr(transactionContext.get()) {
   TRI_ASSERT(_transactionContextPtr != nullptr);
@@ -606,15 +594,11 @@ TransactionCollection* transaction::Methods::trxCollection(TRI_voc_cid_t cid) co
 }
 
 /// @brief order a ditch for a collection
-void transaction::Methods::orderDitch(TRI_voc_cid_t cid) {
+void transaction::Methods::pinData(TRI_voc_cid_t cid) {
   TRI_ASSERT(_state != nullptr);
   TRI_ASSERT(_state->status() == transaction::Status::RUNNING ||
              _state->status() == transaction::Status::CREATED);
 
-  if (_ditchCache.cid == cid) {
-    return;
-  }
-
   TransactionCollection* trxCollection = _state->collection(cid, AccessMode::Type::READ);
 
   if (trxCollection == nullptr) {
@@ -623,19 +607,12 @@ void transaction::Methods::orderDitch(TRI_voc_cid_t cid) {
 
   TRI_ASSERT(trxCollection->collection() != nullptr);
 
-  DocumentDitch* ditch = _transactionContextPtr->orderDitch(trxCollection->collection());
+  _transactionContextPtr->pinData(trxCollection->collection());
-
-  if (ditch == nullptr) {
-    THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
-  }
-
-  _ditchCache.cid = cid;
-  _ditchCache.ditch = ditch;
 }
   
 /// @brief whether or not a ditch has been created for the collection
-bool transaction::Methods::hasDitch(TRI_voc_cid_t cid) const {
+bool transaction::Methods::isPinned(TRI_voc_cid_t cid) const {
-  return (_transactionContextPtr->ditch(cid) != nullptr);
+  return _transactionContextPtr->isPinned(cid);
 }
 
 /// @brief extract the _id attribute from a slice, and convert it into a 
@@ -742,7 +719,7 @@ int transaction::Methods::begin() {
     return TRI_ERROR_NO_ERROR;
   }
 
-  return _state->beginTransaction(_hints, _state->nestingLevel());
+  return _state->beginTransaction(_localHints);
 }
   
 /// @brief commit / finish the transaction
@@ -759,7 +736,7 @@ int transaction::Methods::commit() {
     return TRI_ERROR_NO_ERROR;
   }
 
-  return _state->commitTransaction(this, _state->nestingLevel());
+  return _state->commitTransaction(this);
 }
   
 /// @brief abort the transaction
@@ -777,7 +754,7 @@ int transaction::Methods::abort() {
     return TRI_ERROR_NO_ERROR;
   }
 
-  return _state->abortTransaction(this, _state->nestingLevel());
+  return _state->abortTransaction(this);
 }
   
 /// @brief finish a transaction (commit or abort), based on the previous state
@@ -831,7 +808,7 @@ OperationResult transaction::Methods::anyLocal(std::string const& collectionName
     throwCollectionNotFound(collectionName.c_str());
   }
  
-  orderDitch(cid); // will throw when it fails 
+  pinData(cid); // will throw when it fails 
   
   int res = lock(trxCollection(cid), AccessMode::Type::READ);
 
@@ -850,7 +827,7 @@ OperationResult transaction::Methods::anyLocal(std::string const& collectionName
 
   LogicalCollection* collection = cursor->collection();
   auto cb = [&] (DocumentIdentifierToken const& token) {
-    if (collection->readDocument(this, mmdr, token)) {
+    if (collection->readDocument(this, token, mmdr)) {
       uint8_t const* vpack = mmdr.vpack();
       resultBuilder.add(VPackSlice(vpack));
     }
@@ -945,24 +922,23 @@ void transaction::Methods::invokeOnAllElements(std::string const& collectionName
   if (_state->isCoordinator()) {
     THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
   }
-  
+
   TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName); 
   TransactionCollection* trxCol = trxCollection(cid);
-  LogicalCollection* document = documentCollection(trxCol);
+  LogicalCollection* logical = documentCollection(trxCol);
 
-  orderDitch(cid); // will throw when it fails
+  pinData(cid); // will throw when it fails
 
   int res = lock(trxCol, AccessMode::Type::READ);
-  
+
   if (res != TRI_ERROR_NO_ERROR) {
     THROW_ARANGO_EXCEPTION(res);
   }
 
-  auto primaryIndex = document->primaryIndex();
+  logical->invokeOnAllElements(callback);
-  primaryIndex->invokeOnAllElements(callback);
+
-  
   res = unlock(trxCol, AccessMode::Type::READ);
-  
+
   if (res != TRI_ERROR_NO_ERROR) {
     THROW_ARANGO_EXCEPTION(res);
   }
@@ -1000,7 +976,7 @@ int transaction::Methods::documentFastPath(std::string const& collectionName,
   TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName); 
   LogicalCollection* collection = documentCollection(trxCollection(cid));
 
-  orderDitch(cid); // will throw when it fails
+  pinData(cid); // will throw when it fails
 
   StringRef key(transaction::helpers::extractKeyPart(value));
   if (key.empty()) {
@@ -1022,7 +998,7 @@ int transaction::Methods::documentFastPath(std::string const& collectionName,
     return res;
   }
   
-  TRI_ASSERT(hasDitch(cid));
+  TRI_ASSERT(isPinned(cid));
 
   uint8_t const* vpack = mmdr->vpack();
   TRI_ASSERT(vpack != nullptr);
@@ -1044,7 +1020,7 @@ int transaction::Methods::documentFastPathLocal(std::string const& collectionNam
   TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName);
   LogicalCollection* collection = documentCollection(trxCollection(cid));
 
-  orderDitch(cid);  // will throw when it fails
+  pinData(cid);  // will throw when it fails
 
   if (key.empty()) {
     return TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD;
@@ -1056,7 +1032,7 @@ int transaction::Methods::documentFastPathLocal(std::string const& collectionNam
     return res;
   }
     
-  TRI_ASSERT(hasDitch(cid));
+  TRI_ASSERT(isPinned(cid));
   return TRI_ERROR_NO_ERROR;
 }
 
@@ -1212,7 +1188,7 @@ OperationResult transaction::Methods::documentLocal(std::string const& collectio
   LogicalCollection* collection = documentCollection(trxCollection(cid));
 
   if (!options.silent) {
-    orderDitch(cid); // will throw when it fails
+    pinData(cid); // will throw when it fails
   }
  
   VPackBuilder resultBuilder;
@@ -1242,7 +1218,7 @@ OperationResult transaction::Methods::documentLocal(std::string const& collectio
       return res;
     }
   
-    TRI_ASSERT(hasDitch(cid));
+    TRI_ASSERT(isPinned(cid));
 
     uint8_t const* vpack = result.vpack();
   
@@ -1370,7 +1346,7 @@ OperationResult transaction::Methods::insertLocal(std::string const& collectionN
   LogicalCollection* collection = documentCollection(trxCollection(cid));
 
   if (options.returnNew) {
-    orderDitch(cid); // will throw when it fails 
+    pinData(cid); // will throw when it fails 
   }
 
   VPackBuilder resultBuilder;
@@ -1664,7 +1640,7 @@ OperationResult transaction::Methods::modifyLocal(
   LogicalCollection* collection = documentCollection(trxCollection(cid));
   
   if (options.returnOld || options.returnNew) {
-    orderDitch(cid); // will throw when it fails 
+    pinData(cid); // will throw when it fails 
   }
 
   // Update/replace are a read and a write, let's get the write lock already
@@ -1918,7 +1894,7 @@ OperationResult transaction::Methods::removeLocal(std::string const& collectionN
   LogicalCollection* collection = documentCollection(trxCollection(cid));
   
   if (options.returnOld) {
-    orderDitch(cid); // will throw when it fails 
+    pinData(cid); // will throw when it fails 
   }
  
   VPackBuilder resultBuilder;
@@ -2131,7 +2107,7 @@ OperationResult transaction::Methods::allLocal(std::string const& collectionName
                                       OperationOptions& options) {
   TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName); 
   
-  orderDitch(cid); // will throw when it fails
+  pinData(cid); // will throw when it fails
   
   int res = lock(trxCollection(cid), AccessMode::Type::READ);
 
@@ -2154,7 +2130,7 @@ OperationResult transaction::Methods::allLocal(std::string const& collectionName
 
   LogicalCollection* collection = cursor->collection();
   auto cb = [&] (DocumentIdentifierToken const& token) {
-    if (collection->readDocument(this, mmdr, token)) {
+    if (collection->readDocument(this, token, mmdr)) {
       uint8_t const* vpack = mmdr.vpack();
       resultBuilder.add(VPackSlice(vpack));
     }
@@ -2209,7 +2185,7 @@ OperationResult transaction::Methods::truncateLocal(std::string const& collectio
                                            OperationOptions& options) {
   TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName); 
   
-  orderDitch(cid); // will throw when it fails
+  pinData(cid); // will throw when it fails
   
   int res = lock(trxCollection(cid), AccessMode::Type::WRITE);
 
@@ -2567,35 +2543,19 @@ std::unique_ptr<OperationCursor> transaction::Methods::indexScan(
   }
 
   TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName); 
-  LogicalCollection* document = documentCollection(trxCollection(cid));
+  LogicalCollection* logical = documentCollection(trxCollection(cid));
   
-  orderDitch(cid); // will throw when it fails 
+  pinData(cid); // will throw when it fails 
 
-  std::unique_ptr<IndexIterator> iterator;
+  std::unique_ptr<IndexIterator> iterator = nullptr;
 
   switch (cursorType) {
     case CursorType::ANY: {
-      arangodb::MMFilesPrimaryIndex* idx = document->primaryIndex();
+      iterator = logical->getAnyIterator(this, mmdr);
-
-      if (idx == nullptr) {
-        THROW_ARANGO_EXCEPTION_MESSAGE(
-            TRI_ERROR_ARANGO_INDEX_NOT_FOUND,
-            "Could not find primary index in collection '" + collectionName + "'.");
-      }
-
-      iterator.reset(idx->anyIterator(this, mmdr));
       break;
     }
     case CursorType::ALL: {
-      arangodb::MMFilesPrimaryIndex* idx = document->primaryIndex();
+      iterator = logical->getAllIterator(this, mmdr, reverse);
-
-      if (idx == nullptr) {
-        THROW_ARANGO_EXCEPTION_MESSAGE(
-            TRI_ERROR_ARANGO_INDEX_NOT_FOUND,
-            "Could not find primary index in collection '" + collectionName + "'.");
-      }
-
-      iterator.reset(idx->allIterator(this, mmdr, reverse));
       break;
     }
   }

arangod/Transaction/Methods.h

@@ -55,22 +55,24 @@ struct Variable;
 }
 
 namespace rest {
-  enum class ResponseCode; 
+enum class ResponseCode; 
 }
 
 namespace traverser {
 class BaseTraverserEngine;
 }
 
+namespace transaction {
+class Context;
+}
+
 /// @brief forward declarations
 class CollectionNameResolver;
-class DocumentDitch;
 struct DocumentIdentifierToken;
 class Index;
 class ManagedDocumentResult;
 struct OperationCursor;
 struct OperationOptions;
-class TransactionContext;
 class TransactionState;
 class TransactionCollection;
 
@@ -120,7 +122,7 @@ class Methods {
  protected:
 
   /// @brief create the transaction
-  explicit Methods(std::shared_ptr<TransactionContext> transactionContext);
+  explicit Methods(std::shared_ptr<transaction::Context> transactionContext);
 
  public:
 
@@ -148,16 +150,16 @@ class Methods {
   int resolveId(char const* handle, size_t length, TRI_voc_cid_t& cid, char const*& key, size_t& outLength); 
   
   /// @brief return a pointer to the transaction context
-  std::shared_ptr<TransactionContext> transactionContext() const {
+  std::shared_ptr<transaction::Context> transactionContext() const {
     return _transactionContext;
   }
   
-  inline TransactionContext* transactionContextPtr() const {
+  inline transaction::Context* transactionContextPtr() const {
     return _transactionContextPtr;
   }
   
   /// @brief add a transaction hint
-  void addHint(transaction::Hints::Hint hint) { _hints.set(hint); }
+  void addHint(transaction::Hints::Hint hint) { _localHints.set(hint); }
 
   /// @brief whether or not the transaction consists of a single operation only
   bool isSingleOperationTransaction() const;
@@ -181,10 +183,10 @@ class Methods {
   std::string name(TRI_voc_cid_t cid) const;
 
   /// @brief order a ditch for a collection
-  void orderDitch(TRI_voc_cid_t);
+  void pinData(TRI_voc_cid_t);
   
   /// @brief whether or not a ditch has been created for the collection
-  bool hasDitch(TRI_voc_cid_t cid) const;
+  bool isPinned(TRI_voc_cid_t cid) const;
 
   /// @brief extract the _id attribute from a slice, and convert it into a 
   /// string
@@ -539,36 +541,25 @@ class Methods {
   /// @brief set up a top-level transaction
   void setupToplevel(TRI_vocbase_t*);
 
- private:
-  /// @brief transaction hints
-  transaction::Hints _hints;
-
  protected:
   /// @brief the state 
   TransactionState* _state;
 
   /// @brief the transaction context
-  std::shared_ptr<TransactionContext> _transactionContext;
+  std::shared_ptr<transaction::Context> _transactionContext;
-
+  
-  /// @brief cache for last handed out DocumentDitch
+  /// @brief pointer to transaction context (faster than shared ptr)
-  struct {
+  transaction::Context* const _transactionContextPtr;
-    TRI_voc_cid_t cid = 0;
+ 
-    DocumentDitch* ditch = nullptr;
+ private:
-  }
+  /// @brief transaction hints
-  _ditchCache;
+  transaction::Hints _localHints;
 
   struct {
     TRI_voc_cid_t cid = 0;
     std::string name;
   }
   _collectionCache;
-  
-  /// @brief pointer to transaction context (faster than shared ptr)
-  TransactionContext* const _transactionContextPtr;
-
- public:
-  /// @brief makeNolockHeaders
-  static thread_local std::unordered_set<std::string>* _makeNolockHeaders;
 };
 
 }

arangod/Transaction/StandaloneContext.cpp

@@ -21,21 +21,20 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "StandaloneTransactionContext.h"
+#include "StandaloneContext.h"
 #include "StorageEngine/TransactionState.h"
 #include "Utils/CollectionNameResolver.h"
 
 using namespace arangodb;
 
 /// @brief create the context
-StandaloneTransactionContext::StandaloneTransactionContext(TRI_vocbase_t* vocbase)
+transaction::StandaloneContext::StandaloneContext(TRI_vocbase_t* vocbase)
-    : TransactionContext(vocbase) {
+    : Context(vocbase) {}
-}
 
 /// @brief order a custom type handler for the collection
-std::shared_ptr<VPackCustomTypeHandler> StandaloneTransactionContext::orderCustomTypeHandler() {
+std::shared_ptr<arangodb::velocypack::CustomTypeHandler> transaction::StandaloneContext::orderCustomTypeHandler() {
   if (_customTypeHandler == nullptr) {
-    _customTypeHandler.reset(TransactionContext::createCustomTypeHandler(_vocbase, getResolver()));
+    _customTypeHandler.reset(transaction::Context::createCustomTypeHandler(_vocbase, getResolver()));
     _options.customTypeHandler = _customTypeHandler.get();
     _dumpOptions.customTypeHandler = _customTypeHandler.get();
   }
@@ -45,7 +44,7 @@ std::shared_ptr<VPackCustomTypeHandler> StandaloneTransactionContext::orderCusto
 }
 
 /// @brief return the resolver
-CollectionNameResolver const* StandaloneTransactionContext::getResolver() {
+CollectionNameResolver const* transaction::StandaloneContext::getResolver() {
   if (_resolver == nullptr) {
     createResolver();
   }
@@ -54,7 +53,7 @@ CollectionNameResolver const* StandaloneTransactionContext::getResolver() {
 }
 
 /// @brief create a context, returned in a shared ptr
-std::shared_ptr<StandaloneTransactionContext> StandaloneTransactionContext::Create(TRI_vocbase_t* vocbase) {
+std::shared_ptr<transaction::StandaloneContext> transaction::StandaloneContext::Create(TRI_vocbase_t* vocbase) {
-  return std::make_shared<StandaloneTransactionContext>(vocbase);
+  return std::make_shared<transaction::StandaloneContext>(vocbase);
 }
 

arangod/Transaction/StandaloneContext.h

@@ -21,32 +21,35 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#ifndef ARANGOD_UTILS_STANDALONE_TRANSACTION_CONTEXT_H
+#ifndef ARANGOD_TRANSACTION_STANDALONE_CONTEXT_H
-#define ARANGOD_UTILS_STANDALONE_TRANSACTION_CONTEXT_H 1
+#define ARANGOD_TRANSACTION_STANDALONE_CONTEXT_H 1
 
+#include "Context.h"
 #include "Basics/Common.h"
-#include "Utils/TransactionContext.h"
 
 struct TRI_vocbase_t;
 
 namespace arangodb {
 class TransactionState;
 
-class StandaloneTransactionContext final : public TransactionContext {
+namespace transaction {
+
+class StandaloneContext final : public Context {
 
  public:
 
   /// @brief create the context
-  explicit StandaloneTransactionContext(TRI_vocbase_t*);
+  explicit StandaloneContext(TRI_vocbase_t*);
 
   /// @brief destroy the context
-  ~StandaloneTransactionContext() = default;
+  ~StandaloneContext() = default;
  
  public:
 
   /// @brief order a custom type handler
-  std::shared_ptr<VPackCustomTypeHandler> orderCustomTypeHandler() override final;
+  std::shared_ptr<arangodb::velocypack::CustomTypeHandler>
-  
+  orderCustomTypeHandler() override final;
+
   /// @brief return the resolver
   CollectionNameResolver const* getResolver() override final;
   
@@ -63,9 +66,11 @@ class StandaloneTransactionContext final : public TransactionContext {
   bool isEmbeddable() const override { return false; }
   
   /// @brief create a context, returned in a shared ptr
-  static std::shared_ptr<StandaloneTransactionContext> Create(TRI_vocbase_t*);
+  static std::shared_ptr<transaction::StandaloneContext> Create(TRI_vocbase_t*);
 
 };
+
+}
 }
 
 #endif

arangod/Transaction/V8Context.cpp

@@ -21,7 +21,7 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "V8TransactionContext.h"
+#include "V8Context.h"
 #include "StorageEngine/TransactionState.h"
 #include "Utils/CollectionNameResolver.h"
 
@@ -31,10 +31,10 @@
 using namespace arangodb;
 
 /// @brief create the context
-V8TransactionContext::V8TransactionContext(TRI_vocbase_t* vocbase,
+transaction::V8Context::V8Context(TRI_vocbase_t* vocbase,
                                            bool embeddable)
-    : TransactionContext(vocbase),
+    : Context(vocbase),
-      _sharedTransactionContext(static_cast<V8TransactionContext*>(
+      _sharedTransactionContext(static_cast<transaction::V8Context*>(
           static_cast<TRI_v8_global_t*>(v8::Isolate::GetCurrent()->GetData(
                                             V8PlatformFeature::V8_DATA_SLOT))
               ->_transactionContext)),
@@ -44,15 +44,15 @@ V8TransactionContext::V8TransactionContext(TRI_vocbase_t* vocbase,
 
 /// @brief order a custom type handler for the collection
 std::shared_ptr<VPackCustomTypeHandler>
-V8TransactionContext::orderCustomTypeHandler() {
+transaction::V8Context::orderCustomTypeHandler() {
   if (_customTypeHandler == nullptr) {
-    V8TransactionContext* main = _sharedTransactionContext->_mainScope;
+    transaction::V8Context* main = _sharedTransactionContext->_mainScope;
 
     if (main != nullptr && main != this && !main->isGlobal()) {
       _customTypeHandler = main->orderCustomTypeHandler();
     } else {
       _customTypeHandler.reset(
-          TransactionContext::createCustomTypeHandler(_vocbase, getResolver()));
+          transaction::Context::createCustomTypeHandler(_vocbase, getResolver()));
     }
     _options.customTypeHandler = _customTypeHandler.get();
     _dumpOptions.customTypeHandler = _customTypeHandler.get();
@@ -65,9 +65,9 @@ V8TransactionContext::orderCustomTypeHandler() {
 }
 
 /// @brief return the resolver
-CollectionNameResolver const* V8TransactionContext::getResolver() {
+CollectionNameResolver const* transaction::V8Context::getResolver() {
   if (_resolver == nullptr) {
-    V8TransactionContext* main = _sharedTransactionContext->_mainScope;
+    transaction::V8Context* main = _sharedTransactionContext->_mainScope;
 
     if (main != nullptr && main != this && !main->isGlobal()) {
       _resolver = main->getResolver();
@@ -82,13 +82,13 @@ CollectionNameResolver const* V8TransactionContext::getResolver() {
 }
 
 /// @brief get parent transaction (if any)
-TransactionState* V8TransactionContext::getParentTransaction() const {
+TransactionState* transaction::V8Context::getParentTransaction() const {
   TRI_ASSERT(_sharedTransactionContext != nullptr);
   return _sharedTransactionContext->_currentTransaction;
 }
 
 /// @brief register the transaction in the context
-void V8TransactionContext::registerTransaction(TransactionState* trx) {
+void transaction::V8Context::registerTransaction(TransactionState* trx) {
   TRI_ASSERT(_sharedTransactionContext != nullptr);
   TRI_ASSERT(_sharedTransactionContext->_currentTransaction == nullptr);
   TRI_ASSERT(_sharedTransactionContext->_mainScope == nullptr);
@@ -97,37 +97,37 @@ void V8TransactionContext::registerTransaction(TransactionState* trx) {
 }
 
 /// @brief unregister the transaction from the context
-void V8TransactionContext::unregisterTransaction() noexcept {
+void transaction::V8Context::unregisterTransaction() noexcept {
   TRI_ASSERT(_sharedTransactionContext != nullptr);
   _sharedTransactionContext->_currentTransaction = nullptr;
   _sharedTransactionContext->_mainScope = nullptr;
 }
 
 /// @brief whether or not the transaction is embeddable
-bool V8TransactionContext::isEmbeddable() const { return _embeddable; }
+bool transaction::V8Context::isEmbeddable() const { return _embeddable; }
 
 /// @brief make this context a global context
 /// this is only called upon V8 context initialization
-void V8TransactionContext::makeGlobal() { _sharedTransactionContext = this; }
+void transaction::V8Context::makeGlobal() { _sharedTransactionContext = this; }
 
 /// @brief whether or not the transaction context is a global one
-bool V8TransactionContext::isGlobal() const {
+bool transaction::V8Context::isGlobal() const {
   return _sharedTransactionContext == this;
 }
 
 /// @brief check whether the transaction is embedded
-bool V8TransactionContext::IsEmbedded() {
+bool transaction::V8Context::IsEmbedded() {
   TRI_v8_global_t* v8g = static_cast<TRI_v8_global_t*>(
       v8::Isolate::GetCurrent()->GetData(V8PlatformFeature::V8_DATA_SLOT));
   if (v8g->_transactionContext == nullptr) {
     return false;
   }
-  return static_cast<V8TransactionContext*>(v8g->_transactionContext)
+  return static_cast<transaction::V8Context*>(v8g->_transactionContext)
              ->_currentTransaction != nullptr;
 }
 
 /// @brief create a context, returned in a shared ptr
-std::shared_ptr<V8TransactionContext> V8TransactionContext::Create(
+std::shared_ptr<transaction::V8Context> transaction::V8Context::Create(
     TRI_vocbase_t* vocbase, bool embeddable) {
-  return std::make_shared<V8TransactionContext>(vocbase, embeddable);
+  return std::make_shared<transaction::V8Context>(vocbase, embeddable);
 }

arangod/Transaction/V8Context.h

@@ -21,30 +21,33 @@
 /// @author Jan Steemann
 ////////////////////////////////////////////////////////////////////////////////
 
-#ifndef ARANGOD_UTILS_V8_TRANSACTION_CONTEXT_H
+#ifndef ARANGOD_TRANSACTION_V8_CONTEXT_H
-#define ARANGOD_UTILS_V8_TRANSACTION_CONTEXT_H 1
+#define ARANGOD_TRANSACTION_V8_CONTEXT_H 1
 
+#include "Context.h"
 #include "Basics/Common.h"
-#include "Utils/TransactionContext.h"
 
 struct TRI_vocbase_t;
 
 namespace arangodb {
 class TransactionState;
 
-class V8TransactionContext final : public TransactionContext {
+namespace transaction {
+
+class V8Context final : public Context {
 
  public:
 
   /// @brief create the context
-  V8TransactionContext(TRI_vocbase_t*, bool);
+  V8Context(TRI_vocbase_t*, bool);
 
   /// @brief destroy the context
-  ~V8TransactionContext() = default;
+  ~V8Context() = default;
   
   /// @brief order a custom type handler
-  std::shared_ptr<VPackCustomTypeHandler> orderCustomTypeHandler() override final;
+  std::shared_ptr<arangodb::velocypack::CustomTypeHandler>
-  
+  orderCustomTypeHandler() override final;
+
   /// @brief return the resolver
   CollectionNameResolver const* getResolver() override final;
   
@@ -70,14 +73,14 @@ class V8TransactionContext final : public TransactionContext {
   static bool IsEmbedded();
   
   /// @brief create a context, returned in a shared ptr
-  static std::shared_ptr<V8TransactionContext> Create(TRI_vocbase_t*, bool);
+  static std::shared_ptr<transaction::V8Context> Create(TRI_vocbase_t*, bool);
 
  private:
 
   /// @brief the v8 thread-local "global" transaction context
-  V8TransactionContext* _sharedTransactionContext;
+  transaction::V8Context* _sharedTransactionContext;
 
-  V8TransactionContext* _mainScope;
+  transaction::V8Context* _mainScope;
 
   /// @brief the currently ongoing transaction
   TransactionState* _currentTransaction;
@@ -85,6 +88,8 @@ class V8TransactionContext final : public TransactionContext {
   /// @brief whether or not further transactions can be embedded
   bool const _embeddable;
 };
+
+}
 }
 
 #endif

arangod/Utils/CollectionExport.cpp

@@ -23,16 +23,17 @@
 
 #include "CollectionExport.h"
 #include "Basics/WriteLocker.h"
-#include "MMFiles/MMFilesPrimaryIndex.h"
+#include "MMFiles/MMFilesDitch.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "Utils/CollectionGuard.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "Transaction/Hints.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
+#include "VocBase/PhysicalCollection.h"
 #include "VocBase/vocbase.h"
+#include "MMFiles/MMFilesCollection.h" //TODO -- REMOVE
 
 using namespace arangodb;
 
@@ -54,7 +55,7 @@ CollectionExport::CollectionExport(TRI_vocbase_t* vocbase,
 
 CollectionExport::~CollectionExport() {
   if (_ditch != nullptr) {
-    _ditch->ditches()->freeDocumentDitch(_ditch, false);
+    _ditch->ditches()->freeMMFilesDocumentDitch(_ditch, false);
   }
 }
 
@@ -64,7 +65,9 @@ void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
   // try to acquire the exclusive lock on the compaction
   engine->preventCompaction(_collection->vocbase(), [this](TRI_vocbase_t* vocbase) {
     // create a ditch under the compaction lock
-    _ditch = _collection->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
+    _ditch = arangodb::MMFilesCollection::toMMFilesCollection(_collection)
+                 ->ditches()
+                 ->createMMFilesDocumentDitch(false, __FILE__, __LINE__);
   });
 
   // now we either have a ditch or not
@@ -79,7 +82,7 @@ void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
     uint64_t const maxTries = maxWaitTime / SleepTime;
 
     while (++tries < maxTries) {
-      if (_collection->isFullyCollected()) {
+      if (_collection->getPhysical()->isFullyCollected()) {
         break;
       }
       usleep(SleepTime);
@@ -88,7 +91,7 @@ void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
 
   {
     SingleCollectionTransaction trx(
-        StandaloneTransactionContext::Create(_collection->vocbase()), _name,
+        transaction::StandaloneContext::Create(_collection->vocbase()), _name,
         AccessMode::Type::READ);
 
     // already locked by guard above
@@ -99,7 +102,7 @@ void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
       THROW_ARANGO_EXCEPTION(res);
     }
     
-    size_t maxDocuments = _collection->primaryIndex()->size();
+    size_t maxDocuments = _collection->numberDocuments();
     if (limit > 0 && limit < maxDocuments) {
       maxDocuments = limit;
     } else {
@@ -113,7 +116,7 @@ void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
       if (limit == 0) {
         return false;
       }
-      if (_collection->readDocumentConditional(&trx, mmdr, token, 0, true)) {
+      if (_collection->readDocumentConditional(&trx, token, 0, mmdr)) {
         _vpack.emplace_back(mmdr.vpack());
         --limit;
       }

arangod/Utils/CollectionExport.h

@@ -34,7 +34,7 @@ struct TRI_vocbase_t;
 namespace arangodb {
 
 class CollectionGuard;
-class DocumentDitch;
+class MMFilesDocumentDitch;
 
 class CollectionExport {
   friend class ExportCursor;
@@ -63,7 +63,7 @@ class CollectionExport {
  private:
   std::unique_ptr<arangodb::CollectionGuard> _guard;
   LogicalCollection* _collection;
-  arangodb::DocumentDitch* _ditch;
+  arangodb::MMFilesDocumentDitch* _ditch;
   std::string const _name;
   arangodb::CollectionNameResolver _resolver;
   Restrictions _restrictions;

arangod/Utils/CollectionKeys.cpp

@@ -24,15 +24,15 @@
 #include "CollectionKeys.h"
 #include "Basics/StaticStrings.h"
 #include "Basics/StringRef.h"
-#include "MMFiles/MMFilesDatafileHelper.h"
+#include "MMFiles/MMFilesLogfileManager.h" //TODO -- REMOVE
-#include "MMFiles/MMFilesLogfileManager.h"
+#include "MMFiles/MMFilesCollection.h"
+#include "MMFiles/MMFilesDitch.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "Transaction/Helpers.h"
 #include "Utils/CollectionGuard.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ticks.h"
 #include "VocBase/vocbase.h"
@@ -74,7 +74,7 @@ CollectionKeys::~CollectionKeys() {
   engine->removeCompactionBlocker(_vocbase, _blockerId);
 
   if (_ditch != nullptr) {
-    _ditch->ditches()->freeDocumentDitch(_ditch, false);
+    _ditch->ditches()->freeMMFilesDocumentDitch(_ditch, false);
   }
 }
 
@@ -89,7 +89,9 @@ void CollectionKeys::create(TRI_voc_tick_t maxTick) {
   StorageEngine* engine = EngineSelectorFeature::ENGINE;
   engine->preventCompaction(_collection->vocbase(), [this](TRI_vocbase_t* vocbase) {
     // create a ditch under the compaction lock
-    _ditch = _collection->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
+    _ditch = arangodb::MMFilesCollection::toMMFilesCollection(_collection)
+                 ->ditches()
+                 ->createMMFilesDocumentDitch(false, __FILE__, __LINE__);
   });
 
   // now we either have a ditch or not
@@ -102,7 +104,7 @@ void CollectionKeys::create(TRI_voc_tick_t maxTick) {
   // copy all datafile markers into the result under the read-lock
   {
     SingleCollectionTransaction trx(
-        StandaloneTransactionContext::Create(_collection->vocbase()), _name,
+        transaction::StandaloneContext::Create(_collection->vocbase()), _name,
         AccessMode::Type::READ);
 
     int res = trx.begin();
@@ -114,7 +116,7 @@ void CollectionKeys::create(TRI_voc_tick_t maxTick) {
     ManagedDocumentResult mmdr;
     trx.invokeOnAllElements(
         _collection->name(), [this, &trx, &maxTick, &mmdr](DocumentIdentifierToken const& token) {
-          if (_collection->readDocumentConditional(&trx, mmdr, token, maxTick, true)) {
+          if (_collection->readDocumentConditional(&trx, token, maxTick, mmdr)) {
             _vpack.emplace_back(mmdr.vpack());
           }
           return true;

arangod/Utils/CollectionKeys.h

@@ -41,7 +41,7 @@ class Slice;
 }
 
 class CollectionGuard;
-class DocumentDitch;
+class MMFilesDocumentDitch;
 
 typedef TRI_voc_tick_t CollectionKeysId;
 
@@ -115,7 +115,7 @@ class CollectionKeys {
   TRI_vocbase_t* _vocbase;
   std::unique_ptr<arangodb::CollectionGuard> _guard;
   arangodb::LogicalCollection* _collection;
-  arangodb::DocumentDitch* _ditch;
+  arangodb::MMFilesDocumentDitch* _ditch;
   std::string const _name;
   arangodb::CollectionNameResolver _resolver;
   TRI_voc_tick_t _blockerId;

arangod/Utils/Cursor.cpp

@@ -27,8 +27,8 @@
 #include "Basics/VelocyPackHelper.h"
 #include "Utils/CollectionExport.h"
 #include "Utils/CollectionNameResolver.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
 #include "VocBase/vocbase.h"
 
 #include <velocypack/Builder.h>
@@ -228,7 +228,7 @@ static bool IncludeAttribute(
 
 void ExportCursor::dump(VPackBuilder& builder) {
   auto transactionContext =
-      std::make_shared<StandaloneTransactionContext>(_vocbaseGuard.vocbase());
+      std::make_shared<transaction::StandaloneContext>(_vocbaseGuard.vocbase());
 
   VPackOptions const* oldOptions = builder.options;
 

arangod/Utils/SingleCollectionTransaction.cpp

@@ -27,15 +27,14 @@
 #include "Utils/CollectionNameResolver.h"
 #include "Utils/OperationResult.h"
 #include "Transaction/Methods.h"
-#include "Utils/TransactionContext.h"
+#include "Transaction/Context.h"
-#include "VocBase/Ditch.h"
 #include "VocBase/LogicalCollection.h"
 
 using namespace arangodb;
 
 /// @brief create the transaction, using a collection id
 SingleCollectionTransaction::SingleCollectionTransaction(
-  std::shared_ptr<TransactionContext> transactionContext, TRI_voc_cid_t cid, 
+  std::shared_ptr<transaction::Context> transactionContext, TRI_voc_cid_t cid, 
   AccessMode::Type accessType)
       : transaction::Methods(transactionContext),
         _cid(cid),
@@ -49,7 +48,7 @@ SingleCollectionTransaction::SingleCollectionTransaction(
 
 /// @brief create the transaction, using a collection name
 SingleCollectionTransaction::SingleCollectionTransaction(
-  std::shared_ptr<TransactionContext> transactionContext,
+  std::shared_ptr<transaction::Context> transactionContext,
   std::string const& name, AccessMode::Type accessType)
       : transaction::Methods(transactionContext),
         _cid(0),

arangod/Utils/SingleCollectionTransaction.h

@@ -30,8 +30,11 @@
 #include "VocBase/voc-types.h"
 
 namespace arangodb {
-class DocumentDitch;
+class MMFilesDocumentDitch;
-class TransactionContext;
+
+namespace transaction {
+class Context;
+}
 
 class SingleCollectionTransaction final : public transaction::Methods {
 
@@ -40,14 +43,14 @@ class SingleCollectionTransaction final : public transaction::Methods {
   /// @brief create the transaction, using a collection id
   //////////////////////////////////////////////////////////////////////////////
 
-  SingleCollectionTransaction(std::shared_ptr<TransactionContext>,
+  SingleCollectionTransaction(std::shared_ptr<transaction::Context>,
                               TRI_voc_cid_t, AccessMode::Type);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief create the transaction, using a collection name
   //////////////////////////////////////////////////////////////////////////////
 
-  SingleCollectionTransaction(std::shared_ptr<TransactionContext>,
+  SingleCollectionTransaction(std::shared_ptr<transaction::Context>,
                               std::string const&, AccessMode::Type);
 
   //////////////////////////////////////////////////////////////////////////////

arangod/Utils/UserTransaction.h

@@ -28,14 +28,14 @@
 
 #include "StorageEngine/TransactionState.h"
 #include "Transaction/Methods.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 
 namespace arangodb {
 
 class UserTransaction final : public transaction::Methods {
  public:
   /// @brief create the transaction
-  UserTransaction(std::shared_ptr<V8TransactionContext> transactionContext,
+  UserTransaction(std::shared_ptr<transaction::V8Context> transactionContext,
                       std::vector<std::string> const& readCollections,
                       std::vector<std::string> const& writeCollections,
                       std::vector<std::string> const& exclusiveCollections,

arangod/V8Server/V8DealerFeature.cpp

@@ -31,6 +31,7 @@
 #include "Basics/ConditionLocker.h"
 #include "Basics/FileUtils.h"
 #include "Basics/StringUtils.h"
+#include "Basics/TimedAction.h"
 #include "Basics/WorkMonitor.h"
 #include "Cluster/ServerState.h"
 #include "ProgramOptions/ProgramOptions.h"
@@ -39,7 +40,7 @@
 #include "RestServer/DatabaseFeature.h"
 #include "Scheduler/JobGuard.h"
 #include "Scheduler/SchedulerFeature.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 #include "V8/v8-buffer.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-globals.h"
@@ -457,6 +458,10 @@ V8Context* V8DealerFeature::enterContext(TRI_vocbase_t* vocbase,
     return nullptr;
   }
 
+  TimedAction exitWhenNoContext([](double waitTime) {
+    LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "giving up waiting for V8 context after " << Logger::FIXED(waitTime) << " s";
+  }, 60);
+
 
   V8Context* context = nullptr;
 
@@ -537,6 +542,11 @@ V8Context* V8DealerFeature::enterContext(TRI_vocbase_t* vocbase,
         
         guard.wait();
       }
+
+      if (exitWhenNoContext.tick()) {
+        vocbase->release();
+        return nullptr;
+      }
     }
 
     // in case we are in the shutdown phase, do not enter a context!
@@ -1119,7 +1129,7 @@ void V8DealerFeature::shutdownV8Instance(V8Context* context) {
 
       if (v8g != nullptr) {
         if (v8g->_transactionContext != nullptr) {
-          delete static_cast<V8TransactionContext*>(v8g->_transactionContext);
+          delete static_cast<transaction::V8Context*>(v8g->_transactionContext);
           v8g->_transactionContext = nullptr;
         }
         delete v8g;

arangod/V8Server/v8-collection.cpp

@@ -36,6 +36,7 @@
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/FollowerInfo.h"
 #include "Cluster/ClusterMethods.h"
+#include "MMFiles/MMFilesCollection.h"
 #include "MMFiles/MMFilesEngine.h"
 #include "MMFiles/MMFilesLogfileManager.h"
 #include "RestServer/DatabaseFeature.h"
@@ -45,7 +46,7 @@
 #include "Utils/OperationResult.h"
 #include "Utils/SingleCollectionTransaction.h"
 #include "Transaction/Hints.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
@@ -305,7 +306,7 @@ static void ExistsVocbaseVPack(
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
   }
 
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
 
   VPackBuilder builder;
   std::string collectionName;
@@ -427,7 +428,7 @@ static void DocumentVocbaseCol(
 
 
   TIMER_START(JS_DOCUMENT_CREATE_TRX);
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
 
   SingleCollectionTransaction trx(transactionContext, collectionName, 
                                   AccessMode::Type::READ);
@@ -490,7 +491,7 @@ static void DocumentVocbase(
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
   }
 
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
 
   VPackBuilder builder;
   std::string collectionName;
@@ -606,7 +607,7 @@ static void RemoveVocbaseCol(v8::FunctionCallbackInfo<v8::Value> const& args) {
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
   }
 
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
 
   SingleCollectionTransaction trx(transactionContext, collectionName, AccessMode::Type::WRITE);
   if (!args[0]->IsArray()) {
@@ -727,7 +728,7 @@ static void RemoveVocbase(v8::FunctionCallbackInfo<v8::Value> const& args) {
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
   }
 
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
 
   VPackBuilder builder;
   std::string collectionName;
@@ -924,7 +925,7 @@ static void JS_FiguresVocbaseCol(
     TRI_V8_THROW_EXCEPTION_INTERNAL("cannot extract collection");
   }
     
-  SingleCollectionTransaction trx(V8TransactionContext::Create(collection->vocbase(), true), collection->cid(), 
+  SingleCollectionTransaction trx(transaction::V8Context::Create(collection->vocbase(), true), collection->cid(), 
                                   AccessMode::Type::READ);
   int res = trx.begin();
 
@@ -1152,7 +1153,7 @@ static void JS_LoadVocbaseCol(v8::FunctionCallbackInfo<v8::Value> const& args) {
   }
 
   SingleCollectionTransaction trx(
-    V8TransactionContext::Create(collection->vocbase(), true),
+    transaction::V8Context::Create(collection->vocbase(), true),
     collection->cid(), AccessMode::Type::READ);
   
   int res = trx.begin();
@@ -1254,7 +1255,6 @@ static void JS_PropertiesVocbaseCol(
     v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_BEGIN(isolate);
   v8::HandleScope scope(isolate);
-  TRI_GET_GLOBALS();
 
   arangodb::LogicalCollection* collection =
       TRI_UnwrapClass<arangodb::LogicalCollection>(args.Holder(), WRP_VOCBASE_COL_TYPE);
@@ -1268,7 +1268,6 @@ static void JS_PropertiesVocbaseCol(
     std::shared_ptr<LogicalCollection> info =
         ClusterInfo::instance()->getCollection(
             databaseName, collection->cid_as_string());
-
     if (0 < args.Length()) {
       v8::Handle<v8::Value> par = args[0];
 
@@ -1283,110 +1282,34 @@ static void JS_PropertiesVocbaseCol(
         }
 
         VPackSlice const slice = builder.slice();
-        if (slice.hasKey("journalSize")) {
-          VPackSlice maxSizeSlice = slice.get("journalSize");
-          TRI_voc_size_t maximalSize =
-              maxSizeSlice.getNumericValue<TRI_voc_size_t>();
-          if (maximalSize < TRI_JOURNAL_MINIMAL_SIZE) {
-            TRI_V8_THROW_EXCEPTION_PARAMETER(
-                "<properties>.journalSize too small");
-          }
-        }
-        if (info->isVolatile() !=
-            arangodb::basics::VelocyPackHelper::getBooleanValue(
-                slice, "isVolatile", info->isVolatile())) {
-          TRI_V8_THROW_EXCEPTION_PARAMETER(
-              "isVolatile option cannot be changed at runtime");
-        }
-        if (info->isVolatile() && info->waitForSync()) {
-          TRI_V8_THROW_EXCEPTION_PARAMETER(
-              "volatile collections do not support the waitForSync option");
-        }
-        uint32_t tmp =
-            arangodb::basics::VelocyPackHelper::getNumericValue<uint32_t>(
-                slice, "indexBuckets",
-                2 /*Just for validation, this default Value passes*/);
-        if (tmp == 0 || tmp > 1024) {
-          TRI_V8_THROW_EXCEPTION_PARAMETER(
-              "indexBuckets must be a two-power between 1 and 1024");
-        }
 
-        int res = info->updateProperties(slice, false);
+        CollectionResult res = info->updateProperties(slice, false);
 
-        if (res != TRI_ERROR_NO_ERROR) {
+        if (!res.successful()) {
-          TRI_V8_THROW_EXCEPTION(res);
+          TRI_V8_THROW_EXCEPTION_MESSAGE(res.code, res.errorMessage);
         }
       }
 
     }
 
-    // return the current parameter set
-    v8::Handle<v8::Object> result = v8::Object::New(isolate);
-
-    TRI_GET_GLOBAL_STRING(DoCompactKey);
-    TRI_GET_GLOBAL_STRING(IsSystemKey);
-    TRI_GET_GLOBAL_STRING(IsVolatileKey);
-    TRI_GET_GLOBAL_STRING(JournalSizeKey);
-    TRI_GET_GLOBAL_STRING(WaitForSyncKey);
-    result->Set(DoCompactKey, v8::Boolean::New(isolate, info->doCompact()));
-    result->Set(IsSystemKey, v8::Boolean::New(isolate, info->isSystem()));
-    result->Set(IsVolatileKey, v8::Boolean::New(isolate, info->isVolatile()));
-    result->Set(JournalSizeKey, v8::Number::New(isolate, static_cast<double>(info->journalSize())));
-    result->Set(WaitForSyncKey, v8::Boolean::New(isolate, info->waitForSync()));
-    result->Set(TRI_V8_ASCII_STRING("indexBuckets"),
-                v8::Number::New(isolate, info->indexBuckets()));
-
     auto c = ClusterInfo::instance()->getCollection(
         databaseName, StringUtils::itoa(collection->cid()));
-    v8::Handle<v8::Array> shardKeys = v8::Array::New(isolate);
-    std::vector<std::string> const sks = c->shardKeys();
-    for (size_t i = 0; i < sks.size(); ++i) {
-      shardKeys->Set((uint32_t)i, TRI_V8_STD_STRING(sks[i]));
-    }
-    result->Set(TRI_V8_ASCII_STRING("shardKeys"), shardKeys);
 
-    result->Set(TRI_V8_ASCII_STRING("numberOfShards"),
+    std::unordered_set<std::string> const ignoreKeys{
-                v8::Number::New(isolate, c->numberOfShards()));
+        "allowUserKeys", "cid",  "count",  "deleted", "id",
-    auto keyOpts = info->keyOptions();
+        "indexes",       "name", "path",   "planId",  "shards",
-    if (keyOpts.isObject() && keyOpts.length() > 0) {
+        "status",        "type", "version"};
-      TRI_GET_GLOBAL_STRING(KeyOptionsKey);
+    VPackBuilder vpackProperties = c->toVelocyPackIgnore(ignoreKeys, true);
-      result->Set(KeyOptionsKey, TRI_VPackToV8(isolate, keyOpts)->ToObject());
-    }
-    if (c->isSatellite()) {
-      result->Set(
-          TRI_V8_ASCII_STRING("replicationFactor"),
-          TRI_V8_STD_STRING(std::string("satellite")));
-    } else {
-      result->Set(
-          TRI_V8_ASCII_STRING("replicationFactor"),
-          v8::Number::New(isolate, static_cast<double>(c->replicationFactor())));
-    }
-    std::string shardsLike = c->distributeShardsLike();
-    if (!shardsLike.empty()) {
-      CollectionNameResolver resolver(c->vocbase());
-      TRI_voc_cid_t cid =
-        static_cast<TRI_voc_cid_t>(
-          arangodb::basics::StringUtils::uint64(shardsLike));
-      result->Set(
-        TRI_V8_ASCII_STRING("distributeShardsLike"),
-        TRI_V8_STD_STRING(resolver.getCollectionNameCluster(cid)));
-    }
-
-    std::vector<std::string> const avoidServers = c->avoidServers();
-    if (!avoidServers.empty()) {
-      v8::Handle<v8::Array> avoidKeys = v8::Array::New(isolate);
-      for (size_t i = 0; i < avoidServers.size(); ++i) {
-        avoidKeys->Set((uint32_t)i, TRI_V8_STD_STRING(avoidServers[i]));
-      }
-      result->Set(TRI_V8_ASCII_STRING("avoidServers"), avoidKeys);
-    }
 
+    // return the current parameter set
+    v8::Handle<v8::Object> result =
+                  TRI_VPackToV8(isolate, vpackProperties.slice())->ToObject();
     TRI_V8_RETURN(result);
   }
   
   bool const isModification = (args.Length() != 0);
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(),
       isModification ? AccessMode::Type::WRITE : AccessMode::Type::READ);
 
@@ -1399,7 +1322,7 @@ static void JS_PropertiesVocbaseCol(
   if (res != TRI_ERROR_NO_ERROR) {
     TRI_V8_THROW_EXCEPTION(res);
   }
-        
+
   // check if we want to change some parameters
   if (isModification) {
     v8::Handle<v8::Value> par = args[0];
@@ -1416,31 +1339,15 @@ static void JS_PropertiesVocbaseCol(
 
       // try to write new parameter to file
       bool doSync = application_features::ApplicationServer::getFeature<DatabaseFeature>("Database")->forceSyncProperties();
-      res = collection->updateProperties(slice, doSync);
+      CollectionResult updateRes = collection->updateProperties(slice, doSync);
 
-      if (res != TRI_ERROR_NO_ERROR) {
+      if (!updateRes.successful()) {
-        TRI_V8_THROW_EXCEPTION(res);
+        TRI_V8_THROW_EXCEPTION_MESSAGE(updateRes.code, updateRes.errorMessage);
       }
 
-      try {
+      auto physical = static_cast<MMFilesCollection*>(collection->getPhysical());
-        VPackBuilder infoBuilder;
+      TRI_ASSERT(physical != nullptr);
-        collection->toVelocyPack(infoBuilder, false);
+      res = physical->persistProperties();
-
-        // now log the property changes
-        res = TRI_ERROR_NO_ERROR;
-
-        MMFilesCollectionMarker marker(TRI_DF_MARKER_VPACK_CHANGE_COLLECTION, collection->vocbase()->id(), collection->cid(), infoBuilder.slice());
-        MMFilesWalSlotInfoCopy slotInfo =
-            MMFilesLogfileManager::instance()->allocateAndWrite(marker, false);
-
-        if (slotInfo.errorCode != TRI_ERROR_NO_ERROR) {
-          THROW_ARANGO_EXCEPTION(slotInfo.errorCode);
-        }
-      } catch (arangodb::basics::Exception const& ex) {
-        res = ex.code();
-      } catch (...) {
-        res = TRI_ERROR_INTERNAL;
-      }
 
       if (res != TRI_ERROR_NO_ERROR) {
         // TODO: what to do here
@@ -1450,37 +1357,17 @@ static void JS_PropertiesVocbaseCol(
     }
   }
 
+  std::unordered_set<std::string> const ignoreKeys{
+      "allowUserKeys", "cid", "count", "deleted", "id", "indexes", "name",
+      "path", "planId", "shards", "status", "type", "version",
+      /* These are only relevant for cluster */
+      "distributeShardsLike", "isSmart", "numberOfShards", "replicationFactor",
+      "shardKeys"};
+  VPackBuilder vpackProperties = collection->toVelocyPackIgnore(ignoreKeys, true);
+
   // return the current parameter set
-  v8::Handle<v8::Object> result = v8::Object::New(isolate);
+  v8::Handle<v8::Object> result =
-
+                TRI_VPackToV8(isolate, vpackProperties.slice())->ToObject();
-  TRI_GET_GLOBAL_STRING(DoCompactKey);
-  TRI_GET_GLOBAL_STRING(IsSystemKey);
-  TRI_GET_GLOBAL_STRING(IsVolatileKey);
-  TRI_GET_GLOBAL_STRING(JournalSizeKey);
-  result->Set(DoCompactKey, v8::Boolean::New(isolate, collection->doCompact()));
-  result->Set(IsSystemKey, v8::Boolean::New(isolate, collection->isSystem()));
-  result->Set(IsVolatileKey,
-              v8::Boolean::New(isolate, collection->isVolatile()));
-  result->Set(JournalSizeKey,
-              v8::Number::New(isolate, (double) collection->journalSize()));
-  result->Set(TRI_V8_ASCII_STRING("indexBuckets"),
-              v8::Number::New(isolate, collection->indexBuckets()));
-
-  TRI_GET_GLOBAL_STRING(KeyOptionsKey);
-  try {
-    VPackBuilder optionsBuilder;
-    optionsBuilder.openObject();
-    collection->keyGenerator()->toVelocyPack(optionsBuilder);
-    optionsBuilder.close();
-    result->Set(KeyOptionsKey,
-                TRI_VPackToV8(isolate, optionsBuilder.slice())->ToObject());
-  } catch (...) {
-    // Could not build the VPack
-    result->Set(KeyOptionsKey, v8::Array::New(isolate));
-  }
-  TRI_GET_GLOBAL_STRING(WaitForSyncKey);
-  result->Set(WaitForSyncKey,
-              v8::Boolean::New(isolate, collection->waitForSync()));
 
   trx.finish(res);
 
@@ -1796,7 +1683,7 @@ static void ModifyVocbaseCol(TRI_voc_document_operation_e operation,
   VPackSlice const update = updateBuilder.slice();
 
 
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
   
   // Now start the transaction:
   SingleCollectionTransaction trx(transactionContext, collectionName,
@@ -1905,7 +1792,7 @@ static void ModifyVocbase(TRI_voc_document_operation_e operation,
 
   TRI_vocbase_t* vocbase = GetContextVocBase(isolate);
 
-  auto transactionContext = std::make_shared<V8TransactionContext>(vocbase, true);
+  auto transactionContext = std::make_shared<transaction::V8Context>(vocbase, true);
   
   VPackBuilder updateBuilder;
 
@@ -1994,7 +1881,7 @@ static int GetRevision(arangodb::LogicalCollection* collection, TRI_voc_rid_t& r
   TRI_ASSERT(collection != nullptr);
 
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(), AccessMode::Type::READ);
 
   int res = trx.begin();
@@ -2086,7 +1973,7 @@ static void JS_RotateVocbaseCol(
   TRI_THROW_SHARDING_COLLECTION_NOT_YET_IMPLEMENTED(collection);
 
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(), AccessMode::Type::READ);
 
   int res = trx.begin();
@@ -2095,7 +1982,7 @@ static void JS_RotateVocbaseCol(
     TRI_V8_THROW_EXCEPTION(res);
   }
 
-  res = collection->rotateActiveJournal();
+  res = collection->getPhysical()->rotateActiveJournal();
 
   trx.finish(res);
 
@@ -2171,7 +2058,7 @@ static void JS_SaveVocbase(v8::FunctionCallbackInfo<v8::Value> const& args) {
   }
 
   // load collection
-  auto transactionContext(V8TransactionContext::Create(vocbase, true));
+  auto transactionContext(transaction::V8Context::Create(vocbase, true));
   SingleCollectionTransaction trx(transactionContext,
                                   collectionName, AccessMode::Type::WRITE);
   trx.addHint(transaction::Hints::Hint::SINGLE_OPERATION);
@@ -2332,7 +2219,7 @@ static void JS_InsertVocbaseCol(
   TIMER_START(JS_INSERT_CREATE_TRX);
   // load collection
   auto transactionContext =
-      std::make_shared<V8TransactionContext>(collection->vocbase(), true);
+      std::make_shared<transaction::V8Context>(collection->vocbase(), true);
 
   SingleCollectionTransaction trx(transactionContext, collection->cid(),
                                   AccessMode::Type::WRITE);
@@ -2433,7 +2320,7 @@ static void JS_TruncateVocbaseCol(
   }
 
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(), AccessMode::Type::WRITE);
 
   int res = trx.begin();
@@ -2933,7 +2820,7 @@ static void JS_CountVocbaseCol(
   
   std::string collectionName(col->name());
 
-  SingleCollectionTransaction trx(V8TransactionContext::Create(vocbase, true), collectionName, AccessMode::Type::READ);
+  SingleCollectionTransaction trx(transaction::V8Context::Create(vocbase, true), collectionName, AccessMode::Type::READ);
 
   int res = trx.begin();
 

arangod/V8Server/v8-query.cpp

@@ -31,7 +31,7 @@
 #include "Transaction/Helpers.h"
 #include "Utils/OperationCursor.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
@@ -212,8 +212,8 @@ static void JS_AllQuery(v8::FunctionCallbackInfo<v8::Value> const& args) {
   
   std::string const collectionName(collection->name());
 
-  std::shared_ptr<V8TransactionContext> transactionContext =
+  std::shared_ptr<transaction::V8Context> transactionContext =
-      V8TransactionContext::Create(collection->vocbase(), true);
+      transaction::V8Context::Create(collection->vocbase(), true);
   SingleCollectionTransaction trx(transactionContext, collection->cid(),
                                   AccessMode::Type::READ);
 
@@ -259,7 +259,7 @@ static void JS_AllQuery(v8::FunctionCallbackInfo<v8::Value> const& args) {
   VPackBuilder resultBuilder;
   resultBuilder.openArray();
   auto cb = [&resultBuilder, &mmdr, &trx, &collection](DocumentIdentifierToken const& tkn) {
-   if (collection->readDocument(&trx, mmdr, tkn)) {
+   if (collection->readDocument(&trx, tkn, mmdr)) {
       resultBuilder.add(VPackSlice(mmdr.vpack()));
     }
   };
@@ -306,8 +306,8 @@ static void JS_AnyQuery(v8::FunctionCallbackInfo<v8::Value> const& args) {
 
   std::string const collectionName(col->name());
 
-  std::shared_ptr<V8TransactionContext> transactionContext =
+  std::shared_ptr<transaction::V8Context> transactionContext =
-      V8TransactionContext::Create(col->vocbase(), true);
+      transaction::V8Context::Create(col->vocbase(), true);
   SingleCollectionTransaction trx(transactionContext, col->cid(),
                                   AccessMode::Type::READ);
 
@@ -372,7 +372,7 @@ static void JS_ChecksumCollection(
     }
   }
 
-  SingleCollectionTransaction trx(V8TransactionContext::Create(col->vocbase(), true),
+  SingleCollectionTransaction trx(transaction::V8Context::Create(col->vocbase(), true),
                                           col->cid(), AccessMode::Type::READ);
 
   int res = trx.begin();
@@ -381,7 +381,7 @@ static void JS_ChecksumCollection(
     TRI_V8_THROW_EXCEPTION(res);
   }
 
-  trx.orderDitch(col->cid()); // will throw when it fails
+  trx.pinData(col->cid()); // will throw when it fails
   
   // get last tick
   LogicalCollection* collection = trx.documentCollection();
@@ -392,7 +392,7 @@ static void JS_ChecksumCollection(
         
   ManagedDocumentResult mmdr;
   trx.invokeOnAllElements(col->name(), [&hash, &withData, &withRevisions, &trx, &collection, &mmdr](DocumentIdentifierToken const& token) {
-    collection->readDocument(&trx, mmdr, token);
+    collection->readDocument(&trx, token, mmdr);
     VPackSlice const slice(mmdr.vpack());
 
     uint64_t localHash = transaction::helpers::extractKeyFromDocument(slice).hashString(); 

arangod/V8Server/v8-replication.cpp

@@ -163,7 +163,7 @@ static void JS_LastLoggerReplication(
         "REPLICATION_LOGGER_LAST(<fromTick>, <toTick>)");
   }
     
-  auto transactionContext = std::make_shared<StandaloneTransactionContext>(vocbase);
+  auto transactionContext = std::make_shared<transaction::StandaloneContext>(vocbase);
 
   TRI_replication_dump_t dump(transactionContext, 0, true, 0);
   TRI_voc_tick_t tickStart = TRI_ObjectToUInt64(args[0], true);

arangod/V8Server/v8-vocbase.cpp

@@ -62,7 +62,7 @@
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
 #include "Utils/UserTransaction.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 #include "V8/JSLoader.h"
 #include "V8/V8LineEditor.h"
 #include "V8/v8-conv.h"
@@ -361,7 +361,7 @@ static void JS_Transaction(v8::FunctionCallbackInfo<v8::Value> const& args) {
   }
 
   auto transactionContext =
-      std::make_shared<V8TransactionContext>(vocbase, embed);
+      std::make_shared<transaction::V8Context>(vocbase, embed);
 
   // start actual transaction
   UserTransaction trx(transactionContext, readCollections, writeCollections, exclusiveCollections,
@@ -2856,8 +2856,8 @@ void TRI_InitV8VocBridge(v8::Isolate* isolate, v8::Handle<v8::Context> context,
   TRI_GET_GLOBALS();
 
   TRI_ASSERT(v8g->_transactionContext == nullptr);
-  v8g->_transactionContext = new V8TransactionContext(vocbase, true);
+  v8g->_transactionContext = new transaction::V8Context(vocbase, true);
-  static_cast<V8TransactionContext*>(v8g->_transactionContext)->makeGlobal();
+  static_cast<transaction::V8Context*>(v8g->_transactionContext)->makeGlobal();
 
   // register the query registry
   TRI_ASSERT(queryRegistry != nullptr);

arangod/V8Server/v8-vocbaseprivate.h

@@ -57,7 +57,7 @@ extern int32_t const WRP_VOCBASE_COL_TYPE;
 ////////////////////////////////////////////////////////////////////////////////
 
 #define PREVENT_EMBEDDED_TRANSACTION()                                  \
-  if (arangodb::V8TransactionContext::IsEmbedded()) {                   \
+  if (arangodb::transaction::V8Context::IsEmbedded()) {                   \
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_TRANSACTION_DISALLOWED_OPERATION); \
   }
 

arangod/V8Server/v8-voccursor.cpp

@@ -25,7 +25,7 @@
 #include "Basics/conversions.h"
 #include "Utils/Cursor.h"
 #include "Utils/CursorRepository.h"
-#include "Utils/StandaloneTransactionContext.h"
+#include "Transaction/StandaloneContext.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-vpack.h"
 #include "V8Server/v8-voccursor.h"
@@ -91,7 +91,7 @@ static void JS_CreateCursor(v8::FunctionCallbackInfo<v8::Value> const& args) {
   arangodb::aql::QueryResult result(TRI_ERROR_NO_ERROR);
   result.result = builder;
   result.cached = false;
-  result.context = std::make_shared<arangodb::StandaloneTransactionContext>(vocbase);
+  result.context = std::make_shared<arangodb::transaction::StandaloneContext>(vocbase);
 
   TRI_ASSERT(builder.get() != nullptr);
 

arangod/V8Server/v8-vocindex.cpp

@@ -35,7 +35,7 @@
 #include "Transaction/Hints.h"
 #include "Utils/Events.h"
 #include "Utils/SingleCollectionTransaction.h"
-#include "Utils/V8TransactionContext.h"
+#include "Transaction/V8Context.h"
 #include "V8/v8-conv.h"
 #include "V8/v8-globals.h"
 #include "V8/v8-utils.h"
@@ -163,7 +163,7 @@ static void EnsureIndexLocal(v8::FunctionCallbackInfo<v8::Value> const& args,
   READ_LOCKER(readLocker, collection->vocbase()->_inventoryLock);
 
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(), create ? AccessMode::Type::WRITE : AccessMode::Type::READ);
 
   int res = trx.begin();
@@ -454,7 +454,7 @@ static void JS_DropIndexVocbaseCol(
   READ_LOCKER(readLocker, collection->vocbase()->_inventoryLock);
 
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(), AccessMode::Type::WRITE);
 
   int res = trx.begin();
@@ -476,7 +476,7 @@ static void JS_DropIndexVocbaseCol(
     TRI_V8_THROW_EXCEPTION(TRI_ERROR_FORBIDDEN);
   }
 
-  bool ok = col->dropIndex(idx->id(), true);
+  bool ok = col->dropIndex(idx->id());
 
   if (ok) {
     TRI_V8_RETURN_TRUE();
@@ -548,7 +548,7 @@ static void JS_GetIndexesVocbaseCol(
   }
 
   SingleCollectionTransaction trx(
-      V8TransactionContext::Create(collection->vocbase(), true),
+      transaction::V8Context::Create(collection->vocbase(), true),
       collection->cid(), AccessMode::Type::READ);
     
   trx.addHint(transaction::Hints::Hint::NO_USAGE_LOCK);