Merge branch 'engine-api' of https://github.com/arangodb/arangodb into engine-api

arangod/CMakeLists.txt

@@ -373,87 +373,14 @@ SET(ARANGOD_SOURCES
   ${ADDITIONAL_BIN_ARANGOD_SOURCES}
 )
 
-# add sources for mmfiles engine
-set(ARANGOD_SOURCES
-  ${ARANGOD_SOURCES}
-  MMFiles/mmfiles-fulltext-handles.cpp
-  MMFiles/mmfiles-fulltext-index.cpp
-  MMFiles/mmfiles-fulltext-list.cpp
-  MMFiles/mmfiles-fulltext-query.cpp
-  MMFiles/mmfiles-fulltext-result.cpp
-  MMFiles/mmfiles-geo-index.cpp
-  MMFiles/MMFilesAllocatorThread.cpp
-  MMFiles/MMFilesAqlFunctions.cpp
-  MMFiles/MMFilesCleanupThread.cpp
-  MMFiles/MMFilesCollection.cpp
-  MMFiles/MMFilesCollectionExport.cpp
-  MMFiles/MMFilesCollectionKeys.cpp
-  MMFiles/MMFilesCollectorThread.cpp
-  MMFiles/MMFilesCompactorThread.cpp
-  MMFiles/MMFilesDatafile.cpp
-  MMFiles/MMFilesDatafileStatistics.cpp
-  MMFiles/MMFilesDatafileStatisticsContainer.cpp
-  MMFiles/MMFilesDitch.cpp
-  MMFiles/MMFilesDocumentOperation.cpp
-  MMFiles/MMFilesEdgeIndex.cpp
-  MMFiles/MMFilesEngine.cpp
-  MMFiles/MMFilesExportCursor.cpp
-  MMFiles/MMFilesIndexElement.cpp
-  MMFiles/MMFilesIndexFactory.cpp
-  MMFiles/MMFilesLogfileManager.cpp
-  MMFiles/MMFilesFulltextIndex.cpp
-  MMFiles/MMFilesGeoIndex.cpp
-  MMFiles/MMFilesHashIndex.cpp
-  MMFiles/MMFilesOptimizerRules.cpp
-  MMFiles/MMFilesPathBasedIndex.cpp
-  MMFiles/MMFilesPersistentIndexFeature.cpp
-  MMFiles/MMFilesPersistentIndex.cpp
-  MMFiles/MMFilesPersistentIndexKeyComparator.cpp
-  MMFiles/MMFilesPrimaryIndex.cpp
-  MMFiles/MMFilesRemoverThread.cpp
-  MMFiles/MMFilesRestExportHandler.cpp
-  MMFiles/MMFilesRestHandlers.cpp
-  MMFiles/MMFilesRestWalHandler.cpp
-  MMFiles/MMFilesRevisionsCache.cpp
-  MMFiles/MMFilesSkiplistIndex.cpp
-  MMFiles/MMFilesSynchronizerThread.cpp
-  MMFiles/MMFilesTransactionCollection.cpp
-  MMFiles/MMFilesTransactionContextData.cpp
-  MMFiles/MMFilesTransactionState.cpp
-  MMFiles/MMFilesV8Functions.cpp
-  MMFiles/MMFilesView.cpp
-  MMFiles/MMFilesWalLogfile.cpp
-  MMFiles/MMFilesWalRecoverState.cpp
-  MMFiles/MMFilesWalRecoveryFeature.cpp
-  MMFiles/MMFilesWalSlot.cpp
-  MMFiles/MMFilesWalSlots.cpp
-  MMFiles/mmfiles-replication-dump.cpp
-)
-
-# add sources for rocksdb engine
-set(ARANGOD_SOURCES
-  ${ARANGOD_SOURCES}
-  RocksDBEngine/RocksDBCollection.cpp
-  RocksDBEngine/RocksDBCommon.cpp
-  RocksDBEngine/RocksDBComparator.cpp
-  RocksDBEngine/RocksDBEdgeIndex.cpp
-  RocksDBEngine/RocksDBEngine.cpp
-  RocksDBEngine/RocksDBIndexFactory.cpp
-  RocksDBEngine/RocksDBKey.cpp
-  RocksDBEngine/RocksDBPrimaryIndex.cpp
-  RocksDBEngine/RocksDBPrimaryMockIndex.cpp
-  RocksDBEngine/RocksDBTransactionCollection.cpp
-  RocksDBEngine/RocksDBTransactionState.cpp
-  RocksDBEngine/RocksDBTypes.cpp
-  RocksDBEngine/RocksDBValue.cpp
-  RocksDBEngine/RocksDBView.cpp
-)
-
 if (NOT MSVC)
   set(ARANGOD_SOURCES ${ARANGOD_SOURCES} Scheduler/AcceptorUnixDomain.cpp Scheduler/SocketUnixDomain.cpp)
 endif()
 
-add_library(arangoserver STATIC ${ARANGOD_SOURCES})
+include(RocksDBEngine/CMakeLists.txt)
+include(MMFiles/CMakeLists.txt)
+
+add_library(arangoserver STATIC ${ARANGOD_SOURCES} ${MMFILES_SOURCES} ${ROCKSDB_SOURCES})
 
 if (USE_SSL)
   target_compile_definitions(${BIN_ARANGOD} PUBLIC "ARANGODB_SSL_ENABLED=1")

arangod/MMFiles/CMakeLists.txt

@@ -0,0 +1,58 @@
+# -*- mode: CMAKE; -*-
+
+# add sources for mmfiles engine
+set(MMFILES_SOURCES
+  MMFiles/mmfiles-fulltext-handles.cpp
+  MMFiles/mmfiles-fulltext-index.cpp
+  MMFiles/mmfiles-fulltext-list.cpp
+  MMFiles/mmfiles-fulltext-query.cpp
+  MMFiles/mmfiles-fulltext-result.cpp
+  MMFiles/mmfiles-geo-index.cpp
+  MMFiles/MMFilesAllocatorThread.cpp
+  MMFiles/MMFilesAqlFunctions.cpp
+  MMFiles/MMFilesCleanupThread.cpp
+  MMFiles/MMFilesCollection.cpp
+  MMFiles/MMFilesCollectionExport.cpp
+  MMFiles/MMFilesCollectionKeys.cpp
+  MMFiles/MMFilesCollectorThread.cpp
+  MMFiles/MMFilesCompactorThread.cpp
+  MMFiles/MMFilesDatafile.cpp
+  MMFiles/MMFilesDatafileStatistics.cpp
+  MMFiles/MMFilesDatafileStatisticsContainer.cpp
+  MMFiles/MMFilesDitch.cpp
+  MMFiles/MMFilesDocumentOperation.cpp
+  MMFiles/MMFilesEdgeIndex.cpp
+  MMFiles/MMFilesEngine.cpp
+  MMFiles/MMFilesExportCursor.cpp
+  MMFiles/MMFilesIndexElement.cpp
+  MMFiles/MMFilesIndexFactory.cpp
+  MMFiles/MMFilesLogfileManager.cpp
+  MMFiles/MMFilesFulltextIndex.cpp
+  MMFiles/MMFilesGeoIndex.cpp
+  MMFiles/MMFilesHashIndex.cpp
+  MMFiles/MMFilesOptimizerRules.cpp
+  MMFiles/MMFilesPathBasedIndex.cpp
+  MMFiles/MMFilesPersistentIndexFeature.cpp
+  MMFiles/MMFilesPersistentIndex.cpp
+  MMFiles/MMFilesPersistentIndexKeyComparator.cpp
+  MMFiles/MMFilesPrimaryIndex.cpp
+  MMFiles/MMFilesRemoverThread.cpp
+  MMFiles/MMFilesRestExportHandler.cpp
+  MMFiles/MMFilesRestHandlers.cpp
+  MMFiles/MMFilesRestWalHandler.cpp
+  MMFiles/MMFilesRevisionsCache.cpp
+  MMFiles/MMFilesSkiplistIndex.cpp
+  MMFiles/MMFilesSynchronizerThread.cpp
+  MMFiles/MMFilesTransactionCollection.cpp
+  MMFiles/MMFilesTransactionContextData.cpp
+  MMFiles/MMFilesTransactionState.cpp
+  MMFiles/MMFilesV8Functions.cpp
+  MMFiles/MMFilesView.cpp
+  MMFiles/MMFilesWalLogfile.cpp
+  MMFiles/MMFilesWalRecoverState.cpp
+  MMFiles/MMFilesWalRecoveryFeature.cpp
+  MMFiles/MMFilesWalSlot.cpp
+  MMFiles/MMFilesWalSlots.cpp
+  MMFiles/mmfiles-replication-dump.cpp
+  )
+set(MMFILES_SOURCES ${MMFILES_SOURCES} PARENT_SCOPE)

arangod/MMFiles/MMFilesCollection.cpp

@@ -398,12 +398,12 @@ bool MMFilesCollection::OpenIterator(MMFilesMarker const* marker, MMFilesCollect
     }
     
     if (++data->_operations % 1024 == 0) {
-      data->_mmdr.clear();
+      data->_mmdr.reset();
     }
   } else if (type == TRI_DF_MARKER_VPACK_REMOVE) {
     res = OpenIteratorHandleDeletionMarker(marker, datafile, data);
     if (++data->_operations % 1024 == 0) {
-      data->_mmdr.clear();
+      data->_mmdr.reset();
     }
   } else {
     if (type == TRI_DF_MARKER_HEADER) {
@@ -1837,7 +1837,7 @@ bool MMFilesCollection::readDocument(transaction::Methods* trx,
   TRI_voc_rid_t revisionId = tkn->revisionId();
   uint8_t const* vpack = lookupRevisionVPack(revisionId);
   if (vpack != nullptr) {
-    result.addExisting(vpack, revisionId);
+    result.setUnmanaged(vpack, revisionId);
     return true;
   } 
   return false;
@@ -1852,7 +1852,7 @@ bool MMFilesCollection::readDocumentConditional(
   TRI_ASSERT(revisionId != 0);
   uint8_t const* vpack  = lookupRevisionVPackConditional(revisionId, maxTick, true);
   if (vpack != nullptr) {
-    result.addExisting(vpack, revisionId);
+    result.setUnmanaged(vpack, revisionId);
     return true;
   } 
   return false;
@@ -2689,7 +2689,7 @@ int MMFilesCollection::insert(transaction::Methods* trx,
   if (res == TRI_ERROR_NO_ERROR) {
     uint8_t const* vpack = lookupRevisionVPack(revisionId);
     if (vpack != nullptr) {
-      result.addExisting(vpack, revisionId);
+      result.setUnmanaged(vpack, revisionId);
     }
 
     // store the tick that was used for writing the document
@@ -3014,7 +3014,7 @@ int MMFilesCollection::update(arangodb::transaction::Methods* trx,
 
   if (newSlice.length() <= 1) {
     // no need to do anything
-    result = previous;
+    result = std::move(previous);
     if (_logicalCollection->waitForSync()) {
       options.waitForSync = true;
     }
@@ -3064,7 +3064,7 @@ int MMFilesCollection::update(arangodb::transaction::Methods* trx,
     
     if (oldRevisionId == revisionId) {
       // update with same revision id => can happen if isRestore = true
-      result.clear();
+      result.reset();
     }
 
     res = updateDocument(trx, oldRevisionId, oldDoc, revisionId, newDoc,
@@ -3082,7 +3082,7 @@ int MMFilesCollection::update(arangodb::transaction::Methods* trx,
   } else {
     uint8_t const* vpack = lookupRevisionVPack(revisionId);
     if (vpack != nullptr) {
-      result.addExisting(vpack, revisionId);
+      result.setUnmanaged(vpack, revisionId);
     }
     if (options.waitForSync) {
       // store the tick that was used for writing the new document
@@ -3187,7 +3187,7 @@ int MMFilesCollection::replace(
     
     if (oldRevisionId == revisionId) {
       // update with same revision id => can happen if isRestore = true
-      result.clear();
+      result.reset();
     }
 
     res = updateDocument(trx, oldRevisionId, oldDoc, revisionId, newDoc,
@@ -3205,11 +3205,11 @@ int MMFilesCollection::replace(
   } else {
     if (oldRevisionId == revisionId) {
       // update with same revision id => can happen if isRestore = true
-      result.clear();
+      result.reset();
     }
     uint8_t const* vpack = lookupRevisionVPack(revisionId);
     if (vpack != nullptr) {
-      result.addExisting(vpack, revisionId);
+      result.setUnmanaged(vpack, revisionId);
     }
 
     if (options.waitForSync) {
@@ -3526,7 +3526,7 @@ int MMFilesCollection::lookupDocument(transaction::Methods* trx,
     TRI_voc_rid_t revisionId = element.revisionId();
     uint8_t const* vpack = lookupRevisionVPack(revisionId);
     if (vpack != nullptr) {
-      result.addExisting(vpack, revisionId);
+      result.setUnmanaged(vpack, revisionId);
     }
     return TRI_ERROR_NO_ERROR;
   }

arangod/RocksDBEngine/CMakeLists.txt

@@ -0,0 +1,20 @@
+# -*- mode: CMAKE; -*-
+
+# add sources for rocksdb engine
+set(ROCKSDB_SOURCES
+  RocksDBEngine/RocksDBCollection.cpp
+  RocksDBEngine/RocksDBCommon.cpp
+  RocksDBEngine/RocksDBComparator.cpp
+  RocksDBEngine/RocksDBEdgeIndex.cpp
+  RocksDBEngine/RocksDBEngine.cpp
+  RocksDBEngine/RocksDBIndexFactory.cpp
+  RocksDBEngine/RocksDBKey.cpp
+  RocksDBEngine/RocksDBPrimaryIndex.cpp
+  RocksDBEngine/RocksDBPrimaryMockIndex.cpp
+  RocksDBEngine/RocksDBTransactionCollection.cpp
+  RocksDBEngine/RocksDBTransactionState.cpp
+  RocksDBEngine/RocksDBTypes.cpp
+  RocksDBEngine/RocksDBValue.cpp
+  RocksDBEngine/RocksDBView.cpp
+)
+set(ROCKSDB_SOURCES ${ROCKSDB_SOURCES} PARENT_SCOPE)

arangod/VocBase/ManagedDocumentResult.h

@@ -24,35 +24,86 @@
 #ifndef ARANGOD_VOC_BASE_MANAGED_DOCUMENT_RESULT_H
 #define ARANGOD_VOC_BASE_MANAGED_DOCUMENT_RESULT_H 1
 
+#include "velocypack/Slice.h"
+#include "velocypack/Buffer.h"
+#include "velocypack/velocypack-aliases.h"
 #include "Basics/Common.h"
 
 namespace arangodb {
 
 class ManagedDocumentResult {
  public:
-  ManagedDocumentResult() : _vpack(nullptr), _lastRevisionId(0) {}
+  ManagedDocumentResult() : _length(0), _lastRevisionId(0), _vpack(nullptr),
-  ~ManagedDocumentResult() = default;
+                             _managed(false) {}
+  ~ManagedDocumentResult() { reset(); }
+  ManagedDocumentResult(ManagedDocumentResult const& other) = delete;
+  ManagedDocumentResult& operator=(ManagedDocumentResult const& other) = delete;
 
-  inline uint8_t const* vpack() const { 
+  ManagedDocumentResult& operator=(ManagedDocumentResult&& other){
-    TRI_ASSERT(_vpack != nullptr); 
+    if (other._managed){
-    return _vpack; 
+      reset();
+      _vpack = other._vpack;
+      _length = other._length;
+      _lastRevisionId = other._lastRevisionId;
+      _managed = true;
+      other._managed = false;
+      other.reset();
+    } else {
+      setUnmanaged(other._vpack, other._lastRevisionId);
+    }
+    return *this;
   }
-  
+
-  inline void addExisting(uint8_t const* vpack, TRI_voc_rid_t revisionId) {
+  ManagedDocumentResult(ManagedDocumentResult&& other) = delete;
-    _vpack = vpack;
+
+  inline uint8_t const* vpack() const {
+    TRI_ASSERT(_vpack != nullptr);
+    return _vpack;
+  }
+
+  //add unmanaged vpack 
+  inline void setUnmanaged(uint8_t const* vpack, TRI_voc_rid_t revisionId) {
+    if(_managed) {
+      reset();
+    }
+    TRI_ASSERT(_length == 0);
+    _vpack = const_cast<uint8_t*>(vpack);
     _lastRevisionId = revisionId;
   }
 
+  inline void setManaged(uint8_t const* vpack, TRI_voc_rid_t revisionId) {
+    VPackSlice slice(vpack);
+    auto newLen = slice.byteSize();
+    if (_length >= newLen && _managed){
+      std::memcpy(_vpack, vpack, newLen);
+    } else {
+      reset();
+      _vpack = new uint8_t[newLen];
+      std::memcpy(_vpack, vpack, newLen);
+      _length=newLen;
+    }
+    _lastRevisionId = revisionId;
+    _managed = true;
+  }
+
   inline TRI_voc_rid_t lastRevisionId() const { return _lastRevisionId; }
 
-  void clear() {
+  void reset() noexcept {
+    if(_managed) {
+      delete _vpack;
+    }
     _vpack = nullptr;
     _lastRevisionId = 0;
+    _managed = false;
+    _length = 0;
   }
 
  private:
-  uint8_t const* _vpack;
+  uint64_t _length;
   TRI_voc_rid_t _lastRevisionId;
+  uint8_t* _vpack;
+  bool _managed;
+
 };
 
 }

js/server/upgrade-database.js

@@ -497,7 +497,6 @@
           unique: true,
           sparse: true
         });*/
-
         return true;
       }
     });