added unload() functionality for indexes (untested)

arangod/Agency/v8-agency.cpp

@@ -31,7 +31,6 @@
 #include "V8/v8-globals.h"
 #include "V8/v8-utils.h"
 #include "V8/v8-vpack.h"
-#include "VocBase/server.h"
 
 #include <velocypack/Iterator.h>
 #include <velocypack/velocypack-aliases.h>

arangod/FulltextIndex/fulltext-index.cpp

@@ -24,12 +24,13 @@
 #include "fulltext-index.h"
 
 #include "Basics/locks.h"
+#include "Basics/Exceptions.h"
 #include "Logger/Logger.h"
 
-#include "fulltext-handles.h"
-#include "fulltext-list.h"
-#include "fulltext-query.h"
-#include "fulltext-result.h"
+#include "FulltextIndex/fulltext-handles.h"
+#include "FulltextIndex/fulltext-list.h"
+#include "FulltextIndex/fulltext-query.h"
+#include "FulltextIndex/fulltext-result.h"
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief use padding for pointers in binary data
@@ -1287,6 +1288,45 @@ void TRI_FreeFtsIndex(TRI_fts_index_t* ftx) {
   TRI_Free(TRI_UNKNOWN_MEM_ZONE, idx);
 }
 
+void TRI_TruncateFulltextIndex(TRI_fts_index_t* ftx) {
+  index__t* idx = (index__t*)ftx;
+
+  // free root node (this will recursively free all other nodes)
+  FreeNode(idx, idx->_root);
+  
+  // free handles
+  TRI_FreeHandlesFulltextIndex(idx->_handles);
+  idx->_handles = nullptr;
+  
+  idx->_memoryAllocated = sizeof(index__t);
+#if TRI_FULLTEXT_DEBUG
+  idx->_memoryBase = sizeof(index__t);
+  idx->_memoryNodes = 0;
+  idx->_memoryFollowers = 0;
+  idx->_nodesAllocated = 0;
+#endif
+  
+  // create the root node
+  idx->_root = CreateNode(idx);
+  if (idx->_root == nullptr) {
+    // out of memory
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
+  }
+
+  // create an instance for managing document handles
+  idx->_handles = TRI_CreateHandlesFulltextIndex(2048);
+  if (idx->_handles == nullptr) {
+    // out of memory
+    TRI_Free(TRI_UNKNOWN_MEM_ZONE, idx->_root);
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
+  }
+
+  idx->_memoryAllocated += sizeof(TRI_fulltext_handles_t);
+#if TRI_FULLTEXT_DEBUG
+  idx->_memoryBase += sizeof(TRI_fulltext_handles_t);
+#endif
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief delete a document from the index
 ////////////////////////////////////////////////////////////////////////////////

arangod/FulltextIndex/fulltext-index.h

@@ -76,6 +76,8 @@ TRI_fts_index_t* TRI_CreateFtsIndex(uint32_t, uint32_t, uint32_t);
 
 void TRI_FreeFtsIndex(TRI_fts_index_t*);
 
+void TRI_TruncateFulltextIndex(TRI_fts_index_t*);
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief delete a document from the index
 ////////////////////////////////////////////////////////////////////////////////

arangod/Indexes/EdgeIndex.cpp

@@ -559,6 +559,14 @@ int EdgeIndex::batchInsert(arangodb::Transaction* trx,
   return TRI_ERROR_NO_ERROR;
 }
 
+/// @brief unload the index data from memory
+int EdgeIndex::unload() {
+  _edgesFrom->truncate([](TRI_doc_mptr_t*) -> bool { return true; });
+  _edgesTo->truncate([](TRI_doc_mptr_t*) -> bool { return true; });
+
+  return TRI_ERROR_NO_ERROR;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief provides a size hint for the edge index
 ////////////////////////////////////////////////////////////////////////////////

arangod/Indexes/EdgeIndex.h

@@ -158,6 +158,8 @@ class EdgeIndex final : public Index {
   int batchInsert(arangodb::Transaction*,
                   std::vector<TRI_doc_mptr_t const*> const*,
                   size_t) override final;
+  
+  int unload() override final;
 
   int sizeHint(arangodb::Transaction*, size_t) override final;
 

arangod/Indexes/FulltextIndex.cpp

@@ -134,6 +134,11 @@ int FulltextIndex::remove(arangodb::Transaction*, TRI_doc_mptr_t const* doc,
   return TRI_ERROR_NO_ERROR;
 }
 
+int FulltextIndex::unload() {
+  TRI_TruncateFulltextIndex(_fulltextIndex);
+  return TRI_ERROR_NO_ERROR;
+}
+
 int FulltextIndex::cleanup() {
   LOG(TRACE) << "fulltext cleanup called";
 

arangod/Indexes/FulltextIndex.h

@@ -68,6 +68,8 @@ class FulltextIndex final : public Index {
   int remove(arangodb::Transaction*, struct TRI_doc_mptr_t const*,
              bool) override final;
 
+  int unload() override final;
+
   int cleanup() override final;
 
   bool isSame(std::string const& field, int minWordLength) const {

arangod/Indexes/GeoIndex2.cpp

@@ -242,10 +242,26 @@ int GeoIndex2::remove(arangodb::Transaction*, TRI_doc_mptr_t const* doc, bool) {
   return TRI_ERROR_NO_ERROR;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief looks up all points within a given radius
-////////////////////////////////////////////////////////////////////////////////
+int GeoIndex2::unload() {
+  // create a new, empty index
+  auto empty = GeoIndex_new();
 
+  if (empty == nullptr) {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
+  }
+  
+  // free the old one
+  if (_geoIndex != nullptr) {
+    GeoIndex_free(_geoIndex);
+  }
+
+  // and assign it
+  _geoIndex = empty;
+
+  return TRI_ERROR_NO_ERROR;
+}
+
+/// @brief looks up all points within a given radius
 GeoCoordinates* GeoIndex2::withinQuery(arangodb::Transaction* trx, double lat,
                                        double lon, double radius) const {
   GeoCoordinate gc;

arangod/Indexes/GeoIndex2.h

@@ -90,6 +90,8 @@ class GeoIndex2 final : public Index {
   int remove(arangodb::Transaction*, struct TRI_doc_mptr_t const*,
              bool) override final;
 
+  int unload() override final;
+
   //////////////////////////////////////////////////////////////////////////////
   /// @brief looks up all points within a given radius
   //////////////////////////////////////////////////////////////////////////////

arangod/Indexes/HashIndex.cpp

@@ -604,6 +604,15 @@ int HashIndex::batchInsert(arangodb::Transaction* trx,
 
   return batchInsertMulti(trx, documents, numThreads);
 }
+  
+int HashIndex::unload() {
+  if (_unique) {
+    _uniqueArray->_hashArray->truncate(FreeElement);
+  } else {
+    _multiArray->_hashArray->truncate(FreeElement);
+  }
+  return TRI_ERROR_NO_ERROR;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief provides a size hint for the hash index

arangod/Indexes/HashIndex.h

@@ -186,6 +186,8 @@ class HashIndex final : public PathBasedIndex {
   int batchInsert(arangodb::Transaction*,
                   std::vector<TRI_doc_mptr_t const*> const*,
                   size_t) override final;
+  
+  int unload() override final;
 
   int sizeHint(arangodb::Transaction*, size_t) override final;
 

arangod/Indexes/Index.cpp

@@ -418,60 +418,39 @@ void Index::toVelocyPackFigures(VPackBuilder& builder) const {
   builder.add("memory", VPackValue(memory()));
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for selectivityEstimate
-////////////////////////////////////////////////////////////////////////////////
-
 double Index::selectivityEstimate() const {
   THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for selectivityEstimate
-////////////////////////////////////////////////////////////////////////////////
-
 int Index::batchInsert(arangodb::Transaction*,
                        std::vector<TRI_doc_mptr_t const*> const*, size_t) {
   THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for cleanup
-////////////////////////////////////////////////////////////////////////////////
-
 int Index::cleanup() {
   // do nothing
   return TRI_ERROR_NO_ERROR;
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for drop
-////////////////////////////////////////////////////////////////////////////////
-
 int Index::drop() {
   // do nothing
   return TRI_ERROR_NO_ERROR;
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for sizeHint
-////////////////////////////////////////////////////////////////////////////////
-
 int Index::sizeHint(arangodb::Transaction*, size_t) {
   // do nothing
   return TRI_ERROR_NO_ERROR;
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for hasBatchInsert
-////////////////////////////////////////////////////////////////////////////////
-
 bool Index::hasBatchInsert() const { return false; }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for supportsFilterCondition
-////////////////////////////////////////////////////////////////////////////////
-
 bool Index::supportsFilterCondition(arangodb::aql::AstNode const*,
                                     arangodb::aql::Variable const*,
                                     size_t itemsInIndex, size_t& estimatedItems,
@@ -482,10 +461,7 @@ bool Index::supportsFilterCondition(arangodb::aql::AstNode const*,
   return false;
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief default implementation for supportsSortCondition
-////////////////////////////////////////////////////////////////////////////////
-
 bool Index::supportsSortCondition(arangodb::aql::SortCondition const*,
                                   arangodb::aql::Variable const*,
                                   size_t itemsInIndex,

arangod/Indexes/Index.h

@@ -348,6 +348,8 @@ class Index {
   virtual int batchInsert(arangodb::Transaction*,
                           std::vector<TRI_doc_mptr_t const*> const*, size_t);
 
+  virtual int unload() = 0;
+
   // a garbage collection function for the index
   virtual int cleanup();
   // called when the index is dropped

arangod/Indexes/PrimaryIndex.cpp

@@ -202,10 +202,7 @@ void PrimaryIndex::toVelocyPack(VPackBuilder& builder, bool withFigures) const {
   builder.add("sparse", VPackValue(false));
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief return a VelocyPack representation of the index figures
-////////////////////////////////////////////////////////////////////////////////
-
 void PrimaryIndex::toVelocyPackFigures(VPackBuilder& builder) const {
   Index::toVelocyPackFigures(builder);
   _primaryIndex->appendToVelocyPack(builder);
@@ -219,10 +216,13 @@ int PrimaryIndex::remove(arangodb::Transaction*, TRI_doc_mptr_t const*, bool) {
   THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
 }
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief looks up an element given a key
-////////////////////////////////////////////////////////////////////////////////
+/// @brief unload the index data from memory
+int PrimaryIndex::unload() {
+  _primaryIndex->truncate([](TRI_doc_mptr_t*) -> bool { return true; });
+  return TRI_ERROR_NO_ERROR;
+}
 
+/// @brief looks up an element given a key
 TRI_doc_mptr_t* PrimaryIndex::lookup(arangodb::Transaction* trx,
                                      VPackSlice const& slice) const {
   TRI_ASSERT(slice.isArray() && slice.length() == 1);
@@ -232,10 +232,7 @@ TRI_doc_mptr_t* PrimaryIndex::lookup(arangodb::Transaction* trx,
   return _primaryIndex->findByKey(trx, tmp.begin());
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief looks up an element given a key
-////////////////////////////////////////////////////////////////////////////////
-
 TRI_doc_mptr_t* PrimaryIndex::lookupKey(arangodb::Transaction* trx,
                                         VPackSlice const& key) const {
   TRI_ASSERT(key.isString());

arangod/Indexes/PrimaryIndex.h

@@ -160,6 +160,8 @@ class PrimaryIndex final : public Index {
   int remove(arangodb::Transaction*, TRI_doc_mptr_t const*,
              bool) override final;
 
+  int unload() override final;
+
  public:
   TRI_doc_mptr_t* lookupKey(arangodb::Transaction*, VPackSlice const&) const;
 

arangod/Indexes/RocksDBIndex.cpp

@@ -477,21 +477,19 @@ int RocksDBIndex::remove(arangodb::Transaction* trx, TRI_doc_mptr_t const* doc,
   return res;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief called when the index is dropped
-////////////////////////////////////////////////////////////////////////////////
+int RocksDBIndex::unload() {
+  // nothing to do
+  return TRI_ERROR_NO_ERROR;
+}
 
+/// @brief called when the index is dropped
 int RocksDBIndex::drop() {
   return RocksDBFeature::instance()->dropIndex(_collection->_vocbase->id(), _collection->_info.id(), _iid);
 }
 
-////////////////////////////////////////////////////////////////////////////////
 /// @brief attempts to locate an entry in the index
-///
 /// Warning: who ever calls this function is responsible for destroying
 /// the RocksDBIterator* results
-////////////////////////////////////////////////////////////////////////////////
-
 RocksDBIterator* RocksDBIndex::lookup(arangodb::Transaction* trx,
                                       VPackSlice const searchValues,
                                       bool reverse) const {

arangod/Indexes/RocksDBIndex.h

@@ -56,7 +56,7 @@ class Transaction;
 /// @brief Iterator structure for RocksDB. We require a start and stop node
 ////////////////////////////////////////////////////////////////////////////////
 
-class RocksDBIterator : public IndexIterator {
+class RocksDBIterator final : public IndexIterator {
  private:
   friend class RocksDBIndex;
 
@@ -164,6 +164,8 @@ class RocksDBIndex final : public PathBasedIndex {
   int remove(arangodb::Transaction*, struct TRI_doc_mptr_t const*,
              bool) override final;
 
+  int unload() override final;
+
   int drop() override final;
 
   //////////////////////////////////////////////////////////////////////////////

arangod/Indexes/SkiplistIndex.cpp

@@ -863,6 +863,11 @@ int SkiplistIndex::remove(arangodb::Transaction*, TRI_doc_mptr_t const* doc,
   return res;
 }
 
+int SkiplistIndex::unload() {
+  _skiplistIndex->truncate();
+  return TRI_ERROR_NO_ERROR;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief Checks if the interval is valid. It is declared invalid if
 ///        one border is nullptr or the right is lower than left.

arangod/Indexes/SkiplistIndex.h

@@ -168,7 +168,7 @@ class SkiplistInLookupBuilder : public BaseSkiplistLookupBuilder {
 /// are non-empty.
 ////////////////////////////////////////////////////////////////////////////////
 
-class SkiplistIterator : public IndexIterator {
+class SkiplistIterator final : public IndexIterator {
  private:
   friend class SkiplistIndex;
 
@@ -223,7 +223,7 @@ class SkiplistIterator : public IndexIterator {
 /// are non-empty.
 ////////////////////////////////////////////////////////////////////////////////
 
-class SkiplistIterator2 : public IndexIterator {
+class SkiplistIterator2 final : public IndexIterator {
  private:
   // Shorthand for the skiplist node
   typedef arangodb::basics::SkipListNode<VPackSlice,
@@ -379,6 +379,8 @@ class SkiplistIndex final : public PathBasedIndex {
 
   int remove(arangodb::Transaction*, struct TRI_doc_mptr_t const*,
              bool) override final;
+  
+  int unload() override final;
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief attempts to locate an entry in the skip list index

lib/Basics/AssocMulti.h

@@ -169,6 +169,7 @@ class AssocMulti {
   IsEqualElementElementFuncType const _isEqualElementElementByKey;
 
   std::function<std::string()> _contextCallback;
+  IndexType _initialSize;
 
  public:
   AssocMulti(HashKeyFuncType hashKey, HashElementFuncType hashElement,
@@ -193,7 +194,8 @@ class AssocMulti {
         _isEqualKeyElement(isEqualKeyElement),
         _isEqualElementElement(isEqualElementElement),
         _isEqualElementElementByKey(isEqualElementElementByKey),
-        _contextCallback(contextCallback) {
+        _contextCallback(contextCallback),
+        _initialSize(initialSize) {
 
     // Make the number of buckets a power of two:
     size_t ex = 0;
@@ -508,28 +510,67 @@ class AssocMulti {
     }
     return res.load();
   }
+  
+  void truncate(CallbackElementFuncType callback) {
+    std::vector<EntryType*> empty;
+    empty.reserve(_buckets.size());
+    
+    try {
+      for (size_t i = 0; i < _buckets.size(); ++i) {
+        empty.emplace_back(new EntryType[static_cast<size_t>(_initialSize)]);
+      }
+
+      size_t i = 0;
+      for (auto& b : _buckets) {
+        invokeOnAllElements(callback, b);
+
+        // now bucket is empty
+        delete[] b._table;
+        b._table = empty[i];
+        b._nrAlloc = _initialSize;
+        b._nrUsed = 0;
+
+        empty[i] = nullptr; // pass ownership
+        ++i;
+      }
+    } catch (...) {
+      // prevent leaks
+      for (auto& it : empty) {
+        delete[] it;
+      }
+      throw;
+    }
+  }
 
-  //////////////////////////////////////////////////////////////////////////////
   /// @brief a method to iterate over all elements in the hash
-  //////////////////////////////////////////////////////////////////////////////
-
-  void invokeOnAllElements(CallbackElementFuncType callback) {
+  void invokeOnAllElements(CallbackElementFuncType const& callback) {
     for (auto& b : _buckets) {
       if (b._table == nullptr) {
         continue;
       }
 
-      for (size_t i = 0; i < b._nrAlloc; ++i) {
-        if (b._table[i].ptr == nullptr) {
-          continue;
-        }
-        if (!callback(b._table[i].ptr)) {
-          return;
-        }
+      if (!invokeOnAllElements(callback, b)) {
+        return;
       }
     }
   }
 
+  /// @brief a method to iterate over all elements in the hash
+  bool invokeOnAllElements(CallbackElementFuncType const& callback, Bucket& b) {
+    for (size_t i = 0; i < b._nrAlloc; ++i) {
+      if (b._table[i].ptr == nullptr) {
+        continue;
+      }
+      if (!callback(b._table[i].ptr)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+ private:
+  //////////////////////////////////////////////////////////////////////////////
+
  private:
   //////////////////////////////////////////////////////////////////////////////
   /// @brief adds a key/element to the array

lib/Basics/AssocUnique.h

@@ -330,6 +330,37 @@ class AssocUnique {
   }
 
  public:
+  void truncate(CallbackElementFuncType callback) {
+    std::vector<Element**> empty;
+    empty.reserve(_buckets.size());
+   
+    try { 
+      for (size_t i = 0; i < _buckets.size(); ++i) {
+        empty.emplace_back(new Element*[static_cast<size_t>(initialSize())]);
+      }
+
+      size_t i = 0;
+      for (auto& b : _buckets) {
+        invokeOnAllElements(callback, b);
+
+        // now bucket is empty
+        delete[] b._table;
+        b._table = empty[i];
+        b._nrAlloc = initialSize();
+        b._nrUsed = 0;
+        
+        empty[i] = nullptr; // pass ownership
+        ++i;
+      }
+    } catch (...) {
+      // prevent leaks
+      for (auto& it : empty) {
+        delete[] it;
+      }
+      throw;
+    }
+  }
+
   size_t buckets() const {
     return _buckets.size();
   }
@@ -829,26 +860,32 @@ class AssocUnique {
     return old;
   }
 
-  //////////////////////////////////////////////////////////////////////////////
   /// @brief a method to iterate over all elements in the hash. this method
   /// can NOT be used for deleting elements
-  //////////////////////////////////////////////////////////////////////////////
-
-  void invokeOnAllElements(CallbackElementFuncType callback) {
+  void invokeOnAllElements(CallbackElementFuncType const& callback) {
     for (auto& b : _buckets) {
       if (b._table == nullptr) {
         continue;
       }
-      for (size_t i = 0; i < b._nrAlloc; ++i) {
-        if (b._table[i] == nullptr) {
-          continue;
-        }
-        if (!callback(b._table[i])) {
-          return;
-        }
+      if (!invokeOnAllElements(callback, b)) {
+        return;
       }
     }
   }
+      
+  /// @brief a method to iterate over all elements in a bucket. this method
+  /// can NOT be used for deleting elements
+  bool invokeOnAllElements(CallbackElementFuncType const& callback, Bucket& b) {
+    for (size_t i = 0; i < b._nrAlloc; ++i) {
+      if (b._table[i] == nullptr) {
+        continue;
+      }
+      if (!callback(b._table[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief a method to iterate over all elements in the hash. this method

lib/Basics/SkipList.h

@@ -172,6 +172,10 @@ class SkipList {
   //////////////////////////////////////////////////////////////////////////////
 
   ~SkipList() {
+    truncate();
+  }
+
+  void truncate() {
     Node* p;
     Node* next;
 
@@ -186,6 +190,12 @@ class SkipList {
       p = next;
     }
     freeNode(_start);
+    
+    _memoryUsed = sizeof(SkipList);
+    _nrUsed = 0;
+    _start = allocNode(TRI_SKIPLIST_MAX_HEIGHT);
+    // Note that this can throw
+    _end = _start;
   }
 
   //////////////////////////////////////////////////////////////////////////////