Fixing recovery for geo index

arangod/RocksDBEngine/RocksDBGeoIndex.cpp

@@ -286,7 +286,7 @@ RocksDBGeoIndex::RocksDBGeoIndex(TRI_idx_iid_t iid,
   RocksDBKeyBounds b1 = RocksDBKeyBounds::GeoIndex(_objectId, false);
   iter->SeekForPrev(b1.end());
   if (iter->Valid()
-      && _cmp->Compare(iter->key(), b1.start()) >= 0
+      && _cmp->Compare(b1.start(), iter->key()) < 0
       && _cmp->Compare(iter->key(), b1.end()) < 0) {
     // found a key smaller than bounds end
     std::pair<bool, int32_t> pair = RocksDBKey::geoValues(iter->key());
@@ -298,7 +298,7 @@ RocksDBGeoIndex::RocksDBGeoIndex(TRI_idx_iid_t iid,
   RocksDBKeyBounds b2 = RocksDBKeyBounds::GeoIndex(_objectId, true);
   iter->SeekForPrev(b2.end());
   if (iter->Valid()
-      && _cmp->Compare(iter->key(), b2.start()) >= 0
+      && _cmp->Compare(b2.start(), iter->key()) < 0
       && _cmp->Compare(iter->key(), b2.end()) < 0) {
     // found a key smaller than bounds end
     std::pair<bool, int32_t> pair = RocksDBKey::geoValues(iter->key());
@@ -319,7 +319,12 @@ RocksDBGeoIndex::~RocksDBGeoIndex() {
 }
 
 size_t RocksDBGeoIndex::memory() const {
-  return GeoIndex_MemoryUsage(_geoIndex);
+  rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
+  RocksDBKeyBounds bounds = RocksDBKeyBounds::GeoIndex(_objectId);
+  rocksdb::Range r(bounds.start(), bounds.end());
+  uint64_t out;
+  db->GetApproximateSizes(&r, 1, &out, true);
+  return (size_t)out;
 }
 
 /// @brief return a JSON representation of the index
@@ -552,7 +557,7 @@ int RocksDBGeoIndex::removeRaw(rocksdb::WriteBatch*, TRI_voc_rid_t revisionId,
 
 int RocksDBGeoIndex::unload() {
   // create a new, empty index
-  auto empty = GeoIndex_new(_objectId, 0, 0);
+  /*auto empty = GeoIndex_new(_objectId, 0, 0);
 
   if (empty == nullptr) {
     THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
@@ -564,7 +569,7 @@ int RocksDBGeoIndex::unload() {
   }
 
   // and assign it
-  _geoIndex = empty;
+  _geoIndex = empty;*/
 
   return TRI_ERROR_NO_ERROR;
 }

arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp

@@ -485,15 +485,6 @@ GeoIdx* GeoIndex_new(uint64_t objectId,
     return (GeoIdx*)gix;
   }
 
-  gix->objectId = objectId;
-  if (numPots == 0 || numSlots == 0) { // first run
-    gix->nextFreePot = 2;
-    gix->nextFreeSlot = 1;
-  } else {
-    gix->nextFreePot = numPots + 1;
-    gix->nextFreeSlot = numSlots + 1;
-  }
-
   /* set up the fixed points structure  */
 
   for (i = 0; i < GeoIndexFIXEDPOINTS; i++) {
@@ -609,8 +600,12 @@ GeoIdx* GeoIndex_new(uint64_t objectId,
   }
   /* set up the root pot  */
   
+  gix->objectId = objectId;
+  if (numPots == 0 || numSlots == 0) { // first run
+    gix->nextFreePot = 2;
+    gix->nextFreeSlot = 1;
+    
     GeoPot gp;
-  //j = GeoIndexNewPot(gix);
     gp.LorLeaf = 0;   //leaf pot
     gp.RorPoints = 0; // with no points in it!
     gp.middle = 0ll;
@@ -618,17 +613,11 @@ GeoIdx* GeoIndex_new(uint64_t objectId,
     gp.end = 0x1FFFFFFFFFFFFFll;
     gp.level = 1;
     for (i = 0; i < GeoIndexFIXEDPOINTS; i++) gp.maxdist[i] = 0;
-
     PotWrite(gix, 1, &gp);
-
+  } else {
-  /*j = GeoIndexNewPot(gix);
+    gix->nextFreePot = numPots + 1;
-  gix->ypots[j].LorLeaf = 0;   //leaf pot
+    gix->nextFreeSlot = numSlots + 1;
-  gix->ypots[j].RorPoints = 0; // with no points in it!
+  }
-  gix->ypots[j].middle = 0ll;
-  gix->ypots[j].start = 0ll;
-  gix->ypots[j].end = 0x1FFFFFFFFFFFFFll;
-  gix->ypots[j].level = 1;
-  for (i = 0; i < GeoIndexFIXEDPOINTS; i++) gix->ypots[j].maxdist[i] = 0;*/
   return (GeoIdx*)gix;
 }
 /* =================================================== */
@@ -2445,15 +2434,5 @@ int GeoIndex_INDEXVALID(GeoIdx* gi) {
 }
 
 #endif
-
-  // change to Approximate memory
-size_t GeoIndex_MemoryUsage(void* theIndex) {
-  //GeoIx* geoIndex = (GeoIx*)theIndex;
-  //if (geoIndex != nullptr) {
-  //  return geoIndex->_memoryUsed;
-  //}
-#warning FIXME
-  return 0;
-}
 }}
 /* end of GeoIndex.c  */

arangod/RocksDBEngine/RocksDBGeoIndexImpl.h

@@ -91,8 +91,6 @@ typedef struct {
 typedef void GeoIdx;  /* to keep the structure private  */
 typedef void GeoCursor; /* to keep the structure private  */
 
-size_t GeoIndex_MemoryUsage(void*);
-
 GeoIdx* GeoIndex_new(uint64_t objectId, int slo, int);
 void GeoIndex_free(GeoIdx* gi);
 double GeoIndex_distance(GeoCoordinate* c1, GeoCoordinate* c2);

arangod/RocksDBEngine/RocksDBKey.cpp

@@ -87,13 +87,11 @@ RocksDBKey RocksDBKey::GeoIndexValue(uint64_t indexId, int32_t offset, bool isSl
   RocksDBKey key(RocksDBEntryType::GeoIndexValue);
   size_t length = sizeof(char) + sizeof(indexId) + sizeof(offset);
   key._buffer.reserve(length);
-  key._buffer.push_back(static_cast<char>(RocksDBEntryType::GeoIndexValue));
   uint64ToPersistent(key._buffer, indexId);
 
-  uint64_t norm = offset;
+  uint64_t norm = uint64_t(offset) << 32;
-  if (isSlot) norm |= uint64_t(1) << 63;//encode slot|pot in highest bit
+  norm |= isSlot ? 0xFFU : 0; //encode slot|pot in lowest bit
   uint64ToPersistent(key._buffer, norm);
-  
   return key;
 }
 
@@ -196,7 +194,7 @@ std::pair<bool, int32_t> RocksDBKey::geoValues(rocksdb::Slice const& slice) {
   RocksDBEntryType type = static_cast<RocksDBEntryType>(*slice.data());
   TRI_ASSERT(type == RocksDBEntryType::GeoIndexValue);
   uint64_t val = uint64FromPersistent(slice.data() + sizeof(char) + sizeof(uint64_t));
-  bool isSlot = val & 0xFF;// lowest byte is 0xFF if true
+  bool isSlot = val & 0xFFU;// lowest byte is 0xFF if true
   return std::pair<bool, int32_t>(isSlot, (val >> 32));
 }
 

arangod/RocksDBEngine/RocksDBKeyBounds.cpp

@@ -87,11 +87,13 @@ RocksDBKeyBounds RocksDBKeyBounds::GeoIndex(uint64_t indexId, bool isSlot) {
   b._startBuffer.reserve(length);
   b._startBuffer.push_back(static_cast<char>(RocksDBEntryType::GeoIndexValue));
   uint64ToPersistent(b._startBuffer, indexId);
+  
+  b._endBuffer.clear();
   b._endBuffer.append(b._startBuffer);// append common prefix
   
-  uint64_t norm = isSlot ? 0xFF : 0;//encode slot|pot in lowest bit
+  uint64_t norm = isSlot ? 0xFFU : 0;//encode slot|pot in lowest bit
   uint64ToPersistent(b._startBuffer, norm);// lower endian
-  norm |= 0xFFFFFFFFULL << 32;
+  norm = norm | (0xFFFFFFFFULL << 32);
   uint64ToPersistent(b._endBuffer, norm);
   return b;
 }
@@ -129,8 +131,9 @@ RocksDBKeyBounds RocksDBKeyBounds::FulltextIndexPrefix(uint64_t indexId,
   uint64ToPersistent(bounds._startBuffer, indexId);
   bounds._startBuffer.append(word.data(), word.length());
   
+  bounds._endBuffer.clear();
   bounds._endBuffer.append(bounds._startBuffer);
-  bounds._endBuffer.push_back(0xFF);// invalid UTF-8 character, higher than with memcmp
+  bounds._endBuffer.push_back(0xFFU);// invalid UTF-8 character, higher than with memcmp
   return bounds;
 }
 

tests/Basics/RocksDBKeyTest.cpp

@@ -29,7 +29,9 @@
 
 #include "catch.hpp"
 
+#include "RocksDBEngine/RocksDBComparator.h"
 #include "RocksDBEngine/RocksDBKey.h"
+#include "RocksDBEngine/RocksDBKeyBounds.h"
 #include "RocksDBEngine/RocksDBTypes.h"
 #include "Basics/Exceptions.h"
 
@@ -39,7 +41,7 @@ using namespace arangodb;
 // --SECTION--                                                        test suite
 // -----------------------------------------------------------------------------
 
-/// @brief setup
+/// @brief test RocksDBKey class
 TEST_CASE("RocksDBKeyTest", "[rocksdbkeytest]") {
 
 /// @brief test database
@@ -227,6 +229,26 @@ SECTION("test_edge_index") {
   CHECK(s1 == std::string("5\0\0\0\0\0\0\0\0a/1\0foobar\x06", 20));
   
 }
+}
 
+/// @brief test RocksDBKeyBounds class
+TEST_CASE("RocksDBKeyBoundsTest", "[rocksdbkeybounds]") {
+  
+/// @brief test geo index key and bounds consistency
+SECTION("test_geo_index") {
+  
+  RocksDBComparator cmp;
+  
+  RocksDBKey k1 = RocksDBKey::GeoIndexValue(256, 128, false);
+  RocksDBKeyBounds bb1 = RocksDBKeyBounds::GeoIndex(256, false);
+  
+  CHECK(cmp.Compare(k1.string(), bb1.start()) > 0);
+  CHECK(cmp.Compare(k1.string(), bb1.end()) < 0);
+  
+  RocksDBKey k2 = RocksDBKey::GeoIndexValue(256, 128, true);
+  RocksDBKeyBounds bb2 = RocksDBKeyBounds::GeoIndex(256, true);
+  CHECK(cmp.Compare(k2.string(), bb2.start()) > 0);
+  CHECK(cmp.Compare(k2.string(), bb2.end()) < 0);
+}
   
 }