Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fixing recovery for geo index

This commit is contained in:
Simon Grätzer 2017-05-09 22:29:29 +02:00
parent 3b7ef439c8
commit 42d230e4ae
6 changed files with 63 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
arangod/RocksDBEngine/RocksDBGeoIndex.cpp
View File

@ -286,7 +286,7 @@ RocksDBGeoIndex::RocksDBGeoIndex(TRI_idx_iid_t iid,
RocksDBKeyBounds b1 = RocksDBKeyBounds::GeoIndex(_objectId, false);
iter->SeekForPrev(b1.end());
if (iter->Valid()
&& _cmp->Compare(iter->key(), b1.start()) >= 0
&& _cmp->Compare(b1.start(), iter->key()) < 0
&& _cmp->Compare(iter->key(), b1.end()) < 0) {
// found a key smaller than bounds end
std::pair<bool, int32_t> pair = RocksDBKey::geoValues(iter->key());
@ -298,7 +298,7 @@ RocksDBGeoIndex::RocksDBGeoIndex(TRI_idx_iid_t iid,
RocksDBKeyBounds b2 = RocksDBKeyBounds::GeoIndex(_objectId, true);
iter->SeekForPrev(b2.end());
if (iter->Valid()
&& _cmp->Compare(iter->key(), b2.start()) >= 0
&& _cmp->Compare(b2.start(), iter->key()) < 0
&& _cmp->Compare(iter->key(), b2.end()) < 0) {
// found a key smaller than bounds end
std::pair<bool, int32_t> pair = RocksDBKey::geoValues(iter->key());
@ -319,7 +319,12 @@ RocksDBGeoIndex::~RocksDBGeoIndex() {
}
size_t RocksDBGeoIndex::memory() const {
return GeoIndex_MemoryUsage(_geoIndex);
rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
RocksDBKeyBounds bounds = RocksDBKeyBounds::GeoIndex(_objectId);
rocksdb::Range r(bounds.start(), bounds.end());
uint64_t out;
db->GetApproximateSizes(&r, 1, &out, true);
return (size_t)out;
}
/// @brief return a JSON representation of the index
@ -552,7 +557,7 @@ int RocksDBGeoIndex::removeRaw(rocksdb::WriteBatch*, TRI_voc_rid_t revisionId,
int RocksDBGeoIndex::unload() {
// create a new, empty index
auto empty = GeoIndex_new(_objectId, 0, 0);
/*auto empty = GeoIndex_new(_objectId, 0, 0);
if (empty == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
@ -564,7 +569,7 @@ int RocksDBGeoIndex::unload() {
}
// and assign it
_geoIndex = empty;
_geoIndex = empty;*/
return TRI_ERROR_NO_ERROR;
}

55
arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp
View File

@ -485,15 +485,6 @@ GeoIdx* GeoIndex_new(uint64_t objectId,
return (GeoIdx*)gix;
}
gix->objectId = objectId;
if (numPots == 0 || numSlots == 0) { // first run
gix->nextFreePot = 2;
gix->nextFreeSlot = 1;
} else {
gix->nextFreePot = numPots + 1;
gix->nextFreeSlot = numSlots + 1;
}
/* set up the fixed points structure */
for (i = 0; i < GeoIndexFIXEDPOINTS; i++) {
@ -609,26 +600,24 @@ GeoIdx* GeoIndex_new(uint64_t objectId,
}
/* set up the root pot */
GeoPot gp;
//j = GeoIndexNewPot(gix);
gp.LorLeaf = 0; //leaf pot
gp.RorPoints = 0; // with no points in it!
gp.middle = 0ll;
gp.start = 0ll;
gp.end = 0x1FFFFFFFFFFFFFll;
gp.level = 1;
for (i = 0; i < GeoIndexFIXEDPOINTS; i++) gp.maxdist[i] = 0;
gix->objectId = objectId;
if (numPots == 0 || numSlots == 0) { // first run
gix->nextFreePot = 2;
gix->nextFreeSlot = 1;
PotWrite(gix, 1, &gp);
/*j = GeoIndexNewPot(gix);
gix->ypots[j].LorLeaf = 0; //leaf pot
gix->ypots[j].RorPoints = 0; // with no points in it!
gix->ypots[j].middle = 0ll;
gix->ypots[j].start = 0ll;
gix->ypots[j].end = 0x1FFFFFFFFFFFFFll;
gix->ypots[j].level = 1;
for (i = 0; i < GeoIndexFIXEDPOINTS; i++) gix->ypots[j].maxdist[i] = 0;*/
GeoPot gp;
gp.LorLeaf = 0; //leaf pot
gp.RorPoints = 0; // with no points in it!
gp.middle = 0ll;
gp.start = 0ll;
gp.end = 0x1FFFFFFFFFFFFFll;
gp.level = 1;
for (i = 0; i < GeoIndexFIXEDPOINTS; i++) gp.maxdist[i] = 0;
PotWrite(gix, 1, &gp);
} else {
gix->nextFreePot = numPots + 1;
gix->nextFreeSlot = numSlots + 1;
}
return (GeoIdx*)gix;
}
/* =================================================== */
@ -2445,15 +2434,5 @@ int GeoIndex_INDEXVALID(GeoIdx* gi) {
}
#endif
// change to Approximate memory
size_t GeoIndex_MemoryUsage(void* theIndex) {
//GeoIx* geoIndex = (GeoIx*)theIndex;
//if (geoIndex != nullptr) {
// return geoIndex->_memoryUsed;
//}
#warning FIXME
return 0;
}
}}
/* end of GeoIndex.c */

2
arangod/RocksDBEngine/RocksDBGeoIndexImpl.h
View File

@ -91,8 +91,6 @@ typedef struct {
typedef void GeoIdx; /* to keep the structure private */
typedef void GeoCursor; /* to keep the structure private */
size_t GeoIndex_MemoryUsage(void*);
GeoIdx* GeoIndex_new(uint64_t objectId, int slo, int);
void GeoIndex_free(GeoIdx* gi);
double GeoIndex_distance(GeoCoordinate* c1, GeoCoordinate* c2);

8
arangod/RocksDBEngine/RocksDBKey.cpp
View File

@ -87,13 +87,11 @@ RocksDBKey RocksDBKey::GeoIndexValue(uint64_t indexId, int32_t offset, bool isSl
RocksDBKey key(RocksDBEntryType::GeoIndexValue);
size_t length = sizeof(char) + sizeof(indexId) + sizeof(offset);
key._buffer.reserve(length);
key._buffer.push_back(static_cast<char>(RocksDBEntryType::GeoIndexValue));
uint64ToPersistent(key._buffer, indexId);
uint64_t norm = offset;
if (isSlot) norm |= uint64_t(1) << 63;//encode slot|pot in highest bit
uint64_t norm = uint64_t(offset) << 32;
norm |= isSlot ? 0xFFU : 0; //encode slot|pot in lowest bit
uint64ToPersistent(key._buffer, norm);
return key;
}
@ -196,7 +194,7 @@ std::pair<bool, int32_t> RocksDBKey::geoValues(rocksdb::Slice const& slice) {
RocksDBEntryType type = static_cast<RocksDBEntryType>(*slice.data());
TRI_ASSERT(type == RocksDBEntryType::GeoIndexValue);
uint64_t val = uint64FromPersistent(slice.data() + sizeof(char) + sizeof(uint64_t));
bool isSlot = val & 0xFF;// lowest byte is 0xFF if true
bool isSlot = val & 0xFFU;// lowest byte is 0xFF if true
return std::pair<bool, int32_t>(isSlot, (val >> 32));
}

9
arangod/RocksDBEngine/RocksDBKeyBounds.cpp
View File

@ -87,11 +87,13 @@ RocksDBKeyBounds RocksDBKeyBounds::GeoIndex(uint64_t indexId, bool isSlot) {
b._startBuffer.reserve(length);
b._startBuffer.push_back(static_cast<char>(RocksDBEntryType::GeoIndexValue));
uint64ToPersistent(b._startBuffer, indexId);
b._endBuffer.clear();
b._endBuffer.append(b._startBuffer);// append common prefix
uint64_t norm = isSlot ? 0xFF : 0;//encode slot|pot in lowest bit
uint64_t norm = isSlot ? 0xFFU : 0;//encode slot|pot in lowest bit
uint64ToPersistent(b._startBuffer, norm);// lower endian
norm |= 0xFFFFFFFFULL << 32;
norm = norm | (0xFFFFFFFFULL << 32);
uint64ToPersistent(b._endBuffer, norm);
return b;
}
@ -129,8 +131,9 @@ RocksDBKeyBounds RocksDBKeyBounds::FulltextIndexPrefix(uint64_t indexId,
uint64ToPersistent(bounds._startBuffer, indexId);
bounds._startBuffer.append(word.data(), word.length());
bounds._endBuffer.clear();
bounds._endBuffer.append(bounds._startBuffer);
bounds._endBuffer.push_back(0xFF);// invalid UTF-8 character, higher than with memcmp
bounds._endBuffer.push_back(0xFFU);// invalid UTF-8 character, higher than with memcmp
return bounds;
}

24
tests/Basics/RocksDBKeyTest.cpp
View File

@ -29,7 +29,9 @@
#include "catch.hpp"
#include "RocksDBEngine/RocksDBComparator.h"
#include "RocksDBEngine/RocksDBKey.h"
#include "RocksDBEngine/RocksDBKeyBounds.h"
#include "RocksDBEngine/RocksDBTypes.h"
#include "Basics/Exceptions.h"
@ -39,7 +41,7 @@ using namespace arangodb;
// --SECTION-- test suite
// -----------------------------------------------------------------------------
/// @brief setup
/// @brief test RocksDBKey class
TEST_CASE("RocksDBKeyTest", "[rocksdbkeytest]") {
/// @brief test database
@ -227,6 +229,26 @@ SECTION("test_edge_index") {
CHECK(s1 == std::string("5\0\0\0\0\0\0\0\0a/1\0foobar\x06", 20));
}
}
/// @brief test RocksDBKeyBounds class
TEST_CASE("RocksDBKeyBoundsTest", "[rocksdbkeybounds]") {
/// @brief test geo index key and bounds consistency
SECTION("test_geo_index") {
RocksDBComparator cmp;
RocksDBKey k1 = RocksDBKey::GeoIndexValue(256, 128, false);
RocksDBKeyBounds bb1 = RocksDBKeyBounds::GeoIndex(256, false);
CHECK(cmp.Compare(k1.string(), bb1.start()) > 0);
CHECK(cmp.Compare(k1.string(), bb1.end()) < 0);
RocksDBKey k2 = RocksDBKey::GeoIndexValue(256, 128, true);
RocksDBKeyBounds bb2 = RocksDBKeyBounds::GeoIndex(256, true);
CHECK(cmp.Compare(k2.string(), bb2.start()) > 0);
CHECK(cmp.Compare(k2.string(), bb2.end()) < 0);
}
}