diff --git a/arangod/RocksDBEngine/RocksDBCommon.h b/arangod/RocksDBEngine/RocksDBCommon.h index ffba19c4a6..040d62d5f6 100644 --- a/arangod/RocksDBEngine/RocksDBCommon.h +++ b/arangod/RocksDBEngine/RocksDBCommon.h @@ -78,6 +78,47 @@ class Methods; } namespace rocksutils { +template +typename std::enable_if::value,void>::type +toPersistent(T in, char* out){ + using TT = typename std::decay::type; + std::memcpy(out, &in, sizeof(TT)); + out += sizeof(TT); +} + +template ::type>::value, int>::type = 0 + > +typename std::decay::type fromPersistent(char const* in){ + using TT = typename std::decay::type; + TT out; + std::memcpy(&out, in, sizeof(TT)); + in += sizeof(TT); + return out; +} + +template ::type>::value, int>::type = 2 + > +typename std::decay::type fromPersistent(StringLike& in){ + using TT = typename std::decay::type; + TT out; + std::memcpy(&out, in.data(), sizeof(TT)); + return out; +} + +inline uint64_t doubleToInt(double d){ + uint64_t i; + std::memcpy(&i, &d, sizeof(i)); + return i; +} + +inline double intToDouble(uint64_t i){ + double d; + std::memcpy(&d, &i, sizeof(i)); + return d; +} + uint64_t uint64FromPersistent(char const* p); void uint64ToPersistent(char* p, uint64_t value); void uint64ToPersistent(std::string& out, uint64_t value); diff --git a/arangod/RocksDBEngine/RocksDBGeoIndex.h b/arangod/RocksDBEngine/RocksDBGeoIndex.h index b7b30ac860..12ad184ca3 100644 --- a/arangod/RocksDBEngine/RocksDBGeoIndex.h +++ b/arangod/RocksDBEngine/RocksDBGeoIndex.h @@ -33,6 +33,7 @@ #include #include +#include namespace arangodb { diff --git a/arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp b/arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp index 80e1f11c8f..740ec38a1e 100644 --- a/arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp +++ b/arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp @@ -99,14 +99,14 @@ typedef struct { /* only used for a leaf pot. */ /* =================================================== */ typedef struct { - int LorLeaf; - int RorPoints; + int32_t LorLeaf; + int32_t RorPoints; GeoString middle; GeoFix maxdist[GeoIndexFIXEDPOINTS]; GeoString start; GeoString end; - int level; - int points[GeoIndexPOTSIZE]; + int32_t level; + int32_t points[GeoIndexPOTSIZE]; } GeoPot; /* =================================================== */ /* GeoIx structure */ @@ -266,8 +266,98 @@ typedef struct { #include namespace arangodb { namespace rocksdbengine { - - + +GeoCoordinate& fromPersistent(char const* in, GeoCoordinate& out){ + const char* start = in; + + //convert latituide and longitute to uint64 for network transfer / storage + uint64_t fromStorage = rocksutils::fromPersistent(start); + start += sizeof(uint64_t); + out.latitude = rocksutils::intToDouble(fromStorage); + + fromStorage = rocksutils::fromPersistent(start); + start += sizeof(uint64_t); + out.longitude = rocksutils::intToDouble(fromStorage); + + out.data = rocksutils::fromPersistent(start); + start += sizeof(uint64_t); + + return out; +} + +void toPersistent(GeoCoordinate& in, char* out){ + char* start = out; + + uint64_t toStorage = rocksutils::doubleToInt(in.latitude); + rocksutils::toPersistent(toStorage, start); + start += sizeof(in.latitude); + + toStorage = rocksutils::doubleToInt(in.longitude); + rocksutils::toPersistent(toStorage, start); + start += sizeof(in.longitude); + + rocksutils::toPersistent(in.data, start); + start += sizeof(in.data); +} + +GeoPot& fromPersistent(char const* in, GeoPot& out){ + const char* start = in; + + out.LorLeaf = rocksutils::fromPersistent(start); + start += sizeof(int32_t); + out.RorPoints = rocksutils::fromPersistent(start); + start += sizeof(int32_t); + out.middle = rocksutils::fromPersistent(start); + start += sizeof(GeoString); + + for(std::size_t i = 0; i < GeoIndexFIXEDPOINTS; i++){ + out.maxdist[i] = rocksutils::fromPersistent(start); + start += sizeof(GeoFix); + } + + out.start = rocksutils::fromPersistent(start); + start += sizeof(GeoString); + out.end = rocksutils::fromPersistent(start); + start += sizeof(GeoString); + out.level = rocksutils::fromPersistent(start); + start += sizeof(int32_t); + + for(std::size_t i = 0; i < GeoIndexFIXEDPOINTS; i++){ + out.points[i] = rocksutils::fromPersistent(start); + start += sizeof(int32_t); + } + + return out; +} + +void toPersistent(GeoPot const& in, char* out){ + char* start = out; + + rocksutils::toPersistent(in.LorLeaf, start); + start += sizeof(int32_t); + rocksutils::toPersistent(in.RorPoints, start); + start += sizeof(int32_t); + rocksutils::toPersistent(in.middle, start); + start += sizeof(GeoString); + + for(std::size_t i = 0; i< GeoIndexFIXEDPOINTS; i++){ + rocksutils::toPersistent(in.maxdist[i], start); + start += sizeof(GeoFix); + } + + rocksutils::toPersistent(in.start, start); + start += sizeof(GeoString); + rocksutils::toPersistent(in.end, start); + start += sizeof(GeoString); + rocksutils::toPersistent(in.level, start); + start += sizeof(int32_t); + + for(std::size_t i = 0; i< GeoIndexFIXEDPOINTS; i++){ + rocksutils::toPersistent(in.points[i], start); + start += sizeof(int32_t); + } +} + /* CRUD interface */ void GeoIndex_setRocksMethods(GeoIdx* gi, RocksDBMethods* trx) { @@ -279,14 +369,14 @@ void GeoIndex_clearRocks(GeoIdx* gi) { GeoIx* gix = (GeoIx*)gi; gix->rocksMethods = nullptr; } - + inline void RocksRead(GeoIx * gix, RocksDBKey const& key, std::string *val) { arangodb::Result r = gix->rocksMethods->Get(RocksDBColumnFamily::geo(), key, val); if (!r.ok()) { THROW_ARANGO_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage()); } } - + inline void RocksWrite(GeoIx * gix, RocksDBKey const& key, rocksdb::Slice const& slice) { @@ -322,13 +412,23 @@ void SlotRead(GeoIx * gix, int slot, GeoCoordinate * gc /*out param*/) RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, slot, true); std::string slotValue; RocksRead(gix, key, &slotValue); - memcpy(gc, slotValue.data(), slotValue.size()); + fromPersistent(slotValue.data(),*gc); + //memcpy(gc, slotValue.data(), slotValue.size()); } void SlotWrite(GeoIx * gix,int slot, GeoCoordinate * gc) { RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, slot, true); - RocksWrite(gix, key, rocksdb::Slice((char*)gc, - sizeof(GeoCoordinate))); + char data[sizeof (GeoCoordinate)]; + toPersistent(*gc, &data[0]); + RocksWrite(gix, key, rocksdb::Slice(&data[0], sizeof(GeoCoordinate))); + + GeoCoordinate test; + fromPersistent(&data[0],test); + // RocksWrite(gix, key, rocksdb::Slice((char*)gc, sizeof(GeoCoordinate))); + + TRI_ASSERT(test.longitude == gc->longitude); + TRI_ASSERT(test.latitude == gc->latitude); + TRI_ASSERT(test.data == gc->data); } void PotRead(GeoIx * gix, int pot, GeoPot * gp) @@ -336,12 +436,21 @@ void PotRead(GeoIx * gix, int pot, GeoPot * gp) RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, pot, false); std::string potValue; RocksRead(gix, key, &potValue); - memcpy(gp, potValue.data(), potValue.size()); + TRI_ASSERT(potValue.size() == sizeof(GeoPot)); + fromPersistent(potValue.data(), *gp); + //memcpy(gp, potValue.data(), potValue.size()); } void PotWrite(GeoIx * gix, int pot, GeoPot * gp) { RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, pot, false); - RocksWrite(gix, key, rocksdb::Slice((char*)gp, sizeof(GeoPot))); + char data[sizeof (GeoPot)]; + toPersistent(*gp, &data[0]); + RocksWrite(gix, key, rocksdb::Slice(&data[0], sizeof(GeoPot))); + //RocksWrite(gix, key, rocksdb::Slice((char*)gp, sizeof(GeoPot))); + + GeoPot test; + fromPersistent(&data[0],test); + TRI_ASSERT(test.level == gp->level); } /* =================================================== */ diff --git a/arangod/RocksDBEngine/RocksDBGeoIndexImpl.h b/arangod/RocksDBEngine/RocksDBGeoIndexImpl.h index a7409dd414..e28db88ae2 100644 --- a/arangod/RocksDBEngine/RocksDBGeoIndexImpl.h +++ b/arangod/RocksDBEngine/RocksDBGeoIndexImpl.h @@ -37,7 +37,7 @@ namespace rocksdbengine { /* first the things that a user might want to change */ /* a GeoString - a signed type of at least 64 bits */ -typedef std::uint_fast64_t GeoString; +typedef std::uint64_t GeoString; /* percentage growth of slot or slotslot tables */ #define GeoIndexGROW 50 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9cdf7e4b64..13c95c9bf6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -55,6 +55,7 @@ add_executable( Geo/georeg.cpp Pregel/typedbuffer.cpp RocksDBEngine/IndexEstimatorTest.cpp + RocksDBEngine/TypeConversionTest.cpp main.cpp ) diff --git a/tests/RocksDBEngine/TypeConversionTest.cpp b/tests/RocksDBEngine/TypeConversionTest.cpp new file mode 100644 index 0000000000..9d605973b0 --- /dev/null +++ b/tests/RocksDBEngine/TypeConversionTest.cpp @@ -0,0 +1,81 @@ +//////////////////////////////////////////////////////////////////////////////// +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2004-2012 triagens GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is triAGENS GmbH, Cologne, Germany +/// +/// @author Jan Christoph Uhde +/// @author Copyright 2017, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + + +#include "catch.hpp" +#include "RocksDBEngine/RocksDBCommon.h" + +#include +#include + +using namespace arangodb; + +// ----------------------------------------------------------------------------- +// --SECTION-- test suite +// ----------------------------------------------------------------------------- + +using namespace arangodb::rocksutils; +// @brief setup + +void doFromToTest(double num){ + CHECK( (num == intToDouble(doubleToInt(num))) ); +} + +template +void doFromToTest(T num){ + T x = num , y; + char s[sizeof(x)]; + toPersistent(x,&s[0]); + y = fromPersistent(&s[0]); + CHECK((x == y)); +} + +TEST_CASE("TypeConversion", "[type_conv]") { + +// @brief Test fixme + +SECTION("test_from_to_persist_uint64"){ + doFromToTest(std::numeric_limits::min()); + doFromToTest(std::numeric_limits::max()/2); + doFromToTest(std::numeric_limits::max()); +} + + +SECTION("test_from_to_persist_int32"){ + doFromToTest(std::numeric_limits::min()); + doFromToTest(std::numeric_limits::lowest()); + doFromToTest(std::numeric_limits::max()/2); + doFromToTest(std::numeric_limits::max()); +} + +// @brief generate tests +SECTION("test_from_to_double"){ + doFromToTest(std::numeric_limits::min()); + doFromToTest(std::numeric_limits::lowest()); + doFromToTest(std::numeric_limits::max()/2); + doFromToTest(std::numeric_limits::max()); +} +}