Prepare geodata to be more platform independent

arangod/RocksDBEngine/RocksDBCommon.h

@@ -78,6 +78,47 @@ class Methods;
 }
 namespace rocksutils {
 
+template <typename T>
+typename std::enable_if<std::is_integral<T>::value,void>::type
+toPersistent(T in, char* out){
+  using TT = typename std::decay<T>::type;
+  std::memcpy(out, &in, sizeof(TT));
+  out += sizeof(TT);
+}
+
+template <typename T,
+          typename std::enable_if<std::is_integral<typename std::remove_reference<T>::type>::value, int>::type = 0
+         >
+typename std::decay<T>::type fromPersistent(char const* in){
+  using TT = typename std::decay<T>::type;
+  TT out;
+  std::memcpy(&out, in, sizeof(TT));
+  in += sizeof(TT);
+  return out;
+}
+
+template <typename T, typename StringLike,
+          typename std::enable_if<std::is_integral<typename std::remove_reference<T>::type>::value, int>::type = 2
+         >
+typename std::decay<T>::type fromPersistent(StringLike& in){
+  using TT = typename std::decay<T>::type;
+  TT out;
+  std::memcpy(&out, in.data(), sizeof(TT));
+  return out;
+}
+
+inline uint64_t doubleToInt(double d){
+  uint64_t i;
+  std::memcpy(&i, &d, sizeof(i));
+  return i;
+}
+
+inline double intToDouble(uint64_t i){
+  double d;
+  std::memcpy(&d, &i, sizeof(i));
+  return d;
+}
+
 uint64_t uint64FromPersistent(char const* p);
 void uint64ToPersistent(char* p, uint64_t value);
 void uint64ToPersistent(std::string& out, uint64_t value);

arangod/RocksDBEngine/RocksDBGeoIndex.h

@@ -33,6 +33,7 @@
 
 #include <velocypack/Builder.h>
 #include <velocypack/velocypack-aliases.h>
+#include <type_traits>
 
 namespace arangodb {
 

arangod/RocksDBEngine/RocksDBGeoIndexImpl.cpp

@@ -99,14 +99,14 @@ typedef struct {
 /* only used for a leaf pot.                           */
 /* =================================================== */
 typedef struct {
-  int LorLeaf;
-  int RorPoints;
+  int32_t LorLeaf;
+  int32_t RorPoints;
   GeoString middle;
   GeoFix maxdist[GeoIndexFIXEDPOINTS];
   GeoString start;
   GeoString end;
-  int level;
-  int points[GeoIndexPOTSIZE];
+  int32_t level;
+  int32_t points[GeoIndexPOTSIZE];
 } GeoPot;
 /* =================================================== */
 /*                 GeoIx structure                     */
@@ -266,8 +266,98 @@ typedef struct {
 #include <StorageEngine/EngineSelectorFeature.h>
 
 namespace arangodb { namespace rocksdbengine {
-  
-  
+
+GeoCoordinate& fromPersistent(char const* in, GeoCoordinate& out){
+  const char* start = in;
+
+  //convert latituide and longitute to uint64 for network transfer / storage
+  uint64_t fromStorage = rocksutils::fromPersistent<uint64_t>(start);
+  start += sizeof(uint64_t);
+  out.latitude = rocksutils::intToDouble(fromStorage);
+
+  fromStorage = rocksutils::fromPersistent<uint64_t>(start);
+  start += sizeof(uint64_t);
+  out.longitude = rocksutils::intToDouble(fromStorage);
+
+  out.data = rocksutils::fromPersistent<uint64_t>(start);
+  start += sizeof(uint64_t);
+
+  return out;
+}
+
+void toPersistent(GeoCoordinate& in, char* out){
+  char* start = out;
+
+  uint64_t toStorage = rocksutils::doubleToInt(in.latitude);
+  rocksutils::toPersistent(toStorage, start);
+  start += sizeof(in.latitude);
+ 
+  toStorage = rocksutils::doubleToInt(in.longitude);
+  rocksutils::toPersistent(toStorage, start);
+  start += sizeof(in.longitude);
+
+  rocksutils::toPersistent(in.data, start);
+  start += sizeof(in.data);
+}
+
+GeoPot& fromPersistent(char const* in, GeoPot& out){
+  const char* start = in;
+
+  out.LorLeaf = rocksutils::fromPersistent<int32_t>(start);
+  start += sizeof(int32_t);
+  out.RorPoints = rocksutils::fromPersistent<int32_t>(start);
+  start += sizeof(int32_t);
+  out.middle = rocksutils::fromPersistent<GeoString>(start);
+  start += sizeof(GeoString);
+
+  for(std::size_t i = 0; i < GeoIndexFIXEDPOINTS; i++){
+    out.maxdist[i] = rocksutils::fromPersistent<GeoFix>(start);
+    start += sizeof(GeoFix);
+  }
+
+  out.start = rocksutils::fromPersistent<GeoString>(start);
+  start += sizeof(GeoString);
+  out.end = rocksutils::fromPersistent<GeoString>(start);
+  start += sizeof(GeoString);
+  out.level = rocksutils::fromPersistent<int32_t>(start);
+  start += sizeof(int32_t);
+
+  for(std::size_t i = 0; i < GeoIndexFIXEDPOINTS; i++){
+    out.points[i] = rocksutils::fromPersistent<int32_t>(start);
+    start += sizeof(int32_t);
+  }
+
+  return out;
+}
+
+void toPersistent(GeoPot const& in, char* out){
+  char* start = out;
+
+  rocksutils::toPersistent(in.LorLeaf, start);
+  start += sizeof(int32_t);
+  rocksutils::toPersistent(in.RorPoints, start);
+  start += sizeof(int32_t);
+  rocksutils::toPersistent(in.middle, start);
+  start += sizeof(GeoString);
+
+  for(std::size_t i = 0; i< GeoIndexFIXEDPOINTS; i++){
+    rocksutils::toPersistent(in.maxdist[i], start);
+    start += sizeof(GeoFix);
+  }
+
+  rocksutils::toPersistent(in.start, start);
+  start += sizeof(GeoString);
+  rocksutils::toPersistent(in.end, start);
+  start += sizeof(GeoString);
+  rocksutils::toPersistent(in.level, start);
+  start += sizeof(int32_t);
+
+  for(std::size_t i = 0; i< GeoIndexFIXEDPOINTS; i++){
+    rocksutils::toPersistent(in.points[i], start);
+    start += sizeof(int32_t);
+  }
+}
+
 /* CRUD interface */
 
 void GeoIndex_setRocksMethods(GeoIdx* gi, RocksDBMethods* trx) {
@@ -279,14 +369,14 @@ void GeoIndex_clearRocks(GeoIdx* gi) {
   GeoIx* gix = (GeoIx*)gi;
   gix->rocksMethods = nullptr;
 }
-  
+
 inline void RocksRead(GeoIx * gix, RocksDBKey const& key, std::string *val) {
   arangodb::Result r = gix->rocksMethods->Get(RocksDBColumnFamily::geo(), key, val);
   if (!r.ok()) {
     THROW_ARANGO_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage());
   }
 }
-  
+
 inline void RocksWrite(GeoIx * gix,
                        RocksDBKey const& key,
                        rocksdb::Slice const& slice) {
@@ -322,13 +412,23 @@ void SlotRead(GeoIx * gix, int slot, GeoCoordinate * gc /*out param*/)
   RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, slot, true);
   std::string slotValue;
   RocksRead(gix, key, &slotValue);
-  memcpy(gc, slotValue.data(), slotValue.size());
+  fromPersistent(slotValue.data(),*gc);
+  //memcpy(gc, slotValue.data(), slotValue.size());
 }
 void SlotWrite(GeoIx * gix,int slot, GeoCoordinate * gc)
 {
   RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, slot, true);
-  RocksWrite(gix, key, rocksdb::Slice((char*)gc,
-                                      sizeof(GeoCoordinate)));
+  char data[sizeof (GeoCoordinate)];
+  toPersistent(*gc, &data[0]);
+  RocksWrite(gix, key, rocksdb::Slice(&data[0], sizeof(GeoCoordinate)));
+
+  GeoCoordinate test;
+  fromPersistent(&data[0],test);
+  // RocksWrite(gix, key, rocksdb::Slice((char*)gc, sizeof(GeoCoordinate)));
+
+  TRI_ASSERT(test.longitude == gc->longitude);
+  TRI_ASSERT(test.latitude == gc->latitude);
+  TRI_ASSERT(test.data == gc->data);
 }
 
 void PotRead(GeoIx * gix, int pot, GeoPot * gp)
@@ -336,12 +436,21 @@ void PotRead(GeoIx * gix, int pot, GeoPot * gp)
   RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, pot, false);
   std::string potValue;
   RocksRead(gix, key, &potValue);
-  memcpy(gp, potValue.data(), potValue.size());
+  TRI_ASSERT(potValue.size() == sizeof(GeoPot));
+  fromPersistent(potValue.data(), *gp);
+  //memcpy(gp, potValue.data(), potValue.size());
 }
   
 void PotWrite(GeoIx * gix, int pot, GeoPot * gp) {
   RocksDBKey key = RocksDBKey::GeoIndexValue(gix->objectId, pot, false);
-  RocksWrite(gix, key, rocksdb::Slice((char*)gp, sizeof(GeoPot)));
+  char data[sizeof (GeoPot)];
+  toPersistent(*gp, &data[0]);
+  RocksWrite(gix, key, rocksdb::Slice(&data[0], sizeof(GeoPot)));
+  //RocksWrite(gix, key, rocksdb::Slice((char*)gp, sizeof(GeoPot)));
+  
+  GeoPot test;
+  fromPersistent(&data[0],test);
+  TRI_ASSERT(test.level == gp->level);
 }
 
 /* =================================================== */

arangod/RocksDBEngine/RocksDBGeoIndexImpl.h

@@ -37,7 +37,7 @@ namespace rocksdbengine {
 /* first the things that a user might want to change */
 
 /* a GeoString - a signed type of at least 64 bits   */
-typedef std::uint_fast64_t GeoString;
+typedef std::uint64_t GeoString;
 
 /* percentage growth of slot or slotslot tables     */
 #define GeoIndexGROW 50

tests/CMakeLists.txt

@@ -55,6 +55,7 @@ add_executable(
   Geo/georeg.cpp
   Pregel/typedbuffer.cpp
   RocksDBEngine/IndexEstimatorTest.cpp
+  RocksDBEngine/TypeConversionTest.cpp
   main.cpp
 )
 

tests/RocksDBEngine/TypeConversionTest.cpp

@@ -0,0 +1,81 @@
+////////////////////////////////////////////////////////////////////////////////
+///
+/// @file
+///
+/// DISCLAIMER
+///
+/// Copyright 2004-2012 triagens GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is triAGENS GmbH, Cologne, Germany
+///
+/// @author Jan Christoph Uhde
+/// @author Copyright 2017, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+
+#include "catch.hpp"
+#include "RocksDBEngine/RocksDBCommon.h"
+
+#include <vector>
+#include <limits>
+
+using namespace arangodb;
+
+// -----------------------------------------------------------------------------
+// --SECTION--                                                        test suite
+// -----------------------------------------------------------------------------
+
+using namespace arangodb::rocksutils;
+// @brief setup
+
+void doFromToTest(double num){
+  CHECK( (num == intToDouble(doubleToInt(num))) );
+}
+
+template <typename T>
+void doFromToTest(T num){
+  T x = num , y;
+  char s[sizeof(x)];
+  toPersistent(x,&s[0]);
+  y = fromPersistent<decltype(y)>(&s[0]);
+  CHECK((x == y));
+}
+
+TEST_CASE("TypeConversion", "[type_conv]") {
+
+// @brief Test fixme
+
+SECTION("test_from_to_persist_uint64"){
+  doFromToTest(std::numeric_limits<uint64_t>::min());
+  doFromToTest(std::numeric_limits<uint64_t>::max()/2);
+  doFromToTest(std::numeric_limits<uint64_t>::max());
+}
+
+
+SECTION("test_from_to_persist_int32"){
+  doFromToTest(std::numeric_limits<int32_t>::min());
+  doFromToTest(std::numeric_limits<int32_t>::lowest());
+  doFromToTest(std::numeric_limits<int32_t>::max()/2);
+  doFromToTest(std::numeric_limits<int32_t>::max());
+}
+
+// @brief generate tests
+SECTION("test_from_to_double"){
+  doFromToTest(std::numeric_limits<double>::min());
+  doFromToTest(std::numeric_limits<double>::lowest());
+  doFromToTest(std::numeric_limits<double>::max()/2);
+  doFromToTest(std::numeric_limits<double>::max());
+}
+}