updated vpack library

3rdParty/velocypack/include/velocypack/AttributeTranslator.h

@@ -0,0 +1,74 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Library to build up VPack documents.
+///
+/// DISCLAIMER
+///
+/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+/// @author Jan Steemann
+/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef VELOCYPACK_ATTRIBUTETRANSLATOR_H
+#define VELOCYPACK_ATTRIBUTETRANSLATOR_H 1
+
+#include <cstdint>
+#include <unordered_map>
+#include <memory>
+
+#include "velocypack/velocypack-common.h"
+
+namespace arangodb {
+  namespace velocypack {
+    class Builder;
+
+    class AttributeTranslator {
+
+      public:
+
+        AttributeTranslator (AttributeTranslator const&) = delete;
+        AttributeTranslator& operator= (AttributeTranslator const&) = delete;
+
+        AttributeTranslator () 
+          : _builder() {
+        }
+
+        ~AttributeTranslator () {
+        }
+
+        void add (std::string const& key, uint64_t id);
+
+        void seal ();
+
+        // translate from string to id
+        uint8_t const* translate (std::string const& key) const;
+
+        // translate from id to string
+        uint8_t const* translate (uint64_t id) const;
+    
+      private:
+
+        std::unique_ptr<Builder> _builder;
+        std::unordered_map<std::string, uint8_t const*> _keyToId;
+        std::unordered_map<uint64_t, uint8_t const*> _idToKey;
+    };
+
+  }  // namespace arangodb::velocypack
+}  // namespace arangodb
+
+#endif

3rdParty/velocypack/include/velocypack/Builder.h

@@ -34,6 +34,7 @@
 #include <memory>
 
 #include "velocypack/velocypack-common.h"
+#include "velocypack/AttributeTranslator.h"
 #include "velocypack/Buffer.h"
 #include "velocypack/Exception.h"
 #include "velocypack/Options.h"
@@ -129,7 +130,7 @@ namespace arangodb {
         Options const* options;
 
         // Constructor and destructor:
-        Builder (std::shared_ptr<Buffer<uint8_t>>& buffer, Options const* options = &Options::Defaults)
+        explicit Builder (std::shared_ptr<Buffer<uint8_t>>& buffer, Options const* options = &Options::Defaults)
           : _buffer(buffer),
             _pos(0),
             options(options) {
@@ -142,7 +143,7 @@ namespace arangodb {
           }
         }
 
-        Builder (Options const* options = &Options::Defaults)
+        explicit Builder (Options const* options = &Options::Defaults)
           : _buffer(new Buffer<uint8_t>()),
             _pos(0),
             options(options) {
@@ -442,6 +443,17 @@ namespace arangodb {
             }
             reportAdd(tos);
           }
+
+          if (options->attributeTranslator != nullptr) {
+            // check if a translation for the attribute name exists
+            uint8_t const* translated = options->attributeTranslator->translate(attrName);
+            if (translated != nullptr) {
+              set(Slice(options->attributeTranslator->translate(attrName)));
+              return set(sub);
+            }
+            // otherwise fall through to regular behavior
+          }
+
           set(Value(attrName, ValueType::String));
           return set(sub);
         }
@@ -486,7 +498,8 @@ namespace arangodb {
             vSize++;
             _start[_pos++] = static_cast<uint8_t>(v & 0xff);
             v >>= 8;
-          } while (v != 0);
+          } 
+          while (v != 0);
           _start[save] = base + vSize;
         }
 

3rdParty/velocypack/include/velocypack/Exception.h

@@ -53,13 +53,17 @@ namespace arangodb {
           InvalidValueType            = 17,
           DuplicateAttributeName      = 18,
           NeedCustomTypeHandler       = 19,
-          BuilderNotSealed            = 20,
+          NeedAttributeTranslator     = 20,
-          BuilderNeedOpenObject       = 21,
+          CannotTranslateKey          = 21,
-          BuilderNeedOpenArray        = 22,
+          KeyNotFound                 = 22,
-          BuilderNeedOpenCompound     = 23,
+
-          BuilderUnexpectedType       = 24,
+          BuilderNotSealed            = 30,
-          BuilderUnexpectedValue      = 25,
+          BuilderNeedOpenObject       = 31,
-          BuilderNeedSubvalue         = 26,
+          BuilderNeedOpenArray        = 32,
+          BuilderNeedOpenCompound     = 33,
+          BuilderUnexpectedType       = 34,
+          BuilderUnexpectedValue      = 35,
+          BuilderNeedSubvalue         = 36,
 
           UnknownError                = 999 
         };
@@ -113,6 +117,12 @@ namespace arangodb {
               return "Invalid value type for operation";
             case NeedCustomTypeHandler:
               return "Cannot execute operation without custom type handler";
+            case NeedAttributeTranslator:
+              return "Cannot execute operation without attribute translator";
+            case CannotTranslateKey:
+              return "Cannot translate key";
+            case KeyNotFound:
+              return "Key not found";
             case BuilderNotSealed:
               return "Builder object not yet sealed";
             case BuilderNeedOpenObject:

3rdParty/velocypack/include/velocypack/Iterator.h

@@ -27,6 +27,8 @@
 #ifndef VELOCYPACK_ITERATOR_H
 #define VELOCYPACK_ITERATOR_H 1
 
+#include <ostream>
+
 #include "velocypack/velocypack-common.h"
 #include "velocypack/Exception.h"
 #include "velocypack/Slice.h"
@@ -135,6 +137,10 @@ namespace arangodb {
           return _position;
         }
         
+        inline ValueLength size () const throw() {
+          return _size;
+        }
+
         inline bool isFirst () const throw() {
           return (_position == 0);
         }
@@ -277,6 +283,10 @@ namespace arangodb {
           return _position;
         }
         
+        inline ValueLength size () const throw() {
+          return _size;
+        }
+
         inline bool isFirst () const throw() {
           return (_position == 0);
         }
@@ -296,4 +306,12 @@ namespace arangodb {
   }  // namespace arangodb::velocypack
 }  // namespace arangodb
     
+std::ostream& operator<< (std::ostream&, arangodb::velocypack::ArrayIterator const*);
+
+std::ostream& operator<< (std::ostream&, arangodb::velocypack::ArrayIterator const&);
+    
+std::ostream& operator<< (std::ostream&, arangodb::velocypack::ObjectIterator const*);
+
+std::ostream& operator<< (std::ostream&, arangodb::velocypack::ObjectIterator const&);
+
 #endif

3rdParty/velocypack/include/velocypack/Options.h

@@ -27,10 +27,14 @@
 #ifndef VELOCYPACK_OPTIONS_H
 #define VELOCYPACK_OPTIONS_H 1
 
+#include <string>
+#include <cstdint>
+
 #include "velocypack/velocypack-common.h"
 
 namespace arangodb {
   namespace velocypack {
+    class AttributeTranslator;
     class Dumper;
     class Slice;
 
@@ -65,6 +69,8 @@ namespace arangodb {
       // callback for excluding attributes from being built by the Parser
       AttributeExcludeHandler* attributeExcludeHandler = nullptr;
 
+      AttributeTranslator* attributeTranslator = nullptr;
+
       // custom type handler used for processing custom types by Dumper and Slicer
       CustomTypeHandler* customTypeHandler = nullptr;
 

3rdParty/velocypack/include/velocypack/Slice.h

@@ -236,7 +236,7 @@ namespace arangodb {
           return isType(ValueType::BCD);
         }
         
-        // check if slice is a custom type
+        // check if slice is a Custom type
         bool isCustom () const {
           return isType(ValueType::Custom);
         }
@@ -352,11 +352,15 @@ namespace arangodb {
             throw Exception(Exception::InvalidValueType, "Expecting Object");
           }
           
-          return getNth(index);
+          return getNthKey(index, true);
         }
 
         Slice valueAt (ValueLength index) const {
-          Slice key = keyAt(index);
+          if (! isType(ValueType::Object)) {
+            throw Exception(Exception::InvalidValueType, "Expecting Object");
+          }
+          
+          Slice key = getNthKey(index, false);
           return Slice(key.start() + key.byteSize(), options);
         }
 
@@ -403,9 +407,13 @@ namespace arangodb {
 
         // return the pointer to the data for an External object
         char const* getExternal () const {
+          assertType(ValueType::External);
           return extractValue<char const*>();
         }
 
+        // translates an integer key into a string
+        Slice translate () const;
+
         // return the value for an Int object
         int64_t getInt () const;
 
@@ -635,10 +643,11 @@ namespace arangodb {
           return 0;
         }
         
+        Slice makeKey () const;
+
         int compareString (std::string const& attribute) const;
         bool isEqualString (std::string const& attribute) const;
 
-
         std::string toJson () const;
         std::string toString () const;
         std::string hexType () const;
@@ -671,12 +680,15 @@ namespace arangodb {
           return getNthOffset(index);
         }
         
-        // get the offset for the nth member from an Array or Object type
+        // get the offset for the nth member from an Array type
         ValueLength getNthOffset (ValueLength index) const;
 
-        // extract the nth member from an Array or Object type
+        // extract the nth member from an Array 
         Slice getNth (ValueLength index) const;
 
+        // extract the nth member from an Object 
+        Slice getNthKey (ValueLength index, bool) const;
+
         // get the offset for the nth member from a compact Array or Object type
         ValueLength getNthOffsetFromCompact (ValueLength index) const;
 

3rdParty/velocypack/include/velocypack/velocypack-aliases.h

@@ -30,6 +30,7 @@
 #include "velocypack/vpack.h"
 
 using VPackArrayIterator       = arangodb::velocypack::ArrayIterator;
+using VPackAttributeTranslator = arangodb::velocypack::AttributeTranslator;
 using VPackBuilder             = arangodb::velocypack::Builder;
 using VPackCharBuffer          = arangodb::velocypack::CharBuffer;
 using VPackCharBufferSink      = arangodb::velocypack::CharBufferSink;

3rdParty/velocypack/include/velocypack/vpack.h

@@ -28,6 +28,7 @@
 #define VELOCYPACK_VPACK_H 1
 
 #include "velocypack/velocypack-common.h"
+#include "velocypack/AttributeTranslator.h"
 #include "velocypack/Buffer.h"
 #include "velocypack/Builder.h"
 #include "velocypack/Collection.h"

3rdParty/velocypack/src/AttributeTranslator.cpp

@@ -0,0 +1,83 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Library to build up VPack documents.
+///
+/// DISCLAIMER
+///
+/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+/// @author Jan Steemann
+/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#include "velocypack/velocypack-common.h"
+#include "velocypack/AttributeTranslator.h"
+#include "velocypack/Builder.h"
+#include "velocypack/Exception.h"
+#include "velocypack/Iterator.h"
+#include "velocypack/Value.h"
+
+using namespace arangodb::velocypack;
+        
+void AttributeTranslator::add (std::string const& key, uint64_t id) {
+  if (_builder == nullptr) {
+    _builder.reset(new Builder);
+    _builder->add(Value(ValueType::Object));
+  }
+
+  _builder->add(key, Value(id));
+}
+
+void AttributeTranslator::seal () {
+  if (_builder == nullptr) {
+    return;
+  }
+
+  _builder->close();
+
+  Slice s(_builder->slice());
+
+  ObjectIterator it(s);
+  while (it.valid()) {
+    _keyToId.emplace(it.key().copyString(), it.value().begin());
+    _idToKey.emplace(it.value().getUInt(), it.key().begin());
+    it.next();
+  }
+}
+
+// translate from string to id
+uint8_t const* AttributeTranslator::translate (std::string const& key) const {
+  auto it = _keyToId.find(key);
+
+  if (it == _keyToId.end()) {
+    return nullptr;
+  }
+
+  return (*it).second;
+}
+
+// translate from id to string
+uint8_t const* AttributeTranslator::translate (uint64_t id) const {
+  auto it = _idToKey.find(id);
+
+  if (it == _idToKey.end()) {
+    throw Exception(Exception::KeyNotFound);
+  }
+
+  return (*it).second;
+}
+

3rdParty/velocypack/src/Dumper.cpp

@@ -259,7 +259,7 @@ void Dumper::dumpValue (Slice const* slice, Slice const* base) {
         ++_indentation;
         while (it.valid()) {
           indent();
-          dumpValue(it.key(), base);
+          dumpValue(it.key().makeKey(), base);
           _sink->append(" : ", 3);
           dumpValue(it.value(), base);
           if (! it.isLast()) {
@@ -276,7 +276,7 @@ void Dumper::dumpValue (Slice const* slice, Slice const* base) {
           if (! it.isFirst()) {
             _sink->push_back(',');
           }
-          dumpValue(it.key(), base);
+          dumpValue(it.key().makeKey(), base);
           _sink->push_back(':');
           dumpValue(it.value(), base);
           it.next();

3rdParty/velocypack/src/Iterator.cpp

@@ -0,0 +1,49 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Library to build up VPack documents.
+///
+/// DISCLAIMER
+///
+/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+/// @author Jan Steemann
+/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#include "velocypack/velocypack-common.h"
+#include "velocypack/Iterator.h"
+
+using namespace arangodb::velocypack;
+
+std::ostream& operator<< (std::ostream& stream, ArrayIterator const* it) {
+  stream << "[ArrayIterator " << it->index() << " / " << it->size() << "]";
+  return stream;
+}
+
+std::ostream& operator<< (std::ostream& stream, ArrayIterator const& it) {
+  return operator<<(stream, &it);
+}
+
+std::ostream& operator<< (std::ostream& stream, ObjectIterator const* it) {
+  stream << "[ObjectIterator " << it->index() << " / " << it->size() << "]";
+  return stream;
+}
+
+std::ostream& operator<< (std::ostream& stream, ObjectIterator const& it) {
+  return operator<<(stream, &it);
+}
+

3rdParty/velocypack/src/Slice.cpp

@@ -25,9 +25,11 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "velocypack/velocypack-common.h"
+#include "velocypack/AttributeTranslator.h"
 #include "velocypack/Builder.h"
 #include "velocypack/Dumper.h"
 #include "velocypack/HexDump.h"
+#include "velocypack/Iterator.h"
 #include "velocypack/Slice.h"
 #include "velocypack/ValueType.h"
 
@@ -147,6 +149,18 @@ unsigned int const Slice::FirstSubMap[32] = {
   0
 };
 
+// translates an integer key into a string
+Slice Slice::translate () const {
+  VELOCYPACK_ASSERT(isSmallInt() || isUInt());
+  uint64_t id = getUInt();
+
+  if (options->attributeTranslator == nullptr) {
+    throw Exception(Exception::NeedAttributeTranslator);
+  }
+
+  return Slice(options->attributeTranslator->translate(id), options);
+}
+
 std::string Slice::toJson () const {
   std::string buffer;
   StringSink sink(&buffer);
@@ -213,10 +227,20 @@ Slice Slice::get (std::string const& attribute) const {
     }
 
     Slice key = Slice(_start + dataOffset, options);
-    if (! key.isString()) {
+
+    if (key.isString()) { 
+      if (! key.isEqualString(attribute)) {
         return Slice();
       }
-    if (! key.isEqualString(attribute)) {
+    }
+    else if (key.isInteger()) {
+      // translate key
+      if (! key.translate().isEqualString(attribute)) {
+        return Slice();
+      }
+    }
+    else {
+      // invalid key
       return Slice();
     }
 
@@ -351,17 +375,21 @@ bool Slice::isEqualString (std::string const& attribute) const {
 }
 
 Slice Slice::getFromCompactObject (std::string const& attribute) const {
-  ValueLength n = length();
+  ObjectIterator it(*this);
-  ValueLength current = 0;
+  while (it.valid()) {
-
+    Slice key = it.key();
-  while (current != n) { 
-    Slice key = keyAt(current);
     if (key.isString()) { 
       if (key.isEqualString(attribute)) {
         return Slice(key.start() + key.byteSize(), options);
       }
     }
-    ++current;
+    else if (key.isInteger()) {
+      if (key.translate().isEqualString(attribute)) {
+        return Slice(key.start() + key.byteSize(), options);
+      }
+    }
+
+    it.next();
   }
   // not found
   return Slice();
@@ -422,19 +450,49 @@ ValueLength Slice::getNthOffset (ValueLength index) const {
   return readInteger<ValueLength>(_start + ieBase, offsetSize);
 }
 
-// extract the nth member from an Array or Object type
+// extract the nth member from an Array 
 Slice Slice::getNth (ValueLength index) const {
-  VELOCYPACK_ASSERT(type() == ValueType::Array || type() == ValueType::Object);
+  VELOCYPACK_ASSERT(type() == ValueType::Array);
 
   auto const h = head();
-  if (h == 0x01 || h == 0x0a) {
+  if (h == 0x01) {
-    // special case. empty array or object
+    // special case. empty Array
     throw Exception(Exception::IndexOutOfBounds);
   }
 
   return Slice(_start + getNthOffset(index), options);
 }
 
+// extract the nth member from an Object 
+Slice Slice::getNthKey (ValueLength index, bool translate) const {
+  VELOCYPACK_ASSERT(type() == ValueType::Object);
+
+  auto const h = head();
+  if (h == 0x01) {
+    // special case. empty Object
+    throw Exception(Exception::IndexOutOfBounds);
+  }
+
+  Slice s(_start + getNthOffset(index), options); 
+
+  if (translate) {
+    return s.makeKey();
+  }
+
+  return s;
+}
+
+Slice Slice::makeKey () const {
+  if (isString()) {
+    return *this;
+  }
+  if (isInteger()) {
+    return translate();
+  }
+
+  throw Exception(Exception::InternalError, "Cannot translate key");
+}
+        
 // get the offset for the nth member from a compact Array or Object type
 ValueLength Slice::getNthOffsetFromCompact (ValueLength index) const {
   ValueLength end = readVariableValueLength<false>(_start + 1);
@@ -467,14 +525,23 @@ Slice Slice::searchObjectKeyLinear (std::string const& attribute,
   for (ValueLength index = 0; index < n; ++index) {
     ValueLength offset = ieBase + index * offsetSize;
     Slice key(_start + readInteger<ValueLength>(_start + offset, offsetSize), options);
-    if (! key.isString()) {
-      // invalid object
-      return Slice();
-    }
 
+    if (key.isString()) {
       if (! key.isEqualString(attribute)) {
         continue;
       }
+    }
+    else if (key.isInteger()) {
+      // translate key
+      if (! key.translate().isEqualString(attribute)) {
+        continue;
+      }
+    }
+    else {
+      // invalid key type
+      return Slice();
+    }
+
     // key is identical. now return value
     return Slice(key.start() + key.byteSize(), options);
   }
@@ -499,13 +566,20 @@ Slice Slice::searchObjectKeyBinary (std::string const& attribute,
 
     ValueLength offset = ieBase + index * offsetSize;
     Slice key(_start + readInteger<ValueLength>(_start + offset, offsetSize), options);
-    if (! key.isString()) {
+
-      // invalid object
+    int res;
+    if (key.isString()) { 
+      res = key.compareString(attribute);
+    }
+    else if (key.isInteger()) { 
+      // translate key
+      res = key.translate().compareString(attribute);
+    }
+    else {
+      // invalid key
       return Slice();
     }
 
-    int res = key.compareString(attribute);
-
     if (res == 0) {
       // found
       return Slice(key.start() + key.byteSize(), options);

lib/CMakeLists.txt

@@ -52,11 +52,13 @@ SET(LIB_ARANGO_CONSOLE
 include_directories(../3rdParty/velocypack/include)
 
 SET(LIB_ARANGO_VPACK
+  ../3rdParty/velocypack/src/AttributeTranslator.cpp
   ../3rdParty/velocypack/src/Builder.cpp
   ../3rdParty/velocypack/src/Collection.cpp
   ../3rdParty/velocypack/src/Dumper.cpp
   ../3rdParty/velocypack/src/Exception.cpp
   ../3rdParty/velocypack/src/HexDump.cpp
+  ../3rdParty/velocypack/src/Iterator.cpp
   ../3rdParty/velocypack/src/Options.cpp
   ../3rdParty/velocypack/src/Parser.cpp
   ../3rdParty/velocypack/src/Slice.cpp

lib/Makefile.files

@@ -110,11 +110,13 @@ lib_libarango_a_SOURCES += \
 endif
 
 lib_libarango_a_SOURCES += \
+	3rdParty/velocypack/src/AttributeTranslator.cpp \
 	3rdParty/velocypack/src/Builder.cpp \
 	3rdParty/velocypack/src/Collection.cpp \
 	3rdParty/velocypack/src/Dumper.cpp \
 	3rdParty/velocypack/src/Exception.cpp \
 	3rdParty/velocypack/src/HexDump.cpp \
+	3rdParty/velocypack/src/Iterator.cpp \
 	3rdParty/velocypack/src/Options.cpp \
 	3rdParty/velocypack/src/Parser.cpp \
 	3rdParty/velocypack/src/Slice.cpp \