Merge branch 'devel' of ssh://github.com/ArangoDB/ArangoDB into devel

3rdParty/velocypack/include/velocypack/Basics.h

@@ -55,9 +55,9 @@ class NonCopyable {
 // prevent heap allocation
 struct NonHeapAllocatable {
   void* operator new(std::size_t) throw(std::bad_alloc) = delete; 
-  void operator delete(void*) throw() = delete; 
+  void operator delete(void*) noexcept = delete; 
   void* operator new[](std::size_t) throw(std::bad_alloc) = delete; 
-  void operator delete[](void*) throw() = delete;  
+  void operator delete[](void*) noexcept = delete;  
 };
 
 #ifdef _WIN32

3rdParty/velocypack/include/velocypack/Builder.h

@@ -309,11 +309,11 @@ class Builder {
     return _pos;
   }
 
-  bool isEmpty() const throw() { return _pos == 0; }
+  bool isEmpty() const noexcept { return _pos == 0; }
 
-  bool isClosed() const throw() { return _stack.empty(); }
+  bool isClosed() const noexcept { return _stack.empty(); }
 
-  bool isOpenArray() const throw() {
+  bool isOpenArray() const noexcept {
     if (_stack.empty()) {
       return false;
     }
@@ -321,7 +321,7 @@ class Builder {
     return _start[tos] == 0x06 || _start[tos] == 0x13;
   }
 
-  bool isOpenObject() const throw() {
+  bool isOpenObject() const noexcept {
     if (_stack.empty()) {
       return false;
     }

3rdParty/velocypack/include/velocypack/Exception.h

@@ -68,6 +68,9 @@ struct Exception : std::exception {
     BuilderKeyAlreadyWritten = 38,
     BuilderKeyMustBeString = 39,
 
+    ValidatorInvalidLength = 50,
+    ValidatorInvalidType = 51,
+
     UnknownError = 999
   };
 
@@ -83,11 +86,11 @@ struct Exception : std::exception {
 
   explicit Exception(ExceptionType type) : Exception(type, message(type)) {}
 
-  char const* what() const throw() { return _msg.c_str(); }
+  char const* what() const noexcept { return _msg.c_str(); }
 
-  ExceptionType errorCode() const throw() { return _type; }
+  ExceptionType errorCode() const noexcept { return _type; }
 
-  static char const* message(ExceptionType type) throw() {
+  static char const* message(ExceptionType type) noexcept {
     switch (type) {
       case InternalError:
         return "Internal error";
@@ -139,6 +142,11 @@ struct Exception : std::exception {
         return "The key of the next key/value pair is already written";
       case BuilderKeyMustBeString:
         return "The key of the next key/value pair must be a string";
+    
+      case ValidatorInvalidType:
+        return "Invalid type found in binary data";
+      case ValidatorInvalidLength:
+        return "Invalid length found in binary data";
 
       case UnknownError:
       default:

3rdParty/velocypack/include/velocypack/Iterator.h

@@ -42,9 +42,8 @@ class ArrayIterator {
  public:
   ArrayIterator() = delete;
 
-  ArrayIterator(Slice const& slice, bool allowRandomIteration = false)
+  explicit ArrayIterator(Slice const& slice)
-      : _slice(slice), _size(_slice.length()), _position(0), _current(nullptr), 
+      : _slice(slice), _size(_slice.length()), _position(0), _current(nullptr) { 
-        _allowRandomIteration(allowRandomIteration) {
     if (slice.type() != ValueType::Array) {
       throw Exception(Exception::InvalidValueType, "Expecting Array slice");
     }
@@ -56,15 +55,13 @@ class ArrayIterator {
       : _slice(other._slice),
         _size(other._size),
         _position(other._position),
-        _current(other._current),
+        _current(other._current) {}
-        _allowRandomIteration(other._allowRandomIteration) {}
 
   ArrayIterator& operator=(ArrayIterator const& other) {
     _slice = other._slice;
     _size = other._size;
     _position = other._position;
     _current = other._current;
-    _allowRandomIteration = other._allowRandomIteration;
     return *this;
   }
 
@@ -97,23 +94,23 @@ class ArrayIterator {
     return _slice.at(_position);
   }
 
-  ArrayIterator begin() { return ArrayIterator(_slice, _allowRandomIteration); }
+  ArrayIterator begin() { return ArrayIterator(_slice); }
 
-  ArrayIterator begin() const { return ArrayIterator(_slice, _allowRandomIteration); }
+  ArrayIterator begin() const { return ArrayIterator(_slice); }
 
   ArrayIterator end() {
-    auto it = ArrayIterator(_slice, _allowRandomIteration);
+    auto it = ArrayIterator(_slice);
     it._position = it._size;
     return it;
   }
 
   ArrayIterator end() const {
-    auto it = ArrayIterator(_slice, _allowRandomIteration);
+    auto it = ArrayIterator(_slice);
     it._position = it._size;
     return it;
   }
 
-  inline bool valid() const throw() { return (_position < _size); }
+  inline bool valid() const noexcept { return (_position < _size); }
 
   inline Slice value() const {
     if (_position >= _size) {
@@ -122,18 +119,18 @@ class ArrayIterator {
     return operator*();
   }
 
-  inline bool next() throw() {
+  inline bool next() noexcept {
     operator++();
     return valid();
   }
 
-  inline ValueLength index() const throw() { return _position; }
+  inline ValueLength index() const noexcept { return _position; }
 
-  inline ValueLength size() const throw() { return _size; }
+  inline ValueLength size() const noexcept { return _size; }
 
-  inline bool isFirst() const throw() { return (_position == 0); }
+  inline bool isFirst() const noexcept { return (_position == 0); }
 
-  inline bool isLast() const throw() { return (_position + 1 >= _size); }
+  inline bool isLast() const noexcept { return (_position + 1 >= _size); }
 
   inline void forward(ValueLength count) {
     if (_position + count >= _size) {
@@ -161,7 +158,7 @@ class ArrayIterator {
       auto h = _slice.head();
       if (h == 0x13) {
         _current = _slice.at(0).start();
-      } else if (_allowRandomIteration) {
+      } else {
         _current = _slice.begin() + _slice.findDataOffset(h);
       }
     }
@@ -172,7 +169,6 @@ class ArrayIterator {
   ValueLength _size;
   ValueLength _position;
   uint8_t const* _current;
-  bool _allowRandomIteration;
 };
 
 class ObjectIterator {
@@ -267,7 +263,7 @@ class ObjectIterator {
     return it;
   }
 
-  inline bool valid() const throw() { return (_position < _size); }
+  inline bool valid() const noexcept { return (_position < _size); }
 
   inline Slice key(bool translate = true) const {
     if (_position >= _size) {
@@ -290,18 +286,18 @@ class ObjectIterator {
     return _slice.getNthValue(_position);
   }
 
-  inline bool next() throw() {
+  inline bool next() noexcept {
     operator++();
     return valid();
   }
 
-  inline ValueLength index() const throw() { return _position; }
+  inline ValueLength index() const noexcept { return _position; }
 
-  inline ValueLength size() const throw() { return _size; }
+  inline ValueLength size() const noexcept { return _size; }
 
-  inline bool isFirst() const throw() { return (_position == 0); }
+  inline bool isFirst() const noexcept { return (_position == 0); }
 
-  inline bool isLast() const throw() { return (_position + 1 >= _size); }
+  inline bool isLast() const noexcept { return (_position + 1 >= _size); }
 
  private:
   Slice _slice;

3rdParty/velocypack/include/velocypack/Parser.h

@@ -89,7 +89,7 @@ class Parser {
   Options const* options;
 
   Parser(Parser const&) = delete;
-  Parser(Parser &&) = delete;
+  Parser(Parser&&) = delete;
   Parser& operator=(Parser const&) = delete;
   Parser& operator=(Parser&&) = delete;
   ~Parser() = default;
@@ -209,7 +209,7 @@ class Parser {
 
   ValueLength parseInternal(bool multi);
 
-  inline bool isWhiteSpace(uint8_t i) const throw() {
+  inline bool isWhiteSpace(uint8_t i) const noexcept {
     return (i == ' ' || i == '\t' || i == '\n' || i == '\r');
   }
 

3rdParty/velocypack/include/velocypack/Slice.h

@@ -42,12 +42,18 @@
 #include "velocypack/Value.h"
 #include "velocypack/ValueType.h"
 
-#ifndef VELOCYPACK_HASH
+#ifdef VELOCYPACK_XXHASH
 // forward for XXH64 function declared elsewhere
 extern "C" unsigned long long XXH64(void const*, size_t, unsigned long long);
 
 #define VELOCYPACK_HASH(mem, size, seed) XXH64(mem, size, seed)
+#endif
 
+#ifdef VELOCYPACK_FASTHASH
+// forward for fasthash64 function declared elsewhere
+uint64_t fasthash64(void const*, size_t, uint64_t);
+
+#define VELOCYPACK_HASH(mem, size, seed) fasthash64(mem, size, seed)
 #endif
 
 namespace arangodb {
@@ -355,7 +361,7 @@ class Slice {
   }
   
   // extract the nth key from an Object
-  Slice getNthKey(ValueLength index, bool) const;
+  Slice getNthKey(ValueLength index, bool translate) const;
   
   // extract the nth value from an Object
   Slice getNthValue(ValueLength index) const {
@@ -758,6 +764,9 @@ class Slice {
     }
     return 9;
   }
+  
+  // get the offset for the nth member from an Array type
+  ValueLength getNthOffset(ValueLength index) const;
 
   Slice makeKey() const;
 
@@ -814,9 +823,6 @@ class Slice {
 
   Slice getFromCompactObject(std::string const& attribute) const;
 
-  // get the offset for the nth member from an Array type
-  ValueLength getNthOffset(ValueLength index) const;
-
   // extract the nth member from an Array
   Slice getNth(ValueLength index) const;
 

3rdParty/velocypack/include/velocypack/Validator.h

@@ -0,0 +1,72 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Library to build up VPack documents.
+///
+/// DISCLAIMER
+///
+/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+/// @author Jan Steemann
+/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef VELOCYPACK_VALIDATOR_H
+#define VELOCYPACK_VALIDATOR_H 1
+
+#include "velocypack/velocypack-common.h"
+#include "velocypack/Options.h"
+
+namespace arangodb {
+namespace velocypack {
+class Slice;
+
+class Validator {
+  // This class can validate a binary VelocyPack value.
+
+ public:
+  explicit Validator(Options const* options = &Options::Defaults)
+      : options(options) {
+    if (options == nullptr) {
+      throw Exception(Exception::InternalError, "Options cannot be a nullptr");
+    }
+  }
+
+  ~Validator() = default;
+
+ public:
+  // validates a VelocyPack Slice value starting at ptr, with length bytes length
+  // throws if the data is invalid
+  bool validate(char const* ptr, size_t length, bool isSubPart = false) const {
+    return validate(reinterpret_cast<uint8_t const*>(ptr), length, isSubPart);
+  }
+
+  // validates a VelocyPack Slice value starting at ptr, with length bytes length
+  // throws if the data is invalid
+  bool validate(uint8_t const* ptr, size_t length, bool isSubPart = false) const;
+
+ private:
+  void validateBufferLength(size_t expected, size_t actual, bool isSubPart) const;
+  void validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const;
+
+ public:
+  Options const* options;
+};
+
+}  // namespace arangodb::velocypack
+}  // namespace arangodb
+
+#endif

3rdParty/velocypack/include/velocypack/velocypack-common.h

@@ -88,7 +88,7 @@ static constexpr std::size_t checkOverflow(ValueLength length) {
 #endif
 
 // calculate the length of a variable length integer in unsigned LEB128 format
-static inline ValueLength getVariableValueLength(ValueLength value) throw() {
+static inline ValueLength getVariableValueLength(ValueLength value) noexcept {
   ValueLength len = 1;
   while (value >= 0x80) {
     value >>= 7;
@@ -139,7 +139,7 @@ static inline void storeVariableValueLength(uint8_t* dst, ValueLength value) {
 // returns current value for UTCDate
 int64_t currentUTCDateValue();
 
-static inline uint64_t toUInt64(int64_t v) throw() {
+static inline uint64_t toUInt64(int64_t v) noexcept {
   // If v is negative, we need to add 2^63 to make it positive,
   // before we can cast it to an uint64_t:
   uint64_t shift2 = 1ULL << 63;
@@ -151,7 +151,7 @@ static inline uint64_t toUInt64(int64_t v) throw() {
   // uint64_t is not guaranteed to work for negative values!
 }
 
-static inline int64_t toInt64(uint64_t v) throw() {
+static inline int64_t toInt64(uint64_t v) noexcept {
   uint64_t shift2 = 1ULL << 63;
   int64_t shift = static_cast<int64_t>(shift2 - 1);
   return v >= shift2 ? (static_cast<int64_t>(v - shift2) - shift) - 1
@@ -161,7 +161,7 @@ static inline int64_t toInt64(uint64_t v) throw() {
 // read an unsigned little endian integer value of the
 // specified length, starting at the specified byte offset
 template <typename T>
-static inline T readInteger(uint8_t const* start, ValueLength length) throw() {
+static inline T readInteger(uint8_t const* start, ValueLength length) noexcept {
   uint64_t value = 0;
   uint64_t x = 0;
   uint8_t const* end = start + length;
@@ -172,11 +172,11 @@ static inline T readInteger(uint8_t const* start, ValueLength length) throw() {
   return value;
 }
 
-static inline uint64_t readUInt64(uint8_t const* start) throw() {
+static inline uint64_t readUInt64(uint8_t const* start) noexcept {
   return readInteger<uint64_t>(start, 8);
 }
 
-static inline void storeUInt64(uint8_t* start, uint64_t value) throw() {
+static inline void storeUInt64(uint8_t* start, uint64_t value) noexcept {
   uint8_t const* end = start + 8;
   do {
     *start++ = static_cast<uint8_t>(value & 0xff);

3rdParty/velocypack/src/Dumper.cpp

@@ -309,7 +309,7 @@ void Dumper::dumpValue(Slice const* slice, Slice const* base) {
     }
 
     case ValueType::Array: {
-      ArrayIterator it(*slice, true);
+      ArrayIterator it(*slice);
       _sink->push_back('[');
       if (options->prettyPrint) {
         _sink->push_back('\n');

3rdParty/velocypack/src/Slice.cpp

@@ -423,7 +423,7 @@ uint64_t Slice::normalizedHash(uint64_t seed) const {
     // over all array members
     uint64_t const n = length() ^ 0xba5bedf00d;
     value = VELOCYPACK_HASH(&n, sizeof(n), seed);
-    for (auto const& it : ArrayIterator(*this, true)) {
+    for (auto const& it : ArrayIterator(*this)) {
       value ^= it.normalizedHash(value);
     }
   } else if (isObject()) {

3rdParty/velocypack/src/Validator.cpp

@@ -0,0 +1,318 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Library to build up VPack documents.
+///
+/// DISCLAIMER
+///
+/// Copyright 2015 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Max Neunhoeffer
+/// @author Jan Steemann
+/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#include "velocypack/velocypack-common.h"
+#include "velocypack/Validator.h"
+#include "velocypack/Exception.h"
+#include "velocypack/Slice.h"
+#include "velocypack/ValueType.h"
+
+using namespace arangodb::velocypack;
+
+bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) const {
+  if (length == 0) {
+    throw Exception(Exception::ValidatorInvalidLength, "length 0 is invalid for any VelocyPack value");
+  }
+
+  uint8_t const head = *ptr;
+
+  // type() only reads the first byte, which is safe
+  ValueType const type = Slice(ptr).type();
+
+  if (type == ValueType::None && head != 0x00) {
+    // invalid type
+    throw Exception(Exception::ValidatorInvalidType);
+  }
+
+  // special handling for certain types...
+  switch (type) {
+    case ValueType::None:
+    case ValueType::Null: 
+    case ValueType::Bool: 
+    case ValueType::MinKey: 
+    case ValueType::MaxKey:  
+    case ValueType::SmallInt:
+    case ValueType::Int: 
+    case ValueType::UInt: 
+    case ValueType::Double: 
+    case ValueType::UTCDate: 
+    case ValueType::Binary:
+    case ValueType::Illegal: {
+      break;
+    }
+    
+    case ValueType::String: {
+      if (head == 0xbf) {
+        // long UTF-8 string. must be at least 9 bytes long so we
+        // can read the entire string length safely
+        validateBufferLength(1 + 8, length, true);
+      } 
+      break;
+    }
+
+    case ValueType::Array: {
+      ValueLength byteLength = 0;
+      bool equalSize = false;
+      bool hasIndexTable = false;
+
+      if (head >= 0x02 && head <= 0x05) {
+        // Array without index table, with 1-8 bytes bytelength, all values with same length
+        byteLength = 1 << (head - 0x02);
+        equalSize = true;
+      } else if (head >= 0x06 && head <= 0x09) {
+        // Array with index table, with 1-8 bytes bytelength
+        byteLength = 1 << (head - 0x06);
+        hasIndexTable = true;
+      } 
+      
+      if (head == 0x13) {
+        // compact Array without index table
+        validateBufferLength(2, length, true);
+        uint8_t const* p = ptr + 1;
+        uint8_t const* e = p + length;
+        ValueLength shifter = 0;
+        while (true) {
+          uint8_t c = *p;
+          byteLength += (c & 0x7f) << shifter;
+          shifter += 7;
+          ++p;
+          if (!(c & 0x80)) {
+            break;
+          }
+          if (p == e) {
+            throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds");
+          }
+        }
+        if (byteLength > length || byteLength < 4) {
+          throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds");
+        }
+        uint8_t const* data = p;
+        p = ptr + byteLength - 1;
+        ValueLength nrItems = 0;
+        shifter = 0;
+        while (true) {
+          uint8_t c = *p;
+          nrItems += (c & 0x7f) << shifter;
+          shifter += 7;
+          --p;
+          if (!(c & 0x80)) {
+            break;
+          }
+          if (p == ptr + byteLength) {
+            throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds");
+          }
+        } 
+        if (nrItems == 0) {
+          throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds");
+        }
+        ++p;
+        
+        // validate the array members 
+        e = p;
+        p = data;
+        while (nrItems-- > 0) { 
+          validate(p, e - p, true);
+          p += Slice(p).byteSize();
+        }
+      } else if (byteLength > 0) {
+        ValueLength nrItemsLength = 0;
+        if (head >= 0x06) {
+          nrItemsLength = byteLength;
+        }
+        validateBufferLength(1 + byteLength + nrItemsLength, length, true);
+        ValueLength nrItems = Slice(ptr).length();
+        uint8_t const* p = ptr + 1 + byteLength;
+        if (!equalSize) {
+          p += byteLength;
+        }
+        uint8_t const* e = ptr + length;
+        ValueLength l = 0;
+        while (nrItems > 0) {
+          if (p >= e) {
+            throw Exception(Exception::ValidatorInvalidLength, "Array value is out of bounds");
+          }
+          // validate sub value
+          validate(p, e - p, true);
+          ValueLength al = Slice(p).byteSize();
+          if (equalSize) {
+            if (l == 0) {
+              l = al;
+            } else if (l != al) {
+              throw Exception(Exception::ValidatorInvalidLength, "Unexpected Array value length");
+            }
+          }
+          p += al;
+          --nrItems;
+        }
+        
+        if (hasIndexTable) {
+          // now also validate index table
+          nrItems = Slice(ptr).length();
+          for (ValueLength i = 0; i < nrItems; ++i) {
+            ValueLength offset = Slice(ptr).getNthOffset(i);
+            if (offset < 1 + byteLength + nrItemsLength ||
+                offset >= Slice(ptr).byteSize() - nrItems * byteLength) {
+              throw Exception(Exception::ValidatorInvalidLength, "Array value offset is out of bounds");
+            }
+            validate(ptr + offset, length - offset, true);
+          }
+        }
+      }
+      break;
+    }
+    
+    case ValueType::Object: {
+      ValueLength byteLength = 0;
+      if (head >= 0x0b && head <= 0x0e) {
+        // Object with index table, with 1-8 bytes bytelength, sorted
+        byteLength = 1 << (head - 0x0b);
+      } else if (head >= 0x0f && head <= 0x12) {
+        // Object with index table, with 1-8 bytes bytelength, unsorted
+        byteLength = 1 << (head - 0x0f);
+      } else if (head == 0x14) {
+        // compact Object without index table
+        // TODO
+      }
+
+      if (byteLength > 0) {
+        validateBufferLength(1 + byteLength, length, true);
+        ValueLength nrItems = Slice(ptr).length();
+        uint8_t const* p = ptr + 1 + byteLength;
+        uint8_t const* e = ptr + length;
+        while (nrItems > 0) {
+          if (p >= e) {
+            throw Exception(Exception::ValidatorInvalidLength, "Object key offset is out of bounds");
+          }
+          // validate key 
+          validate(p, e - p, true);
+          // skip over key
+          p += Slice(p).byteSize();
+          
+          if (p >= e) {
+            throw Exception(Exception::ValidatorInvalidLength, "Object value offset is out of bounds");
+          }
+          // validate value
+          validate(p, e - p, true);
+          // skip over value
+          p += Slice(p).byteSize();
+
+          --nrItems;
+        }
+        
+        // now also validate index table
+        for (ValueLength i = 0; i < nrItems; ++i) {
+          // get offset to key
+          ValueLength offset = Slice(ptr).getNthOffset(i);
+          if (offset >= length) {
+            throw Exception(Exception::ValidatorInvalidLength, "Object key offset is out of bounds");
+          }
+          // validate length of key
+          validate(ptr + offset, length - offset, true);
+          // skip over key
+          offset += Slice(ptr + offset).byteSize();
+          if (offset >= length) {
+            throw Exception(Exception::ValidatorInvalidLength, "Object value offset is out of bounds");
+          }
+          // validate length of value
+          validate(ptr + offset, length - offset, true);
+        }
+      }
+      break;
+    }
+
+    case ValueType::BCD: {
+      throw Exception(Exception::NotImplemented);
+    }
+
+    case ValueType::External: {
+      // check if Externals are forbidden
+      if (options->disallowExternals) {
+        throw Exception(Exception::BuilderExternalsDisallowed);
+      }
+      // validate if Slice length exceeds the given buffer
+      validateBufferLength(1 + sizeof(void*), length, true);
+      // do not perform pointer validation
+      break;
+    }
+
+    case ValueType::Custom: {
+      ValueLength byteSize = 0;
+
+      if (head == 0xf0) {
+        byteSize = 1 + 1;
+      } else if (head == 0xf1) {
+        byteSize = 1 + 2;
+      } else if (head == 0xf2) {
+        byteSize = 1 + 4;
+      } else if (head == 0xf3) {
+        byteSize = 1 + 8;
+      } else if (head >= 0xf4 && head <= 0xf6) {
+        validateBufferLength(1 + 1, length, true);
+        byteSize = 1 + 1 + readInteger<ValueLength>(ptr + 1, 1);
+        if (byteSize == 1 + 1) {
+          throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
+        }
+      } else if (head >= 0xf7 && head <= 0xf9) {
+        validateBufferLength(1 + 2, length, true);
+        byteSize = 1 + 2 + readInteger<ValueLength>(ptr + 1, 2); 
+        if (byteSize == 1 + 2) {
+          throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
+        }
+      } else if (head >= 0xfa && head <= 0xfc) {
+        validateBufferLength(1 + 4, length, true);
+        byteSize = 1 + 4 + readInteger<ValueLength>(ptr + 1, 4); 
+        if (byteSize == 1 + 4) {
+          throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
+        }
+      } else if (head >= 0xfd) {
+        validateBufferLength(1 + 8, length, true);
+        byteSize = 1 + 8 + readInteger<ValueLength>(ptr + 1, 8); 
+        if (byteSize == 1 + 8) {
+          throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
+        }
+      }
+  
+      validateSliceLength(ptr, byteSize, isSubPart);
+      break;
+    }
+  }
+     
+  // common validation that must happen for all types 
+  validateSliceLength(ptr, length, isSubPart);
+  return true;
+}
+
+void Validator::validateBufferLength(size_t expected, size_t actual, bool isSubPart) const {
+  if ((expected > actual) ||
+      (expected != actual && !isSubPart)) {
+    throw Exception(Exception::ValidatorInvalidLength, "given buffer length is unequal to actual length of Slice in buffer");
+  }
+}
+        
+void Validator::validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const {
+  size_t actual = static_cast<size_t>(Slice(ptr).byteSize());
+  validateBufferLength(actual, length, isSubPart);
+}

3rdParty/velocypack/src/fasthash.cpp

@@ -1,82 +0,0 @@
-/* The MIT License
-
-   Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com)
-
-   Permission is hereby granted, free of charge, to any person
-   obtaining a copy of this software and associated documentation
-   files (the "Software"), to deal in the Software without
-   restriction, including without limitation the rights to use, copy,
-   modify, merge, publish, distribute, sublicense, and/or sell copies
-   of the Software, and to permit persons to whom the Software is
-   furnished to do so, subject to the following conditions:
-
-   The above copyright notice and this permission notice shall be
-   included in all copies or substantial portions of the Software.
-
-   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-   SOFTWARE.
-*/
-
-#include <cstdint>
-#include <cstdlib>
-
-namespace arangodb {
-namespace velocypack {
-
-// Compression function for Merkle-Damgard construction.
-// This function is generated using the framework provided.
-static inline uint64_t mix(uint64_t h) {
-  h ^= h >> 23;
-  h *= 0x2127599bf4325c37ULL;
-  h ^= h >> 47;
-
-  return h;
-}
-
-uint64_t fasthash64(const void *buf, size_t len, uint64_t seed) {
-  const uint64_t m = 0x880355f21e6d1965ULL;
-  const uint64_t *pos = (const uint64_t *)buf;
-  const uint64_t *end = pos + (len / 8);
-  const unsigned char *pos2;
-  uint64_t h = seed ^ (len * m);
-  uint64_t v;
-
-  while (pos != end) {
-    v = *pos++;
-    h ^= mix(v);
-    h *= m;
-  }
-
-  pos2 = (const unsigned char *)pos;
-  v = 0;
-
-  switch (len & 7) {
-    case 7:
-      v ^= (uint64_t)pos2[6] << 48;
-    case 6:
-      v ^= (uint64_t)pos2[5] << 40;
-    case 5:
-      v ^= (uint64_t)pos2[4] << 32;
-    case 4:
-      v ^= (uint64_t)pos2[3] << 24;
-    case 3:
-      v ^= (uint64_t)pos2[2] << 16;
-    case 2:
-      v ^= (uint64_t)pos2[1] << 8;
-    case 1:
-      v ^= (uint64_t)pos2[0];
-      h ^= mix(v);
-      h *= m;
-  }
-
-  return mix(h);
-}
-
-}  // namespace arangodb::velocypack
-}  // namespace arangodb

3rdParty/velocypack/src/velocypack-common.cpp

@@ -59,8 +59,7 @@ static_assert(sizeof(std::size_t) == sizeof(uint64_t),
 
 int64_t arangodb::velocypack::currentUTCDateValue() {
   return static_cast<int64_t>(
-      std::chrono::system_clock::now().time_since_epoch().count() /
+      std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count());
-      std::chrono::milliseconds(1).count());
 }
 
 static_assert(sizeof(arangodb::velocypack::ValueLength) >= sizeof(SIZE_MAX),

CMakeLists.txt

@@ -625,6 +625,12 @@ endif ()
 
 add_subdirectory(3rdParty)
 
+################################################################################
+## VELOCYPACK
+################################################################################
+
+add_definitions("-DVELOCYPACK_XXHASH=1")
+
 set(BOOST_VERSION 1.61.0b1)
 
 foreach (LINK_DIR ${LINK_DIRECTORIES})

Documentation/Books/AQL/Functions/String.mdpp

@@ -267,9 +267,9 @@ RANDOM_TOKEN(8) // "zGl09z42"
 RANDOM_TOKEN(8) // "m9w50Ft9"
 ```
 
-!SUBSECTION REGEX_MATCH()
+!SUBSECTION REGEX_TEST()
 
-`REGEX_MATCH(text, search, caseInsensitive) → bool`
+`REGEX_TEST(text, search, caseInsensitive) → bool`
 
 Check whether the pattern *search* is contained in the string *text*,
 using regular expression matching.
@@ -327,9 +327,9 @@ If the regular expression in *search* is invalid, a warning will be raised
 and the function will return *false*.
 
 ```js
-REGEX_MATCH("the quick brown fox", "the.*fox") // true
+REGEX_TEST("the quick brown fox", "the.*fox") // true
-REGEX_MATCH("the quick brown fox", "^(a|the)\s+(quick|slow).*f.x$") // true
+REGEX_TEST("the quick brown fox", "^(a|the)\s+(quick|slow).*f.x$") // true
-REGEX_MATCH("the\nquick\nbrown\nfox", "^the(\n[a-w]+)+\nfox$") // true
+REGEX_TEST("the\nquick\nbrown\nfox", "^the(\n[a-w]+)+\nfox$") // true
 ```
 
 !SUBSECTION REVERSE()

Documentation/Books/Manual/ReleaseNotes/NewFeatures30.mdpp

@@ -89,6 +89,49 @@ var q = `FOR doc IN ´collection´ RETURN doc.´name´`;
 
 The following AQL functions have been added in 3.0:
 
+- *REGEX_TEST(value, regex)*: tests whether the string *value* matches the regular expression
+  specified in *regex*. Returns *true* if it matches, and *false* otherwise.
+
+  The regular expression may consist of literal characters and the following 
+  characters and sequences:
+
+  - `.` – the dot matches any single character except line terminators.
+    To include line terminators, use `[\s\S]` instead to simulate `.` with *DOTALL* flag.
+  - `\d` – matches a single digit, equivalent to `[0-9]`
+  - `\s` – matches a single whitespace character
+  - `\S` – matches a single non-whitespace character
+  - `\t` – matches a tab character
+  - `\r` – matches a carriage return
+  - `\n` – matches a line-feed character
+  - `[xyz]` – set of characters. matches any of the enclosed characters (i.e.
+    *x*, *y* or *z* in this case
+  - `[^xyz]` – negated set of characters. matches any other character than the
+    enclosed ones (i.e. anything but *x*, *y* or *z* in this case)
+  - `[x-z]` – range of characters. Matches any of the characters in the 
+    specified range, e.g. `[0-9A-F]` to match any character in
+    *0123456789ABCDEF*
+  - `[^x-z]` – negated range of characters. Matches any other character than the
+    ones specified in the range
+  - `(xyz)` – defines and matches a pattern group
+  - `(x|y)` – matches either *x* or *y*
+  - `^` – matches the beginning of the string (e.g. `^xyz`)
+  - `$` – matches the end of the string (e.g. `xyz$`)
+
+  Note that the characters `.`, `*`, `?`, `[`, `]`, `(`, `)`, `{`, `}`, `^`, 
+  and `$` have a special meaning in regular expressions and may need to be 
+  escaped using a backslash (`\\`). A literal backslash should also be escaped
+  using another backslash, i.e. `\\\\`.
+
+  Characters and sequences may optionally be repeated using the following
+  quantifiers:
+
+  - `x*` – matches zero or more occurrences of *x*
+  - `x+` – matches one or more occurrences of *x*
+  - `x?` – matches one or zero occurrences of *x*
+  - `x{y}` – matches exactly *y* occurrences of *x*
+  - `x{y,z}` – matches between *y* and *z* occurrences of *x*
+  - `x{y,}` – matches at least *y* occurences of *x*
+
 - *HASH(value)*: Calculates a hash value for *value*. *value* is not required to be a 
   string, but can have any data type. The calculated hash value will take the data type
   of *value* into account, so for example the number *1* and the string *"1"* will have 

arangod/Aql/BindParameters.cpp

@@ -85,7 +85,7 @@ VPackBuilder BindParameters::StripCollectionNames(VPackSlice const& keys,
   TRI_ASSERT(keys.isArray());
   VPackBuilder result;
   result.openArray();
-  for (auto const& element : VPackArrayIterator(keys, false)) {
+  for (auto const& element : VPackArrayIterator(keys)) {
     if (element.isString()) {
       VPackValueLength l;
       char const* s = element.getString(l);

arangod/Aql/EnumerateCollectionBlock.cpp

@@ -90,7 +90,7 @@ bool EnumerateCollectionBlock::moreDocuments(size_t hint) {
   _documents = _scanner.scan(hint);
 
   TRI_ASSERT(_documents.isArray());
-  _iterator = VPackArrayIterator(_documents, true);
+  _iterator = VPackArrayIterator(_documents);
 
   VPackValueLength count = _iterator.size();
 

arangod/Aql/FunctionDefinitions.cpp

@@ -177,8 +177,8 @@ struct FunctionDefiner {
                           false, true, true, &Functions::Contains});
     add({"LIKE", "AQL_LIKE", "s,r|b", true, true, false, true,
                       true, &Functions::Like});
-    add({"REGEX_MATCH", "AQL_REGEX_MATCH", "s,r|b", true, true, false, true,
+    add({"REGEX_TEST", "AQL_REGEX_TEST", "s,r|b", true, true, false, true,
-                       true, &Functions::RegexMatch});
+                       true, &Functions::RegexTest});
     add({"LEFT", "AQL_LEFT", "s,n", true, true, false, true, true});
     add({"RIGHT", "AQL_RIGHT", "s,n", true, true, false, true, true});
     add({"TRIM", "AQL_TRIM", "s|ns", true, true, false, true, true});

arangod/Aql/Functions.cpp

@@ -351,7 +351,7 @@ static void ExtractKeys(std::unordered_set<std::string>& names,
       AqlValueMaterializer materializer(trx);
       VPackSlice s = materializer.slice(param, false);
 
-      for (auto const& v : VPackArrayIterator(s, true)) {
+      for (auto const& v : VPackArrayIterator(s)) {
         if (v.isString()) {
           names.emplace(v.copyString());
         } else {
@@ -418,7 +418,7 @@ static bool ListContainsElement(arangodb::AqlTransaction* trx,
   AqlValueMaterializer testeeMaterializer(trx);
   VPackSlice testeeSlice = testeeMaterializer.slice(testee, false);
 
-  VPackArrayIterator it(slice, true);
+  VPackArrayIterator it(slice);
   while (it.valid()) {
     if (arangodb::basics::VelocyPackHelper::compare(testeeSlice, it.value(), false, options) == 0) {
       index = static_cast<size_t>(it.index());
@@ -467,7 +467,7 @@ static bool Variance(arangodb::AqlTransaction* trx,
   AqlValueMaterializer materializer(trx);
   VPackSlice slice = materializer.slice(values, false);
 
-  for (auto const& element : VPackArrayIterator(slice, true)) {
+  for (auto const& element : VPackArrayIterator(slice)) {
     if (!element.isNull()) {
       if (!element.isNumber()) {
         return false;
@@ -494,7 +494,7 @@ static bool SortNumberList(arangodb::AqlTransaction* trx,
   AqlValueMaterializer materializer(trx);
   VPackSlice slice = materializer.slice(values, false);
 
-  for (auto const& element : VPackArrayIterator(slice, true)) {
+  for (auto const& element : VPackArrayIterator(slice)) {
     if (!element.isNull()) {
       if (!element.isNumber()) {
         return false;
@@ -547,7 +547,7 @@ static void RequestEdges(VPackSlice vertexSlice,
     VPackSlice edges = opRes->slice();
     TRI_ASSERT(edges.isArray());
     if (includeVertices) {
-      for (auto const& edge : VPackArrayIterator(edges, true)) {
+      for (auto const& edge : VPackArrayIterator(edges)) {
         VPackObjectBuilder guard(&result);
         if (matcher == nullptr || matcher->matches(edge)) {
           result.add("edge", edge);
@@ -597,7 +597,7 @@ static void RequestEdges(VPackSlice vertexSlice,
         }
       }
     } else {
-      for (auto const& edge : VPackArrayIterator(edges, true)) {
+      for (auto const& edge : VPackArrayIterator(edges)) {
         if (matcher == nullptr || matcher->matches(edge)) {
           result.add(edge);
         }
@@ -724,7 +724,7 @@ static AqlValue MergeParameters(arangodb::aql::Query* query,
       THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
     }
     // merge in all other arguments
-    for (auto const& it : VPackArrayIterator(initialSlice, true)) {
+    for (auto const& it : VPackArrayIterator(initialSlice)) {
       if (!it.isObject()) {
         RegisterInvalidArgumentWarning(query, funcName);
         builder.clear();
@@ -905,7 +905,7 @@ static AqlValue buildGeoResult(arangodb::AqlTransaction* trx,
 static void FlattenList(VPackSlice const& array, size_t maxDepth,
                         size_t curDepth, VPackBuilder& result) {
   TRI_ASSERT(result.isOpenArray());
-  for (auto const& tmp : VPackArrayIterator(array, true)) {
+  for (auto const& tmp : VPackArrayIterator(array)) {
     if (tmp.isArray() && curDepth < maxDepth) {
       FlattenList(tmp, maxDepth, curDepth + 1, result);
     } else {
@@ -1258,7 +1258,7 @@ AqlValue Functions::Concat(arangodb::aql::Query* query,
       AqlValueMaterializer materializer(trx);
       VPackSlice slice = materializer.slice(member, false);
 
-      for (auto const& it : VPackArrayIterator(slice, true)) {
+      for (auto const& it : VPackArrayIterator(slice)) {
         if (it.isNull()) {
           continue;
         }
@@ -1313,7 +1313,7 @@ AqlValue Functions::ConcatSeparator(arangodb::aql::Query* query,
       AqlValueMaterializer materializer(trx);
       VPackSlice slice = materializer.slice(member, false);
 
-      for (auto const& it : VPackArrayIterator(slice, true)) {
+      for (auto const& it : VPackArrayIterator(slice)) {
         if (it.isNull()) {
           continue;
         }
@@ -1419,11 +1419,11 @@ AqlValue Functions::Like(arangodb::aql::Query* query,
   return AqlValue(result);
 }
 
-/// @brief function REGEX_MATCH
+/// @brief function REGEX_TEST
-AqlValue Functions::RegexMatch(arangodb::aql::Query* query,
+AqlValue Functions::RegexTest(arangodb::aql::Query* query,
-                               arangodb::AqlTransaction* trx,
+                              arangodb::AqlTransaction* trx,
-                               VPackFunctionParameters const& parameters) {
+                              VPackFunctionParameters const& parameters) {
-  ValidateParameters(parameters, "REGEX_MATCH", 2, 3);
+  ValidateParameters(parameters, "REGEX_TEST", 2, 3);
   bool const caseInsensitive = GetBooleanParameter(trx, parameters, 2, false);
   StringBufferLeaser buffer(trx);
   arangodb::basics::VPackStringBufferAdapter adapter(buffer->stringBuffer());
@@ -1464,7 +1464,7 @@ AqlValue Functions::RegexMatch(arangodb::aql::Query* query,
 
   if (matcher == nullptr) {
     // compiling regular expression failed
-    RegisterWarning(query, "REGEX_MATCH", TRI_ERROR_QUERY_INVALID_REGEX);
+    RegisterWarning(query, "REGEX_TEST", TRI_ERROR_QUERY_INVALID_REGEX);
     return AqlValue(arangodb::basics::VelocyPackHelper::NullValue());
   }
 
@@ -1479,7 +1479,7 @@ AqlValue Functions::RegexMatch(arangodb::aql::Query* query,
 
   if (error) {
     // compiling regular expression failed
-    RegisterWarning(query, "REGEX_MATCH", TRI_ERROR_QUERY_INVALID_REGEX);
+    RegisterWarning(query, "REGEX_TEST", TRI_ERROR_QUERY_INVALID_REGEX);
     return AqlValue(arangodb::basics::VelocyPackHelper::NullValue());
   } 
   
@@ -1752,7 +1752,7 @@ AqlValue Functions::Min(arangodb::aql::Query* query,
   VPackSlice slice = materializer.slice(value, false);
 
   VPackSlice minValue;
-  for (auto const& it : VPackArrayIterator(slice, true)) {
+  for (auto const& it : VPackArrayIterator(slice)) {
     if (it.isNull()) {
       continue;
     }
@@ -1781,7 +1781,7 @@ AqlValue Functions::Max(arangodb::aql::Query* query,
   AqlValueMaterializer materializer(trx);
   VPackSlice slice = materializer.slice(value, false);
   VPackSlice maxValue;
-  for (auto const& it : VPackArrayIterator(slice, true)) {
+  for (auto const& it : VPackArrayIterator(slice)) {
     if (maxValue.isNone() || arangodb::basics::VelocyPackHelper::compare(it, maxValue, true) > 0) {
       maxValue = it;
     }
@@ -1807,7 +1807,7 @@ AqlValue Functions::Sum(arangodb::aql::Query* query,
   AqlValueMaterializer materializer(trx);
   VPackSlice slice = materializer.slice(value, false);
   double sum = 0.0;
-  for (auto const& it : VPackArrayIterator(slice, true)) {
+  for (auto const& it : VPackArrayIterator(slice)) {
     if (it.isNull()) {
       continue;
     }
@@ -1841,7 +1841,7 @@ AqlValue Functions::Average(arangodb::aql::Query* query,
 
   double sum = 0.0;
   size_t count = 0;
-  for (auto const& v : VPackArrayIterator(slice, true)) {
+  for (auto const& v : VPackArrayIterator(slice)) {
     if (v.isNull()) {
       continue;
     }
@@ -1976,7 +1976,7 @@ AqlValue Functions::Unique(arangodb::aql::Query* query,
       values(512, arangodb::basics::VelocyPackHelper::VPackHash(),
              arangodb::basics::VelocyPackHelper::VPackEqual(&options));
 
-  for (auto const& s : VPackArrayIterator(slice, true)) {
+  for (auto const& s : VPackArrayIterator(slice)) {
     if (!s.isNone()) {
       values.emplace(s);
     }
@@ -2013,7 +2013,7 @@ AqlValue Functions::SortedUnique(arangodb::aql::Query* query,
 
   arangodb::basics::VelocyPackHelper::VPackLess<true> less(trx->transactionContext()->getVPackOptions(), &slice, &slice);
   std::set<VPackSlice, arangodb::basics::VelocyPackHelper::VPackLess<true>> values(less);
-  for (auto const& it : VPackArrayIterator(slice, true)) {
+  for (auto const& it : VPackArrayIterator(slice)) {
     if (!it.isNone()) {
       values.insert(it);
     }
@@ -2059,7 +2059,7 @@ AqlValue Functions::Union(arangodb::aql::Query* query,
       VPackSlice slice = materializer.slice(value, false);
 
       // this passes ownership for the JSON contens into result
-      for (auto const& it : VPackArrayIterator(slice, true)) {
+      for (auto const& it : VPackArrayIterator(slice)) {
         builder->add(it);
         TRI_IF_FAILURE("AqlFunctions::OutOfMemory2") {
           THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
@@ -2110,7 +2110,7 @@ AqlValue Functions::UnionDistinct(arangodb::aql::Query* query,
     materializers.emplace_back(trx);
     VPackSlice slice = materializers.back().slice(value, false);
 
-    for (auto const& v : VPackArrayIterator(slice, true)) {
+    for (auto const& v : VPackArrayIterator(slice)) {
       if (values.find(v) == values.end()) {
         TRI_IF_FAILURE("AqlFunctions::OutOfMemory1") {
           THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
@@ -2176,7 +2176,7 @@ AqlValue Functions::Intersection(arangodb::aql::Query* query,
     materializers.emplace_back(trx);
     VPackSlice slice = materializers.back().slice(value, false);
 
-    for (auto const& it : VPackArrayIterator(slice, true)) {
+    for (auto const& it : VPackArrayIterator(slice)) {
       if (i == 0) {
         // round one
 
@@ -2678,7 +2678,7 @@ AqlValue Functions::Slice(arangodb::aql::Query* query,
   builder->openArray();
  
   int64_t pos = 0; 
-  VPackArrayIterator it(arraySlice, true);
+  VPackArrayIterator it(arraySlice);
   while (it.valid()) {
     if (pos >= from && pos < to) {
       builder->add(it.value());
@@ -2722,7 +2722,7 @@ AqlValue Functions::Minus(arangodb::aql::Query* query,
   AqlValueMaterializer materializer(trx);
   VPackSlice arraySlice = materializer.slice(baseArray, false);
   
-  VPackArrayIterator it(arraySlice, true);
+  VPackArrayIterator it(arraySlice);
   while (it.valid()) {
     contains.emplace(it.value(), it.index());
     it.next();
@@ -2741,7 +2741,7 @@ AqlValue Functions::Minus(arangodb::aql::Query* query,
     AqlValueMaterializer materializer(trx);
     VPackSlice arraySlice = materializer.slice(next, false);
 
-    for (auto const& search : VPackArrayIterator(arraySlice, true)) {
+    for (auto const& search : VPackArrayIterator(arraySlice)) {
       auto find = contains.find(search);
 
       if (find != contains.end()) {
@@ -2787,7 +2787,7 @@ AqlValue Functions::Document(arangodb::aql::Query* query,
       AqlValueMaterializer materializer(trx);
       VPackSlice idSlice = materializer.slice(id, false);
       builder->openArray();
-      for (auto const& next : VPackArrayIterator(idSlice, true)) {
+      for (auto const& next : VPackArrayIterator(idSlice)) {
         if (next.isString()) {
           std::string identifier = next.copyString();
           std::string colName;
@@ -2829,7 +2829,7 @@ AqlValue Functions::Document(arangodb::aql::Query* query,
     if (!notFound) {
       AqlValueMaterializer materializer(trx);
       VPackSlice idSlice = materializer.slice(id, false);
-      for (auto const& next : VPackArrayIterator(idSlice, true)) {
+      for (auto const& next : VPackArrayIterator(idSlice)) {
         if (next.isString()) {
           std::string identifier(next.copyString());
           GetDocumentByIdentifier(trx, collectionName, identifier, true, *builder.get());
@@ -2943,7 +2943,7 @@ AqlValue Functions::Edges(arangodb::aql::Query* query,
   builder->openArray();
     
   if (vertexSlice.isArray()) {
-    for (auto const& v : VPackArrayIterator(vertexSlice, true)) {
+    for (auto const& v : VPackArrayIterator(vertexSlice)) {
       RequestEdges(v, trx, collectionName, indexId, direction,
                    matcher.get(), includeVertices, *builder.get());
     }
@@ -3323,7 +3323,7 @@ AqlValue Functions::Push(arangodb::aql::Query* query,
   AqlValueMaterializer materializer(trx);
   VPackSlice l = materializer.slice(list, false);
 
-  for (auto const& it : VPackArrayIterator(l, true)) {
+  for (auto const& it : VPackArrayIterator(l)) {
     builder->add(it);
   }
   VPackOptions options;
@@ -3364,7 +3364,7 @@ AqlValue Functions::Pop(arangodb::aql::Query* query,
 
     TransactionBuilderLeaser builder(trx);
     builder->openArray();
-    auto iterator = VPackArrayIterator(slice, true);
+    auto iterator = VPackArrayIterator(slice);
     while (iterator.valid() && !iterator.isLast()) {
       builder->add(iterator.value());
       iterator.next();
@@ -3412,7 +3412,7 @@ AqlValue Functions::Append(arangodb::aql::Query* query,
       trx->transactionContext()->orderCustomTypeHandler().get();
   if (!list.isNull(true)) {
     if (list.isArray()) {
-      for (auto const& it : VPackArrayIterator(l, true)) {
+      for (auto const& it : VPackArrayIterator(l)) {
         builder->add(it);
       }
     }
@@ -3427,7 +3427,7 @@ AqlValue Functions::Append(arangodb::aql::Query* query,
     if (unique) {
       std::unordered_set<VPackSlice> added;
       added.reserve(static_cast<size_t>(slice.length()));
-      for (auto const& it : VPackArrayIterator(slice, true)) {
+      for (auto const& it : VPackArrayIterator(slice)) {
         if (added.find(it) == added.end() &&
             !ListContainsElement(&options, l, it)) {
           builder->add(it);
@@ -3435,7 +3435,7 @@ AqlValue Functions::Append(arangodb::aql::Query* query,
         }
       }
     } else {
-      for (auto const& it : VPackArrayIterator(slice, true)) {
+      for (auto const& it : VPackArrayIterator(slice)) {
         builder->add(it);
       }
     }
@@ -3483,7 +3483,7 @@ AqlValue Functions::Unshift(arangodb::aql::Query* query,
   if (list.isArray()) {
     AqlValueMaterializer materializer(trx);
     VPackSlice v = materializer.slice(list, false);
-    for (auto const& it : VPackArrayIterator(v, true)) {
+    for (auto const& it : VPackArrayIterator(v)) {
       builder->add(it);
     }
   }
@@ -3514,7 +3514,7 @@ AqlValue Functions::Shift(arangodb::aql::Query* query,
     AqlValueMaterializer materializer(trx);
     VPackSlice l = materializer.slice(list, false);
 
-    auto iterator = VPackArrayIterator(l, true);
+    auto iterator = VPackArrayIterator(l);
     // This jumps over the first element
     while (iterator.next()) {
       builder->add(iterator.value());
@@ -3563,7 +3563,7 @@ AqlValue Functions::RemoveValue(arangodb::aql::Query* query,
     AqlValueMaterializer materializer(trx);
     VPackSlice v = materializer.slice(list, false);
 
-    for (auto const& it : VPackArrayIterator(v, true)) {
+    for (auto const& it : VPackArrayIterator(v)) {
       if (useLimit && limit == 0) {
         // Just copy
         builder->add(it);
@@ -3616,7 +3616,7 @@ AqlValue Functions::RemoveValues(arangodb::aql::Query* query,
 
     TransactionBuilderLeaser builder(trx);
     builder->openArray();
-    for (auto const& it : VPackArrayIterator(l, true)) {
+    for (auto const& it : VPackArrayIterator(l)) {
       if (!ListContainsElement(&options, v, it)) {
         builder->add(it);
       }
@@ -3665,7 +3665,7 @@ AqlValue Functions::RemoveNth(arangodb::aql::Query* query,
     size_t target = static_cast<size_t>(p);
     size_t cur = 0;
     builder->openArray();
-    for (auto const& it : VPackArrayIterator(v, true)) {
+    for (auto const& it : VPackArrayIterator(v)) {
       if (cur != target) {
         builder->add(it);
       }
@@ -3996,7 +3996,7 @@ AqlValue Functions::Range(arangodb::aql::Query* query,
 
   AqlValue stepValue = ExtractFunctionParameterValue(trx, parameters, 2);
   if (stepValue.isNull(true)) {
-    // no step specified
+    // no step specified. return a real range object
     return AqlValue(left.toInt64(), right.toInt64());
   } 
   

arangod/Aql/Functions.h

@@ -97,7 +97,7 @@ struct Functions {
                                   VPackFunctionParameters const&);
   static AqlValue Like(arangodb::aql::Query*, arangodb::AqlTransaction*,
                        VPackFunctionParameters const&);
-  static AqlValue RegexMatch(arangodb::aql::Query*, arangodb::AqlTransaction*,
+  static AqlValue RegexTest(arangodb::aql::Query*, arangodb::AqlTransaction*,
                         VPackFunctionParameters const&);
   static AqlValue Passthru(arangodb::aql::Query*, arangodb::AqlTransaction*,
                            VPackFunctionParameters const&);

arangod/Aql/IndexBlock.cpp

@@ -410,7 +410,7 @@ bool IndexBlock::readIndex(size_t atMost) {
       THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
     }
     
-    for (auto const& doc : VPackArrayIterator(slice, true)) {
+    for (auto const& doc : VPackArrayIterator(slice)) {
       if (!hasMultipleIndexes) {
         _documents.emplace_back(doc);
       } else {

arangod/Aql/ModificationBlocks.cpp

@@ -453,7 +453,7 @@ AqlItemBlock* InsertBlock::work(std::vector<AqlItemBlock*>& blocks) {
         dstRow -= n;
         VPackSlice resultList = opRes.slice();
         TRI_ASSERT(resultList.isArray());
-        for (auto const& elm: VPackArrayIterator(resultList, false)) {
+        for (auto const& elm: VPackArrayIterator(resultList)) {
           bool wasError = arangodb::basics::VelocyPackHelper::getBooleanValue(
               elm, "error", false);
           if (!wasError) {

arangod/Indexes/EdgeIndex.h

@@ -55,7 +55,7 @@ class EdgeIndexIterator final : public IndexIterator {
         _index(index),
         _searchValues(searchValues),
         _keys(_searchValues.slice()),
-        _iterator(_keys, true),
+        _iterator(_keys),
         _posInBuffer(0),
         _batchSize(1000) {}
 
@@ -66,7 +66,7 @@ class EdgeIndexIterator final : public IndexIterator {
         _index(index),
         _searchValues(arangodb::velocypack::Builder::clone(searchValues)),
         _keys(_searchValues.slice()),
-        _iterator(_keys, true),
+        _iterator(_keys),
         _posInBuffer(0),
         _batchSize(1000) {}
 

arangod/Indexes/HashIndex.cpp

@@ -599,7 +599,7 @@ void HashIndex::transformSearchValues(VPackSlice const values,
   }
 
   VPackArrayBuilder guard(&result);
-  for (auto const& v : VPackArrayIterator(values, true)) {
+  for (auto const& v : VPackArrayIterator(values)) {
     if (!v.isObject() || !v.hasKey(TRI_SLICE_KEY_EQUAL)) {
       THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "Hash index only allows == comparison.");
     }

arangod/Indexes/HashIndex.h

@@ -123,7 +123,7 @@ class HashIndexIteratorVPack final : public IndexIterator {
       : _trx(trx),
         _index(index),
         _searchValues(searchValues.get()),
-        _iterator(_searchValues->slice(), true),
+        _iterator(_searchValues->slice()),
         _buffer(),
         _posInBuffer(0) {
     searchValues.release(); // now we have ownership for searchValues

arangod/Indexes/PrimaryIndex.h

@@ -47,7 +47,7 @@ class PrimaryIndexIterator final : public IndexIterator {
       : _trx(trx), 
         _index(index), 
         _keys(keys.get()), 
-        _iterator(_keys->slice(), true) {
+        _iterator(_keys->slice()) {
 
         keys.release(); // now we have ownership for _keys
         TRI_ASSERT(_keys->slice().isArray());

arangod/RestHandler/RestEdgesHandler.cpp

@@ -108,7 +108,7 @@ bool RestEdgesHandler::getEdgesForVertexList(
     // generate result
     scannedIndex += static_cast<size_t>(edges.length());
 
-    for (auto const& edge : VPackArrayIterator(edges, true)) {
+    for (auto const& edge : VPackArrayIterator(edges)) {
       bool add = true;
       if (!expressions.empty()) {
         for (auto& exp : expressions) {
@@ -164,7 +164,7 @@ bool RestEdgesHandler::getEdgesForVertex(
     // generate result
     scannedIndex += static_cast<size_t>(edges.length());
 
-    for (auto const& edge : VPackArrayIterator(edges, true)) {
+    for (auto const& edge : VPackArrayIterator(edges)) {
       bool add = true;
       if (!expressions.empty()) {
         for (auto& exp : expressions) {

arangod/RestHandler/RestVocbaseBaseHandler.cpp

@@ -637,13 +637,15 @@ bool RestVocbaseBaseHandler::extractBooleanParameter(char const* name,
 
 std::shared_ptr<VPackBuilder> RestVocbaseBaseHandler::parseVelocyPackBody(
     VPackOptions const* options, bool& success) {
-  bool found;
-  std::string const& contentType =
-      _request->header(StaticStrings::ContentTypeHeader, found);
-
   try {
     success = true;
 
+#if 0
+    // currently deactivated...
+    bool found;
+    std::string const& contentType =
+        _request->header(StaticStrings::ContentTypeHeader, found);
+
     if (found && contentType.size() == StaticStrings::MimeTypeVPack.size() &&
         contentType == StaticStrings::MimeTypeVPack) {
       VPackSlice slice{_request->body().c_str()};
@@ -653,6 +655,9 @@ std::shared_ptr<VPackBuilder> RestVocbaseBaseHandler::parseVelocyPackBody(
     } else {
       return _request->toVelocyPack(options);
     }
+#else
+    return _request->toVelocyPack(options);
+#endif
   } catch (std::bad_alloc const&) {
     generateOOMError();
   } catch (VPackException const& e) {

arangod/Utils/Cursor.cpp

@@ -70,7 +70,7 @@ VelocyPackCursor::VelocyPackCursor(TRI_vocbase_t* vocbase, CursorId id,
     : Cursor(id, batchSize, extra, ttl, hasCount),
       _vocbase(vocbase),
       _result(std::move(result)),
-      _iterator(_result.result->slice(), true),
+      _iterator(_result.result->slice()),
       _cached(_result.cached) {
   TRI_ASSERT(_result.result->slice().isArray());
   TRI_UseVocBase(vocbase);

arangod/Utils/Transaction.cpp

@@ -1146,7 +1146,7 @@ OperationResult Transaction::anyLocal(std::string const& collectionName,
     }
   
     VPackSlice docs = result->slice();
-    VPackArrayIterator it(docs, true);
+    VPackArrayIterator it(docs);
     while (it.valid()) {
       resultBuilder.add(it.value());
       it.next();

js/apps/system/_admin/aardvark/APP/frontend/src/mode-aql.js

@@ -88,14 +88,14 @@ var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
 var AqlHighlightRules = function() {
 
     var keywords = (
-        "for|return|filter|sort|limit|let|collect|asc|desc|in|into|insert|update|remove|replace|upsert|options|with|and|or|not|distinct|graph|outbound|inbound|any|all|none|aggregate|like|count"
+        "for|return|filter|sort|limit|let|collect|asc|desc|in|into|insert|update|remove|replace|upsert|options|with|and|or|not|distinct|graph|shortest_path|outbound|inbound|any|all|none|aggregate|like"
     );
 
     var builtinFunctions = (
         "(to_bool|to_number|to_string|to_list|is_null|is_bool|is_number|is_string|is_list|is_document|typename|" +
         "concat|concat_separator|char_length|lower|upper|substring|left|right|trim|reverse|contains|" +
-        "log|log2|log10|exp|exp2|sin|cos|tan|asin|acos|atan|atan2|radians|degrees|pi|regex|" +
+        "log|log2|log10|exp|exp2|sin|cos|tan|asin|acos|atan|atan2|radians|degrees|pi|regex_test|" +
-        "like|floor|ceil|round|abs|rand|sqrt|pow|length|min|max|average|sum|median|variance_population|" +
+        "like|floor|ceil|round|abs|rand|sqrt|pow|length|count|min|max|average|sum|median|variance_population|" +
         "variance_sample|first|last|unique|matches|merge|merge_recursive|has|attributes|values|unset|unset_recursive|keep|" +
         "near|within|within_rectangle|is_in_polygon|fulltext|paths|traversal|traversal_tree|edges|stddev_sample|stddev_population|" +
         "slice|nth|position|translate|zip|call|apply|push|append|pop|shift|unshift|remove_value|remove_values|" + 

js/common/modules/@arangodb/foxx/manager-utils.js

@@ -54,7 +54,7 @@ var getStorage = function() {
   var c = db._collection("_apps");
   if (c === null) {
     c = db._create("_apps", {isSystem: true, replicationFactor: 1,
-                   distributeShardsLike: "_graphs"});
+                   distributeShardsLike: "_graphs", journalSize: 4 * 1024 * 1024});
     c.ensureIndex({ type: "hash", fields: [ "mount" ], unique: true });
   }
   return c;

js/server/modules/@arangodb/aql.js

@@ -2301,7 +2301,7 @@ function AQL_LIKE (value, regex, caseInsensitive) {
 /// @brief searches a substring in a string, using a regex
 ////////////////////////////////////////////////////////////////////////////////
 
-function AQL_REGEX_MATCH (value, regex, caseInsensitive) {
+function AQL_REGEX_TEST (value, regex, caseInsensitive) {
   'use strict';
 
   var modifiers = '';
@@ -8358,7 +8358,7 @@ exports.AQL_UPPER = AQL_UPPER;
 exports.AQL_SUBSTRING = AQL_SUBSTRING;
 exports.AQL_CONTAINS = AQL_CONTAINS;
 exports.AQL_LIKE = AQL_LIKE;
-exports.AQL_REGEX_MATCH = AQL_REGEX_MATCH;
+exports.AQL_REGEX_TEST = AQL_REGEX_TEST;
 exports.AQL_LEFT = AQL_LEFT;
 exports.AQL_RIGHT = AQL_RIGHT;
 exports.AQL_TRIM = AQL_TRIM;

js/server/modules/@arangodb/statistics.js

@@ -49,10 +49,11 @@ function createStatisticsCollection (name) {
 
   if (collection === null) {
     var r = null;
-
+                             
     try {
       r = db._create(name, { isSystem: true, waitForSync: false,
                              replicationFactor: 1,
+                             journalSize: 8 * 1024 * 1024,
                              distributeShardsLike: "_graphs" });
     }
     catch (err) {
@@ -424,7 +425,6 @@ exports.STATISTICS_INTERVAL = 10;
 
 exports.STATISTICS_HISTORY_INTERVAL = 15 * 60;
 
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief createCollections
 ///

js/server/tests/aql/aql-functions-string.js

@@ -62,18 +62,18 @@ function ahuacatlStringFunctionsTestSuite () {
 ////////////////////////////////////////////////////////////////////////////////
     
     testRegexInvalid : function () {
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_MATCH()"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_TEST()"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_MATCH(\"test\")"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_TEST(\"test\")"); 
-      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_MATCH(\"test\", \"meow\", \"foo\", \"bar\")"); 
+      assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_TEST(\"test\", \"meow\", \"foo\", \"bar\")"); 
       
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"[\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"[\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"[^\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"[^\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"a.(\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"a.(\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"(a\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"(a\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"(a]\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"(a]\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"**\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"**\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"?\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"?\")");
-      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"*\")");
+      assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"*\")");
     },
 
     testRegex : function () {
@@ -217,13 +217,13 @@ function ahuacatlStringFunctionsTestSuite () {
       ];
 
       values.forEach(function(v) {
-        var query = "RETURN REGEX_MATCH(@what, @re)";
+        var query = "RETURN REGEX_TEST(@what, @re)";
         assertEqual(v[2], getQueryResults(query, { what: v[0], re: v[1] })[0], v);
         
-        query = "RETURN NOOPT(REGEX_MATCH(@what, @re))";
+        query = "RETURN NOOPT(REGEX_TEST(@what, @re))";
         assertEqual(v[2], getQueryResults(query, { what: v[0], re: v[1] })[0], v);
         
-        query = "RETURN NOOPT(V8(REGEX_MATCH(@what, @re)))";
+        query = "RETURN NOOPT(V8(REGEX_TEST(@what, @re)))";
         assertEqual(v[2], getQueryResults(query, { what: v[0], re: v[1] })[0], v);
       });
 

js/server/upgrade-database.js

@@ -625,7 +625,7 @@
       task: function() {
         // needs to be big enough for assets
         return createSystemCollection("_routing", {
-          journalSize: 8 * 1024 * 1024,
+          journalSize: 4 * 1024 * 1024,
           replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM,
           distributeShardsLike: "_graphs"
         });
@@ -692,7 +692,7 @@
 
       task: function() {
         return createSystemCollection("_aqlfunctions", {
-          journalSize: 2 * 1024 * 1024,
+          journalSize: 1 * 1024 * 1024,
           replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM,
           distributeShardsLike: "_graphs"
         });
@@ -763,7 +763,7 @@
 
       task: function() {
         return createSystemCollection("_jobs", {
-          journalSize: 4 * 1024 * 1024,
+          journalSize: 2 * 1024 * 1024,
           replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM,
           distributeShardsLike: "_graphs"
         });

lib/Basics/VelocyPackDumper.cpp

@@ -366,7 +366,7 @@ void VelocyPackDumper::dumpValue(VPackSlice const* slice, VPackSlice const* base
     }
 
     case VPackValueType::Array: {
-      VPackArrayIterator it(*slice, true);
+      VPackArrayIterator it(*slice);
       TRI_AppendCharUnsafeStringBuffer(buffer, '[');
       while (it.valid()) {
         if (!it.isFirst()) {

lib/CMakeLists.txt

@@ -87,11 +87,12 @@ set(LIB_ARANGO_VPACK
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Parser.cpp
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Slice.cpp
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/ValueType.cpp
+  ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Validator.cpp
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Version.cpp
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/asm-functions.cpp
-  ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/fasthash.cpp
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/fpconv.cpp
   ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/velocypack-common.cpp
+  ${PROJECT_SOURCE_DIR}/lib/Basics/xxhash.cpp
 )
 
 if (ASM_OPTIMIZATIONS AND CMAKE_TARGET_ARCHITECTURE_CODE MATCHES "x86_64")

lib/Rest/GeneralRequest.cpp

@@ -258,7 +258,12 @@ void GeneralRequest::setArrayValue(char* key, size_t length, char const* value)
   _arrayValues[std::string(key, length)].emplace_back(value);
 }
 
-bool GeneralRequest::velocyPackResponse () const {
+bool GeneralRequest::velocyPackResponse() const {
+#if 0
+  // currently deactivated
   std::string const& result = header(StaticStrings::Accept);
   return (std::string::npos != result.find(StaticStrings::MimeTypeVPack));
+#else
+  return false;
+#endif
 }

lib/V8/v8-vpack.cpp

@@ -138,7 +138,7 @@ static v8::Handle<v8::Value> ObjectVPackArray(v8::Isolate* isolate,
   }
 
   uint32_t j = 0;
-  VPackArrayIterator it(slice, true);
+  VPackArrayIterator it(slice);
 
   while (it.valid()) {
     v8::Handle<v8::Value> val =