update velocypack (#8074)

3rdParty/velocypack/include/velocypack/Builder.h

@@ -385,7 +385,7 @@ class Builder {
       }
     }
     try {
-      checkKeyIsString(Slice(sub).isString());
+      checkKeyIsString(Slice(sub));
       auto oldPos = _pos;
       reserveSpace(1 + sizeof(void*));
       // store pointer. this doesn't need to be portable
@@ -576,6 +576,18 @@ class Builder {
     }
   }
 
+  inline void checkKeyIsString(Slice const& item) {
+    if (!_stack.empty()) {
+      ValueLength const& tos = _stack.back();
+      if (_start[tos] == 0x0b || _start[tos] == 0x14) {
+        if (!_keyWritten && !item.isString()) {
+          throw Exception(Exception::BuilderKeyMustBeString);
+        }
+        _keyWritten = !_keyWritten;
+      }
+    }
+  }
+
   inline void addArray(bool unindexed = false) {
     addCompoundValue(unindexed ? 0x13 : 0x06);
   }

3rdParty/velocypack/include/velocypack/Exception.h

@@ -66,6 +66,7 @@ struct Exception : std::exception {
     BuilderExternalsDisallowed = 37,
     BuilderKeyAlreadyWritten = 38,
     BuilderKeyMustBeString = 39,
+    BuilderCustomDisallowed = 40,
 
     ValidatorInvalidLength = 50,
     ValidatorInvalidType = 51,
@@ -140,6 +141,8 @@ struct Exception : std::exception {
         return "The key of the next key/value pair is already written";
       case BuilderKeyMustBeString:
         return "The key of the next key/value pair must be a string";
+      case BuilderCustomDisallowed:
+        return "Custom types are not allowed in this configuration";
     
       case ValidatorInvalidType:
         return "Invalid type found in binary data";

3rdParty/velocypack/include/velocypack/Options.h

@@ -115,6 +115,10 @@ struct Options {
   // values as a security precaution)
   bool disallowExternals = false;
 
+  // disallow using type Custom (to prevent injection of arbitrary opaque
+  // values as a security precaution)
+  bool disallowCustom = false;
+
   // default options with the above settings
   static Options Defaults;
 };

3rdParty/velocypack/include/velocypack/Validator.h

@@ -39,7 +39,7 @@ class Validator {
 
  public:
   explicit Validator(Options const* options = &Options::Defaults)
-      : options(options) {
+      : options(options), _level(0) {
     if (options == nullptr) {
       throw Exception(Exception::InternalError, "Options cannot be a nullptr");
     }
@@ -50,28 +50,31 @@ class Validator {
  public:
   // validates a VelocyPack Slice value starting at ptr, with length bytes length
   // throws if the data is invalid
-  bool validate(char const* ptr, size_t length, bool isSubPart = false) const {
+  bool validate(char const* ptr, size_t length, bool isSubPart = false) {
     return validate(reinterpret_cast<uint8_t const*>(ptr), length, isSubPart);
   }
 
   // validates a VelocyPack Slice value starting at ptr, with length bytes length
   // throws if the data is invalid
-  bool validate(uint8_t const* ptr, size_t length, bool isSubPart = false) const;
+  bool validate(uint8_t const* ptr, size_t length, bool isSubPart = false);
 
  private:
-  void validateArray(uint8_t const* ptr, size_t length) const;
+  void validateArray(uint8_t const* ptr, size_t length);
-  void validateCompactArray(uint8_t const* ptr, size_t length) const;
+  void validateCompactArray(uint8_t const* ptr, size_t length);
-  void validateUnindexedArray(uint8_t const* ptr, size_t length) const;
+  void validateUnindexedArray(uint8_t const* ptr, size_t length);
-  void validateIndexedArray(uint8_t const* ptr, size_t length) const;
+  void validateIndexedArray(uint8_t const* ptr, size_t length);
-  void validateObject(uint8_t const* ptr, size_t length) const;
+  void validateObject(uint8_t const* ptr, size_t length);
-  void validateCompactObject(uint8_t const* ptr, size_t length) const;
+  void validateCompactObject(uint8_t const* ptr, size_t length);
-  void validateIndexedObject(uint8_t const* ptr, size_t length) const;
+  void validateIndexedObject(uint8_t const* ptr, size_t length);
-  void validateBufferLength(size_t expected, size_t actual, bool isSubPart) const;
+  void validateBufferLength(size_t expected, size_t actual, bool isSubPart);
-  void validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const;
+  void validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart);
-  ValueLength readByteSize(uint8_t const*& ptr, uint8_t const* end) const;
+  ValueLength readByteSize(uint8_t const*& ptr, uint8_t const* end);
 
  public:
   Options const* options;
+
+ private:
+  int _level;
 };
 
 }  // namespace arangodb::velocypack

3rdParty/velocypack/src/Builder.cpp

@@ -794,6 +794,10 @@ uint8_t* Builder::set(Value const& item) {
       throw Exception(Exception::NotImplemented);
     }
     case ValueType::Custom: {
+      if (options->disallowCustom) {
+        // Custom values explicitly disallowed as a security precaution
+        throw Exception(Exception::BuilderCustomDisallowed);
+      }
       throw Exception(Exception::BuilderUnexpectedType,
                       "Cannot set a ValueType::Custom with this method");
     }
@@ -802,7 +806,12 @@ uint8_t* Builder::set(Value const& item) {
 }
 
 uint8_t* Builder::set(Slice const& item) {
-  checkKeyIsString(item.isString());
+  checkKeyIsString(item);
+
+  if (options->disallowCustom && item.isCustom()) {
+    // Custom values explicitly disallowed as a security precaution
+    throw Exception(Exception::BuilderCustomDisallowed);
+  }
 
   ValueLength const l = item.byteSize();
   reserveSpace(l);
@@ -846,6 +855,10 @@ uint8_t* Builder::set(ValuePair const& pair) {
     }
     return _start + oldPos;
   } else if (pair.valueType() == ValueType::Custom) {
+    if (options->disallowCustom) {
+      // Custom values explicitly disallowed as a security precaution
+      throw Exception(Exception::BuilderCustomDisallowed);
+    }
     // We only reserve space here, the caller has to fill in the custom type
     uint64_t size = pair.getSize();
     reserveSpace(size);

3rdParty/velocypack/src/Validator.cpp

@@ -24,6 +24,9 @@
 /// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
 ////////////////////////////////////////////////////////////////////////////////
 
+#include <unordered_set>
+#include <memory>
+
 #include "velocypack/velocypack-common.h"
 #include "velocypack/Validator.h"
 #include "velocypack/Exception.h"
@@ -56,7 +59,7 @@ static ValueLength ReadVariableLengthValue(uint8_t const*& p, uint8_t const* end
   return value;
 }
 
-bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) const {
+bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) {
   if (length == 0) {
     throw Exception(Exception::ValidatorInvalidLength, "length 0 is invalid for any VelocyPack value");
   }
@@ -74,20 +77,20 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons
   // special handling for certain types...
   switch (type) {
     case ValueType::None:
-    case ValueType::Null: 
+    case ValueType::Null:
-    case ValueType::Bool: 
+    case ValueType::Bool:
-    case ValueType::MinKey: 
+    case ValueType::MinKey:
-    case ValueType::MaxKey:  
+    case ValueType::MaxKey:
     case ValueType::SmallInt:
-    case ValueType::Int: 
+    case ValueType::Int:
-    case ValueType::UInt: 
+    case ValueType::UInt:
-    case ValueType::Double: 
+    case ValueType::Double:
-    case ValueType::UTCDate: 
+    case ValueType::UTCDate:
     case ValueType::Binary:
     case ValueType::Illegal: {
       break;
     }
-    
+
     case ValueType::String: {
       uint8_t const* p;
       ValueLength len;
@@ -111,12 +114,16 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons
     }
 
     case ValueType::Array: {
+      ++_level;
       validateArray(ptr, length);
+      --_level;
       break;
     }
-    
+
     case ValueType::Object: {
+      ++_level;
       validateObject(ptr, length);
+      --_level;
       break;
     }
 
@@ -154,35 +161,35 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons
         }
       } else if (head >= 0xf7U && head <= 0xf9U) {
         validateBufferLength(1 + 2, length, true);
-        byteSize = 1 + 2 + readIntegerNonEmpty<ValueLength>(ptr + 1, 2); 
+        byteSize = 1 + 2 + readIntegerNonEmpty<ValueLength>(ptr + 1, 2);
         if (byteSize == 1 + 2) {
           throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
         }
       } else if (head >= 0xfaU && head <= 0xfcU) {
         validateBufferLength(1 + 4, length, true);
-        byteSize = 1 + 4 + readIntegerNonEmpty<ValueLength>(ptr + 1, 4); 
+        byteSize = 1 + 4 + readIntegerNonEmpty<ValueLength>(ptr + 1, 4);
         if (byteSize == 1 + 4) {
           throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
         }
       } else if (head >= 0xfdU) {
         validateBufferLength(1 + 8, length, true);
-        byteSize = 1 + 8 + readIntegerNonEmpty<ValueLength>(ptr + 1, 8); 
+        byteSize = 1 + 8 + readIntegerNonEmpty<ValueLength>(ptr + 1, 8);
         if (byteSize == 1 + 8) {
           throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type");
         }
       }
-  
+
       validateSliceLength(ptr, byteSize, isSubPart);
       break;
     }
   }
-     
+
-  // common validation that must happen for all types 
+  // common validation that must happen for all types
   validateSliceLength(ptr, length, isSubPart);
   return true;
 }
 
-void Validator::validateArray(uint8_t const* ptr, size_t length) const {
+void Validator::validateArray(uint8_t const* ptr, size_t length) {
   uint8_t head = *ptr;
 
   if (head == 0x13U) {
@@ -199,7 +206,7 @@ void Validator::validateArray(uint8_t const* ptr, size_t length) const {
   }
 }
 
-void Validator::validateCompactArray(uint8_t const* ptr, size_t length) const {
+void Validator::validateCompactArray(uint8_t const* ptr, size_t length) {
   // compact Array without index table
   validateBufferLength(4, length, true);
 
@@ -218,31 +225,31 @@ void Validator::validateCompactArray(uint8_t const* ptr, size_t length) const {
     throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds");
   }
   ++p;
-  
+
-  // validate the array members 
+  // validate the array members
   uint8_t const* e = p;
   p = data;
-  while (nrItems-- > 0) { 
+  while (nrItems-- > 0) {
     validate(p, e - p, true);
     p += Slice(p).byteSize();
   }
 }
 
-void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const {
+void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) {
   // Array without index table, with 1-8 bytes lengths, all values with same length
   uint8_t head = *ptr;
   ValueLength const byteSizeLength = 1ULL << (static_cast<ValueLength>(head) - 0x02U);
   validateBufferLength(1 + byteSizeLength + 1, length, true);
   ValueLength const byteSize = readIntegerNonEmpty<ValueLength>(ptr + 1, byteSizeLength);
-  
+
   if (byteSize > length) {
     throw Exception(Exception::ValidatorInvalidLength, "Array length is out of bounds");
   }
 
   // look up first member
   uint8_t const* p = ptr + 1 + byteSizeLength;
-  uint8_t const* e = ptr + 1 + byteSizeLength + (8 - byteSizeLength);
+  uint8_t const* e = p + (8 - byteSizeLength);
-  
+
   if (e > ptr + byteSize) {
     e = ptr + byteSize;
   }
@@ -253,19 +260,28 @@ void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const
   if (p >= ptr + byteSize) {
     throw Exception(Exception::ValidatorInvalidLength, "Array structure is invalid");
   }
+
+  // check if padding is correct
+  if (p != ptr + 1 + byteSizeLength &&
+      p != ptr + 1 + byteSizeLength + (8 - byteSizeLength)) {
+    throw Exception(Exception::ValidatorInvalidLength, "Array padding is invalid");
+  }
   
   validate(p, length - (p - ptr), true);
   ValueLength itemSize = Slice(p).byteSize();
   if (itemSize == 0) {
     throw Exception(Exception::ValidatorInvalidLength, "Array itemSize value is invalid");
   }
-  ValueLength nrItems = (byteSize - (p - ptr)) / itemSize; 
+  ValueLength nrItems = (byteSize - (p - ptr)) / itemSize;
 
   if (nrItems == 0) {
     throw Exception(Exception::ValidatorInvalidLength, "Array nrItems value is invalid");
   }
-
+  // we already validated p, so move it forward
+  p += itemSize;
   e = ptr + length;
+  --nrItems;
+
   while (nrItems > 0) {
     if (p >= e) {
       throw Exception(Exception::ValidatorInvalidLength, "Array value is out of bounds");
@@ -278,10 +294,10 @@ void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const
     }
     p += itemSize;
     --nrItems;
-  }
+  } 
 }
 
-void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
+void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) {
   // Array with index table, with 1-8 bytes lengths
   uint8_t head = *ptr;
   ValueLength const byteSizeLength = 1ULL << (static_cast<ValueLength>(head) - 0x06U);
@@ -293,13 +309,13 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
   }
 
   ValueLength nrItems;
-  ValueLength dataOffset;
   uint8_t const* indexTable;
+  uint8_t const* firstMember;
 
   if (head == 0x09U) {
     // byte length = 8
     nrItems = readIntegerNonEmpty<ValueLength>(ptr + byteSize - byteSizeLength, byteSizeLength);
-    
+
     if (nrItems == 0) {
       throw Exception(Exception::ValidatorInvalidLength, "Array nrItems value is invalid");
     }
@@ -309,18 +325,18 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
       throw Exception(Exception::ValidatorInvalidLength, "Array index table is out of bounds");
     }
     
-    dataOffset = 1 + byteSizeLength;
+    firstMember = ptr + 1 + byteSizeLength; 
   } else {
     // byte length = 1, 2 or 4
     nrItems = readIntegerNonEmpty<ValueLength>(ptr + 1 + byteSizeLength, byteSizeLength);
-    
+
     if (nrItems == 0) {
       throw Exception(Exception::ValidatorInvalidLength, "Array nrItems value is invalid");
     }
-    
+
     // look up first member
     uint8_t const* p = ptr + 1 + byteSizeLength + byteSizeLength;
-    uint8_t const* e = ptr + 1 + (8 - byteSizeLength - byteSizeLength);
+    uint8_t const* e = p + (8 - byteSizeLength - byteSizeLength);
     if (e > ptr + byteSize) {
       e = ptr + byteSize;
     }
@@ -328,26 +344,42 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
       ++p;
     }
 
+    // check if padding is correct
+    if (p != ptr + 1 + byteSizeLength + byteSizeLength &&
+        p != ptr + 1 + byteSizeLength + byteSizeLength + (8 - byteSizeLength - byteSizeLength)) {
+      throw Exception(Exception::ValidatorInvalidLength, "Array padding is invalid");
+    }
+  
     indexTable = ptr + byteSize - (nrItems * byteSizeLength);
     if (indexTable < ptr + byteSizeLength + byteSizeLength || indexTable < p) {
       throw Exception(Exception::ValidatorInvalidLength, "Array index table is out of bounds");
     }
-    
+
-    dataOffset = 1 + byteSizeLength + byteSizeLength;
+    firstMember = p;
+  }
+   
+  VELOCYPACK_ASSERT(nrItems > 0); 
+  
+  ValueLength actualNrItems = 0;
+  uint8_t const* member = firstMember;
+  while (member < indexTable) {
+    validate(member, indexTable - member, true);
+    ValueLength offset = readIntegerNonEmpty<ValueLength>(
+        indexTable + actualNrItems * byteSizeLength, byteSizeLength);
+    if (offset != static_cast<ValueLength>(member - ptr)) {
+      throw Exception(Exception::ValidatorInvalidLength, "Array index table is wrong");
+    }
+  
+    member += Slice(member).byteSize();
+    ++actualNrItems;
   }
 
-  while (nrItems > 0) {
+  if (actualNrItems != nrItems) {
-    ValueLength const offset = readIntegerNonEmpty<ValueLength>(indexTable, byteSizeLength);
+    throw Exception(Exception::ValidatorInvalidLength, "Array has more items than in index");
-    if (offset < dataOffset || offset >= static_cast<ValueLength>(indexTable - ptr)) {
-      throw Exception(Exception::ValidatorInvalidLength, "Array index table entry is out of bounds");
-    }
-    validate(ptr + offset, length - offset, true);
-    indexTable += byteSizeLength; 
-    --nrItems;
   }
 }
 
-void Validator::validateObject(uint8_t const* ptr, size_t length) const {
+void Validator::validateObject(uint8_t const* ptr, size_t length) {
   uint8_t head = *ptr;
 
   if (head == 0x14U) {
@@ -361,7 +393,7 @@ void Validator::validateObject(uint8_t const* ptr, size_t length) const {
   }
 }
 
-void Validator::validateCompactObject(uint8_t const* ptr, size_t length) const {
+void Validator::validateCompactObject(uint8_t const* ptr, size_t length) {
   // compact Object without index table
   validateBufferLength(5, length, true);
 
@@ -380,12 +412,12 @@ void Validator::validateCompactObject(uint8_t const* ptr, size_t length) const {
     throw Exception(Exception::ValidatorInvalidLength, "Object length value is out of bounds");
   }
   ++p;
-  
+
-  // validate the object members 
+  // validate the object members
   uint8_t const* e = p;
   p = data;
   while (nrItems-- > 0) {
-    // validate key 
+    // validate key
     validate(p, e - p, true);
     Slice key(p);
     if (!key.isString() && !key.isInteger()) {
@@ -397,9 +429,14 @@ void Validator::validateCompactObject(uint8_t const* ptr, size_t length) const {
     validate(p, e - p, true);
     p += Slice(p).byteSize();
   }
+
+  // finally check if we are now pointing at the end or not
+  if (p != e) {
+    throw Exception(Exception::ValidatorInvalidLength, "Object has more members than specified");
+  }
 }
 
-void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
+void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) {
   // Object with index table, with 1-8 bytes lengths
   uint8_t head = *ptr;
   ValueLength const byteSizeLength = 1ULL << (static_cast<ValueLength>(head) - 0x0bU);
@@ -411,13 +448,14 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
   }
 
   ValueLength nrItems;
-  ValueLength dataOffset;
   uint8_t const* indexTable;
+  uint8_t const* firstMember;
 
-  if (head == 0x12U) {
+  //if (head == 0x12U) {
+  if (head == 0x0eU || head == 0x12U) {
     // byte length = 8
     nrItems = readIntegerNonEmpty<ValueLength>(ptr + byteSize - byteSizeLength, byteSizeLength);
-    
+
     if (nrItems == 0) {
       throw Exception(Exception::ValidatorInvalidLength, "Object nrItems value is invalid");
     }
@@ -427,18 +465,18 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
       throw Exception(Exception::ValidatorInvalidLength, "Object index table is out of bounds");
     }
     
-    dataOffset = 1 + byteSizeLength;
+    firstMember = ptr + byteSize;
   } else {
     // byte length = 1, 2 or 4
     nrItems = readIntegerNonEmpty<ValueLength>(ptr + 1 + byteSizeLength, byteSizeLength);
-    
+
     if (nrItems == 0) {
       throw Exception(Exception::ValidatorInvalidLength, "Object nrItems value is invalid");
     }
-    
+
     // look up first member
     uint8_t const* p = ptr + 1 + byteSizeLength + byteSizeLength;
-    uint8_t const* e = ptr + 1 + (8 - byteSizeLength - byteSizeLength);
+    uint8_t const* e = p + (8 - byteSizeLength - byteSizeLength);
     if (e > ptr + byteSize) {
       e = ptr + byteSize;
     }
@@ -446,44 +484,116 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
       ++p;
     }
 
+    // check if padding is correct
+    if (p != ptr + 1 + byteSizeLength + byteSizeLength &&
+        p != ptr + 1 + byteSizeLength + byteSizeLength + (8 - byteSizeLength - byteSizeLength)) {
+      throw Exception(Exception::ValidatorInvalidLength, "Object padding is invalid");
+    }
+  
     indexTable = ptr + byteSize - (nrItems * byteSizeLength);
     if (indexTable < ptr + byteSizeLength + byteSizeLength || indexTable < p) {
       throw Exception(Exception::ValidatorInvalidLength, "Object index table is out of bounds");
     }
-    
+
-    dataOffset = 1 + byteSizeLength + byteSizeLength;
+    firstMember = p;
   }
 
-  while (nrItems > 0) {
+  VELOCYPACK_ASSERT(nrItems > 0);
-    ValueLength offset = readIntegerNonEmpty<ValueLength>(indexTable, byteSizeLength);
+  
-    if (offset < dataOffset || offset >= static_cast<ValueLength>(indexTable - ptr)) {
+  ValueLength tableBuf[16];    // Fixed space to save offsets found sequentially
-      throw Exception(Exception::ValidatorInvalidLength, "Object index table entry is out of bounds");
+  ValueLength* table = tableBuf;
+  std::unique_ptr<ValueLength[]> tableGuard;
+  std::unique_ptr<std::unordered_set<ValueLength>> offsetSet;
+  if (nrItems > 16) {
+    if (nrItems <= 128) {
+      table = new ValueLength[nrItems];  // throws if bad_alloc
+      tableGuard.reset(table);   // for automatic deletion
+    } else {
+      // if we have even more items, we directly create an unordered_set
+      offsetSet.reset(new std::unordered_set<ValueLength>());
     }
-    // validate key
+  }
-    validate(ptr + offset, length - offset, true);
+  ValueLength actualNrItems = 0;
-    Slice key(ptr + offset);
+  uint8_t const* member = firstMember;
+  while (member < indexTable) {
+    validate(member, indexTable - member, true);
+
+    Slice key(member);
     if (!key.isString() && !key.isInteger()) {
       throw Exception(Exception::ValidatorInvalidLength, "Invalid object key type");
     }
-    
-    // validate value
-    offset += key.byteSize();
-    validate(ptr + offset, length - offset, true);
 
-    indexTable += byteSizeLength; 
+    ValueLength const keySize = key.byteSize();
-    --nrItems;
+    uint8_t const* value = member + keySize;
+    if (value >= indexTable) {
+      throw Exception(Exception::ValidatorInvalidLength, "Object value leaking into index table");
+    }
+    validate(value, indexTable - value, true);
+
+    ValueLength offset = static_cast<ValueLength>(member - ptr);
+    if (nrItems <= 128) {
+      table[actualNrItems] = offset;
+    } else {
+      offsetSet->emplace(offset);
+    }
+
+    member += keySize + Slice(value).byteSize();
+    ++actualNrItems;
+
+    if (actualNrItems > nrItems) {
+      throw Exception(Exception::ValidatorInvalidLength, "Object value has more key/value pairs than announced");
+    }
+  }
+
+  if (actualNrItems < nrItems) {
+    throw Exception(Exception::ValidatorInvalidLength, "Object has fewer items than in index");
+  }
+
+  // Finally verify each offset in the index:
+  if (nrItems <= 128) {
+    for (ValueLength pos = 0; pos < nrItems; ++pos) {
+      ValueLength offset = readIntegerNonEmpty<ValueLength>(
+          indexTable + pos * byteSizeLength, byteSizeLength);
+      // Binary search in sorted index list:
+      ValueLength low = 0;
+      ValueLength high = nrItems;
+      bool found = false;
+      while (low < high) {
+        ValueLength mid = (low + high) / 2;
+        if (offset == table[mid]) {
+          found = true;
+          break;
+        } else if (offset < table[mid]) {
+          high = mid;
+        } else {   // offset > table[mid]
+          low = mid + 1;
+        }
+      }
+      if (!found) {
+        throw Exception(Exception::ValidatorInvalidLength, "Object has invalid index offset");
+      }
+    }
+  } else {
+    for (ValueLength pos = 0; pos < nrItems; ++pos) {
+      ValueLength offset = readIntegerNonEmpty<ValueLength>(
+          indexTable + pos * byteSizeLength, byteSizeLength);
+      auto i = offsetSet->find(offset);
+      if (i == offsetSet->end()) {
+        throw Exception(Exception::ValidatorInvalidLength, "Object has invalid index offset");
+      }
+      offsetSet->erase(i);
+    }
   }
 }
 
-void Validator::validateBufferLength(size_t expected, size_t actual, bool isSubPart) const {
+void Validator::validateBufferLength(size_t expected, size_t actual, bool isSubPart) {
   if ((expected > actual) ||
       (expected != actual && !isSubPart)) {
     throw Exception(Exception::ValidatorInvalidLength, "given buffer length is unequal to actual length of Slice in buffer");
   }
 }
-        
+
-void Validator::validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const {
+void Validator::validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) {
   size_t actual = static_cast<size_t>(Slice(ptr).byteSize());
   validateBufferLength(actual, length, isSubPart);
 }
-

arangod/GeneralServer/VstCommTask.cpp

@@ -39,20 +39,25 @@
 #include "RestServer/ServerFeature.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
-#include "Utils/Events.h"
 #include "VocBase/ticks.h"
 
-#include <iostream>
-#include <limits>
 #include <stdexcept>
 
+#include <velocypack/Options.h>
+#include <velocypack/Validator.h>
+#include <velocypack/velocypack-aliases.h>
+
 using namespace arangodb;
 using namespace arangodb::basics;
 using namespace arangodb::rest;
 
 inline std::size_t validateAndCount(char const* vpStart, char const* vpEnd) {
+  // intentional copy
   VPackOptions validationOptions = VPackOptions::Defaults;
   validationOptions.validateUtf8Strings = true;
+  validationOptions.disallowExternals = true;
+  validationOptions.disallowCustom = true;
+  validationOptions.checkAttributeUniqueness = true;
   VPackValidator validator(&validationOptions);
 
   try {

arangod/GeneralServer/VstNetwork.h

@@ -30,7 +30,6 @@
 
 #include <velocypack/Options.h>
 #include <velocypack/Slice.h>
-#include <velocypack/Validator.h>
 #include <velocypack/velocypack-aliases.h>
 
 #include <memory>

arangod/V8Server/v8-actions.cpp

@@ -51,7 +51,6 @@
 #include <velocypack/Buffer.h>
 #include <velocypack/Builder.h>
 #include <velocypack/Parser.h>
-#include <velocypack/Validator.h>
 #include <velocypack/velocypack-aliases.h>
 
 using namespace arangodb;

lib/Rest/HttpRequest.cpp

@@ -740,9 +740,12 @@ VPackSlice HttpRequest::payload(VPackOptions const* options) {
   } else /*VPACK*/ {
     VPackOptions validationOptions = *options;  // intentional copy
     validationOptions.validateUtf8Strings = true;
+    validationOptions.checkAttributeUniqueness = true;
+    validationOptions.disallowExternals = true;
+    validationOptions.disallowCustom = true;
     VPackValidator validator(&validationOptions);
-    validator.validate(_body.c_str(), _body.length());
+    validator.validate(_body.data(), _body.size());
-    return VPackSlice(_body.c_str());
+    return VPackSlice(_body.data());
   }
 }
 

lib/V8/v8-utils.cpp

@@ -3981,13 +3981,18 @@ static void JS_VPackToV8(v8::FunctionCallbackInfo<v8::Value> const& args) {
   if (args.Length() != 1) {
     TRI_V8_THROW_EXCEPTION_USAGE("VPACK_TO_V8(value)");
   }
+    
+  VPackOptions validationOptions = VPackOptions::Defaults;
+  validationOptions.validateUtf8Strings = true;
+  validationOptions.disallowExternals = true;
+  validationOptions.checkAttributeUniqueness = true;
+  VPackValidator validator(&validationOptions);
 
   if (args[0]->IsString() || args[0]->IsStringObject()) {
     // supplied argument is a string
     std::string const value = TRI_ObjectToString(isolate, args[0]);
 
-    VPackValidator validator;
+    validator.validate(value.data(), value.size(), false);
-    validator.validate(value.c_str(), value.size(), false);
 
     VPackSlice slice(value.c_str());
     v8::Handle<v8::Value> result = TRI_VPackToV8(isolate, slice);
@@ -3997,7 +4002,6 @@ static void JS_VPackToV8(v8::FunctionCallbackInfo<v8::Value> const& args) {
     char const* data = V8Buffer::data(args[0].As<v8::Object>());
     size_t size = V8Buffer::length(args[0].As<v8::Object>());
 
-    VPackValidator validator;
     validator.validate(data, size, false);
 
     VPackSlice slice(data);