Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update velocypack (#8074)

This commit is contained in:
Jan 2019-01-31 17:32:10 +01:00 committed by Frank Celler
parent 8046192cdf
commit 2e29b08b9f
11 changed files with 258 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
3rdParty/velocypack/include/velocypack/Builder.h vendored
View File

@ -385,7 +385,7 @@ class Builder {
}
}
try {
checkKeyIsString(Slice(sub).isString());
checkKeyIsString(Slice(sub));
auto oldPos = _pos;
reserveSpace(1 + sizeof(void*));
// store pointer. this doesn't need to be portable
@ -576,6 +576,18 @@ class Builder {
}
}
inline void checkKeyIsString(Slice const& item) {
if (!_stack.empty()) {
ValueLength const& tos = _stack.back();
if (_start[tos] == 0x0b || _start[tos] == 0x14) {
if (!_keyWritten && !item.isString()) {
throw Exception(Exception::BuilderKeyMustBeString);
}
_keyWritten = !_keyWritten;
}
}
}
inline void addArray(bool unindexed = false) {
addCompoundValue(unindexed ? 0x13 : 0x06);
}

3
3rdParty/velocypack/include/velocypack/Exception.h vendored
View File

@ -66,6 +66,7 @@ struct Exception : std::exception {
BuilderExternalsDisallowed = 37,
BuilderKeyAlreadyWritten = 38,
BuilderKeyMustBeString = 39,
BuilderCustomDisallowed = 40,
ValidatorInvalidLength = 50,
ValidatorInvalidType = 51,
@ -140,6 +141,8 @@ struct Exception : std::exception {
return "The key of the next key/value pair is already written";
case BuilderKeyMustBeString:
return "The key of the next key/value pair must be a string";
case BuilderCustomDisallowed:
return "Custom types are not allowed in this configuration";
case ValidatorInvalidType:
return "Invalid type found in binary data";

4
3rdParty/velocypack/include/velocypack/Options.h vendored
View File

@ -115,6 +115,10 @@ struct Options {
// values as a security precaution)
bool disallowExternals = false;
// disallow using type Custom (to prevent injection of arbitrary opaque
// values as a security precaution)
bool disallowCustom = false;
// default options with the above settings
static Options Defaults;
};

29
3rdParty/velocypack/include/velocypack/Validator.h vendored
View File

@ -39,7 +39,7 @@ class Validator {
public:
explicit Validator(Options const* options = &Options::Defaults)
: options(options) {
: options(options), _level(0) {
if (options == nullptr) {
throw Exception(Exception::InternalError, "Options cannot be a nullptr");
}
@ -50,28 +50,31 @@ class Validator {
public:
// validates a VelocyPack Slice value starting at ptr, with length bytes length
// throws if the data is invalid
bool validate(char const* ptr, size_t length, bool isSubPart = false) const {
bool validate(char const* ptr, size_t length, bool isSubPart = false) {
return validate(reinterpret_cast<uint8_t const*>(ptr), length, isSubPart);
}
// validates a VelocyPack Slice value starting at ptr, with length bytes length
// throws if the data is invalid
bool validate(uint8_t const* ptr, size_t length, bool isSubPart = false) const;
bool validate(uint8_t const* ptr, size_t length, bool isSubPart = false);
private:
void validateArray(uint8_t const* ptr, size_t length) const;
void validateCompactArray(uint8_t const* ptr, size_t length) const;
void validateUnindexedArray(uint8_t const* ptr, size_t length) const;
void validateIndexedArray(uint8_t const* ptr, size_t length) const;
void validateObject(uint8_t const* ptr, size_t length) const;
void validateCompactObject(uint8_t const* ptr, size_t length) const;
void validateIndexedObject(uint8_t const* ptr, size_t length) const;
void validateBufferLength(size_t expected, size_t actual, bool isSubPart) const;
void validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const;
ValueLength readByteSize(uint8_t const*& ptr, uint8_t const* end) const;
void validateArray(uint8_t const* ptr, size_t length);
void validateCompactArray(uint8_t const* ptr, size_t length);
void validateUnindexedArray(uint8_t const* ptr, size_t length);
void validateIndexedArray(uint8_t const* ptr, size_t length);
void validateObject(uint8_t const* ptr, size_t length);
void validateCompactObject(uint8_t const* ptr, size_t length);
void validateIndexedObject(uint8_t const* ptr, size_t length);
void validateBufferLength(size_t expected, size_t actual, bool isSubPart);
void validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart);
ValueLength readByteSize(uint8_t const*& ptr, uint8_t const* end);
public:
Options const* options;
private:
int _level;
};
} // namespace arangodb::velocypack

15
3rdParty/velocypack/src/Builder.cpp vendored
View File

@ -794,6 +794,10 @@ uint8_t* Builder::set(Value const& item) {
throw Exception(Exception::NotImplemented);
}
case ValueType::Custom: {
if (options->disallowCustom) {
// Custom values explicitly disallowed as a security precaution
throw Exception(Exception::BuilderCustomDisallowed);
}
throw Exception(Exception::BuilderUnexpectedType,
"Cannot set a ValueType::Custom with this method");
}
@ -802,7 +806,12 @@ uint8_t* Builder::set(Value const& item) {
}
uint8_t* Builder::set(Slice const& item) {
checkKeyIsString(item.isString());
checkKeyIsString(item);
if (options->disallowCustom && item.isCustom()) {
// Custom values explicitly disallowed as a security precaution
throw Exception(Exception::BuilderCustomDisallowed);
}
ValueLength const l = item.byteSize();
reserveSpace(l);
@ -846,6 +855,10 @@ uint8_t* Builder::set(ValuePair const& pair) {
}
return _start + oldPos;
} else if (pair.valueType() == ValueType::Custom) {
if (options->disallowCustom) {
// Custom values explicitly disallowed as a security precaution
throw Exception(Exception::BuilderCustomDisallowed);
}
// We only reserve space here, the caller has to fill in the custom type
uint64_t size = pair.getSize();
reserveSpace(size);

192
3rdParty/velocypack/src/Validator.cpp vendored
View File

@ -24,6 +24,9 @@
/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include <unordered_set>
#include <memory>
#include "velocypack/velocypack-common.h"
#include "velocypack/Validator.h"
#include "velocypack/Exception.h"
@ -56,7 +59,7 @@ static ValueLength ReadVariableLengthValue(uint8_t const*& p, uint8_t const* end
return value;
}
bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) const {
bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) {
if (length == 0) {
throw Exception(Exception::ValidatorInvalidLength, "length 0 is invalid for any VelocyPack value");
}
@ -111,12 +114,16 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons
}
case ValueType::Array: {
++_level;
validateArray(ptr, length);
--_level;
break;
}
case ValueType::Object: {
++_level;
validateObject(ptr, length);
--_level;
break;
}
@ -182,7 +189,7 @@ bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) cons
return true;
}
void Validator::validateArray(uint8_t const* ptr, size_t length) const {
void Validator::validateArray(uint8_t const* ptr, size_t length) {
uint8_t head = *ptr;
if (head == 0x13U) {
@ -199,7 +206,7 @@ void Validator::validateArray(uint8_t const* ptr, size_t length) const {
}
}
void Validator::validateCompactArray(uint8_t const* ptr, size_t length) const {
void Validator::validateCompactArray(uint8_t const* ptr, size_t length) {
// compact Array without index table
validateBufferLength(4, length, true);
@ -228,7 +235,7 @@ void Validator::validateCompactArray(uint8_t const* ptr, size_t length) const {
}
}
void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const {
void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) {
// Array without index table, with 1-8 bytes lengths, all values with same length
uint8_t head = *ptr;
ValueLength const byteSizeLength = 1ULL << (static_cast<ValueLength>(head) - 0x02U);
@ -241,7 +248,7 @@ void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const
// look up first member
uint8_t const* p = ptr + 1 + byteSizeLength;
uint8_t const* e = ptr + 1 + byteSizeLength + (8 - byteSizeLength);
uint8_t const* e = p + (8 - byteSizeLength);
if (e > ptr + byteSize) {
e = ptr + byteSize;
@ -254,6 +261,12 @@ void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const
throw Exception(Exception::ValidatorInvalidLength, "Array structure is invalid");
}
// check if padding is correct
if (p != ptr + 1 + byteSizeLength &&
p != ptr + 1 + byteSizeLength + (8 - byteSizeLength)) {
throw Exception(Exception::ValidatorInvalidLength, "Array padding is invalid");
}
validate(p, length - (p - ptr), true);
ValueLength itemSize = Slice(p).byteSize();
if (itemSize == 0) {
@ -264,8 +277,11 @@ void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const
if (nrItems == 0) {
throw Exception(Exception::ValidatorInvalidLength, "Array nrItems value is invalid");
}
// we already validated p, so move it forward
p += itemSize;
e = ptr + length;
--nrItems;
while (nrItems > 0) {
if (p >= e) {
throw Exception(Exception::ValidatorInvalidLength, "Array value is out of bounds");
@ -281,7 +297,7 @@ void Validator::validateUnindexedArray(uint8_t const* ptr, size_t length) const
}
}
void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) {
// Array with index table, with 1-8 bytes lengths
uint8_t head = *ptr;
ValueLength const byteSizeLength = 1ULL << (static_cast<ValueLength>(head) - 0x06U);
@ -293,8 +309,8 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
}
ValueLength nrItems;
ValueLength dataOffset;
uint8_t const* indexTable;
uint8_t const* firstMember;
if (head == 0x09U) {
// byte length = 8
@ -309,7 +325,7 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
throw Exception(Exception::ValidatorInvalidLength, "Array index table is out of bounds");
}
dataOffset = 1 + byteSizeLength;
firstMember = ptr + 1 + byteSizeLength;
} else {
// byte length = 1, 2 or 4
nrItems = readIntegerNonEmpty<ValueLength>(ptr + 1 + byteSizeLength, byteSizeLength);
@ -320,7 +336,7 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
// look up first member
uint8_t const* p = ptr + 1 + byteSizeLength + byteSizeLength;
uint8_t const* e = ptr + 1 + (8 - byteSizeLength - byteSizeLength);
uint8_t const* e = p + (8 - byteSizeLength - byteSizeLength);
if (e > ptr + byteSize) {
e = ptr + byteSize;
}
@ -328,26 +344,42 @@ void Validator::validateIndexedArray(uint8_t const* ptr, size_t length) const {
++p;
}
// check if padding is correct
if (p != ptr + 1 + byteSizeLength + byteSizeLength &&
p != ptr + 1 + byteSizeLength + byteSizeLength + (8 - byteSizeLength - byteSizeLength)) {
throw Exception(Exception::ValidatorInvalidLength, "Array padding is invalid");
}
indexTable = ptr + byteSize - (nrItems * byteSizeLength);
if (indexTable < ptr + byteSizeLength + byteSizeLength || indexTable < p) {
throw Exception(Exception::ValidatorInvalidLength, "Array index table is out of bounds");
}
dataOffset = 1 + byteSizeLength + byteSizeLength;
firstMember = p;
}
while (nrItems > 0) {
ValueLength const offset = readIntegerNonEmpty<ValueLength>(indexTable, byteSizeLength);
if (offset < dataOffset || offset >= static_cast<ValueLength>(indexTable - ptr)) {
throw Exception(Exception::ValidatorInvalidLength, "Array index table entry is out of bounds");
VELOCYPACK_ASSERT(nrItems > 0);
ValueLength actualNrItems = 0;
uint8_t const* member = firstMember;
while (member < indexTable) {
validate(member, indexTable - member, true);
ValueLength offset = readIntegerNonEmpty<ValueLength>(
indexTable + actualNrItems * byteSizeLength, byteSizeLength);
if (offset != static_cast<ValueLength>(member - ptr)) {
throw Exception(Exception::ValidatorInvalidLength, "Array index table is wrong");
}
validate(ptr + offset, length - offset, true);
indexTable += byteSizeLength;
--nrItems;
member += Slice(member).byteSize();
++actualNrItems;
}
if (actualNrItems != nrItems) {
throw Exception(Exception::ValidatorInvalidLength, "Array has more items than in index");
}
}
void Validator::validateObject(uint8_t const* ptr, size_t length) const {
void Validator::validateObject(uint8_t const* ptr, size_t length) {
uint8_t head = *ptr;
if (head == 0x14U) {
@ -361,7 +393,7 @@ void Validator::validateObject(uint8_t const* ptr, size_t length) const {
}
}
void Validator::validateCompactObject(uint8_t const* ptr, size_t length) const {
void Validator::validateCompactObject(uint8_t const* ptr, size_t length) {
// compact Object without index table
validateBufferLength(5, length, true);
@ -397,9 +429,14 @@ void Validator::validateCompactObject(uint8_t const* ptr, size_t length) const {
validate(p, e - p, true);
p += Slice(p).byteSize();
}
// finally check if we are now pointing at the end or not
if (p != e) {
throw Exception(Exception::ValidatorInvalidLength, "Object has more members than specified");
}
}
void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) {
// Object with index table, with 1-8 bytes lengths
uint8_t head = *ptr;
ValueLength const byteSizeLength = 1ULL << (static_cast<ValueLength>(head) - 0x0bU);
@ -411,10 +448,11 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
}
ValueLength nrItems;
ValueLength dataOffset;
uint8_t const* indexTable;
uint8_t const* firstMember;
if (head == 0x12U) {
//if (head == 0x12U) {
if (head == 0x0eU || head == 0x12U) {
// byte length = 8
nrItems = readIntegerNonEmpty<ValueLength>(ptr + byteSize - byteSizeLength, byteSizeLength);
@ -427,7 +465,7 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
throw Exception(Exception::ValidatorInvalidLength, "Object index table is out of bounds");
}
dataOffset = 1 + byteSizeLength;
firstMember = ptr + byteSize;
} else {
// byte length = 1, 2 or 4
nrItems = readIntegerNonEmpty<ValueLength>(ptr + 1 + byteSizeLength, byteSizeLength);
@ -438,7 +476,7 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
// look up first member
uint8_t const* p = ptr + 1 + byteSizeLength + byteSizeLength;
uint8_t const* e = ptr + 1 + (8 - byteSizeLength - byteSizeLength);
uint8_t const* e = p + (8 - byteSizeLength - byteSizeLength);
if (e > ptr + byteSize) {
e = ptr + byteSize;
}
@ -446,44 +484,116 @@ void Validator::validateIndexedObject(uint8_t const* ptr, size_t length) const {
++p;
}
// check if padding is correct
if (p != ptr + 1 + byteSizeLength + byteSizeLength &&
p != ptr + 1 + byteSizeLength + byteSizeLength + (8 - byteSizeLength - byteSizeLength)) {
throw Exception(Exception::ValidatorInvalidLength, "Object padding is invalid");
}
indexTable = ptr + byteSize - (nrItems * byteSizeLength);
if (indexTable < ptr + byteSizeLength + byteSizeLength || indexTable < p) {
throw Exception(Exception::ValidatorInvalidLength, "Object index table is out of bounds");
}
dataOffset = 1 + byteSizeLength + byteSizeLength;
firstMember = p;
}
while (nrItems > 0) {
ValueLength offset = readIntegerNonEmpty<ValueLength>(indexTable, byteSizeLength);
if (offset < dataOffset || offset >= static_cast<ValueLength>(indexTable - ptr)) {
throw Exception(Exception::ValidatorInvalidLength, "Object index table entry is out of bounds");
VELOCYPACK_ASSERT(nrItems > 0);
ValueLength tableBuf[16]; // Fixed space to save offsets found sequentially
ValueLength* table = tableBuf;
std::unique_ptr<ValueLength[]> tableGuard;
std::unique_ptr<std::unordered_set<ValueLength>> offsetSet;
if (nrItems > 16) {
if (nrItems <= 128) {
table = new ValueLength[nrItems]; // throws if bad_alloc
tableGuard.reset(table); // for automatic deletion
} else {
// if we have even more items, we directly create an unordered_set
offsetSet.reset(new std::unordered_set<ValueLength>());
}
// validate key
validate(ptr + offset, length - offset, true);
Slice key(ptr + offset);
}
ValueLength actualNrItems = 0;
uint8_t const* member = firstMember;
while (member < indexTable) {
validate(member, indexTable - member, true);
Slice key(member);
if (!key.isString() && !key.isInteger()) {
throw Exception(Exception::ValidatorInvalidLength, "Invalid object key type");
}
// validate value
offset += key.byteSize();
validate(ptr + offset, length - offset, true);
ValueLength const keySize = key.byteSize();
uint8_t const* value = member + keySize;
if (value >= indexTable) {
throw Exception(Exception::ValidatorInvalidLength, "Object value leaking into index table");
}
validate(value, indexTable - value, true);
indexTable += byteSizeLength;
--nrItems;
ValueLength offset = static_cast<ValueLength>(member - ptr);
if (nrItems <= 128) {
table[actualNrItems] = offset;
} else {
offsetSet->emplace(offset);
}
member += keySize + Slice(value).byteSize();
++actualNrItems;
if (actualNrItems > nrItems) {
throw Exception(Exception::ValidatorInvalidLength, "Object value has more key/value pairs than announced");
}
}
void Validator::validateBufferLength(size_t expected, size_t actual, bool isSubPart) const {
if (actualNrItems < nrItems) {
throw Exception(Exception::ValidatorInvalidLength, "Object has fewer items than in index");
}
// Finally verify each offset in the index:
if (nrItems <= 128) {
for (ValueLength pos = 0; pos < nrItems; ++pos) {
ValueLength offset = readIntegerNonEmpty<ValueLength>(
indexTable + pos * byteSizeLength, byteSizeLength);
// Binary search in sorted index list:
ValueLength low = 0;
ValueLength high = nrItems;
bool found = false;
while (low < high) {
ValueLength mid = (low + high) / 2;
if (offset == table[mid]) {
found = true;
break;
} else if (offset < table[mid]) {
high = mid;
} else { // offset > table[mid]
low = mid + 1;
}
}
if (!found) {
throw Exception(Exception::ValidatorInvalidLength, "Object has invalid index offset");
}
}
} else {
for (ValueLength pos = 0; pos < nrItems; ++pos) {
ValueLength offset = readIntegerNonEmpty<ValueLength>(
indexTable + pos * byteSizeLength, byteSizeLength);
auto i = offsetSet->find(offset);
if (i == offsetSet->end()) {
throw Exception(Exception::ValidatorInvalidLength, "Object has invalid index offset");
}
offsetSet->erase(i);
}
}
}
void Validator::validateBufferLength(size_t expected, size_t actual, bool isSubPart) {
if ((expected > actual) ||
(expected != actual && !isSubPart)) {
throw Exception(Exception::ValidatorInvalidLength, "given buffer length is unequal to actual length of Slice in buffer");
}
}
void Validator::validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const {
void Validator::validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) {
size_t actual = static_cast<size_t>(Slice(ptr).byteSize());
validateBufferLength(actual, length, isSubPart);
}

11
arangod/GeneralServer/VstCommTask.cpp
View File

@ -39,20 +39,25 @@
#include "RestServer/ServerFeature.h"
#include "Scheduler/Scheduler.h"
#include "Scheduler/SchedulerFeature.h"
#include "Utils/Events.h"
#include "VocBase/ticks.h"
#include <iostream>
#include <limits>
#include <stdexcept>
#include <velocypack/Options.h>
#include <velocypack/Validator.h>
#include <velocypack/velocypack-aliases.h>
using namespace arangodb;
using namespace arangodb::basics;
using namespace arangodb::rest;
inline std::size_t validateAndCount(char const* vpStart, char const* vpEnd) {
// intentional copy
VPackOptions validationOptions = VPackOptions::Defaults;
validationOptions.validateUtf8Strings = true;
validationOptions.disallowExternals = true;
validationOptions.disallowCustom = true;
validationOptions.checkAttributeUniqueness = true;
VPackValidator validator(&validationOptions);
try {

1
arangod/GeneralServer/VstNetwork.h
View File

@ -30,7 +30,6 @@
#include <velocypack/Options.h>
#include <velocypack/Slice.h>
#include <velocypack/Validator.h>
#include <velocypack/velocypack-aliases.h>
#include <memory>

1
arangod/V8Server/v8-actions.cpp
View File

@ -51,7 +51,6 @@
#include <velocypack/Buffer.h>
#include <velocypack/Builder.h>
#include <velocypack/Parser.h>
#include <velocypack/Validator.h>
#include <velocypack/velocypack-aliases.h>
using namespace arangodb;

7
lib/Rest/HttpRequest.cpp
View File

@ -740,9 +740,12 @@ VPackSlice HttpRequest::payload(VPackOptions const* options) {
} else /*VPACK*/ {
VPackOptions validationOptions = *options; // intentional copy
validationOptions.validateUtf8Strings = true;
validationOptions.checkAttributeUniqueness = true;
validationOptions.disallowExternals = true;
validationOptions.disallowCustom = true;
VPackValidator validator(&validationOptions);
validator.validate(_body.c_str(), _body.length());
return VPackSlice(_body.c_str());
validator.validate(_body.data(), _body.size());
return VPackSlice(_body.data());
}
}

10
lib/V8/v8-utils.cpp
View File

@ -3982,12 +3982,17 @@ static void JS_VPackToV8(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_THROW_EXCEPTION_USAGE("VPACK_TO_V8(value)");
}
VPackOptions validationOptions = VPackOptions::Defaults;
validationOptions.validateUtf8Strings = true;
validationOptions.disallowExternals = true;
validationOptions.checkAttributeUniqueness = true;
VPackValidator validator(&validationOptions);
if (args[0]->IsString() || args[0]->IsStringObject()) {
// supplied argument is a string
std::string const value = TRI_ObjectToString(isolate, args[0]);
VPackValidator validator;
validator.validate(value.c_str(), value.size(), false);
validator.validate(value.data(), value.size(), false);
VPackSlice slice(value.c_str());
v8::Handle<v8::Value> result = TRI_VPackToV8(isolate, slice);
@ -3997,7 +4002,6 @@ static void JS_VPackToV8(v8::FunctionCallbackInfo<v8::Value> const& args) {
char const* data = V8Buffer::data(args[0].As<v8::Object>());
size_t size = V8Buffer::length(args[0].As<v8::Object>());
VPackValidator validator;
validator.validate(data, size, false);
VPackSlice slice(data);