From 325aa137950443fa5a6e556fa8e9011396419719 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 6 Jun 2016 16:59:36 +0200 Subject: [PATCH 1/4] merge with velocypack --- .../velocypack/include/velocypack/Basics.h | 4 +- .../velocypack/include/velocypack/Builder.h | 8 +- .../velocypack/include/velocypack/Exception.h | 14 +- .../velocypack/include/velocypack/Iterator.h | 44 ++- .../velocypack/include/velocypack/Parser.h | 4 +- .../velocypack/include/velocypack/Slice.h | 16 +- .../velocypack/include/velocypack/Validator.h | 72 ++++ .../include/velocypack/velocypack-common.h | 12 +- 3rdParty/velocypack/src/Dumper.cpp | 2 +- 3rdParty/velocypack/src/Slice.cpp | 2 +- 3rdParty/velocypack/src/Validator.cpp | 318 ++++++++++++++++++ 3rdParty/velocypack/src/fasthash.cpp | 82 ----- 3rdParty/velocypack/src/velocypack-common.cpp | 3 +- CMakeLists.txt | 6 + arangod/Aql/BindParameters.cpp | 2 +- arangod/Aql/EnumerateCollectionBlock.cpp | 2 +- arangod/Aql/Functions.cpp | 70 ++-- arangod/Aql/IndexBlock.cpp | 2 +- arangod/Aql/ModificationBlocks.cpp | 2 +- arangod/Indexes/EdgeIndex.h | 4 +- arangod/Indexes/HashIndex.cpp | 2 +- arangod/Indexes/HashIndex.h | 2 +- arangod/Indexes/PrimaryIndex.h | 2 +- arangod/RestHandler/RestEdgesHandler.cpp | 4 +- .../RestHandler/RestVocbaseBaseHandler.cpp | 13 +- arangod/Utils/Cursor.cpp | 2 +- arangod/Utils/Transaction.cpp | 2 +- lib/Basics/VelocyPackDumper.cpp | 2 +- lib/CMakeLists.txt | 3 +- lib/Rest/GeneralRequest.cpp | 7 +- lib/V8/v8-vpack.cpp | 2 +- 31 files changed, 522 insertions(+), 188 deletions(-) create mode 100644 3rdParty/velocypack/include/velocypack/Validator.h create mode 100644 3rdParty/velocypack/src/Validator.cpp delete mode 100644 3rdParty/velocypack/src/fasthash.cpp diff --git a/3rdParty/velocypack/include/velocypack/Basics.h b/3rdParty/velocypack/include/velocypack/Basics.h index 894922c6c6..2b13c68c8c 100644 --- a/3rdParty/velocypack/include/velocypack/Basics.h +++ b/3rdParty/velocypack/include/velocypack/Basics.h @@ -55,9 +55,9 @@ class NonCopyable { // prevent heap allocation struct NonHeapAllocatable { void* operator new(std::size_t) throw(std::bad_alloc) = delete; - void operator delete(void*) throw() = delete; + void operator delete(void*) noexcept = delete; void* operator new[](std::size_t) throw(std::bad_alloc) = delete; - void operator delete[](void*) throw() = delete; + void operator delete[](void*) noexcept = delete; }; #ifdef _WIN32 diff --git a/3rdParty/velocypack/include/velocypack/Builder.h b/3rdParty/velocypack/include/velocypack/Builder.h index 2e6ebb8349..d6ab7748c7 100644 --- a/3rdParty/velocypack/include/velocypack/Builder.h +++ b/3rdParty/velocypack/include/velocypack/Builder.h @@ -309,11 +309,11 @@ class Builder { return _pos; } - bool isEmpty() const throw() { return _pos == 0; } + bool isEmpty() const noexcept { return _pos == 0; } - bool isClosed() const throw() { return _stack.empty(); } + bool isClosed() const noexcept { return _stack.empty(); } - bool isOpenArray() const throw() { + bool isOpenArray() const noexcept { if (_stack.empty()) { return false; } @@ -321,7 +321,7 @@ class Builder { return _start[tos] == 0x06 || _start[tos] == 0x13; } - bool isOpenObject() const throw() { + bool isOpenObject() const noexcept { if (_stack.empty()) { return false; } diff --git a/3rdParty/velocypack/include/velocypack/Exception.h b/3rdParty/velocypack/include/velocypack/Exception.h index d3f7cf5964..d6fef91894 100644 --- a/3rdParty/velocypack/include/velocypack/Exception.h +++ b/3rdParty/velocypack/include/velocypack/Exception.h @@ -68,6 +68,9 @@ struct Exception : std::exception { BuilderKeyAlreadyWritten = 38, BuilderKeyMustBeString = 39, + ValidatorInvalidLength = 50, + ValidatorInvalidType = 51, + UnknownError = 999 }; @@ -83,11 +86,11 @@ struct Exception : std::exception { explicit Exception(ExceptionType type) : Exception(type, message(type)) {} - char const* what() const throw() { return _msg.c_str(); } + char const* what() const noexcept { return _msg.c_str(); } - ExceptionType errorCode() const throw() { return _type; } + ExceptionType errorCode() const noexcept { return _type; } - static char const* message(ExceptionType type) throw() { + static char const* message(ExceptionType type) noexcept { switch (type) { case InternalError: return "Internal error"; @@ -139,6 +142,11 @@ struct Exception : std::exception { return "The key of the next key/value pair is already written"; case BuilderKeyMustBeString: return "The key of the next key/value pair must be a string"; + + case ValidatorInvalidType: + return "Invalid type found in binary data"; + case ValidatorInvalidLength: + return "Invalid length found in binary data"; case UnknownError: default: diff --git a/3rdParty/velocypack/include/velocypack/Iterator.h b/3rdParty/velocypack/include/velocypack/Iterator.h index ff045bf68d..64437d6ca9 100644 --- a/3rdParty/velocypack/include/velocypack/Iterator.h +++ b/3rdParty/velocypack/include/velocypack/Iterator.h @@ -42,9 +42,8 @@ class ArrayIterator { public: ArrayIterator() = delete; - ArrayIterator(Slice const& slice, bool allowRandomIteration = false) - : _slice(slice), _size(_slice.length()), _position(0), _current(nullptr), - _allowRandomIteration(allowRandomIteration) { + explicit ArrayIterator(Slice const& slice) + : _slice(slice), _size(_slice.length()), _position(0), _current(nullptr) { if (slice.type() != ValueType::Array) { throw Exception(Exception::InvalidValueType, "Expecting Array slice"); } @@ -56,15 +55,13 @@ class ArrayIterator { : _slice(other._slice), _size(other._size), _position(other._position), - _current(other._current), - _allowRandomIteration(other._allowRandomIteration) {} + _current(other._current) {} ArrayIterator& operator=(ArrayIterator const& other) { _slice = other._slice; _size = other._size; _position = other._position; _current = other._current; - _allowRandomIteration = other._allowRandomIteration; return *this; } @@ -97,23 +94,23 @@ class ArrayIterator { return _slice.at(_position); } - ArrayIterator begin() { return ArrayIterator(_slice, _allowRandomIteration); } + ArrayIterator begin() { return ArrayIterator(_slice); } - ArrayIterator begin() const { return ArrayIterator(_slice, _allowRandomIteration); } + ArrayIterator begin() const { return ArrayIterator(_slice); } ArrayIterator end() { - auto it = ArrayIterator(_slice, _allowRandomIteration); + auto it = ArrayIterator(_slice); it._position = it._size; return it; } ArrayIterator end() const { - auto it = ArrayIterator(_slice, _allowRandomIteration); + auto it = ArrayIterator(_slice); it._position = it._size; return it; } - inline bool valid() const throw() { return (_position < _size); } + inline bool valid() const noexcept { return (_position < _size); } inline Slice value() const { if (_position >= _size) { @@ -122,18 +119,18 @@ class ArrayIterator { return operator*(); } - inline bool next() throw() { + inline bool next() noexcept { operator++(); return valid(); } - inline ValueLength index() const throw() { return _position; } + inline ValueLength index() const noexcept { return _position; } - inline ValueLength size() const throw() { return _size; } + inline ValueLength size() const noexcept { return _size; } - inline bool isFirst() const throw() { return (_position == 0); } + inline bool isFirst() const noexcept { return (_position == 0); } - inline bool isLast() const throw() { return (_position + 1 >= _size); } + inline bool isLast() const noexcept { return (_position + 1 >= _size); } inline void forward(ValueLength count) { if (_position + count >= _size) { @@ -161,7 +158,7 @@ class ArrayIterator { auto h = _slice.head(); if (h == 0x13) { _current = _slice.at(0).start(); - } else if (_allowRandomIteration) { + } else { _current = _slice.begin() + _slice.findDataOffset(h); } } @@ -172,7 +169,6 @@ class ArrayIterator { ValueLength _size; ValueLength _position; uint8_t const* _current; - bool _allowRandomIteration; }; class ObjectIterator { @@ -267,7 +263,7 @@ class ObjectIterator { return it; } - inline bool valid() const throw() { return (_position < _size); } + inline bool valid() const noexcept { return (_position < _size); } inline Slice key(bool translate = true) const { if (_position >= _size) { @@ -290,18 +286,18 @@ class ObjectIterator { return _slice.getNthValue(_position); } - inline bool next() throw() { + inline bool next() noexcept { operator++(); return valid(); } - inline ValueLength index() const throw() { return _position; } + inline ValueLength index() const noexcept { return _position; } - inline ValueLength size() const throw() { return _size; } + inline ValueLength size() const noexcept { return _size; } - inline bool isFirst() const throw() { return (_position == 0); } + inline bool isFirst() const noexcept { return (_position == 0); } - inline bool isLast() const throw() { return (_position + 1 >= _size); } + inline bool isLast() const noexcept { return (_position + 1 >= _size); } private: Slice _slice; diff --git a/3rdParty/velocypack/include/velocypack/Parser.h b/3rdParty/velocypack/include/velocypack/Parser.h index 616894f14c..dad363cf5c 100644 --- a/3rdParty/velocypack/include/velocypack/Parser.h +++ b/3rdParty/velocypack/include/velocypack/Parser.h @@ -89,7 +89,7 @@ class Parser { Options const* options; Parser(Parser const&) = delete; - Parser(Parser &&) = delete; + Parser(Parser&&) = delete; Parser& operator=(Parser const&) = delete; Parser& operator=(Parser&&) = delete; ~Parser() = default; @@ -209,7 +209,7 @@ class Parser { ValueLength parseInternal(bool multi); - inline bool isWhiteSpace(uint8_t i) const throw() { + inline bool isWhiteSpace(uint8_t i) const noexcept { return (i == ' ' || i == '\t' || i == '\n' || i == '\r'); } diff --git a/3rdParty/velocypack/include/velocypack/Slice.h b/3rdParty/velocypack/include/velocypack/Slice.h index 6108853c81..e03991a06f 100644 --- a/3rdParty/velocypack/include/velocypack/Slice.h +++ b/3rdParty/velocypack/include/velocypack/Slice.h @@ -42,12 +42,18 @@ #include "velocypack/Value.h" #include "velocypack/ValueType.h" -#ifndef VELOCYPACK_HASH +#ifdef VELOCYPACK_XXHASH // forward for XXH64 function declared elsewhere extern "C" unsigned long long XXH64(void const*, size_t, unsigned long long); #define VELOCYPACK_HASH(mem, size, seed) XXH64(mem, size, seed) +#endif +#ifdef VELOCYPACK_FASTHASH +// forward for fasthash64 function declared elsewhere +uint64_t fasthash64(void const*, size_t, uint64_t); + +#define VELOCYPACK_HASH(mem, size, seed) fasthash64(mem, size, seed) #endif namespace arangodb { @@ -355,7 +361,7 @@ class Slice { } // extract the nth key from an Object - Slice getNthKey(ValueLength index, bool) const; + Slice getNthKey(ValueLength index, bool translate) const; // extract the nth value from an Object Slice getNthValue(ValueLength index) const { @@ -758,6 +764,9 @@ class Slice { } return 9; } + + // get the offset for the nth member from an Array type + ValueLength getNthOffset(ValueLength index) const; Slice makeKey() const; @@ -814,9 +823,6 @@ class Slice { Slice getFromCompactObject(std::string const& attribute) const; - // get the offset for the nth member from an Array type - ValueLength getNthOffset(ValueLength index) const; - // extract the nth member from an Array Slice getNth(ValueLength index) const; diff --git a/3rdParty/velocypack/include/velocypack/Validator.h b/3rdParty/velocypack/include/velocypack/Validator.h new file mode 100644 index 0000000000..bac6b893a3 --- /dev/null +++ b/3rdParty/velocypack/include/velocypack/Validator.h @@ -0,0 +1,72 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief Library to build up VPack documents. +/// +/// DISCLAIMER +/// +/// Copyright 2015 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#ifndef VELOCYPACK_VALIDATOR_H +#define VELOCYPACK_VALIDATOR_H 1 + +#include "velocypack/velocypack-common.h" +#include "velocypack/Options.h" + +namespace arangodb { +namespace velocypack { +class Slice; + +class Validator { + // This class can validate a binary VelocyPack value. + + public: + explicit Validator(Options const* options = &Options::Defaults) + : options(options) { + if (options == nullptr) { + throw Exception(Exception::InternalError, "Options cannot be a nullptr"); + } + } + + ~Validator() = default; + + public: + // validates a VelocyPack Slice value starting at ptr, with length bytes length + // throws if the data is invalid + bool validate(char const* ptr, size_t length, bool isSubPart = false) const { + return validate(reinterpret_cast(ptr), length, isSubPart); + } + + // validates a VelocyPack Slice value starting at ptr, with length bytes length + // throws if the data is invalid + bool validate(uint8_t const* ptr, size_t length, bool isSubPart = false) const; + + private: + void validateBufferLength(size_t expected, size_t actual, bool isSubPart) const; + void validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const; + + public: + Options const* options; +}; + +} // namespace arangodb::velocypack +} // namespace arangodb + +#endif diff --git a/3rdParty/velocypack/include/velocypack/velocypack-common.h b/3rdParty/velocypack/include/velocypack/velocypack-common.h index e16292eb1c..70ba87b8dc 100644 --- a/3rdParty/velocypack/include/velocypack/velocypack-common.h +++ b/3rdParty/velocypack/include/velocypack/velocypack-common.h @@ -88,7 +88,7 @@ static constexpr std::size_t checkOverflow(ValueLength length) { #endif // calculate the length of a variable length integer in unsigned LEB128 format -static inline ValueLength getVariableValueLength(ValueLength value) throw() { +static inline ValueLength getVariableValueLength(ValueLength value) noexcept { ValueLength len = 1; while (value >= 0x80) { value >>= 7; @@ -139,7 +139,7 @@ static inline void storeVariableValueLength(uint8_t* dst, ValueLength value) { // returns current value for UTCDate int64_t currentUTCDateValue(); -static inline uint64_t toUInt64(int64_t v) throw() { +static inline uint64_t toUInt64(int64_t v) noexcept { // If v is negative, we need to add 2^63 to make it positive, // before we can cast it to an uint64_t: uint64_t shift2 = 1ULL << 63; @@ -151,7 +151,7 @@ static inline uint64_t toUInt64(int64_t v) throw() { // uint64_t is not guaranteed to work for negative values! } -static inline int64_t toInt64(uint64_t v) throw() { +static inline int64_t toInt64(uint64_t v) noexcept { uint64_t shift2 = 1ULL << 63; int64_t shift = static_cast(shift2 - 1); return v >= shift2 ? (static_cast(v - shift2) - shift) - 1 @@ -161,7 +161,7 @@ static inline int64_t toInt64(uint64_t v) throw() { // read an unsigned little endian integer value of the // specified length, starting at the specified byte offset template -static inline T readInteger(uint8_t const* start, ValueLength length) throw() { +static inline T readInteger(uint8_t const* start, ValueLength length) noexcept { uint64_t value = 0; uint64_t x = 0; uint8_t const* end = start + length; @@ -172,11 +172,11 @@ static inline T readInteger(uint8_t const* start, ValueLength length) throw() { return value; } -static inline uint64_t readUInt64(uint8_t const* start) throw() { +static inline uint64_t readUInt64(uint8_t const* start) noexcept { return readInteger(start, 8); } -static inline void storeUInt64(uint8_t* start, uint64_t value) throw() { +static inline void storeUInt64(uint8_t* start, uint64_t value) noexcept { uint8_t const* end = start + 8; do { *start++ = static_cast(value & 0xff); diff --git a/3rdParty/velocypack/src/Dumper.cpp b/3rdParty/velocypack/src/Dumper.cpp index eb43ecd943..a1babc57d4 100644 --- a/3rdParty/velocypack/src/Dumper.cpp +++ b/3rdParty/velocypack/src/Dumper.cpp @@ -309,7 +309,7 @@ void Dumper::dumpValue(Slice const* slice, Slice const* base) { } case ValueType::Array: { - ArrayIterator it(*slice, true); + ArrayIterator it(*slice); _sink->push_back('['); if (options->prettyPrint) { _sink->push_back('\n'); diff --git a/3rdParty/velocypack/src/Slice.cpp b/3rdParty/velocypack/src/Slice.cpp index 1e994c78e0..9dcfb3485a 100644 --- a/3rdParty/velocypack/src/Slice.cpp +++ b/3rdParty/velocypack/src/Slice.cpp @@ -423,7 +423,7 @@ uint64_t Slice::normalizedHash(uint64_t seed) const { // over all array members uint64_t const n = length() ^ 0xba5bedf00d; value = VELOCYPACK_HASH(&n, sizeof(n), seed); - for (auto const& it : ArrayIterator(*this, true)) { + for (auto const& it : ArrayIterator(*this)) { value ^= it.normalizedHash(value); } } else if (isObject()) { diff --git a/3rdParty/velocypack/src/Validator.cpp b/3rdParty/velocypack/src/Validator.cpp new file mode 100644 index 0000000000..fa2f940773 --- /dev/null +++ b/3rdParty/velocypack/src/Validator.cpp @@ -0,0 +1,318 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief Library to build up VPack documents. +/// +/// DISCLAIMER +/// +/// Copyright 2015 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "velocypack/velocypack-common.h" +#include "velocypack/Validator.h" +#include "velocypack/Exception.h" +#include "velocypack/Slice.h" +#include "velocypack/ValueType.h" + +using namespace arangodb::velocypack; + +bool Validator::validate(uint8_t const* ptr, size_t length, bool isSubPart) const { + if (length == 0) { + throw Exception(Exception::ValidatorInvalidLength, "length 0 is invalid for any VelocyPack value"); + } + + uint8_t const head = *ptr; + + // type() only reads the first byte, which is safe + ValueType const type = Slice(ptr).type(); + + if (type == ValueType::None && head != 0x00) { + // invalid type + throw Exception(Exception::ValidatorInvalidType); + } + + // special handling for certain types... + switch (type) { + case ValueType::None: + case ValueType::Null: + case ValueType::Bool: + case ValueType::MinKey: + case ValueType::MaxKey: + case ValueType::SmallInt: + case ValueType::Int: + case ValueType::UInt: + case ValueType::Double: + case ValueType::UTCDate: + case ValueType::Binary: + case ValueType::Illegal: { + break; + } + + case ValueType::String: { + if (head == 0xbf) { + // long UTF-8 string. must be at least 9 bytes long so we + // can read the entire string length safely + validateBufferLength(1 + 8, length, true); + } + break; + } + + case ValueType::Array: { + ValueLength byteLength = 0; + bool equalSize = false; + bool hasIndexTable = false; + + if (head >= 0x02 && head <= 0x05) { + // Array without index table, with 1-8 bytes bytelength, all values with same length + byteLength = 1 << (head - 0x02); + equalSize = true; + } else if (head >= 0x06 && head <= 0x09) { + // Array with index table, with 1-8 bytes bytelength + byteLength = 1 << (head - 0x06); + hasIndexTable = true; + } + + if (head == 0x13) { + // compact Array without index table + validateBufferLength(2, length, true); + uint8_t const* p = ptr + 1; + uint8_t const* e = p + length; + ValueLength shifter = 0; + while (true) { + uint8_t c = *p; + byteLength += (c & 0x7f) << shifter; + shifter += 7; + ++p; + if (!(c & 0x80)) { + break; + } + if (p == e) { + throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds"); + } + } + if (byteLength > length || byteLength < 4) { + throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds"); + } + uint8_t const* data = p; + p = ptr + byteLength - 1; + ValueLength nrItems = 0; + shifter = 0; + while (true) { + uint8_t c = *p; + nrItems += (c & 0x7f) << shifter; + shifter += 7; + --p; + if (!(c & 0x80)) { + break; + } + if (p == ptr + byteLength) { + throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds"); + } + } + if (nrItems == 0) { + throw Exception(Exception::ValidatorInvalidLength, "Array length value is out of bounds"); + } + ++p; + + // validate the array members + e = p; + p = data; + while (nrItems-- > 0) { + validate(p, e - p, true); + p += Slice(p).byteSize(); + } + } else if (byteLength > 0) { + ValueLength nrItemsLength = 0; + if (head >= 0x06) { + nrItemsLength = byteLength; + } + validateBufferLength(1 + byteLength + nrItemsLength, length, true); + ValueLength nrItems = Slice(ptr).length(); + uint8_t const* p = ptr + 1 + byteLength; + if (!equalSize) { + p += byteLength; + } + uint8_t const* e = ptr + length; + ValueLength l = 0; + while (nrItems > 0) { + if (p >= e) { + throw Exception(Exception::ValidatorInvalidLength, "Array value is out of bounds"); + } + // validate sub value + validate(p, e - p, true); + ValueLength al = Slice(p).byteSize(); + if (equalSize) { + if (l == 0) { + l = al; + } else if (l != al) { + throw Exception(Exception::ValidatorInvalidLength, "Unexpected Array value length"); + } + } + p += al; + --nrItems; + } + + if (hasIndexTable) { + // now also validate index table + nrItems = Slice(ptr).length(); + for (ValueLength i = 0; i < nrItems; ++i) { + ValueLength offset = Slice(ptr).getNthOffset(i); + if (offset < 1 + byteLength + nrItemsLength || + offset >= Slice(ptr).byteSize() - nrItems * byteLength) { + throw Exception(Exception::ValidatorInvalidLength, "Array value offset is out of bounds"); + } + validate(ptr + offset, length - offset, true); + } + } + } + break; + } + + case ValueType::Object: { + ValueLength byteLength = 0; + if (head >= 0x0b && head <= 0x0e) { + // Object with index table, with 1-8 bytes bytelength, sorted + byteLength = 1 << (head - 0x0b); + } else if (head >= 0x0f && head <= 0x12) { + // Object with index table, with 1-8 bytes bytelength, unsorted + byteLength = 1 << (head - 0x0f); + } else if (head == 0x14) { + // compact Object without index table + // TODO + } + + if (byteLength > 0) { + validateBufferLength(1 + byteLength, length, true); + ValueLength nrItems = Slice(ptr).length(); + uint8_t const* p = ptr + 1 + byteLength; + uint8_t const* e = ptr + length; + while (nrItems > 0) { + if (p >= e) { + throw Exception(Exception::ValidatorInvalidLength, "Object key offset is out of bounds"); + } + // validate key + validate(p, e - p, true); + // skip over key + p += Slice(p).byteSize(); + + if (p >= e) { + throw Exception(Exception::ValidatorInvalidLength, "Object value offset is out of bounds"); + } + // validate value + validate(p, e - p, true); + // skip over value + p += Slice(p).byteSize(); + + --nrItems; + } + + // now also validate index table + for (ValueLength i = 0; i < nrItems; ++i) { + // get offset to key + ValueLength offset = Slice(ptr).getNthOffset(i); + if (offset >= length) { + throw Exception(Exception::ValidatorInvalidLength, "Object key offset is out of bounds"); + } + // validate length of key + validate(ptr + offset, length - offset, true); + // skip over key + offset += Slice(ptr + offset).byteSize(); + if (offset >= length) { + throw Exception(Exception::ValidatorInvalidLength, "Object value offset is out of bounds"); + } + // validate length of value + validate(ptr + offset, length - offset, true); + } + } + break; + } + + case ValueType::BCD: { + throw Exception(Exception::NotImplemented); + } + + case ValueType::External: { + // check if Externals are forbidden + if (options->disallowExternals) { + throw Exception(Exception::BuilderExternalsDisallowed); + } + // validate if Slice length exceeds the given buffer + validateBufferLength(1 + sizeof(void*), length, true); + // do not perform pointer validation + break; + } + + case ValueType::Custom: { + ValueLength byteSize = 0; + + if (head == 0xf0) { + byteSize = 1 + 1; + } else if (head == 0xf1) { + byteSize = 1 + 2; + } else if (head == 0xf2) { + byteSize = 1 + 4; + } else if (head == 0xf3) { + byteSize = 1 + 8; + } else if (head >= 0xf4 && head <= 0xf6) { + validateBufferLength(1 + 1, length, true); + byteSize = 1 + 1 + readInteger(ptr + 1, 1); + if (byteSize == 1 + 1) { + throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type"); + } + } else if (head >= 0xf7 && head <= 0xf9) { + validateBufferLength(1 + 2, length, true); + byteSize = 1 + 2 + readInteger(ptr + 1, 2); + if (byteSize == 1 + 2) { + throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type"); + } + } else if (head >= 0xfa && head <= 0xfc) { + validateBufferLength(1 + 4, length, true); + byteSize = 1 + 4 + readInteger(ptr + 1, 4); + if (byteSize == 1 + 4) { + throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type"); + } + } else if (head >= 0xfd) { + validateBufferLength(1 + 8, length, true); + byteSize = 1 + 8 + readInteger(ptr + 1, 8); + if (byteSize == 1 + 8) { + throw Exception(Exception::ValidatorInvalidLength, "Invalid size for Custom type"); + } + } + + validateSliceLength(ptr, byteSize, isSubPart); + break; + } + } + + // common validation that must happen for all types + validateSliceLength(ptr, length, isSubPart); + return true; +} + +void Validator::validateBufferLength(size_t expected, size_t actual, bool isSubPart) const { + if ((expected > actual) || + (expected != actual && !isSubPart)) { + throw Exception(Exception::ValidatorInvalidLength, "given buffer length is unequal to actual length of Slice in buffer"); + } +} + +void Validator::validateSliceLength(uint8_t const* ptr, size_t length, bool isSubPart) const { + size_t actual = static_cast(Slice(ptr).byteSize()); + validateBufferLength(actual, length, isSubPart); +} diff --git a/3rdParty/velocypack/src/fasthash.cpp b/3rdParty/velocypack/src/fasthash.cpp deleted file mode 100644 index bb0627eba2..0000000000 --- a/3rdParty/velocypack/src/fasthash.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* The MIT License - - Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com) - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, copy, - modify, merge, publish, distribute, sublicense, and/or sell copies - of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#include -#include - -namespace arangodb { -namespace velocypack { - -// Compression function for Merkle-Damgard construction. -// This function is generated using the framework provided. -static inline uint64_t mix(uint64_t h) { - h ^= h >> 23; - h *= 0x2127599bf4325c37ULL; - h ^= h >> 47; - - return h; -} - -uint64_t fasthash64(const void *buf, size_t len, uint64_t seed) { - const uint64_t m = 0x880355f21e6d1965ULL; - const uint64_t *pos = (const uint64_t *)buf; - const uint64_t *end = pos + (len / 8); - const unsigned char *pos2; - uint64_t h = seed ^ (len * m); - uint64_t v; - - while (pos != end) { - v = *pos++; - h ^= mix(v); - h *= m; - } - - pos2 = (const unsigned char *)pos; - v = 0; - - switch (len & 7) { - case 7: - v ^= (uint64_t)pos2[6] << 48; - case 6: - v ^= (uint64_t)pos2[5] << 40; - case 5: - v ^= (uint64_t)pos2[4] << 32; - case 4: - v ^= (uint64_t)pos2[3] << 24; - case 3: - v ^= (uint64_t)pos2[2] << 16; - case 2: - v ^= (uint64_t)pos2[1] << 8; - case 1: - v ^= (uint64_t)pos2[0]; - h ^= mix(v); - h *= m; - } - - return mix(h); -} - -} // namespace arangodb::velocypack -} // namespace arangodb diff --git a/3rdParty/velocypack/src/velocypack-common.cpp b/3rdParty/velocypack/src/velocypack-common.cpp index 295d81e8e8..6f5e5885ba 100644 --- a/3rdParty/velocypack/src/velocypack-common.cpp +++ b/3rdParty/velocypack/src/velocypack-common.cpp @@ -59,8 +59,7 @@ static_assert(sizeof(std::size_t) == sizeof(uint64_t), int64_t arangodb::velocypack::currentUTCDateValue() { return static_cast( - std::chrono::system_clock::now().time_since_epoch().count() / - std::chrono::milliseconds(1).count()); + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); } static_assert(sizeof(arangodb::velocypack::ValueLength) >= sizeof(SIZE_MAX), diff --git a/CMakeLists.txt b/CMakeLists.txt index 001a44e9dd..943fa7ab37 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -625,6 +625,12 @@ endif () add_subdirectory(3rdParty) +################################################################################ +## VELOCYPACK +################################################################################ + +add_definitions("-DVELOCYPACK_XXHASH=1") + set(BOOST_VERSION 1.61.0b1) foreach (LINK_DIR ${LINK_DIRECTORIES}) diff --git a/arangod/Aql/BindParameters.cpp b/arangod/Aql/BindParameters.cpp index 9fbf00c311..dfe481549c 100644 --- a/arangod/Aql/BindParameters.cpp +++ b/arangod/Aql/BindParameters.cpp @@ -85,7 +85,7 @@ VPackBuilder BindParameters::StripCollectionNames(VPackSlice const& keys, TRI_ASSERT(keys.isArray()); VPackBuilder result; result.openArray(); - for (auto const& element : VPackArrayIterator(keys, false)) { + for (auto const& element : VPackArrayIterator(keys)) { if (element.isString()) { VPackValueLength l; char const* s = element.getString(l); diff --git a/arangod/Aql/EnumerateCollectionBlock.cpp b/arangod/Aql/EnumerateCollectionBlock.cpp index 0079d5dd60..69906c8a98 100644 --- a/arangod/Aql/EnumerateCollectionBlock.cpp +++ b/arangod/Aql/EnumerateCollectionBlock.cpp @@ -90,7 +90,7 @@ bool EnumerateCollectionBlock::moreDocuments(size_t hint) { _documents = _scanner.scan(hint); TRI_ASSERT(_documents.isArray()); - _iterator = VPackArrayIterator(_documents, true); + _iterator = VPackArrayIterator(_documents); VPackValueLength count = _iterator.size(); diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index fea4910d3f..13244f22f3 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -351,7 +351,7 @@ static void ExtractKeys(std::unordered_set& names, AqlValueMaterializer materializer(trx); VPackSlice s = materializer.slice(param, false); - for (auto const& v : VPackArrayIterator(s, true)) { + for (auto const& v : VPackArrayIterator(s)) { if (v.isString()) { names.emplace(v.copyString()); } else { @@ -418,7 +418,7 @@ static bool ListContainsElement(arangodb::AqlTransaction* trx, AqlValueMaterializer testeeMaterializer(trx); VPackSlice testeeSlice = testeeMaterializer.slice(testee, false); - VPackArrayIterator it(slice, true); + VPackArrayIterator it(slice); while (it.valid()) { if (arangodb::basics::VelocyPackHelper::compare(testeeSlice, it.value(), false, options) == 0) { index = static_cast(it.index()); @@ -467,7 +467,7 @@ static bool Variance(arangodb::AqlTransaction* trx, AqlValueMaterializer materializer(trx); VPackSlice slice = materializer.slice(values, false); - for (auto const& element : VPackArrayIterator(slice, true)) { + for (auto const& element : VPackArrayIterator(slice)) { if (!element.isNull()) { if (!element.isNumber()) { return false; @@ -494,7 +494,7 @@ static bool SortNumberList(arangodb::AqlTransaction* trx, AqlValueMaterializer materializer(trx); VPackSlice slice = materializer.slice(values, false); - for (auto const& element : VPackArrayIterator(slice, true)) { + for (auto const& element : VPackArrayIterator(slice)) { if (!element.isNull()) { if (!element.isNumber()) { return false; @@ -547,7 +547,7 @@ static void RequestEdges(VPackSlice vertexSlice, VPackSlice edges = opRes->slice(); TRI_ASSERT(edges.isArray()); if (includeVertices) { - for (auto const& edge : VPackArrayIterator(edges, true)) { + for (auto const& edge : VPackArrayIterator(edges)) { VPackObjectBuilder guard(&result); if (matcher == nullptr || matcher->matches(edge)) { result.add("edge", edge); @@ -597,7 +597,7 @@ static void RequestEdges(VPackSlice vertexSlice, } } } else { - for (auto const& edge : VPackArrayIterator(edges, true)) { + for (auto const& edge : VPackArrayIterator(edges)) { if (matcher == nullptr || matcher->matches(edge)) { result.add(edge); } @@ -724,7 +724,7 @@ static AqlValue MergeParameters(arangodb::aql::Query* query, THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); } // merge in all other arguments - for (auto const& it : VPackArrayIterator(initialSlice, true)) { + for (auto const& it : VPackArrayIterator(initialSlice)) { if (!it.isObject()) { RegisterInvalidArgumentWarning(query, funcName); builder.clear(); @@ -905,7 +905,7 @@ static AqlValue buildGeoResult(arangodb::AqlTransaction* trx, static void FlattenList(VPackSlice const& array, size_t maxDepth, size_t curDepth, VPackBuilder& result) { TRI_ASSERT(result.isOpenArray()); - for (auto const& tmp : VPackArrayIterator(array, true)) { + for (auto const& tmp : VPackArrayIterator(array)) { if (tmp.isArray() && curDepth < maxDepth) { FlattenList(tmp, maxDepth, curDepth + 1, result); } else { @@ -1258,7 +1258,7 @@ AqlValue Functions::Concat(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice slice = materializer.slice(member, false); - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (it.isNull()) { continue; } @@ -1313,7 +1313,7 @@ AqlValue Functions::ConcatSeparator(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice slice = materializer.slice(member, false); - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (it.isNull()) { continue; } @@ -1752,7 +1752,7 @@ AqlValue Functions::Min(arangodb::aql::Query* query, VPackSlice slice = materializer.slice(value, false); VPackSlice minValue; - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (it.isNull()) { continue; } @@ -1781,7 +1781,7 @@ AqlValue Functions::Max(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice slice = materializer.slice(value, false); VPackSlice maxValue; - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (maxValue.isNone() || arangodb::basics::VelocyPackHelper::compare(it, maxValue, true) > 0) { maxValue = it; } @@ -1807,7 +1807,7 @@ AqlValue Functions::Sum(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice slice = materializer.slice(value, false); double sum = 0.0; - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (it.isNull()) { continue; } @@ -1841,7 +1841,7 @@ AqlValue Functions::Average(arangodb::aql::Query* query, double sum = 0.0; size_t count = 0; - for (auto const& v : VPackArrayIterator(slice, true)) { + for (auto const& v : VPackArrayIterator(slice)) { if (v.isNull()) { continue; } @@ -1976,7 +1976,7 @@ AqlValue Functions::Unique(arangodb::aql::Query* query, values(512, arangodb::basics::VelocyPackHelper::VPackHash(), arangodb::basics::VelocyPackHelper::VPackEqual(&options)); - for (auto const& s : VPackArrayIterator(slice, true)) { + for (auto const& s : VPackArrayIterator(slice)) { if (!s.isNone()) { values.emplace(s); } @@ -2013,7 +2013,7 @@ AqlValue Functions::SortedUnique(arangodb::aql::Query* query, arangodb::basics::VelocyPackHelper::VPackLess less(trx->transactionContext()->getVPackOptions(), &slice, &slice); std::set> values(less); - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (!it.isNone()) { values.insert(it); } @@ -2059,7 +2059,7 @@ AqlValue Functions::Union(arangodb::aql::Query* query, VPackSlice slice = materializer.slice(value, false); // this passes ownership for the JSON contens into result - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { builder->add(it); TRI_IF_FAILURE("AqlFunctions::OutOfMemory2") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -2110,7 +2110,7 @@ AqlValue Functions::UnionDistinct(arangodb::aql::Query* query, materializers.emplace_back(trx); VPackSlice slice = materializers.back().slice(value, false); - for (auto const& v : VPackArrayIterator(slice, true)) { + for (auto const& v : VPackArrayIterator(slice)) { if (values.find(v) == values.end()) { TRI_IF_FAILURE("AqlFunctions::OutOfMemory1") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -2176,7 +2176,7 @@ AqlValue Functions::Intersection(arangodb::aql::Query* query, materializers.emplace_back(trx); VPackSlice slice = materializers.back().slice(value, false); - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (i == 0) { // round one @@ -2678,7 +2678,7 @@ AqlValue Functions::Slice(arangodb::aql::Query* query, builder->openArray(); int64_t pos = 0; - VPackArrayIterator it(arraySlice, true); + VPackArrayIterator it(arraySlice); while (it.valid()) { if (pos >= from && pos < to) { builder->add(it.value()); @@ -2722,7 +2722,7 @@ AqlValue Functions::Minus(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice arraySlice = materializer.slice(baseArray, false); - VPackArrayIterator it(arraySlice, true); + VPackArrayIterator it(arraySlice); while (it.valid()) { contains.emplace(it.value(), it.index()); it.next(); @@ -2741,7 +2741,7 @@ AqlValue Functions::Minus(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice arraySlice = materializer.slice(next, false); - for (auto const& search : VPackArrayIterator(arraySlice, true)) { + for (auto const& search : VPackArrayIterator(arraySlice)) { auto find = contains.find(search); if (find != contains.end()) { @@ -2787,7 +2787,7 @@ AqlValue Functions::Document(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice idSlice = materializer.slice(id, false); builder->openArray(); - for (auto const& next : VPackArrayIterator(idSlice, true)) { + for (auto const& next : VPackArrayIterator(idSlice)) { if (next.isString()) { std::string identifier = next.copyString(); std::string colName; @@ -2829,7 +2829,7 @@ AqlValue Functions::Document(arangodb::aql::Query* query, if (!notFound) { AqlValueMaterializer materializer(trx); VPackSlice idSlice = materializer.slice(id, false); - for (auto const& next : VPackArrayIterator(idSlice, true)) { + for (auto const& next : VPackArrayIterator(idSlice)) { if (next.isString()) { std::string identifier(next.copyString()); GetDocumentByIdentifier(trx, collectionName, identifier, true, *builder.get()); @@ -2943,7 +2943,7 @@ AqlValue Functions::Edges(arangodb::aql::Query* query, builder->openArray(); if (vertexSlice.isArray()) { - for (auto const& v : VPackArrayIterator(vertexSlice, true)) { + for (auto const& v : VPackArrayIterator(vertexSlice)) { RequestEdges(v, trx, collectionName, indexId, direction, matcher.get(), includeVertices, *builder.get()); } @@ -3323,7 +3323,7 @@ AqlValue Functions::Push(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice l = materializer.slice(list, false); - for (auto const& it : VPackArrayIterator(l, true)) { + for (auto const& it : VPackArrayIterator(l)) { builder->add(it); } VPackOptions options; @@ -3364,7 +3364,7 @@ AqlValue Functions::Pop(arangodb::aql::Query* query, TransactionBuilderLeaser builder(trx); builder->openArray(); - auto iterator = VPackArrayIterator(slice, true); + auto iterator = VPackArrayIterator(slice); while (iterator.valid() && !iterator.isLast()) { builder->add(iterator.value()); iterator.next(); @@ -3412,7 +3412,7 @@ AqlValue Functions::Append(arangodb::aql::Query* query, trx->transactionContext()->orderCustomTypeHandler().get(); if (!list.isNull(true)) { if (list.isArray()) { - for (auto const& it : VPackArrayIterator(l, true)) { + for (auto const& it : VPackArrayIterator(l)) { builder->add(it); } } @@ -3427,7 +3427,7 @@ AqlValue Functions::Append(arangodb::aql::Query* query, if (unique) { std::unordered_set added; added.reserve(static_cast(slice.length())); - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { if (added.find(it) == added.end() && !ListContainsElement(&options, l, it)) { builder->add(it); @@ -3435,7 +3435,7 @@ AqlValue Functions::Append(arangodb::aql::Query* query, } } } else { - for (auto const& it : VPackArrayIterator(slice, true)) { + for (auto const& it : VPackArrayIterator(slice)) { builder->add(it); } } @@ -3483,7 +3483,7 @@ AqlValue Functions::Unshift(arangodb::aql::Query* query, if (list.isArray()) { AqlValueMaterializer materializer(trx); VPackSlice v = materializer.slice(list, false); - for (auto const& it : VPackArrayIterator(v, true)) { + for (auto const& it : VPackArrayIterator(v)) { builder->add(it); } } @@ -3514,7 +3514,7 @@ AqlValue Functions::Shift(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice l = materializer.slice(list, false); - auto iterator = VPackArrayIterator(l, true); + auto iterator = VPackArrayIterator(l); // This jumps over the first element while (iterator.next()) { builder->add(iterator.value()); @@ -3563,7 +3563,7 @@ AqlValue Functions::RemoveValue(arangodb::aql::Query* query, AqlValueMaterializer materializer(trx); VPackSlice v = materializer.slice(list, false); - for (auto const& it : VPackArrayIterator(v, true)) { + for (auto const& it : VPackArrayIterator(v)) { if (useLimit && limit == 0) { // Just copy builder->add(it); @@ -3616,7 +3616,7 @@ AqlValue Functions::RemoveValues(arangodb::aql::Query* query, TransactionBuilderLeaser builder(trx); builder->openArray(); - for (auto const& it : VPackArrayIterator(l, true)) { + for (auto const& it : VPackArrayIterator(l)) { if (!ListContainsElement(&options, v, it)) { builder->add(it); } @@ -3665,7 +3665,7 @@ AqlValue Functions::RemoveNth(arangodb::aql::Query* query, size_t target = static_cast(p); size_t cur = 0; builder->openArray(); - for (auto const& it : VPackArrayIterator(v, true)) { + for (auto const& it : VPackArrayIterator(v)) { if (cur != target) { builder->add(it); } diff --git a/arangod/Aql/IndexBlock.cpp b/arangod/Aql/IndexBlock.cpp index e234dd2373..0bfdf8e58f 100644 --- a/arangod/Aql/IndexBlock.cpp +++ b/arangod/Aql/IndexBlock.cpp @@ -410,7 +410,7 @@ bool IndexBlock::readIndex(size_t atMost) { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - for (auto const& doc : VPackArrayIterator(slice, true)) { + for (auto const& doc : VPackArrayIterator(slice)) { if (!hasMultipleIndexes) { _documents.emplace_back(doc); } else { diff --git a/arangod/Aql/ModificationBlocks.cpp b/arangod/Aql/ModificationBlocks.cpp index c1e972a9c1..1a47c6c4e5 100644 --- a/arangod/Aql/ModificationBlocks.cpp +++ b/arangod/Aql/ModificationBlocks.cpp @@ -453,7 +453,7 @@ AqlItemBlock* InsertBlock::work(std::vector& blocks) { dstRow -= n; VPackSlice resultList = opRes.slice(); TRI_ASSERT(resultList.isArray()); - for (auto const& elm: VPackArrayIterator(resultList, false)) { + for (auto const& elm: VPackArrayIterator(resultList)) { bool wasError = arangodb::basics::VelocyPackHelper::getBooleanValue( elm, "error", false); if (!wasError) { diff --git a/arangod/Indexes/EdgeIndex.h b/arangod/Indexes/EdgeIndex.h index 8cc8ff6e82..50981e23aa 100644 --- a/arangod/Indexes/EdgeIndex.h +++ b/arangod/Indexes/EdgeIndex.h @@ -55,7 +55,7 @@ class EdgeIndexIterator final : public IndexIterator { _index(index), _searchValues(searchValues), _keys(_searchValues.slice()), - _iterator(_keys, true), + _iterator(_keys), _posInBuffer(0), _batchSize(1000) {} @@ -66,7 +66,7 @@ class EdgeIndexIterator final : public IndexIterator { _index(index), _searchValues(arangodb::velocypack::Builder::clone(searchValues)), _keys(_searchValues.slice()), - _iterator(_keys, true), + _iterator(_keys), _posInBuffer(0), _batchSize(1000) {} diff --git a/arangod/Indexes/HashIndex.cpp b/arangod/Indexes/HashIndex.cpp index 155c66e57e..e5294e858a 100644 --- a/arangod/Indexes/HashIndex.cpp +++ b/arangod/Indexes/HashIndex.cpp @@ -599,7 +599,7 @@ void HashIndex::transformSearchValues(VPackSlice const values, } VPackArrayBuilder guard(&result); - for (auto const& v : VPackArrayIterator(values, true)) { + for (auto const& v : VPackArrayIterator(values)) { if (!v.isObject() || !v.hasKey(TRI_SLICE_KEY_EQUAL)) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, "Hash index only allows == comparison."); } diff --git a/arangod/Indexes/HashIndex.h b/arangod/Indexes/HashIndex.h index 3d6027d7b7..2a282b5dec 100644 --- a/arangod/Indexes/HashIndex.h +++ b/arangod/Indexes/HashIndex.h @@ -123,7 +123,7 @@ class HashIndexIteratorVPack final : public IndexIterator { : _trx(trx), _index(index), _searchValues(searchValues.get()), - _iterator(_searchValues->slice(), true), + _iterator(_searchValues->slice()), _buffer(), _posInBuffer(0) { searchValues.release(); // now we have ownership for searchValues diff --git a/arangod/Indexes/PrimaryIndex.h b/arangod/Indexes/PrimaryIndex.h index e9bc969843..f75ac0a894 100644 --- a/arangod/Indexes/PrimaryIndex.h +++ b/arangod/Indexes/PrimaryIndex.h @@ -47,7 +47,7 @@ class PrimaryIndexIterator final : public IndexIterator { : _trx(trx), _index(index), _keys(keys.get()), - _iterator(_keys->slice(), true) { + _iterator(_keys->slice()) { keys.release(); // now we have ownership for _keys TRI_ASSERT(_keys->slice().isArray()); diff --git a/arangod/RestHandler/RestEdgesHandler.cpp b/arangod/RestHandler/RestEdgesHandler.cpp index bffb591e42..eb75cf9b70 100644 --- a/arangod/RestHandler/RestEdgesHandler.cpp +++ b/arangod/RestHandler/RestEdgesHandler.cpp @@ -108,7 +108,7 @@ bool RestEdgesHandler::getEdgesForVertexList( // generate result scannedIndex += static_cast(edges.length()); - for (auto const& edge : VPackArrayIterator(edges, true)) { + for (auto const& edge : VPackArrayIterator(edges)) { bool add = true; if (!expressions.empty()) { for (auto& exp : expressions) { @@ -164,7 +164,7 @@ bool RestEdgesHandler::getEdgesForVertex( // generate result scannedIndex += static_cast(edges.length()); - for (auto const& edge : VPackArrayIterator(edges, true)) { + for (auto const& edge : VPackArrayIterator(edges)) { bool add = true; if (!expressions.empty()) { for (auto& exp : expressions) { diff --git a/arangod/RestHandler/RestVocbaseBaseHandler.cpp b/arangod/RestHandler/RestVocbaseBaseHandler.cpp index 04b27930e0..48c352273c 100644 --- a/arangod/RestHandler/RestVocbaseBaseHandler.cpp +++ b/arangod/RestHandler/RestVocbaseBaseHandler.cpp @@ -637,13 +637,15 @@ bool RestVocbaseBaseHandler::extractBooleanParameter(char const* name, std::shared_ptr RestVocbaseBaseHandler::parseVelocyPackBody( VPackOptions const* options, bool& success) { - bool found; - std::string const& contentType = - _request->header(StaticStrings::ContentTypeHeader, found); - try { success = true; +#if 0 + // currently deactivated... + bool found; + std::string const& contentType = + _request->header(StaticStrings::ContentTypeHeader, found); + if (found && contentType.size() == StaticStrings::MimeTypeVPack.size() && contentType == StaticStrings::MimeTypeVPack) { VPackSlice slice{_request->body().c_str()}; @@ -653,6 +655,9 @@ std::shared_ptr RestVocbaseBaseHandler::parseVelocyPackBody( } else { return _request->toVelocyPack(options); } +#else + return _request->toVelocyPack(options); +#endif } catch (std::bad_alloc const&) { generateOOMError(); } catch (VPackException const& e) { diff --git a/arangod/Utils/Cursor.cpp b/arangod/Utils/Cursor.cpp index fc224ebb26..b80de5506f 100644 --- a/arangod/Utils/Cursor.cpp +++ b/arangod/Utils/Cursor.cpp @@ -70,7 +70,7 @@ VelocyPackCursor::VelocyPackCursor(TRI_vocbase_t* vocbase, CursorId id, : Cursor(id, batchSize, extra, ttl, hasCount), _vocbase(vocbase), _result(std::move(result)), - _iterator(_result.result->slice(), true), + _iterator(_result.result->slice()), _cached(_result.cached) { TRI_ASSERT(_result.result->slice().isArray()); TRI_UseVocBase(vocbase); diff --git a/arangod/Utils/Transaction.cpp b/arangod/Utils/Transaction.cpp index 080084fad2..38a66496e8 100644 --- a/arangod/Utils/Transaction.cpp +++ b/arangod/Utils/Transaction.cpp @@ -1146,7 +1146,7 @@ OperationResult Transaction::anyLocal(std::string const& collectionName, } VPackSlice docs = result->slice(); - VPackArrayIterator it(docs, true); + VPackArrayIterator it(docs); while (it.valid()) { resultBuilder.add(it.value()); it.next(); diff --git a/lib/Basics/VelocyPackDumper.cpp b/lib/Basics/VelocyPackDumper.cpp index 537b4ec60a..449a7b0a75 100644 --- a/lib/Basics/VelocyPackDumper.cpp +++ b/lib/Basics/VelocyPackDumper.cpp @@ -366,7 +366,7 @@ void VelocyPackDumper::dumpValue(VPackSlice const* slice, VPackSlice const* base } case VPackValueType::Array: { - VPackArrayIterator it(*slice, true); + VPackArrayIterator it(*slice); TRI_AppendCharUnsafeStringBuffer(buffer, '['); while (it.valid()) { if (!it.isFirst()) { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 906aa136bb..152633a910 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -87,11 +87,12 @@ set(LIB_ARANGO_VPACK ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Parser.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Slice.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/ValueType.cpp + ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Validator.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/Version.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/asm-functions.cpp - ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/fasthash.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/fpconv.cpp ${PROJECT_SOURCE_DIR}/3rdParty/velocypack/src/velocypack-common.cpp + ${PROJECT_SOURCE_DIR}/lib/Basics/xxhash.cpp ) if (ASM_OPTIMIZATIONS AND CMAKE_TARGET_ARCHITECTURE_CODE MATCHES "x86_64") diff --git a/lib/Rest/GeneralRequest.cpp b/lib/Rest/GeneralRequest.cpp index 06e91bb8ae..d693cf30ea 100644 --- a/lib/Rest/GeneralRequest.cpp +++ b/lib/Rest/GeneralRequest.cpp @@ -258,7 +258,12 @@ void GeneralRequest::setArrayValue(char* key, size_t length, char const* value) _arrayValues[std::string(key, length)].emplace_back(value); } -bool GeneralRequest::velocyPackResponse () const { +bool GeneralRequest::velocyPackResponse() const { +#if 0 + // currently deactivated std::string const& result = header(StaticStrings::Accept); return (std::string::npos != result.find(StaticStrings::MimeTypeVPack)); +#else + return false; +#endif } diff --git a/lib/V8/v8-vpack.cpp b/lib/V8/v8-vpack.cpp index 131c5fe81c..9db9b33360 100644 --- a/lib/V8/v8-vpack.cpp +++ b/lib/V8/v8-vpack.cpp @@ -138,7 +138,7 @@ static v8::Handle ObjectVPackArray(v8::Isolate* isolate, } uint32_t j = 0; - VPackArrayIterator it(slice, true); + VPackArrayIterator it(slice); while (it.valid()) { v8::Handle val = From eca908359ad68a669a921dd54d2738b073004527 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 6 Jun 2016 17:16:55 +0200 Subject: [PATCH 2/4] make some system collections smaller --- js/common/modules/@arangodb/foxx/manager-utils.js | 2 +- js/server/modules/@arangodb/statistics.js | 4 ++-- js/server/upgrade-database.js | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/js/common/modules/@arangodb/foxx/manager-utils.js b/js/common/modules/@arangodb/foxx/manager-utils.js index ffcda165d5..a0ecec5ebb 100644 --- a/js/common/modules/@arangodb/foxx/manager-utils.js +++ b/js/common/modules/@arangodb/foxx/manager-utils.js @@ -54,7 +54,7 @@ var getStorage = function() { var c = db._collection("_apps"); if (c === null) { c = db._create("_apps", {isSystem: true, replicationFactor: 1, - distributeShardsLike: "_graphs"}); + distributeShardsLike: "_graphs", journalSize: 4 * 1024 * 1024}); c.ensureIndex({ type: "hash", fields: [ "mount" ], unique: true }); } return c; diff --git a/js/server/modules/@arangodb/statistics.js b/js/server/modules/@arangodb/statistics.js index 4544b5b053..dd87ce57d9 100644 --- a/js/server/modules/@arangodb/statistics.js +++ b/js/server/modules/@arangodb/statistics.js @@ -49,10 +49,11 @@ function createStatisticsCollection (name) { if (collection === null) { var r = null; - + try { r = db._create(name, { isSystem: true, waitForSync: false, replicationFactor: 1, + journalSize: 8 * 1024 * 1024, distributeShardsLike: "_graphs" }); } catch (err) { @@ -424,7 +425,6 @@ exports.STATISTICS_INTERVAL = 10; exports.STATISTICS_HISTORY_INTERVAL = 15 * 60; - //////////////////////////////////////////////////////////////////////////////// /// @brief createCollections /// diff --git a/js/server/upgrade-database.js b/js/server/upgrade-database.js index a23c6d2967..2a433c89c5 100644 --- a/js/server/upgrade-database.js +++ b/js/server/upgrade-database.js @@ -625,7 +625,7 @@ task: function() { // needs to be big enough for assets return createSystemCollection("_routing", { - journalSize: 8 * 1024 * 1024, + journalSize: 4 * 1024 * 1024, replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM, distributeShardsLike: "_graphs" }); @@ -692,7 +692,7 @@ task: function() { return createSystemCollection("_aqlfunctions", { - journalSize: 2 * 1024 * 1024, + journalSize: 1 * 1024 * 1024, replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM, distributeShardsLike: "_graphs" }); @@ -763,7 +763,7 @@ task: function() { return createSystemCollection("_jobs", { - journalSize: 4 * 1024 * 1024, + journalSize: 2 * 1024 * 1024, replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM, distributeShardsLike: "_graphs" }); From 031094763b1d7ebe64f3938f629c5e390f0c2b9e Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 6 Jun 2016 23:37:29 +0200 Subject: [PATCH 3/4] renamed REGEX_MATCH to REGEX_TEST --- Documentation/Books/AQL/Functions/String.mdpp | 10 +++---- arangod/Aql/FunctionDefinitions.cpp | 4 +-- arangod/Aql/Functions.cpp | 16 +++++------ arangod/Aql/Functions.h | 2 +- .../aardvark/APP/frontend/src/mode-aql.js | 6 ++-- js/server/modules/@arangodb/aql.js | 4 +-- js/server/tests/aql/aql-functions-string.js | 28 +++++++++---------- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Documentation/Books/AQL/Functions/String.mdpp b/Documentation/Books/AQL/Functions/String.mdpp index cb6c316373..261390675d 100644 --- a/Documentation/Books/AQL/Functions/String.mdpp +++ b/Documentation/Books/AQL/Functions/String.mdpp @@ -267,9 +267,9 @@ RANDOM_TOKEN(8) // "zGl09z42" RANDOM_TOKEN(8) // "m9w50Ft9" ``` -!SUBSECTION REGEX_MATCH() +!SUBSECTION REGEX_TEST() -`REGEX_MATCH(text, search, caseInsensitive) → bool` +`REGEX_TEST(text, search, caseInsensitive) → bool` Check whether the pattern *search* is contained in the string *text*, using regular expression matching. @@ -327,9 +327,9 @@ If the regular expression in *search* is invalid, a warning will be raised and the function will return *false*. ```js -REGEX_MATCH("the quick brown fox", "the.*fox") // true -REGEX_MATCH("the quick brown fox", "^(a|the)\s+(quick|slow).*f.x$") // true -REGEX_MATCH("the\nquick\nbrown\nfox", "^the(\n[a-w]+)+\nfox$") // true +REGEX_TEST("the quick brown fox", "the.*fox") // true +REGEX_TEST("the quick brown fox", "^(a|the)\s+(quick|slow).*f.x$") // true +REGEX_TEST("the\nquick\nbrown\nfox", "^the(\n[a-w]+)+\nfox$") // true ``` !SUBSECTION REVERSE() diff --git a/arangod/Aql/FunctionDefinitions.cpp b/arangod/Aql/FunctionDefinitions.cpp index d63fb5d2dc..93c5ded434 100644 --- a/arangod/Aql/FunctionDefinitions.cpp +++ b/arangod/Aql/FunctionDefinitions.cpp @@ -177,8 +177,8 @@ struct FunctionDefiner { false, true, true, &Functions::Contains}); add({"LIKE", "AQL_LIKE", "s,r|b", true, true, false, true, true, &Functions::Like}); - add({"REGEX_MATCH", "AQL_REGEX_MATCH", "s,r|b", true, true, false, true, - true, &Functions::RegexMatch}); + add({"REGEX_TEST", "AQL_REGEX_TEST", "s,r|b", true, true, false, true, + true, &Functions::RegexTest}); add({"LEFT", "AQL_LEFT", "s,n", true, true, false, true, true}); add({"RIGHT", "AQL_RIGHT", "s,n", true, true, false, true, true}); add({"TRIM", "AQL_TRIM", "s|ns", true, true, false, true, true}); diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index 61de94d90c..cf4791f167 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -1419,11 +1419,11 @@ AqlValue Functions::Like(arangodb::aql::Query* query, return AqlValue(result); } -/// @brief function REGEX_MATCH -AqlValue Functions::RegexMatch(arangodb::aql::Query* query, - arangodb::AqlTransaction* trx, - VPackFunctionParameters const& parameters) { - ValidateParameters(parameters, "REGEX_MATCH", 2, 3); +/// @brief function REGEX_TEST +AqlValue Functions::RegexTest(arangodb::aql::Query* query, + arangodb::AqlTransaction* trx, + VPackFunctionParameters const& parameters) { + ValidateParameters(parameters, "REGEX_TEST", 2, 3); bool const caseInsensitive = GetBooleanParameter(trx, parameters, 2, false); StringBufferLeaser buffer(trx); arangodb::basics::VPackStringBufferAdapter adapter(buffer->stringBuffer()); @@ -1464,7 +1464,7 @@ AqlValue Functions::RegexMatch(arangodb::aql::Query* query, if (matcher == nullptr) { // compiling regular expression failed - RegisterWarning(query, "REGEX_MATCH", TRI_ERROR_QUERY_INVALID_REGEX); + RegisterWarning(query, "REGEX_TEST", TRI_ERROR_QUERY_INVALID_REGEX); return AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); } @@ -1479,7 +1479,7 @@ AqlValue Functions::RegexMatch(arangodb::aql::Query* query, if (error) { // compiling regular expression failed - RegisterWarning(query, "REGEX_MATCH", TRI_ERROR_QUERY_INVALID_REGEX); + RegisterWarning(query, "REGEX_TEST", TRI_ERROR_QUERY_INVALID_REGEX); return AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); } @@ -3996,7 +3996,7 @@ AqlValue Functions::Range(arangodb::aql::Query* query, AqlValue stepValue = ExtractFunctionParameterValue(trx, parameters, 2); if (stepValue.isNull(true)) { - // no step specified + // no step specified. return a real range object return AqlValue(left.toInt64(), right.toInt64()); } diff --git a/arangod/Aql/Functions.h b/arangod/Aql/Functions.h index 63ae304902..fcee83d9cc 100644 --- a/arangod/Aql/Functions.h +++ b/arangod/Aql/Functions.h @@ -97,7 +97,7 @@ struct Functions { VPackFunctionParameters const&); static AqlValue Like(arangodb::aql::Query*, arangodb::AqlTransaction*, VPackFunctionParameters const&); - static AqlValue RegexMatch(arangodb::aql::Query*, arangodb::AqlTransaction*, + static AqlValue RegexTest(arangodb::aql::Query*, arangodb::AqlTransaction*, VPackFunctionParameters const&); static AqlValue Passthru(arangodb::aql::Query*, arangodb::AqlTransaction*, VPackFunctionParameters const&); diff --git a/js/apps/system/_admin/aardvark/APP/frontend/src/mode-aql.js b/js/apps/system/_admin/aardvark/APP/frontend/src/mode-aql.js index 3878552615..f41c208ee3 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/src/mode-aql.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/src/mode-aql.js @@ -88,14 +88,14 @@ var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules; var AqlHighlightRules = function() { var keywords = ( - "for|return|filter|sort|limit|let|collect|asc|desc|in|into|insert|update|remove|replace|upsert|options|with|and|or|not|distinct|graph|outbound|inbound|any|all|none|aggregate|like|count" + "for|return|filter|sort|limit|let|collect|asc|desc|in|into|insert|update|remove|replace|upsert|options|with|and|or|not|distinct|graph|shortest_path|outbound|inbound|any|all|none|aggregate|like" ); var builtinFunctions = ( "(to_bool|to_number|to_string|to_list|is_null|is_bool|is_number|is_string|is_list|is_document|typename|" + "concat|concat_separator|char_length|lower|upper|substring|left|right|trim|reverse|contains|" + - "log|log2|log10|exp|exp2|sin|cos|tan|asin|acos|atan|atan2|radians|degrees|pi|regex|" + - "like|floor|ceil|round|abs|rand|sqrt|pow|length|min|max|average|sum|median|variance_population|" + + "log|log2|log10|exp|exp2|sin|cos|tan|asin|acos|atan|atan2|radians|degrees|pi|regex_test|" + + "like|floor|ceil|round|abs|rand|sqrt|pow|length|count|min|max|average|sum|median|variance_population|" + "variance_sample|first|last|unique|matches|merge|merge_recursive|has|attributes|values|unset|unset_recursive|keep|" + "near|within|within_rectangle|is_in_polygon|fulltext|paths|traversal|traversal_tree|edges|stddev_sample|stddev_population|" + "slice|nth|position|translate|zip|call|apply|push|append|pop|shift|unshift|remove_value|remove_values|" + diff --git a/js/server/modules/@arangodb/aql.js b/js/server/modules/@arangodb/aql.js index 0dfe22d2a6..545cf575cc 100644 --- a/js/server/modules/@arangodb/aql.js +++ b/js/server/modules/@arangodb/aql.js @@ -2301,7 +2301,7 @@ function AQL_LIKE (value, regex, caseInsensitive) { /// @brief searches a substring in a string, using a regex //////////////////////////////////////////////////////////////////////////////// -function AQL_REGEX_MATCH (value, regex, caseInsensitive) { +function AQL_REGEX_TEST (value, regex, caseInsensitive) { 'use strict'; var modifiers = ''; @@ -8358,7 +8358,7 @@ exports.AQL_UPPER = AQL_UPPER; exports.AQL_SUBSTRING = AQL_SUBSTRING; exports.AQL_CONTAINS = AQL_CONTAINS; exports.AQL_LIKE = AQL_LIKE; -exports.AQL_REGEX_MATCH = AQL_REGEX_MATCH; +exports.AQL_REGEX_TEST = AQL_REGEX_TEST; exports.AQL_LEFT = AQL_LEFT; exports.AQL_RIGHT = AQL_RIGHT; exports.AQL_TRIM = AQL_TRIM; diff --git a/js/server/tests/aql/aql-functions-string.js b/js/server/tests/aql/aql-functions-string.js index f529bb16e0..c806d901aa 100644 --- a/js/server/tests/aql/aql-functions-string.js +++ b/js/server/tests/aql/aql-functions-string.js @@ -62,18 +62,18 @@ function ahuacatlStringFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testRegexInvalid : function () { - assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_MATCH()"); - assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_MATCH(\"test\")"); - assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_MATCH(\"test\", \"meow\", \"foo\", \"bar\")"); + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_TEST()"); + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_TEST(\"test\")"); + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN REGEX_TEST(\"test\", \"meow\", \"foo\", \"bar\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"[\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"[^\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"a.(\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"(a\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"(a]\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"**\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"?\")"); - assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_MATCH(\"test\", \"*\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"[\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"[^\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"a.(\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"(a\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"(a]\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"**\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"?\")"); + assertQueryWarningAndFalse(errors.ERROR_QUERY_INVALID_REGEX.code, "RETURN REGEX_TEST(\"test\", \"*\")"); }, testRegex : function () { @@ -217,13 +217,13 @@ function ahuacatlStringFunctionsTestSuite () { ]; values.forEach(function(v) { - var query = "RETURN REGEX_MATCH(@what, @re)"; + var query = "RETURN REGEX_TEST(@what, @re)"; assertEqual(v[2], getQueryResults(query, { what: v[0], re: v[1] })[0], v); - query = "RETURN NOOPT(REGEX_MATCH(@what, @re))"; + query = "RETURN NOOPT(REGEX_TEST(@what, @re))"; assertEqual(v[2], getQueryResults(query, { what: v[0], re: v[1] })[0], v); - query = "RETURN NOOPT(V8(REGEX_MATCH(@what, @re)))"; + query = "RETURN NOOPT(V8(REGEX_TEST(@what, @re)))"; assertEqual(v[2], getQueryResults(query, { what: v[0], re: v[1] })[0], v); }); From fad6de811031d1e960b3fb51c90a6de95b224f91 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 6 Jun 2016 23:51:57 +0200 Subject: [PATCH 4/4] updated new features --- .../Manual/ReleaseNotes/NewFeatures30.mdpp | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Documentation/Books/Manual/ReleaseNotes/NewFeatures30.mdpp b/Documentation/Books/Manual/ReleaseNotes/NewFeatures30.mdpp index d29ecd0dc0..f20a9f7aca 100644 --- a/Documentation/Books/Manual/ReleaseNotes/NewFeatures30.mdpp +++ b/Documentation/Books/Manual/ReleaseNotes/NewFeatures30.mdpp @@ -89,6 +89,49 @@ var q = `FOR doc IN ´collection´ RETURN doc.´name´`; The following AQL functions have been added in 3.0: +- *REGEX_TEST(value, regex)*: tests whether the string *value* matches the regular expression + specified in *regex*. Returns *true* if it matches, and *false* otherwise. + + The regular expression may consist of literal characters and the following + characters and sequences: + + - `.` – the dot matches any single character except line terminators. + To include line terminators, use `[\s\S]` instead to simulate `.` with *DOTALL* flag. + - `\d` – matches a single digit, equivalent to `[0-9]` + - `\s` – matches a single whitespace character + - `\S` – matches a single non-whitespace character + - `\t` – matches a tab character + - `\r` – matches a carriage return + - `\n` – matches a line-feed character + - `[xyz]` – set of characters. matches any of the enclosed characters (i.e. + *x*, *y* or *z* in this case + - `[^xyz]` – negated set of characters. matches any other character than the + enclosed ones (i.e. anything but *x*, *y* or *z* in this case) + - `[x-z]` – range of characters. Matches any of the characters in the + specified range, e.g. `[0-9A-F]` to match any character in + *0123456789ABCDEF* + - `[^x-z]` – negated range of characters. Matches any other character than the + ones specified in the range + - `(xyz)` – defines and matches a pattern group + - `(x|y)` – matches either *x* or *y* + - `^` – matches the beginning of the string (e.g. `^xyz`) + - `$` – matches the end of the string (e.g. `xyz$`) + + Note that the characters `.`, `*`, `?`, `[`, `]`, `(`, `)`, `{`, `}`, `^`, + and `$` have a special meaning in regular expressions and may need to be + escaped using a backslash (`\\`). A literal backslash should also be escaped + using another backslash, i.e. `\\\\`. + + Characters and sequences may optionally be repeated using the following + quantifiers: + + - `x*` – matches zero or more occurrences of *x* + - `x+` – matches one or more occurrences of *x* + - `x?` – matches one or zero occurrences of *x* + - `x{y}` – matches exactly *y* occurrences of *x* + - `x{y,z}` – matches between *y* and *z* occurrences of *x* + - `x{y,}` – matches at least *y* occurences of *x* + - *HASH(value)*: Calculates a hash value for *value*. *value* is not required to be a string, but can have any data type. The calculated hash value will take the data type of *value* into account, so for example the number *1* and the string *"1"* will have