diff --git a/3rdParty/velocypack/include/velocypack/AttributeTranslator.h b/3rdParty/velocypack/include/velocypack/AttributeTranslator.h index 76d2d83bd8..6b53be5e14 100644 --- a/3rdParty/velocypack/include/velocypack/AttributeTranslator.h +++ b/3rdParty/velocypack/include/velocypack/AttributeTranslator.h @@ -33,6 +33,7 @@ #include #include "velocypack/velocypack-common.h" +#include "velocypack/Options.h" #include "velocypack/Slice.h" namespace arangodb { @@ -79,8 +80,8 @@ class AttributeTranslatorScope { public: explicit AttributeTranslatorScope(AttributeTranslator* translator) - : _old(Slice::attributeTranslator) { - Slice::attributeTranslator = translator; + : _old(Options::Defaults.attributeTranslator) { + Options::Defaults.attributeTranslator = translator; } ~AttributeTranslatorScope() { @@ -89,7 +90,7 @@ class AttributeTranslatorScope { // prematurely revert the change void revert() { - Slice::attributeTranslator = _old; + Options::Defaults.attributeTranslator = _old; } private: diff --git a/3rdParty/velocypack/include/velocypack/Exception.h b/3rdParty/velocypack/include/velocypack/Exception.h index 4da304a20e..d3f7cf5964 100644 --- a/3rdParty/velocypack/include/velocypack/Exception.h +++ b/3rdParty/velocypack/include/velocypack/Exception.h @@ -55,7 +55,7 @@ struct Exception : std::exception { NeedCustomTypeHandler = 19, NeedAttributeTranslator = 20, CannotTranslateKey = 21, - KeyNotFound = 22, + KeyNotFound = 22, // not used anymore BuilderNotSealed = 30, BuilderNeedOpenObject = 31, diff --git a/3rdParty/velocypack/include/velocypack/Slice.h b/3rdParty/velocypack/include/velocypack/Slice.h index ff4dce0eab..7ba6211d34 100644 --- a/3rdParty/velocypack/include/velocypack/Slice.h +++ b/3rdParty/velocypack/include/velocypack/Slice.h @@ -48,7 +48,6 @@ namespace velocypack { // forward for fasthash64 function declared elsewhere uint64_t fasthash64(void const*, size_t, uint64_t); -class AttributeTranslator; class SliceScope; class Slice { @@ -62,8 +61,6 @@ class Slice { public: - static AttributeTranslator* attributeTranslator; - // constructor for an empty Value of type None Slice() : Slice("\x00") {} @@ -220,7 +217,7 @@ class Slice { // check if slice is any Number-type object bool isNumber() const throw() { return isInteger() || isDouble(); } - bool isSorted() const { + bool isSorted() const throw() { auto const h = head(); return (h >= 0x0b && h <= 0x0e); } @@ -750,6 +747,10 @@ class Slice { std::string hexType() const; private: + // return the value for a UInt object, without checks + // returns 0 for invalid values/types + uint64_t getUIntUnchecked() const; + // translates an integer key into a string, without checks Slice translateUnchecked() const; @@ -783,7 +784,7 @@ class Slice { // get the offset for the nth member from a compact Array or Object type ValueLength getNthOffsetFromCompact(ValueLength index) const; - ValueLength indexEntrySize(uint8_t head) const { + inline ValueLength indexEntrySize(uint8_t head) const throw() { VELOCYPACK_ASSERT(head <= 0x12); return static_cast(WidthMap[head]); } diff --git a/3rdParty/velocypack/src/AttributeTranslator.cpp b/3rdParty/velocypack/src/AttributeTranslator.cpp index 1323b1ad5b..11b9deed39 100644 --- a/3rdParty/velocypack/src/AttributeTranslator.cpp +++ b/3rdParty/velocypack/src/AttributeTranslator.cpp @@ -93,7 +93,7 @@ uint8_t const* AttributeTranslator::translate(uint64_t id) const { auto it = _idToKey.find(id); if (it == _idToKey.end()) { - throw Exception(Exception::KeyNotFound); + return nullptr; } return (*it).second; diff --git a/3rdParty/velocypack/src/Builder.cpp b/3rdParty/velocypack/src/Builder.cpp index 83c573b862..444a99f469 100644 --- a/3rdParty/velocypack/src/Builder.cpp +++ b/3rdParty/velocypack/src/Builder.cpp @@ -593,18 +593,17 @@ uint8_t* Builder::set(Value const& item) { break; } case ValueType::String: { - if (ctype != Value::CType::String && ctype != Value::CType::CharPtr) { - throw Exception( - Exception::BuilderUnexpectedValue, - "Must give a string or char const* for ValueType::String"); - } std::string const* s; std::string value; if (ctype == Value::CType::String) { s = item.getString(); - } else { + } else if (ctype == Value::CType::CharPtr) { value = item.getCharPtr(); s = &value; + } else { + throw Exception( + Exception::BuilderUnexpectedValue, + "Must give a string or char const* for ValueType::String"); } size_t const size = s->size(); if (size <= 126) { diff --git a/3rdParty/velocypack/src/Slice.cpp b/3rdParty/velocypack/src/Slice.cpp index 89e77b7345..1153ce85cc 100644 --- a/3rdParty/velocypack/src/Slice.cpp +++ b/3rdParty/velocypack/src/Slice.cpp @@ -39,8 +39,6 @@ using namespace arangodb::velocypack; using VT = arangodb::velocypack::ValueType; -AttributeTranslator* Slice::attributeTranslator = nullptr; - VT const Slice::TypeMap[256] = { /* 0x00 */ VT::None, /* 0x01 */ VT::Array, /* 0x02 */ VT::Array, /* 0x03 */ VT::Array, @@ -231,19 +229,35 @@ Slice Slice::translate() const { throw Exception(Exception::InvalidValueType, "Cannot translate key of this type"); } - if (attributeTranslator == nullptr) { + if (Options::Defaults.attributeTranslator == nullptr) { throw Exception(Exception::NeedAttributeTranslator); } return translateUnchecked(); } +// return the value for a UInt object, without checks! +// returns 0 for invalid values/types +uint64_t Slice::getUIntUnchecked() const { + uint8_t const h = head(); + if (h >= 0x28 && h <= 0x2f) { + // UInt + return readInteger(_start + 1, h - 0x27); + } + + if (h >= 0x30 && h <= 0x39) { + // Smallint >= 0 + return static_cast(h - 0x30); + } + return 0; +} + // translates an integer key into a string, without checks Slice Slice::translateUnchecked() const { - uint8_t const* result = attributeTranslator->translate(getUInt()); - if (result == nullptr) { - return Slice(); + uint8_t const* result = Options::Defaults.attributeTranslator->translate(getUIntUnchecked()); + if (result != nullptr) { + return Slice(result); } - return Slice(result); + return Slice(); } // check if two Slices are equal on the binary level @@ -336,7 +350,6 @@ Slice Slice::get(std::string const& attribute) const { ValueLength const offsetSize = indexEntrySize(h); ValueLength end = readInteger(_start + 1, offsetSize); - ValueLength dataOffset = 0; // read number of items ValueLength n; @@ -348,19 +361,19 @@ Slice Slice::get(std::string const& attribute) const { if (n == 1) { // Just one attribute, there is no index table! - if (dataOffset == 0) { - dataOffset = findDataOffset(h); - } + Slice key = Slice(_start + findDataOffset(h)); - Slice key = Slice(_start + dataOffset); - - if (key.isString() && key.isEqualString(attribute)) { - return Slice(key.start() + key.byteSize()); - } - - if (key.isSmallInt() || key.isUInt()) { + if (key.isString()) { + if (key.isEqualString(attribute)) { + return Slice(key.start() + key.byteSize()); + } + // fall through to returning None Slice below + } else if (key.isSmallInt() || key.isUInt()) { // translate key - if (key.translate().isEqualString(attribute)) { + if (Options::Defaults.attributeTranslator == nullptr) { + throw Exception(Exception::NeedAttributeTranslator); + } + if (key.translateUnchecked().isEqualString(attribute)) { return Slice(key.start() + key.byteSize()); } } @@ -376,7 +389,8 @@ Slice Slice::get(std::string const& attribute) const { // otherwise we'll always use the linear search static ValueLength const SortedSearchEntriesThreshold = 4; - if (isSorted() && n >= SortedSearchEntriesThreshold) { + bool const isSorted = (h >= 0x0b && h <= 0x0e); + if (isSorted && n >= SortedSearchEntriesThreshold) { // This means, we have to handle the special case n == 1 only // in the linear search! return searchObjectKeyBinary(attribute, ieBase, offsetSize, n); @@ -516,24 +530,25 @@ ValueLength Slice::getNthOffset(ValueLength index) const { auto const h = head(); + if (h == 0x13 || h == 0x14) { + // compact Array or Object + return getNthOffsetFromCompact(index); + } + if (h == 0x01 || h == 0x0a) { // special case: empty Array or empty Object throw Exception(Exception::IndexOutOfBounds); } - if (h == 0x13 || h == 0x14) { - // compact Array or Object - return getNthOffsetFromCompact(index); - } - ValueLength const offsetSize = indexEntrySize(h); ValueLength end = readInteger(_start + 1, offsetSize); - ValueLength dataOffset = findDataOffset(h); + ValueLength dataOffset = 0; // find the number of items ValueLength n; if (h <= 0x05) { // No offset table or length, need to compute: + dataOffset = findDataOffset(h); Slice first(_start + dataOffset); n = (end - dataOffset) / first.byteSize(); } else if (offsetSize < 8) { @@ -588,7 +603,7 @@ Slice Slice::makeKey() const { return *this; } if (isSmallInt() || isUInt()) { - if (attributeTranslator == nullptr) { + if (Options::Defaults.attributeTranslator == nullptr) { throw Exception(Exception::NeedAttributeTranslator); } return translateUnchecked(); @@ -613,8 +628,7 @@ ValueLength Slice::getNthOffsetFromCompact(ValueLength index) const { uint8_t const* s = _start + offset; offset += Slice(s).byteSize(); if (h == 0x14) { - Slice value = Slice(_start + offset); - offset += value.byteSize(); + offset += Slice(_start + offset).byteSize(); } ++current; } @@ -625,7 +639,7 @@ ValueLength Slice::getNthOffsetFromCompact(ValueLength index) const { Slice Slice::searchObjectKeyLinear(std::string const& attribute, ValueLength ieBase, ValueLength offsetSize, ValueLength n) const { - bool const useTranslator = (attributeTranslator != nullptr); + bool const useTranslator = (Options::Defaults.attributeTranslator != nullptr); for (ValueLength index = 0; index < n; ++index) { ValueLength offset = ieBase + index * offsetSize; @@ -661,7 +675,7 @@ Slice Slice::searchObjectKeyLinear(std::string const& attribute, Slice Slice::searchObjectKeyBinary(std::string const& attribute, ValueLength ieBase, ValueLength offsetSize, ValueLength n) const { - bool const useTranslator = (attributeTranslator != nullptr); + bool const useTranslator = (Options::Defaults.attributeTranslator != nullptr); VELOCYPACK_ASSERT(n > 0); ValueLength l = 0; diff --git a/CHANGELOG b/CHANGELOG index 9bb080102c..b751475b94 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,18 @@ v3.0.0 (XXXX-XX-XX) ------------------- +* added AQL string comparison operator `LIKE` + + The operator can be used to compare strings like this: + + value LIKE search + + The operator is currently implemented by calling the already existing AQL + function `LIKE`. + + This change also makes `LIKE` an AQL keyword. Using `like` in either case as + an attribute or collection name in AQL thus requires quoting. + * make AQL optimizer rule "remove-unnecessary-calculations" fire in more cases The rule will now remove calculations that are used exactly once in other diff --git a/Documentation/Books/Users/Aql/Operators.mdpp b/Documentation/Books/Users/Aql/Operators.mdpp index 5115ac40f7..a3ccb4269a 100644 --- a/Documentation/Books/Users/Aql/Operators.mdpp +++ b/Documentation/Books/Users/Aql/Operators.mdpp @@ -18,13 +18,17 @@ The following comparison operators are supported: - *>=* greater or equal - *IN* test if a value is contained in an array - *NOT IN* test if a value is not contained in an array - -These operators accept any data types for the first and second operands. +- *LIKE* tests if a string value matches a pattern Each of the comparison operators returns a boolean value if the comparison can be evaluated and returns *true* if the comparison evaluates to true, and *false* -otherwise. Please note that the comparison operators will not perform any -implicit type casts if the compared operands have different types. +otherwise. + +The comparison operators accept any data types for the first and second operands. +However, *IN* and *NOT IN* will only return a meaningful result if their right-hand +operand is a string, and *LIKE* will only execute if both operands are string values. +The comparison operators will not perform any implicit type casts if the compared +operands have different or non-sensible types. Some examples for comparison operations in AQL: @@ -39,8 +43,26 @@ true != null // true 1.5 IN [ 2, 3, 1.5 ] // true "foo" IN null // false 42 NOT IN [ 17, 40, 50 ] // true +"abc" == "abc" // true +"abc" == "ABC" // false +"foo" LIKE "f%" // true ``` +The *LIKE* operator checks whether its left operand matches the pattern specified +in its right operand. The pattern can consist of regular characters and wildcards. +The supported wildcards are *_* to match a single arbitrary character, and *%* to +match any number of arbitrary characters. Literal *%* and *_* need to be escaped +with a backslash. + +``` +"abc" LIKE "a%" // true +"abc" LIKE "_bc" // true +"a_b_foo" LIKE "a\\_b\\_f%" // true +``` + +The pattern matching performed by the *LIKE* operator is case-sensitive. + + !SUBSUBSECTION Array comparison operators The comparison operators also exist as *array variant*. In the array diff --git a/Documentation/Books/Users/Aql/Syntax.mdpp b/Documentation/Books/Users/Aql/Syntax.mdpp index 1863eacd15..d0644aa2e4 100644 --- a/Documentation/Books/Users/Aql/Syntax.mdpp +++ b/Documentation/Books/Users/Aql/Syntax.mdpp @@ -109,6 +109,7 @@ The current list of keywords is: - NOT - AND - OR +- LIKE - NULL - TRUE - FALSE diff --git a/js/server/tests/aql/aql-functions-string.js b/js/server/tests/aql/aql-functions-string.js index f8181cd4c5..0783f3ae6a 100644 --- a/js/server/tests/aql/aql-functions-string.js +++ b/js/server/tests/aql/aql-functions-string.js @@ -60,6 +60,11 @@ function ahuacatlStringFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testLikeInvalid : function () { + assertQueryError(errors.ERROR_QUERY_PARSE.code, "RETURN LIKE"); + assertQueryError(errors.ERROR_QUERY_PARSE.code, "RETURN \"test\" LIKE"); + assertQueryError(errors.ERROR_QUERY_PARSE.code, "RETURN LIKE \"test\""); + assertQueryError(errors.ERROR_QUERY_PARSE.code, "RETURN \"test\" LIKE \"meow\", \"foo\", \"bar\")"); + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN LIKE()"); assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN LIKE(\"test\")"); assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN LIKE(\"test\", \"meow\", \"foo\", \"bar\")"); @@ -70,6 +75,21 @@ function ahuacatlStringFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testLike : function () { + // containment + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"test\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"%test\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"test%\"")); + assertEqual([ true ], getQueryResults("RETURN \"this is a test string\" LIKE \"%test%\"")); + assertEqual([ true ], getQueryResults("RETURN \"this is a test string\" LIKE \"this%test%\"")); + assertEqual([ true ], getQueryResults("RETURN \"this is a test string\" LIKE \"this%is%test%\"")); + assertEqual([ true ], getQueryResults("RETURN \"this is a test string\" LIKE \"this%g\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"this%n\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"This%n\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"his%\"")); + assertEqual([ true ], getQueryResults("RETURN \"this is a test string\" LIKE \"%g\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"%G\"")); + assertEqual([ false ], getQueryResults("RETURN \"this is a test string\" LIKE \"this%test%is%\"")); + assertEqual([ false ], getQueryResults("RETURN LIKE(\"this is a test string\", \"test\")")); assertEqual([ false ], getQueryResults("RETURN LIKE(\"this is a test string\", \"%test\")")); assertEqual([ false ], getQueryResults("RETURN LIKE(\"this is a test string\", \"test%\")")); @@ -83,6 +103,27 @@ function ahuacatlStringFunctionsTestSuite () { assertEqual([ true ], getQueryResults("RETURN LIKE(\"this is a test string\", \"%g\")")); assertEqual([ false ], getQueryResults("RETURN LIKE(\"this is a test string\", \"%G\")")); assertEqual([ false ], getQueryResults("RETURN LIKE(\"this is a test string\", \"this%test%is%\")")); + + // special characters + assertEqual([ true ], getQueryResults("RETURN \"%\" LIKE \"\\%\"")); + assertEqual([ true ], getQueryResults("RETURN \"a%c\" LIKE \"a%c\"")); + assertEqual([ false ], getQueryResults("RETURN \"a%c\" LIKE \"ac\"")); + assertEqual([ false ], getQueryResults("RETURN \"a%c\" LIKE \"a\\\\%\"")); + assertEqual([ false ], getQueryResults("RETURN \"a%c\" LIKE \"\\\\%a%\"")); + assertEqual([ false ], getQueryResults("RETURN \"a%c\" LIKE \"\\\\%\\\\%\"")); + assertEqual([ true ], getQueryResults("RETURN \"%%\" LIKE \"\\\\%\\\\%\"")); + assertEqual([ true ], getQueryResults("RETURN \"_\" LIKE \"\\\\_\"")); + assertEqual([ true ], getQueryResults("RETURN \"_\" LIKE \"\\\\_%\"")); + assertEqual([ true ], getQueryResults("RETURN \"abcd\" LIKE \"_bcd\"")); + assertEqual([ true ], getQueryResults("RETURN \"abcde\" LIKE \"_bcd%\"")); + assertEqual([ false ], getQueryResults("RETURN \"abcde\" LIKE \"\\\\_bcd%\"")); + assertEqual([ true ], getQueryResults("RETURN \"\\\\abc\" LIKE \"\\\\\\\\%\"")); + assertEqual([ true ], getQueryResults("RETURN \"\\abc\" LIKE \"\\a%\"")); + assertEqual([ true ], getQueryResults("RETURN \"[ ] ( ) % * . + -\" LIKE \"[%\"")); + assertEqual([ true ], getQueryResults("RETURN \"[ ] ( ) % * . + -\" LIKE \"[ ] ( ) \\% * . + -\"")); + assertEqual([ true ], getQueryResults("RETURN \"[ ] ( ) % * . + -\" LIKE \"%. +%\"")); + assertEqual([ true ], getQueryResults("RETURN \"abc^def$g\" LIKE \"abc^def$g\"")); + assertEqual([ true ], getQueryResults("RETURN \"abc^def$g\" LIKE \"%^%$g\"")); assertEqual([ true ], getQueryResults("RETURN LIKE(\"%\", \"\\%\")")); assertEqual([ true ], getQueryResults("RETURN LIKE(\"a%c\", \"a%c\")")); @@ -103,6 +144,11 @@ function ahuacatlStringFunctionsTestSuite () { assertEqual([ true ], getQueryResults("RETURN LIKE(\"[ ] ( ) % * . + -\", \"%. +%\")")); assertEqual([ true ], getQueryResults("RETURN LIKE(\"abc^def$g\", \"abc^def$g\")")); assertEqual([ true ], getQueryResults("RETURN LIKE(\"abc^def$g\", \"%^%$g\")")); + + // case-sensivity + assertEqual([ false ], getQueryResults("RETURN \"ABCD\" LIKE \"abcd\"")); + assertEqual([ false ], getQueryResults("RETURN \"abcd\" LIKE \"ABCD\"")); + assertEqual([ false ], getQueryResults("RETURN \"MÖterTräNenMÜtterSöhne\" LIKE \"MÖTERTRÄNENMÜTTERSÖHNE\"")); assertEqual([ false ], getQueryResults("RETURN LIKE(\"ABCD\", \"abcd\", false)")); assertEqual([ true ], getQueryResults("RETURN LIKE(\"ABCD\", \"abcd\", true)")); @@ -130,6 +176,9 @@ function ahuacatlStringFunctionsTestSuite () { actual = getQueryResults("RETURN LIKE(" + JSON.stringify(value) + ", " + JSON.stringify(value) + ")"); assertEqual([ true ], actual); + + actual = getQueryResults("RETURN " + JSON.stringify(value) + " LIKE " + JSON.stringify(value)); + assertEqual([ true ], actual); }); }, diff --git a/lib/Basics/VelocyPackHelper.cpp b/lib/Basics/VelocyPackHelper.cpp index 1d4dfd4691..7d2f6f79d0 100644 --- a/lib/Basics/VelocyPackHelper.cpp +++ b/lib/Basics/VelocyPackHelper.cpp @@ -89,7 +89,6 @@ void VelocyPackHelper::initialize() { // set the attribute translator in the global options VPackOptions::Defaults.attributeTranslator = Translator.get(); - VPackSlice::attributeTranslator = Translator.get(); // VPackOptions::Defaults.unsupportedTypeBehavior = VPackOptions::ConvertUnsupportedType; // initialize exclude handler for system attributes