Merge branch 'devel' of ssh://github.com/ArangoDB/ArangoDB into devel

arangod/Aql/Ast.cpp

@@ -1012,7 +1012,7 @@ AstNode* Ast::createNodeValueString (char const* value,
     THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
   }
 
-  if (*value == '\0') {
+  if (length == 0) {
     // performance optimization:
     // return a pointer to the singleton empty string node
     // note: these nodes are never registered nor freed

arangod/Aql/AstNode.cpp

@@ -1255,7 +1255,7 @@ AstNode* AstNode::castToBool (Ast* ast) {
       case VALUE_TYPE_DOUBLE:
         return ast->createNodeValueBool(value.value._double != 0.0);
       case VALUE_TYPE_STRING:
-        return ast->createNodeValueBool(*(value.value._string) != '\0');
+        return ast->createNodeValueBool(value.length > 0);
       default: {
       }
     }
@@ -2192,7 +2192,7 @@ void AstNode::stringify (triagens::basics::StringBuffer* buffer,
     auto member = getMember(0);
     member->stringify(buffer, verbose, failIfLong);
     buffer->appendChar('.');
-    buffer->appendText(getStringValue());
+    buffer->appendText(getStringValue(), getStringLength());
     return;
   }
 
@@ -2207,7 +2207,7 @@ void AstNode::stringify (triagens::basics::StringBuffer* buffer,
   if (type == NODE_TYPE_PARAMETER) {
     // not used by V8
     buffer->appendChar('@');
-    buffer->appendText(getStringValue());
+    buffer->appendText(getStringValue(), getStringLength());
     return;
   }
     

arangod/Aql/BindParameters.cpp

@@ -140,7 +140,8 @@ void BindParameters::StripCollectionNames (TRI_json_t* keys,
       auto s = key->_value._string.data;
       auto p = strchr(s, '/');
 
-      if (p != nullptr && strncmp(s, collectionName, p - s) == 0) {
+      if (p != nullptr && 
+          strncmp(s, collectionName, p - s) == 0) {
         size_t pos = static_cast<size_t>(p - s);
         // key begins with collection name + '/', now strip it in place for further comparisons
         memmove(s, p + 1, key->_value._string.length - 2 - pos);

arangod/Aql/ExecutionBlock.cpp

@@ -30,20 +30,6 @@
 #include "Aql/ExecutionBlock.h"
 #include "Aql/ExecutionEngine.h"
 #include "Basics/json-utilities.h"
-/*
-#include "Basics/Exceptions.h"
-#include "Basics/StringBuffer.h"
-#include "Basics/StringUtils.h"
-#include "Basics/Traverser.h"
-#include "Cluster/ClusterMethods.h"
-#include "Dispatcher/DispatcherThread.h"
-#include "Indexes/EdgeIndex.h"
-#include "Indexes/HashIndex.h"
-#include "Utils/ShapedJsonTransformer.h"
-#include "V8/v8-globals.h"
-#include "VocBase/edge-collection.h"
-#include "VocBase/vocbase.h"
-*/
 
 using namespace std;
 using namespace triagens::basics;

arangod/Utils/ShapedJsonTransformer.cpp

@@ -34,12 +34,10 @@
 using Json = triagens::basics::Json;
 using CollectionNameResolver = triagens::arango::CollectionNameResolver;
 
-Json TRI_ExpandShapedJson (
+Json TRI_ExpandShapedJson (VocShaper* shaper,
-  VocShaper* shaper,
                            CollectionNameResolver const* resolver,
                            TRI_voc_cid_t const& cid,
-  TRI_df_marker_t const* marker
+                           TRI_df_marker_t const* marker) {
-) {
   TRI_shaped_json_t shaped;
   TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, marker);
   Json json(shaper->memoryZone(), TRI_JsonShapedJson(shaper, &shaped));
@@ -73,12 +71,10 @@ Json TRI_ExpandShapedJson (
   return json;
 }
 
-Json TRI_ExpandShapedJson (
+Json TRI_ExpandShapedJson (VocShaper* shaper,
-  VocShaper* shaper,
                            CollectionNameResolver const* resolver,
                            TRI_voc_cid_t const& cid,
-  TRI_doc_mptr_t const* mptr
+                           TRI_doc_mptr_t const* mptr) {
-) {
   TRI_df_marker_t const* marker = static_cast<TRI_df_marker_t const*>(mptr->getDataPtr());
   return TRI_ExpandShapedJson(shaper, resolver, cid, marker);
 }

js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/@arangodb/arango-query-cursor.js

@@ -100,8 +100,8 @@ ArangoQueryCursor.prototype.toString = function () {
   if (this.data.hasOwnProperty("extra") && 
       this.data.extra.hasOwnProperty("warnings")) {
     for (var j = 0; j < this.data.extra.warnings.length; j++) {
-      result += ", WARNING: " + this.data.extra.warnings[j].code +
+      result += ", warning: " + this.data.extra.warnings[j].code +
-        " - " + this.data.extra.warnings[j].message
+        " - " + this.data.extra.warnings[j].message;
     }
   }
   result += "]";

js/common/tests/shell-attributes.js

@@ -485,6 +485,102 @@ function AttributesSuite () {
       assertEqual("foo\u0001bar\u0000baz", result[0].abc);
       assertEqual("\u0000test\u0000test", result[0].def);
       assertEqual("abc\u0000", result[0]["123"]);
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test values containing NULL bytes
+////////////////////////////////////////////////////////////////////////////////
+
+    testNullBytesValues : function () {
+      var docs = [
+        { value: "\u0000zero\u0000", _key: "first" },
+        { value: "\u0000\r\nnew line", _key: "second" },
+        { value: "\u0000zero", _key: "third" },
+        { value: "0\r\nxxx", _key: "fourth" }
+      ];
+
+      docs.forEach(function(doc) {
+        c.insert(doc);
+      });
+
+      // test return value of document()
+      docs.forEach(function(doc) {
+        var d = c.document(doc._key);
+        assertEqual(doc.value, d.value);
+      });
+
+      // test return value of AQL, using literals
+      docs.forEach(function(doc) {
+        var result = db._query("FOR doc IN @@collection FILTER doc.value == " + 
+                               JSON.stringify(doc.value) + " RETURN doc", { 
+          "@collection" : c.name()
+        }).toArray();
+
+        assertEqual(1, result.length, doc);
+        assertEqual(doc.value, result[0].value);
+        assertEqual(doc._key, result[0]._key);
+      });
+
+      // test return value of AQL, using bind parameters
+      docs.forEach(function(doc) {
+        var result = db._query("FOR doc IN @@collection FILTER doc.value == @value RETURN doc", { 
+          "@collection" : c.name(),
+          value: doc.value 
+        }).toArray();
+
+        assertEqual(1, result.length, doc);
+        assertEqual(doc.value, result[0].value);
+        assertEqual(doc._key, result[0]._key);
+      });
+
+      // test return value of AQL
+      var result;
+      result = db._query("FOR doc IN @@collection FILTER SUBSTRING(doc.value, 0, 5) == @value RETURN doc._key", { 
+        "@collection" : c.name(),
+        value: "\u0000zero" 
+      }).toArray().sort();
+
+      assertEqual(2, result.length);
+      assertEqual([ "first", "third" ], result);
+
+      result = db._query("FOR doc IN @@collection FILTER SUBSTRING(doc.value, 0, 6) == @value RETURN doc._key", { 
+        "@collection" : c.name(),
+        value: "\u0000zero\u0000" 
+      }).toArray();
+
+      assertEqual(1, result.length);
+      assertEqual([ "first" ], result);
+
+      result = db._query("FOR doc IN @@collection FILTER SUBSTRING(doc.value, 0, 1) == " + 
+                         JSON.stringify("\0") + " RETURN doc._key", { 
+        "@collection" : c.name()
+      }).toArray().sort();
+
+      assertEqual(3, result.length);
+      assertEqual([ "first", "second", "third" ], result);
+
+      result = db._query("FOR doc IN @@collection FILTER SUBSTRING(doc.value, 0, 1) == @value RETURN doc._key", { 
+        "@collection" : c.name(),
+        value: "\0",
+      }).toArray().sort();
+
+      assertEqual(3, result.length);
+      assertEqual([ "first", "second", "third" ], result);
+
+      result = db._query("FOR doc IN @@collection FILTER LIKE(doc.value, '\u0000%') SORT doc._key RETURN doc._key", { 
+        "@collection" : c.name()
+      }).toArray().sort();
+
+      assertEqual(3, result.length);
+      assertEqual([ "first", "second", "third" ], result);
+      
+      result = db._query("FOR doc IN @@collection FILTER V8(LIKE(doc.value, '\u0000%')) " + 
+                         "SORT doc._key RETURN doc._key", { 
+        "@collection" : c.name()
+      }).toArray().sort();
+
+      assertEqual(3, result.length);
+      assertEqual([ "first", "second", "third" ], result);
     }
 
   };

lib/Basics/JsonHelper.cpp

@@ -28,7 +28,6 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "Basics/JsonHelper.h"
-
 #include "Basics/conversions.h"
 #include "Basics/string-buffer.h"
 
@@ -226,7 +225,7 @@ std::string JsonHelper::toString (TRI_json_t const* json) {
     return "";
   }
 
-  string out(TRI_BeginStringBuffer(&buffer), TRI_LengthStringBuffer(&buffer));
+  std::string out(TRI_BeginStringBuffer(&buffer), TRI_LengthStringBuffer(&buffer));
   TRI_DestroyStringBuffer(&buffer);
 
   return out;
@@ -252,7 +251,7 @@ TRI_json_t* JsonHelper::getObjectElement (TRI_json_t const* json,
 std::string JsonHelper::getStringValue (TRI_json_t const* json,
                                         std::string const& defaultValue) {
   if (isString(json)) {
-    return string(json->_value._string.data, json->_value._string.length - 1);
+    return std::string(json->_value._string.data, json->_value._string.length - 1);
   }
   return defaultValue;
 }
@@ -267,7 +266,7 @@ std::string JsonHelper::getStringValue (TRI_json_t const* json,
   TRI_json_t const* sub = getObjectElement(json, name);
 
   if (isString(sub)) {
-    return string(sub->_value._string.data, sub->_value._string.length - 1);
+    return std::string(sub->_value._string.data, sub->_value._string.length - 1);
   }
   return defaultValue;
 }
@@ -320,7 +319,7 @@ std::string JsonHelper::checkAndGetStringValue (TRI_json_t const* json,
       + "' was not found or is not a string.";
     THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, msg);
   }
-  return string(sub->_value._string.data, sub->_value._string.length - 1);
+  return std::string(sub->_value._string.data, sub->_value._string.length - 1);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -379,7 +378,6 @@ std::ostream& operator<< (std::ostream& stream,
   return stream;
 }
 
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief append the JSON contents to an output stream
 ////////////////////////////////////////////////////////////////////////////////

lib/Basics/StringBuffer.h

@@ -544,15 +544,6 @@ namespace triagens {
           return *this;
         }
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief appends string
-////////////////////////////////////////////////////////////////////////////////
-
-        StringBuffer& appendText (std::string&& str) {
-          TRI_AppendString2StringBuffer(&_buffer, str.c_str(), str.length());
-          return *this;
-        }
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief appends a string buffer
 ////////////////////////////////////////////////////////////////////////////////

lib/Basics/Utf8Helper.cpp

@@ -695,7 +695,7 @@ bool Utf8Helper::matches (RegexMatcher* matcher,
                           bool& error) {
 
   TRI_ASSERT(value != nullptr);
-  UnicodeString v = UnicodeString::fromUTF8(value);
+  UnicodeString v = UnicodeString::fromUTF8(StringPiece(value, valueLength));
 
   matcher->reset(v);
 

lib/Basics/json-utilities.cpp

@@ -244,14 +244,18 @@ int TRI_CompareValuesJson (TRI_json_t const* lhs,
     case TRI_JSON_STRING_REFERENCE: {
       // same for STRING and STRING_REFERENCE
       int res;
+      size_t const nl = lhs->_value._string.length - 1;   
+      size_t const nr = rhs->_value._string.length - 1;   
       if (useUTF8) {
         res = TRI_compare_utf8(lhs->_value._string.data,
-                               lhs->_value._string.length - 1, 
+                               nl,
                                rhs->_value._string.data,
-                               rhs->_value._string.length - 1);
+                               nr);
       }
       else {
-        res = strcmp(lhs->_value._string.data, rhs->_value._string.data);
+        // beware of strings containing NUL bytes
+        size_t len = nl < nr ? nl : nr;
+        res = memcmp(lhs->_value._string.data, rhs->_value._string.data, len);
       }
       if (res < 0) {
         return -1;
@@ -259,8 +263,13 @@ int TRI_CompareValuesJson (TRI_json_t const* lhs,
       else if (res > 0) {
         return 1;
       }
+      // res == 0
+      if (nl == nr) {
         return 0;
       }
+      // res == 0, but different string lengths
+      return nl < nr ? -1 : 1;
+    }
 
     case TRI_JSON_ARRAY: {
       size_t const nl = TRI_LengthVector(&lhs->_value._objects);

lib/Basics/json.cpp

@@ -1095,8 +1095,8 @@ int TRI_CopyToJson (TRI_memory_zone_t* zone,
       }
 
       for (size_t i = 0;  i < n;  ++i) {
-        TRI_json_t const* v = static_cast<TRI_json_t const*>(TRI_AtVector(&src->_value._objects, i));
+        auto const* v = static_cast<TRI_json_t const*>(TRI_AtVector(&src->_value._objects, i));
-        TRI_json_t* w = static_cast<TRI_json_t*>(TRI_AtVector(&dst->_value._objects, i));
+        auto* w = static_cast<TRI_json_t*>(TRI_AtVector(&dst->_value._objects, i));
 
         res = TRI_CopyToJson(zone, w, v);