Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

Documentation/Books/Users/IndexHandling/IndexBasics.mdpp

@@ -307,6 +307,10 @@ The following query will then use the array index:
       FILTER 'foobar' IN doc.tags[*].name
       RETURN doc
 
+If you store a document having the array which does contain elements not having
+the subattributes this document will also be indexed with the value `null`, which
+in ArangoDB is equal to attribute not existing.
+
 ArangoDB supports creating array indexes with a single <i>[\*]</i> operator per index 
 attribute. For example, creating an index as follows is not supported:
 
@@ -322,8 +326,46 @@ value `bar` will be inserted only once:
 db.posts.insert({ tags: [ "foobar", "bar", "bar" ] });
 ```
 
-If an array index is declared unique, the de-duplication of array values will happen before 
+If an array index is declared **unique**, the de-duplication of array values will happen before 
 inserting the values into the index, so the above insert operation will not necessarily fail.
 It will fail if the index already contains an instance of the `bar` value, but will succeed 
 if the value `bar` is not already present in the index.
 
+If an array index is declared and you store documents that do not have an array at the specified attribute
+this document will not be inserted in the index. Hence the following objects will not be indexed:
+
+```js
+db.posts.ensureIndex({ type: "hash", fields: [ "tags[*]" ] });
+db.posts.insert({ something: "else" });
+db.posts.insert({ tags: null });
+db.posts.insert({ tags: "this is no array" });
+db.posts.insert({ tags: { content: [1, 2, 3] } });
+```
+
+An array index is able to index an explicit `null` value and when queried for it, it will only
+return those documents having explicitly `null` stored in the array, it will not return any
+documents that do not have the array at all.
+
+```js
+db.posts.ensureIndex({ type: "hash", fields: [ "tags[*]" ] });
+db.posts.insert({tags: null}) // Will not be indexed
+db.posts.insert({tags: []})  // Will not be indexed
+db.posts.insert({tags: [null]}); // Will be indexed for null
+db.posts.insert({tags: [null, 1, 2]}); // Will be indexed for null, 1 and 2
+```
+
+Declaring an array index as **sparse** does not have an effect on the array part of the index,
+this in particular means that explicit `null` values are also indexed in the **sparse** version.
+If an index is combined from an array and a normal attribute the sparsity will apply for the attribute e.g.:
+
+```js
+db.posts.ensureIndex({ type: "hash", fields: [ "tags[*]", "name" ], sparse: true });
+db.posts.insert({tags: null, name: "alice"}) // Will not be indexed
+db.posts.insert({tags: [], name: "alice"}) // Will not be indexed
+db.posts.insert({tags: [1, 2, 3]}) // Will not be indexed
+db.posts.insert({tags: [1, 2, 3], name: null}) // Will not be indexed
+db.posts.insert({tags: [1, 2, 3], name: "alice"})
+// Will be indexed for [1, "alice"], [2, "alice"], [3, "alice"]
+db.posts.insert({tags: [null], name: "bob"})
+// Will be indexed for [null, "bob"] 
+```

LICENSES-OTHER-COMPONENTS.md

@@ -48,11 +48,18 @@
 * https://coreos.com/using-coreos/etcd/
 * [Apache 2 License](https://github.com/coreos/etcd/blob/master/LICENSE)
 
+### autotools
+
+* http://www.gnu.org/software/autoconf/autoconf.html
+* https://www.gnu.org/software/automake/
+* only used to generate code, not part of the distribution
+* parts generated are free as-is license
+
 ### Bison
 
 * https://www.gnu.org/software/bison/
 * only used to generate code, not part of the distribution
-* parts used see https://github.com/arangodb/arangodb/blob/devel/arangod/Aql/grammar.cpp#L20
+* parts generated use see https://github.com/arangodb/arangodb/blob/devel/arangod/Aql/grammar.cpp#L20
 
 ### Flex
 

arangod/Indexes/Index.cpp

@@ -568,9 +568,11 @@ bool Index::canUseConditionPart (triagens::aql::AstNode const* access,
         return false;
       }
 
+      /* A sparse index will store null in Array
       if (access->isNullValue()) {
         return false;
       }
+      */
     }
     else if (op->type == triagens::aql::NODE_TYPE_OPERATOR_BINARY_IN &&
              access->type == triagens::aql::NODE_TYPE_EXPANSION) {
@@ -579,9 +581,11 @@ bool Index::canUseConditionPart (triagens::aql::AstNode const* access,
         return false;
       }
 
+      /* A sparse index will store null in Array
       if (other->isNullValue()) {
         return false;
       }
+      */
     }
     else if (access->type == triagens::aql::NODE_TYPE_ATTRIBUTE_ACCESS) {
       // a.b == value  OR  a.b IN values

arangod/Indexes/PathBasedIndex.cpp

@@ -310,13 +310,14 @@ void PathBasedIndex::buildIndexValues (TRI_shaped_json_t const* documentShape,
       if (! check || shape == nullptr || shapedJson._sid == BasicShapes::TRI_SHAPE_SID_NULL) {
         // attribute not, found
         bool expandAnywhere = false;
-        for (size_t k = 0; k < n; ++k) {
+        size_t k = 0;
+        for (; k < n; ++k) {
           if (_paths[level][k].second) {
             expandAnywhere = true;
             break;
           }
         }
-        if (expandAnywhere && (i < n - 1 || ! check || shape == nullptr ) ) {
+        if (expandAnywhere && i <= k) {
           // We have an array index and we are not evaluating the indexed attribute.
           // Do not index this attribute at all
           if (level == 0) {

js/apps/system/_admin/aardvark/APP/frontend/js/views/graphManagementView.js

@@ -602,10 +602,18 @@
       );
 
       if (graph) {
+
+        $('.modal-body table').css('border-collapse', 'separate');
         var i;
+
+        $('.modal-body .spacer').remove();
         for (i = 0; i <= this.counter; i++) {
           $('#row_fromCollections' + i).show();
           $('#row_toCollections' + i).show();
+          $('#row_newEdgeDefinitions' + i).addClass('first');
+          $('#row_fromCollections' + i).addClass('middle');
+          $('#row_toCollections' + i).addClass('last');
+          $('#row_toCollections' + i).after('<tr id="spacer'+ i +'" class="spacer"></tr>');
         }
       }
 
@@ -663,6 +671,17 @@
         });
         window.modalView.undelegateEvents();
         window.modalView.delegateEvents(this.events);
+        
+        var i;
+        $('.modal-body .spacer').remove();
+        for (i = 0; i <= this.counter; i++) {
+          $('#row_fromCollections' + i).show();
+          $('#row_toCollections' + i).show();
+          $('#row_newEdgeDefinitions' + i).addClass('first');
+          $('#row_fromCollections' + i).addClass('middle');
+          $('#row_toCollections' + i).addClass('last');
+          $('#row_toCollections' + i).after('<tr id="spacer'+ i +'" class="spacer"></tr>');
+        }
         return;
       }
       if (id.indexOf("remove_newEdgeDefinitions") !== -1 ) {
@@ -670,6 +689,7 @@
         $('#row_newEdgeDefinitions' + number).remove();
         $('#row_fromCollections' + number).remove();
         $('#row_toCollections' + number).remove();
+        $('#spacer' + number).remove();
       }
     },
 

js/apps/system/_admin/aardvark/APP/frontend/scss/_modals.scss

@@ -114,6 +114,56 @@
     line-height: 17px;
   }
 
+  tr {
+
+    &.spacer {
+      height: 10px;
+    }
+
+    &.first {
+      background-color: $c-lightgreen-2-bg;
+
+      th:first-child {
+        border-top-left-radius: 3px;
+      }
+
+      th:last-child {
+        border-top-right-radius: 3px;
+      }
+    }
+
+    &.middle {
+      background-color: $c-lightgreen-2-bg;
+      padding-left: 10px;
+      padding-right: 10px;
+    }
+
+    &.last {
+      background-color: $c-lightgreen-2-bg;
+
+      th:first-child {
+        border-bottom-left-radius: 3px;
+      }
+
+      th:last-child {
+        border-bottom-right-radius: 3px;
+      }
+    }
+
+    &.first,
+    &.middle,
+    &.last {
+      th:first-child {
+        padding-left: 10px;
+      }
+
+      th:last-child {
+        padding-right: 10px;
+      }
+    }
+
+  }
+
   th {
     %cell-centered {
       text-align: center;
@@ -267,6 +317,7 @@
   border: 0 !important;
   border-radius: 3px !important;
   box-shadow: 0;
+  width: 580px;
 
   .fade.in {
     top: 12.1% !important;

js/server/tests/aql-queries-array.js

@@ -30,6 +30,7 @@
 
 var jsunity = require("jsunity");
 var db = require("org/arangodb").db;
+var isCluster = require("org/arangodb/cluster").isCluster();
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test suite
@@ -84,7 +85,7 @@ function arrayIndexSuite () {
                 "index used for: " + query);
   };
 
-  var validateResults = function (query, sparse) {
+  var validateResults = function (query) {
     var bindVars = {};
     bindVars.tag = "tenth";
 
@@ -113,12 +114,7 @@ function arrayIndexSuite () {
     }
 
     bindVars.tag = null;
-    if (!sparse) {
     checkIsOptimizedQuery(query, bindVars);
-    }
-    else {
-      validateIndexNotUsed(query, bindVars);
-    }
     actual = AQL_EXECUTE(query, bindVars);
     // We check if we found the Arrays with NULL in it
     assertNotEqual(-1, actual.json.indexOf("0t"), "Did not find the null array");
@@ -316,9 +312,9 @@ function arrayIndexSuite () {
       col.save({_key: "noArray", a: "NoArray"});
       col.save({_key: "null", a: null});
       const query = `FOR x IN ${cName} FILTER @tag IN x.a[*] SORT x._key RETURN x._key`;
-      validateResults(query, true);
+      validateResults(query);
       const orQuery = `FOR x IN ${cName} FILTER @tag1 IN x.a[*] || @tag2 IN x.a[*] SORT x._key RETURN x._key`;
-      validateResultsOr(orQuery, true);
+      validateResultsOr(orQuery);
     },
 
     testSkiplistPlainArray : function () {
@@ -429,9 +425,9 @@ function arrayIndexSuite () {
       col.save({_key: "noArray", a: "NoArray"});
       col.save({_key: "null", a: null});
       const query = `FOR x IN ${cName} FILTER @tag IN x.a[*] SORT x._key RETURN x._key`;
-      validateResults(query, true);
+      validateResults(query);
       const orQuery = `FOR x IN ${cName} FILTER @tag1 IN x.a[*] || @tag2 IN x.a[*] SORT x._key RETURN x._key`;
-      validateResultsOr(orQuery, true);
+      validateResultsOr(orQuery);
     }
 
   };
@@ -447,6 +443,7 @@ function arrayIndexNonArraySuite () {
     var allIndexes = col.getIndexes(true);
     assertEqual(allIndexes.length, 2, "We have more than one index!");
     var idx = allIndexes[1];
+    if (! isCluster) {
       switch (idx.type) {
         case "hash":
           assertEqual(idx.figures.totalUsed, count);
@@ -457,6 +454,7 @@ function arrayIndexNonArraySuite () {
         default:
           assertTrue(false, "Unexpected index type");
       }
+    }
   };
 
   return {
@@ -616,13 +614,16 @@ function arrayIndexNonArraySuite () {
       col.save({ a: [], b: 1, c: 1 });  // Empty Array. no indexing
       checkElementsInIndex(inserted);
 
-      col.save({ a: [1, 2, 3, 3, 2, 1] }); // a does not have any nested value. Handled equal to a: []
+      col.save({ a: [1, 2, 3, 3, 2, 1] }); // a does not have any nested value. Index as one null
+      inserted += 1;
       checkElementsInIndex(inserted);
 
-      col.save({ a: [1, 2, 3, 3, 2, 1], b: 1 }); // a does not have any nested value. Handled equal to a: []
+      col.save({ a: [1, 2, 3, 3, 2, 1], b: 1 }); // a does not have any nested value. Index as one null
+      inserted += 1;
       checkElementsInIndex(inserted);
 
-      col.save({ a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 }); // a does not have any nested value. Handled equal to a: []
+      col.save({_key: "null1", a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 }); // a does not have any nested value. Index as one null
+      inserted += 1;
       checkElementsInIndex(inserted);
 
       col.save({ a: [{d: 1}, {d: 2}, {d: 3}, {d: 3}, {d: 2}, {d: 1}] });
@@ -637,23 +638,24 @@ function arrayIndexNonArraySuite () {
       inserted += 3; // We index b: 1, c: 1 and 3 values for a[*].d
       checkElementsInIndex(inserted);
 
-      col.save({_key: "null1", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] });
+      col.save({_key: "null2", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] });
       inserted += 4;
       checkElementsInIndex(inserted); // b: null a: "a", "b", "c", null c:null
 
-      col.save({_key: "null2", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 });
+      col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 });
       inserted += 4;
       checkElementsInIndex(inserted);
 
-      col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 });
+      col.save({_key: "null4", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 });
       inserted += 4;
       checkElementsInIndex(inserted);
 
       const query = `FOR x IN ${cName} FILTER @tag IN x.a[*].d && 1 == x.b && 1 == x.c SORT x._key RETURN x._key`;
       var actual = AQL_EXECUTE(query, { tag : null }).json;
       // We expect that we can only find the Array that has stored exactly the null value
-      assertEqual(actual.length, 1);
+      assertEqual(actual.length, 2);
-      assertEqual(actual[0], "null3");
+      assertEqual(actual[0], "null1");
+      assertEqual(actual[1], "null4");
     },
 
     testHashIndexSubAttributeArray : function () {
@@ -667,6 +669,66 @@ function arrayIndexNonArraySuite () {
       // Do not find anything
     },
 
+    testHashIndexMultiArray : function () {
+      col.ensureHashIndex("a[*]", "b[*]");
+
+      col.save({a: [1, 2, 3]}); // Do not index
+      checkElementsInIndex(0);
+
+      col.save({a: [1, 2, 3], b: null}); // Do not index
+      checkElementsInIndex(0);
+
+      col.save({a: [1, 2, 3], b: "this is no array"}); // Do not index
+      checkElementsInIndex(0);
+
+      col.save({a: "this is no array", b: ["a", "b", "c"]}); // Do not index
+      checkElementsInIndex(0);
+
+      col.save({a: [1, 2, null, null, 2, 1], b: ["a", "b", null, "b", "a"]});
+      checkElementsInIndex(9); // 3*3 many combinations
+
+      const query = `FOR x IN ${cName} FILTER @tag IN x.a[*] && @tag IN x.b[*] SORT x._key RETURN x._key`;
+      var actual = AQL_EXECUTE(query, { tag : null }).json;
+      assertEqual(actual.length, 1);
+    },
+
+    testHashIndexArraySparse : function () {
+      col.ensureHashIndex("a[*]", "b", {sparse: true});
+      var inserted = 0;
+
+      col.save({a: [1, 2, 3]}); // Do not index, b is not set
+      checkElementsInIndex(inserted);
+
+      col.save({a: [1, 2, 3], b: null}); // Do not index, b is null
+      checkElementsInIndex(inserted);
+
+      col.save({a: [1, 2, 3], b: 1}); // Do index
+      inserted += 3;
+      checkElementsInIndex(inserted);
+
+      col.save({a: [null, 4], b: 1}); // Do index
+      inserted += 2;
+      checkElementsInIndex(inserted);
+
+      const query = `FOR x IN ${cName} FILTER null IN x.a[*] && 1 == x.b SORT x._key RETURN x._key`;
+      // We can use the index for null in SPARSE
+      var actual = AQL_EXECUTE(query).json;
+      assertEqual(actual.length, 1);
+      var plan = AQL_EXPLAIN(query).plan;
+      var nodeTypes = plan.nodes.map(function(node) {
+        return node.type;
+      });
+      assertNotEqual(-1, nodeTypes.indexOf("IndexNode"));
+
+      const query2 = `FOR x IN ${cName} FILTER null IN x.a[*] SORT x._key RETURN x._key`;
+      plan = AQL_EXPLAIN(query2).plan;
+      nodeTypes = plan.nodes.map(function(node) {
+        return node.type;
+      });
+      // Cannot use the index for sub attribute a
+      assertEqual(-1, nodeTypes.indexOf("IndexNode"));
+    },
+
     testSkiplistSingleAttribute : function () {
       col.ensureSkiplist("a[*]");
       col.save({}); // a is not set
@@ -876,16 +938,16 @@ function arrayIndexNonArraySuite () {
       checkElementsInIndex(inserted);
 
       col.save({ a: [1, 2, 3, 3, 2, 1] });
-      inserted += 1; // We index b: null But a does not have any nested value. Handled equal to a: []
+      inserted += 1; // We index b: null But a does not have any nested value. Index one for null
       checkElementsInIndex(inserted);
 
-      col.save({ a: [1, 2, 3, 3, 2, 1], b: 1 }); // b: 1 a: 1,2,3 c: null
+      col.save({ _key: "null1", a: [1, 2, 3, 3, 2, 1], b: 1 }); // b: 1 a: 1,2,3 c: null
-      inserted += 1; // We index b: 1 But a does not have any nested value. Handled equal to a: []
+      inserted += 1; // We index b: 1 But a does not have any nested value. Index one for null
       insertedB += 1;
       checkElementsInIndex(inserted);
 
-      col.save({ a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 });
+      col.save({ _key: "null2", a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 });
-      inserted += 1; // We index b: 1, c: 1 But a does not have any nested value. Handled equal to a: []
+      inserted += 1; // We index b: 1, c: 1 But a does not have any nested value. Index one for null
       insertedB += 1;
       checkElementsInIndex(inserted);
 
@@ -903,26 +965,29 @@ function arrayIndexNonArraySuite () {
       insertedB += 3;
       checkElementsInIndex(inserted);
 
-      col.save({_key: "null1", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] });
+      col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] });
       inserted += 4;
       checkElementsInIndex(inserted); // b: null a: "a", "b", "c", null c:null
 
-      col.save({_key: "null2", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 });
+      col.save({_key: "null4", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 });
       inserted += 4;
       insertedB += 4;
       checkElementsInIndex(inserted);
 
-      col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 });
+      col.save({_key: "null5", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 });
       inserted += 4;
       insertedB += 4;
       checkElementsInIndex(inserted);
 
       const query = `FOR x IN ${cName} FILTER @tag IN x.a[*].d && 1 == x.b SORT x._key RETURN x._key`;
       var actual = AQL_EXECUTE(query, { tag : null }).json;
-      // We expect that we can only find the Array that has stored exactly the null value
+      // We expect that we can only find the array that stores exactly the null value
-      assertEqual(actual.length, 2);
+      // And the arrays that do not have the sub attribute.
-      assertEqual(actual[0], "null2");
+      assertEqual(actual.length, 4);
-      assertEqual(actual[1], "null3");
+      assertEqual(actual[0], "null1");
+      assertEqual(actual[1], "null2");
+      assertEqual(actual[2], "null4");
+      assertEqual(actual[3], "null5");
 
       const query2 = `FOR x IN ${cName} FILTER 1 == x.b RETURN x._key`;
       actual = AQL_EXECUTE(query2).json;
@@ -942,7 +1007,45 @@ function arrayIndexNonArraySuite () {
       var actual = AQL_EXECUTE(query, { tag : null }).json;
       assertEqual(actual.length, 0);
       // Do not find anything
-    }
+    },
+
+    testSkiplistIndexArraySparse : function () {
+      col.ensureSkiplist("a[*]", "b", {sparse: true});
+      var inserted = 0;
+
+      col.save({a: [1, 2, 3]}); // Do not index, b is not set
+      checkElementsInIndex(inserted);
+
+      col.save({a: [1, 2, 3], b: null}); // Do not index, b is null
+      checkElementsInIndex(inserted);
+
+      col.save({a: [1, 2, 3], b: 1}); // Do index
+      inserted += 3;
+      checkElementsInIndex(inserted);
+
+      col.save({a: [null, 4], b: 1}); // Do index
+      inserted += 2;
+      checkElementsInIndex(inserted);
+
+      const query = `FOR x IN ${cName} FILTER null IN x.a[*] && 1 == x.b SORT x._key RETURN x._key`;
+      // We can use the index for null in SPARSE
+      var actual = AQL_EXECUTE(query).json;
+      assertEqual(actual.length, 1);
+      var plan = AQL_EXPLAIN(query).plan;
+      var nodeTypes = plan.nodes.map(function(node) {
+        return node.type;
+      });
+      assertNotEqual(-1, nodeTypes.indexOf("IndexNode"));
+
+      const query2 = `FOR x IN ${cName} FILTER null IN x.a[*] SORT x._key RETURN x._key`;
+      plan = AQL_EXPLAIN(query2).plan;
+      nodeTypes = plan.nodes.map(function(node) {
+        return node.type;
+      });
+      // Cannot use the index for sub attribute a
+      assertEqual(-1, nodeTypes.indexOf("IndexNode"));
+    },
+
 
   };