diff --git a/Documentation/Books/Users/IndexHandling/IndexBasics.mdpp b/Documentation/Books/Users/IndexHandling/IndexBasics.mdpp index 12b84ea0f6..6ad0b20689 100644 --- a/Documentation/Books/Users/IndexHandling/IndexBasics.mdpp +++ b/Documentation/Books/Users/IndexHandling/IndexBasics.mdpp @@ -307,6 +307,10 @@ The following query will then use the array index: FILTER 'foobar' IN doc.tags[*].name RETURN doc +If you store a document having the array which does contain elements not having +the subattributes this document will also be indexed with the value `null`, which +in ArangoDB is equal to attribute not existing. + ArangoDB supports creating array indexes with a single [\*] operator per index attribute. For example, creating an index as follows is not supported: @@ -322,8 +326,46 @@ value `bar` will be inserted only once: db.posts.insert({ tags: [ "foobar", "bar", "bar" ] }); ``` -If an array index is declared unique, the de-duplication of array values will happen before +If an array index is declared **unique**, the de-duplication of array values will happen before inserting the values into the index, so the above insert operation will not necessarily fail. It will fail if the index already contains an instance of the `bar` value, but will succeed if the value `bar` is not already present in the index. +If an array index is declared and you store documents that do not have an array at the specified attribute +this document will not be inserted in the index. Hence the following objects will not be indexed: + +```js +db.posts.ensureIndex({ type: "hash", fields: [ "tags[*]" ] }); +db.posts.insert({ something: "else" }); +db.posts.insert({ tags: null }); +db.posts.insert({ tags: "this is no array" }); +db.posts.insert({ tags: { content: [1, 2, 3] } }); +``` + +An array index is able to index an explicit `null` value and when queried for it, it will only +return those documents having explicitly `null` stored in the array, it will not return any +documents that do not have the array at all. + +```js +db.posts.ensureIndex({ type: "hash", fields: [ "tags[*]" ] }); +db.posts.insert({tags: null}) // Will not be indexed +db.posts.insert({tags: []}) // Will not be indexed +db.posts.insert({tags: [null]}); // Will be indexed for null +db.posts.insert({tags: [null, 1, 2]}); // Will be indexed for null, 1 and 2 +``` + +Declaring an array index as **sparse** does not have an effect on the array part of the index, +this in particular means that explicit `null` values are also indexed in the **sparse** version. +If an index is combined from an array and a normal attribute the sparsity will apply for the attribute e.g.: + +```js +db.posts.ensureIndex({ type: "hash", fields: [ "tags[*]", "name" ], sparse: true }); +db.posts.insert({tags: null, name: "alice"}) // Will not be indexed +db.posts.insert({tags: [], name: "alice"}) // Will not be indexed +db.posts.insert({tags: [1, 2, 3]}) // Will not be indexed +db.posts.insert({tags: [1, 2, 3], name: null}) // Will not be indexed +db.posts.insert({tags: [1, 2, 3], name: "alice"}) +// Will be indexed for [1, "alice"], [2, "alice"], [3, "alice"] +db.posts.insert({tags: [null], name: "bob"}) +// Will be indexed for [null, "bob"] +``` diff --git a/LICENSES-OTHER-COMPONENTS.md b/LICENSES-OTHER-COMPONENTS.md index b01b122f98..08f82d06f6 100644 --- a/LICENSES-OTHER-COMPONENTS.md +++ b/LICENSES-OTHER-COMPONENTS.md @@ -48,11 +48,18 @@ * https://coreos.com/using-coreos/etcd/ * [Apache 2 License](https://github.com/coreos/etcd/blob/master/LICENSE) +### autotools + +* http://www.gnu.org/software/autoconf/autoconf.html +* https://www.gnu.org/software/automake/ +* only used to generate code, not part of the distribution +* parts generated are free as-is license + ### Bison * https://www.gnu.org/software/bison/ * only used to generate code, not part of the distribution -* parts used see https://github.com/arangodb/arangodb/blob/devel/arangod/Aql/grammar.cpp#L20 +* parts generated use see https://github.com/arangodb/arangodb/blob/devel/arangod/Aql/grammar.cpp#L20 ### Flex diff --git a/arangod/Indexes/Index.cpp b/arangod/Indexes/Index.cpp index 663981ca20..e575cccbf6 100644 --- a/arangod/Indexes/Index.cpp +++ b/arangod/Indexes/Index.cpp @@ -568,9 +568,11 @@ bool Index::canUseConditionPart (triagens::aql::AstNode const* access, return false; } + /* A sparse index will store null in Array if (access->isNullValue()) { return false; } + */ } else if (op->type == triagens::aql::NODE_TYPE_OPERATOR_BINARY_IN && access->type == triagens::aql::NODE_TYPE_EXPANSION) { @@ -579,9 +581,11 @@ bool Index::canUseConditionPart (triagens::aql::AstNode const* access, return false; } + /* A sparse index will store null in Array if (other->isNullValue()) { return false; } + */ } else if (access->type == triagens::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { // a.b == value OR a.b IN values diff --git a/arangod/Indexes/PathBasedIndex.cpp b/arangod/Indexes/PathBasedIndex.cpp index 85ca0a5232..f0f61deff6 100644 --- a/arangod/Indexes/PathBasedIndex.cpp +++ b/arangod/Indexes/PathBasedIndex.cpp @@ -310,13 +310,14 @@ void PathBasedIndex::buildIndexValues (TRI_shaped_json_t const* documentShape, if (! check || shape == nullptr || shapedJson._sid == BasicShapes::TRI_SHAPE_SID_NULL) { // attribute not, found bool expandAnywhere = false; - for (size_t k = 0; k < n; ++k) { + size_t k = 0; + for (; k < n; ++k) { if (_paths[level][k].second) { expandAnywhere = true; break; } } - if (expandAnywhere && (i < n - 1 || ! check || shape == nullptr ) ) { + if (expandAnywhere && i <= k) { // We have an array index and we are not evaluating the indexed attribute. // Do not index this attribute at all if (level == 0) { diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphManagementView.js b/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphManagementView.js index 1a0b71375b..1ce8c4f746 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphManagementView.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/views/graphManagementView.js @@ -602,10 +602,18 @@ ); if (graph) { + + $('.modal-body table').css('border-collapse', 'separate'); var i; + + $('.modal-body .spacer').remove(); for (i = 0; i <= this.counter; i++) { $('#row_fromCollections' + i).show(); $('#row_toCollections' + i).show(); + $('#row_newEdgeDefinitions' + i).addClass('first'); + $('#row_fromCollections' + i).addClass('middle'); + $('#row_toCollections' + i).addClass('last'); + $('#row_toCollections' + i).after(''); } } @@ -663,6 +671,17 @@ }); window.modalView.undelegateEvents(); window.modalView.delegateEvents(this.events); + + var i; + $('.modal-body .spacer').remove(); + for (i = 0; i <= this.counter; i++) { + $('#row_fromCollections' + i).show(); + $('#row_toCollections' + i).show(); + $('#row_newEdgeDefinitions' + i).addClass('first'); + $('#row_fromCollections' + i).addClass('middle'); + $('#row_toCollections' + i).addClass('last'); + $('#row_toCollections' + i).after(''); + } return; } if (id.indexOf("remove_newEdgeDefinitions") !== -1 ) { @@ -670,6 +689,7 @@ $('#row_newEdgeDefinitions' + number).remove(); $('#row_fromCollections' + number).remove(); $('#row_toCollections' + number).remove(); + $('#spacer' + number).remove(); } }, diff --git a/js/apps/system/_admin/aardvark/APP/frontend/scss/_modals.scss b/js/apps/system/_admin/aardvark/APP/frontend/scss/_modals.scss index 320e66a227..3cec653648 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/scss/_modals.scss +++ b/js/apps/system/_admin/aardvark/APP/frontend/scss/_modals.scss @@ -114,6 +114,56 @@ line-height: 17px; } + tr { + + &.spacer { + height: 10px; + } + + &.first { + background-color: $c-lightgreen-2-bg; + + th:first-child { + border-top-left-radius: 3px; + } + + th:last-child { + border-top-right-radius: 3px; + } + } + + &.middle { + background-color: $c-lightgreen-2-bg; + padding-left: 10px; + padding-right: 10px; + } + + &.last { + background-color: $c-lightgreen-2-bg; + + th:first-child { + border-bottom-left-radius: 3px; + } + + th:last-child { + border-bottom-right-radius: 3px; + } + } + + &.first, + &.middle, + &.last { + th:first-child { + padding-left: 10px; + } + + th:last-child { + padding-right: 10px; + } + } + + } + th { %cell-centered { text-align: center; @@ -267,6 +317,7 @@ border: 0 !important; border-radius: 3px !important; box-shadow: 0; + width: 580px; .fade.in { top: 12.1% !important; diff --git a/js/server/tests/aql-queries-array.js b/js/server/tests/aql-queries-array.js index 88ab6a4bc1..9f03a73e1a 100644 --- a/js/server/tests/aql-queries-array.js +++ b/js/server/tests/aql-queries-array.js @@ -30,6 +30,7 @@ var jsunity = require("jsunity"); var db = require("org/arangodb").db; +var isCluster = require("org/arangodb/cluster").isCluster(); //////////////////////////////////////////////////////////////////////////////// /// @brief test suite @@ -84,7 +85,7 @@ function arrayIndexSuite () { "index used for: " + query); }; - var validateResults = function (query, sparse) { + var validateResults = function (query) { var bindVars = {}; bindVars.tag = "tenth"; @@ -113,12 +114,7 @@ function arrayIndexSuite () { } bindVars.tag = null; - if (!sparse) { - checkIsOptimizedQuery(query, bindVars); - } - else { - validateIndexNotUsed(query, bindVars); - } + checkIsOptimizedQuery(query, bindVars); actual = AQL_EXECUTE(query, bindVars); // We check if we found the Arrays with NULL in it assertNotEqual(-1, actual.json.indexOf("0t"), "Did not find the null array"); @@ -316,9 +312,9 @@ function arrayIndexSuite () { col.save({_key: "noArray", a: "NoArray"}); col.save({_key: "null", a: null}); const query = `FOR x IN ${cName} FILTER @tag IN x.a[*] SORT x._key RETURN x._key`; - validateResults(query, true); + validateResults(query); const orQuery = `FOR x IN ${cName} FILTER @tag1 IN x.a[*] || @tag2 IN x.a[*] SORT x._key RETURN x._key`; - validateResultsOr(orQuery, true); + validateResultsOr(orQuery); }, testSkiplistPlainArray : function () { @@ -429,9 +425,9 @@ function arrayIndexSuite () { col.save({_key: "noArray", a: "NoArray"}); col.save({_key: "null", a: null}); const query = `FOR x IN ${cName} FILTER @tag IN x.a[*] SORT x._key RETURN x._key`; - validateResults(query, true); + validateResults(query); const orQuery = `FOR x IN ${cName} FILTER @tag1 IN x.a[*] || @tag2 IN x.a[*] SORT x._key RETURN x._key`; - validateResultsOr(orQuery, true); + validateResultsOr(orQuery); } }; @@ -447,15 +443,17 @@ function arrayIndexNonArraySuite () { var allIndexes = col.getIndexes(true); assertEqual(allIndexes.length, 2, "We have more than one index!"); var idx = allIndexes[1]; - switch (idx.type) { - case "hash": - assertEqual(idx.figures.totalUsed, count); - break; - case "skiplist": - assertEqual(idx.figures.nrUsed, count); - break; - default: - assertTrue(false, "Unexpected index type"); + if (! isCluster) { + switch (idx.type) { + case "hash": + assertEqual(idx.figures.totalUsed, count); + break; + case "skiplist": + assertEqual(idx.figures.nrUsed, count); + break; + default: + assertTrue(false, "Unexpected index type"); + } } }; @@ -616,13 +614,16 @@ function arrayIndexNonArraySuite () { col.save({ a: [], b: 1, c: 1 }); // Empty Array. no indexing checkElementsInIndex(inserted); - col.save({ a: [1, 2, 3, 3, 2, 1] }); // a does not have any nested value. Handled equal to a: [] + col.save({ a: [1, 2, 3, 3, 2, 1] }); // a does not have any nested value. Index as one null + inserted += 1; checkElementsInIndex(inserted); - col.save({ a: [1, 2, 3, 3, 2, 1], b: 1 }); // a does not have any nested value. Handled equal to a: [] + col.save({ a: [1, 2, 3, 3, 2, 1], b: 1 }); // a does not have any nested value. Index as one null + inserted += 1; checkElementsInIndex(inserted); - col.save({ a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 }); // a does not have any nested value. Handled equal to a: [] + col.save({_key: "null1", a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 }); // a does not have any nested value. Index as one null + inserted += 1; checkElementsInIndex(inserted); col.save({ a: [{d: 1}, {d: 2}, {d: 3}, {d: 3}, {d: 2}, {d: 1}] }); @@ -637,23 +638,24 @@ function arrayIndexNonArraySuite () { inserted += 3; // We index b: 1, c: 1 and 3 values for a[*].d checkElementsInIndex(inserted); - col.save({_key: "null1", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] }); + col.save({_key: "null2", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] }); inserted += 4; checkElementsInIndex(inserted); // b: null a: "a", "b", "c", null c:null - col.save({_key: "null2", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 }); + col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 }); inserted += 4; checkElementsInIndex(inserted); - col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 }); + col.save({_key: "null4", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 }); inserted += 4; checkElementsInIndex(inserted); const query = `FOR x IN ${cName} FILTER @tag IN x.a[*].d && 1 == x.b && 1 == x.c SORT x._key RETURN x._key`; var actual = AQL_EXECUTE(query, { tag : null }).json; // We expect that we can only find the Array that has stored exactly the null value - assertEqual(actual.length, 1); - assertEqual(actual[0], "null3"); + assertEqual(actual.length, 2); + assertEqual(actual[0], "null1"); + assertEqual(actual[1], "null4"); }, testHashIndexSubAttributeArray : function () { @@ -667,6 +669,66 @@ function arrayIndexNonArraySuite () { // Do not find anything }, + testHashIndexMultiArray : function () { + col.ensureHashIndex("a[*]", "b[*]"); + + col.save({a: [1, 2, 3]}); // Do not index + checkElementsInIndex(0); + + col.save({a: [1, 2, 3], b: null}); // Do not index + checkElementsInIndex(0); + + col.save({a: [1, 2, 3], b: "this is no array"}); // Do not index + checkElementsInIndex(0); + + col.save({a: "this is no array", b: ["a", "b", "c"]}); // Do not index + checkElementsInIndex(0); + + col.save({a: [1, 2, null, null, 2, 1], b: ["a", "b", null, "b", "a"]}); + checkElementsInIndex(9); // 3*3 many combinations + + const query = `FOR x IN ${cName} FILTER @tag IN x.a[*] && @tag IN x.b[*] SORT x._key RETURN x._key`; + var actual = AQL_EXECUTE(query, { tag : null }).json; + assertEqual(actual.length, 1); + }, + + testHashIndexArraySparse : function () { + col.ensureHashIndex("a[*]", "b", {sparse: true}); + var inserted = 0; + + col.save({a: [1, 2, 3]}); // Do not index, b is not set + checkElementsInIndex(inserted); + + col.save({a: [1, 2, 3], b: null}); // Do not index, b is null + checkElementsInIndex(inserted); + + col.save({a: [1, 2, 3], b: 1}); // Do index + inserted += 3; + checkElementsInIndex(inserted); + + col.save({a: [null, 4], b: 1}); // Do index + inserted += 2; + checkElementsInIndex(inserted); + + const query = `FOR x IN ${cName} FILTER null IN x.a[*] && 1 == x.b SORT x._key RETURN x._key`; + // We can use the index for null in SPARSE + var actual = AQL_EXECUTE(query).json; + assertEqual(actual.length, 1); + var plan = AQL_EXPLAIN(query).plan; + var nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + assertNotEqual(-1, nodeTypes.indexOf("IndexNode")); + + const query2 = `FOR x IN ${cName} FILTER null IN x.a[*] SORT x._key RETURN x._key`; + plan = AQL_EXPLAIN(query2).plan; + nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + // Cannot use the index for sub attribute a + assertEqual(-1, nodeTypes.indexOf("IndexNode")); + }, + testSkiplistSingleAttribute : function () { col.ensureSkiplist("a[*]"); col.save({}); // a is not set @@ -876,16 +938,16 @@ function arrayIndexNonArraySuite () { checkElementsInIndex(inserted); col.save({ a: [1, 2, 3, 3, 2, 1] }); - inserted += 1; // We index b: null But a does not have any nested value. Handled equal to a: [] + inserted += 1; // We index b: null But a does not have any nested value. Index one for null checkElementsInIndex(inserted); - col.save({ a: [1, 2, 3, 3, 2, 1], b: 1 }); // b: 1 a: 1,2,3 c: null - inserted += 1; // We index b: 1 But a does not have any nested value. Handled equal to a: [] + col.save({ _key: "null1", a: [1, 2, 3, 3, 2, 1], b: 1 }); // b: 1 a: 1,2,3 c: null + inserted += 1; // We index b: 1 But a does not have any nested value. Index one for null insertedB += 1; checkElementsInIndex(inserted); - col.save({ a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 }); - inserted += 1; // We index b: 1, c: 1 But a does not have any nested value. Handled equal to a: [] + col.save({ _key: "null2", a: [1, 2, 3, 3, 2, 1], b: 1, c: 1 }); + inserted += 1; // We index b: 1, c: 1 But a does not have any nested value. Index one for null insertedB += 1; checkElementsInIndex(inserted); @@ -903,26 +965,29 @@ function arrayIndexNonArraySuite () { insertedB += 3; checkElementsInIndex(inserted); - col.save({_key: "null1", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] }); + col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}] }); inserted += 4; checkElementsInIndex(inserted); // b: null a: "a", "b", "c", null c:null - col.save({_key: "null2", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 }); + col.save({_key: "null4", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1 }); inserted += 4; insertedB += 4; checkElementsInIndex(inserted); - col.save({_key: "null3", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 }); + col.save({_key: "null5", a: [{d: null}, {d: "a"}, {d: "b"}, {d: "c"}, {d: "b"}, {d: "a"}, {d: null}], b: 1, c: 1 }); inserted += 4; insertedB += 4; checkElementsInIndex(inserted); const query = `FOR x IN ${cName} FILTER @tag IN x.a[*].d && 1 == x.b SORT x._key RETURN x._key`; var actual = AQL_EXECUTE(query, { tag : null }).json; - // We expect that we can only find the Array that has stored exactly the null value - assertEqual(actual.length, 2); - assertEqual(actual[0], "null2"); - assertEqual(actual[1], "null3"); + // We expect that we can only find the array that stores exactly the null value + // And the arrays that do not have the sub attribute. + assertEqual(actual.length, 4); + assertEqual(actual[0], "null1"); + assertEqual(actual[1], "null2"); + assertEqual(actual[2], "null4"); + assertEqual(actual[3], "null5"); const query2 = `FOR x IN ${cName} FILTER 1 == x.b RETURN x._key`; actual = AQL_EXECUTE(query2).json; @@ -942,7 +1007,45 @@ function arrayIndexNonArraySuite () { var actual = AQL_EXECUTE(query, { tag : null }).json; assertEqual(actual.length, 0); // Do not find anything - } + }, + + testSkiplistIndexArraySparse : function () { + col.ensureSkiplist("a[*]", "b", {sparse: true}); + var inserted = 0; + + col.save({a: [1, 2, 3]}); // Do not index, b is not set + checkElementsInIndex(inserted); + + col.save({a: [1, 2, 3], b: null}); // Do not index, b is null + checkElementsInIndex(inserted); + + col.save({a: [1, 2, 3], b: 1}); // Do index + inserted += 3; + checkElementsInIndex(inserted); + + col.save({a: [null, 4], b: 1}); // Do index + inserted += 2; + checkElementsInIndex(inserted); + + const query = `FOR x IN ${cName} FILTER null IN x.a[*] && 1 == x.b SORT x._key RETURN x._key`; + // We can use the index for null in SPARSE + var actual = AQL_EXECUTE(query).json; + assertEqual(actual.length, 1); + var plan = AQL_EXPLAIN(query).plan; + var nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + assertNotEqual(-1, nodeTypes.indexOf("IndexNode")); + + const query2 = `FOR x IN ${cName} FILTER null IN x.a[*] SORT x._key RETURN x._key`; + plan = AQL_EXPLAIN(query2).plan; + nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + // Cannot use the index for sub attribute a + assertEqual(-1, nodeTypes.indexOf("IndexNode")); + }, + };