diff --git a/Documentation/Books/Users/Aql/Optimizer.mdpp b/Documentation/Books/Users/Aql/Optimizer.mdpp index 4012ff1cfc..d5b4068a1c 100644 --- a/Documentation/Books/Users/Aql/Optimizer.mdpp +++ b/Documentation/Books/Users/Aql/Optimizer.mdpp @@ -12,29 +12,46 @@ meaning that an optimization should not modify the result of a query. A notable to this is that the optimizer is allowed to change the order of results for queries that do not explicitly specify how results should be sorted. - !SUBSECTION Execution plans The `explain` command can be used to query the optimal executed plan or even all plans the optimizer has generated. Additionally, `explain` can reveal some more information about the optimizer's view of the query. -Here's an example that shows the execution plan for a simple query, using the `explain` -method of `ArangoStatement`: +!SUBSECTION Inspecting plans using the explain helper - @startDocuBlockInline AQLEXP_01_explainCreate - @EXAMPLE_ARANGOSH_OUTPUT{AQLEXP_01_explainCreate} +The `explain` method of `ArangoStatement` as shown in the next chapters creates very verbose output. +You can work on the output programaticaly, or use this handsome tool that we created +to generate a more human readable representation. + +You may use it like this: (we disable syntax highlighting here) + + @startDocuBlockInline AQLEXP_01_axplainer + @EXAMPLE_ARANGOSH_OUTPUT{AQLEXP_01_axplainer} ~addIgnoreCollection("test") db._create("test"); for (i = 0; i < 100; ++i) { db.test.save({ value: i }); } db.test.ensureIndex({ type: "skiplist", fields: [ "value" ] }); + var explain = require("org/arangodb/aql/explainer").explain; + explain("FOR i IN test FILTER i.value > 97 SORT i.value RETURN i.value", {colors:false}); + @END_EXAMPLE_ARANGOSH_OUTPUT + @endDocuBlock AQLEXP_01_axplainer + + +!SUBSECTION Execution plans in detail + +Lets have a look at the raw json output of the same execution plan +using the `explain` method of `ArangoStatement`: + + @startDocuBlockInline AQLEXP_01_explainCreate + @EXAMPLE_ARANGOSH_OUTPUT{AQLEXP_01_explainCreate} stmt = db._createStatement("FOR i IN test FILTER i.value > 97 SORT i.value RETURN i.value"); stmt.explain(); @END_EXAMPLE_ARANGOSH_OUTPUT @endDocuBlock AQLEXP_01_explainCreate -The result details will be very verbose so they are not shown here in full. Instead, -let's take a closer look at the results step by step. +The as you can see, result details are very verbose so we will not shown them in full in the next +sections. Instead, lets take a closer look at the results step by step. !SUBSUBSECTION Execution nodes diff --git a/Documentation/Examples/AQLEXP_01_axplainer.generated b/Documentation/Examples/AQLEXP_01_axplainer.generated new file mode 100644 index 0000000000..51f8f13575 --- /dev/null +++ b/Documentation/Examples/AQLEXP_01_axplainer.generated @@ -0,0 +1,44 @@ +arangosh> db._create("test"); +[ArangoCollection 1092291992475, "test" (type document, status loaded)] +arangosh> for (i = 0; i < 100; ++i) { db.test.save({ value: i }); } +arangosh> db.test.ensureSkiplist("value"); +{ + "id" : "test/1092311980955", + "type" : "skiplist", + "fields" : [ + "value" + ], + "unique" : false, + "sparse" : false, + "isNewlyCreated" : true, + "code" : 201 +} +arangosh> var explain = require("org/arangodb/aql/explainer").explain; +arangosh> explain("FOR i IN test FILTER i.value > 97 SORT i.value RETURN i.value", {colors:false}); +Query string: + FOR i IN test FILTER i.value > 97 SORT i.value RETURN i.value + +Execution plan: + Id NodeType Est. Comment + 1 SingletonNode 1 * ROOT + 9 IndexNode 50 - FOR i IN test /* skiplist index scan */ + 5 CalculationNode 50 - LET #3 = i.`value` /* attribute expression */ /* collections used: i : test */ + 8 ReturnNode 50 - RETURN #3 + +Indexes used: + By Type Collection Unique Sparse Selectivity Fields Ranges + 9 skiplist test false false n/a [ `value` ] i.`value` > 97 + +Optimization rules applied: + Id RuleName + 1 move-calculations-up + 2 move-filters-up + 3 remove-redundant-calculations + 4 remove-unnecessary-calculations + 5 move-calculations-up-2 + 6 move-filters-up-2 + 7 use-indexes + 8 remove-filter-covered-by-index + 9 use-index-for-sort + + diff --git a/Documentation/Examples/graph_create_knows_sample.generated b/Documentation/Examples/graph_create_knows_sample.generated new file mode 100644 index 0000000000..7d1f9f60c8 --- /dev/null +++ b/Documentation/Examples/graph_create_knows_sample.generated @@ -0,0 +1,75 @@ +arangosh> var examples = require("org/arangodb/graph-examples/example-graph.js"); +arangosh> var g = examples.loadGraph("knows_graph"); +arangosh> db.persons.toArray() +[ + { + "name" : "Charlie", + "_id" : "persons/charlie", + "_rev" : "1092164590491", + "_key" : "charlie" + }, + { + "name" : "Bob", + "_id" : "persons/bob", + "_rev" : "1092164328347", + "_key" : "bob" + }, + { + "name" : "Eve", + "_id" : "persons/eve", + "_rev" : "1092164983707", + "_key" : "eve" + }, + { + "name" : "Dave", + "_id" : "persons/dave", + "_rev" : "1092164787099", + "_key" : "dave" + }, + { + "name" : "Alice", + "_id" : "persons/alice", + "_rev" : "1092164131739", + "_key" : "alice" + } +] +arangosh> db.knows.toArray(); +[ + { + "_id" : "knows/1092165835675", + "_rev" : "1092165835675", + "_key" : "1092165835675", + "_from" : "persons/eve", + "_to" : "persons/alice" + }, + { + "_id" : "knows/1092165639067", + "_rev" : "1092165639067", + "_key" : "1092165639067", + "_from" : "persons/bob", + "_to" : "persons/dave" + }, + { + "_id" : "knows/1092165245851", + "_rev" : "1092165245851", + "_key" : "1092165245851", + "_from" : "persons/alice", + "_to" : "persons/bob" + }, + { + "_id" : "knows/1092165442459", + "_rev" : "1092165442459", + "_key" : "1092165442459", + "_from" : "persons/bob", + "_to" : "persons/charlie" + }, + { + "_id" : "knows/1092166032283", + "_rev" : "1092166032283", + "_key" : "1092166032283", + "_from" : "persons/eve", + "_to" : "persons/bob" + } +] +arangosh> examples.dropGraph("knows_graph"); +true diff --git a/Documentation/Examples/graph_create_social_sample.generated b/Documentation/Examples/graph_create_social_sample.generated new file mode 100644 index 0000000000..977349a549 --- /dev/null +++ b/Documentation/Examples/graph_create_social_sample.generated @@ -0,0 +1,69 @@ +arangosh> var examples = require("org/arangodb/graph-examples/example-graph.js"); +arangosh> var graph = examples.loadGraph("social"); +arangosh> db.female.toArray() +[ + { + "name" : "Diana", + "_id" : "female/diana", + "_rev" : "1092171340699", + "_key" : "diana" + }, + { + "name" : "Alice", + "_id" : "female/alice", + "_rev" : "1092170619803", + "_key" : "alice" + } +] +arangosh> db.male.toArray() +[ + { + "name" : "Bob", + "_id" : "male/bob", + "_rev" : "1092170947483", + "_key" : "bob" + }, + { + "name" : "Charly", + "_id" : "male/charly", + "_rev" : "1092171144091", + "_key" : "charly" + } +] +arangosh> db.relation.toArray() +[ + { + "type" : "friend", + "_id" : "relation/bobAndDiana", + "_rev" : "1092172323739", + "_key" : "bobAndDiana", + "_from" : "male/bob", + "_to" : "female/diana" + }, + { + "type" : "married", + "_id" : "relation/charlyAndDiana", + "_rev" : "1092172127131", + "_key" : "charlyAndDiana", + "_from" : "male/charly", + "_to" : "female/diana" + }, + { + "type" : "friend", + "_id" : "relation/aliceAndCharly", + "_rev" : "1092171930523", + "_key" : "aliceAndCharly", + "_from" : "female/alice", + "_to" : "male/charly" + }, + { + "type" : "married", + "_id" : "relation/aliceAndBob", + "_rev" : "1092171668379", + "_key" : "aliceAndBob", + "_from" : "female/alice", + "_to" : "male/bob" + } +] +arangosh> examples.dropGraph("social"); +true diff --git a/Installation/travisCI/build.sh b/Installation/travisCI/build.sh index 3ec6c27145..f8f0800532 100755 --- a/Installation/travisCI/build.sh +++ b/Installation/travisCI/build.sh @@ -29,7 +29,7 @@ make jslint || exit 1 echo echo "$0: testing ArangoDB" -./scripts/unittest all --skipRanges true --skipTimeCritical true --skipSsl true || exit 1 +./scripts/unittest all --skipRanges true --skipTimeCritical true --skipSsl true --skipBoost true --skipGeo true || exit 1 success=`cat out/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json` if test "$success" == "false"; then exit 1 diff --git a/UnitTests/Makefile.unittests b/UnitTests/Makefile.unittests index eaada604a9..e653be9673 100755 --- a/UnitTests/Makefile.unittests +++ b/UnitTests/Makefile.unittests @@ -644,6 +644,7 @@ SHELL_SERVER_AQL = @top_srcdir@/js/server/tests/aql-arithmetic.js \ @top_srcdir@/js/server/tests/aql-parse.js \ @top_srcdir@/js/server/tests/aql-primary-index-noncluster.js \ @top_srcdir@/js/server/tests/aql-queries-array.js \ + @top_srcdir@/js/server/tests/aql-queries-array-nested.js \ @top_srcdir@/js/server/tests/aql-queries-collection.js \ @top_srcdir@/js/server/tests/aql-queries-fulltext.js \ @top_srcdir@/js/server/tests/aql-queries-geo.js \ diff --git a/js/server/tests/aql-queries-array-nested.js b/js/server/tests/aql-queries-array-nested.js new file mode 100644 index 0000000000..9efed9f558 --- /dev/null +++ b/js/server/tests/aql-queries-array-nested.js @@ -0,0 +1,251 @@ +/*jshint globalstrict:false, strict:false, maxlen: 500 */ +/*global assertEqual, assertFalse, assertTrue, AQL_EXPLAIN, AQL_EXECUTE */ + +//////////////////////////////////////////////////////////////////////////////// +/// @brief tests for array indexes in AQL +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2015 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2015, ArangoDB GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +var jsunity = require("jsunity"); +var db = require("org/arangodb").db; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite +//////////////////////////////////////////////////////////////////////////////// + +function nestedArrayIndexSuite () { + var cn = "UnitTestsArray"; + var c; + + var indexUsed = function (query, bindVars) { + var plan = AQL_EXPLAIN(query, bindVars || {}).plan; + var nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + return (nodeTypes.indexOf("IndexNode") !== -1); + }; + + return { + + setUp : function () { + db._drop(cn); + c = db._create(cn); + }, + + tearDown : function () { + db._drop(cn); + }, + + testNestedSubAttribute : function () { + c.insert({ tags: [ { name: "foo" }, { name: "bar" }, { name: "baz" } ] }); + c.insert({ tags: [ { name: "quux" } ] }); + c.insert({ tags: [ "foo", "bar", "baz" ] }); + c.insert({ tags: [ "foobar" ] }); + c.insert({ tags: [ { name: "foobar" } ] }); + c.insert({ tags: [ { name: "baz" }, { name: "bark" } ] }); + c.insert({ tags: [ { name: "baz" }, "bark" ] }); + c.insert({ tags: [ { name: "q0rk" }, { name: "foo" } ] }); + + var query = "FOR doc IN " + cn + " FILTER @value IN doc.tags[*].name RETURN doc"; + + var tests = [ + [ "foo", 2 ], + [ "bar", 1 ], + [ "baz", 3 ], + [ "quux", 1 ], + [ "foobar", 1 ], + [ "bark", 1 ], + [ "q0rk", 1 ], + [ "sn0rk", 0 ], + [ "Quetzalcoatl", 0 ] + ]; + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length); + assertFalse(indexUsed(query, { value: value[0] })); + }); + + // now try again with an the index + c.ensureIndex({ type: "hash", fields: [ "tags[*].name" ] }); + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length); + assertTrue(indexUsed(query, { value: value[0] })); + }); + }, + + testNestedAnotherSubAttribute : function () { + c.insert({ persons: [ { name: { first: "Jane", last: "Doe" }, gender: "f" }, { name: { first: "Joe", last: "Public" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Jack", last: "White" }, gender: "m" }, { name: { first: "John", last: "Black" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Janet", last: "Smith" }, gender: "f" } ] }); + c.insert({ persons: [ { name: { first: "Jill", last: "Jones" }, gender: "f" }, { name: { first: "Jeff", last: "Jackson" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Jimbo", last: "Jonas" }, gender: "m" }, { name: { first: "Jay", last: "Jameson" } } ] }); + c.insert({ persons: [ { name: { first: "Jack", last: "Rabbit" }, gender: "m" }, { name: { first: "Jane", last: "Jackson" }, gender: "f" } ] }); + c.insert({ persons: [ { name: { first: "Janet", last: "Black" }, gender: "f" }, { name: { first: "James", last: "Smith" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Jill", last: "Doe" }, gender: "f" }, { name: { first: "Jeff", last: "Jones" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Janet", last: "Doe" }, gender: "f" }, { name: { first: "Julia", last: "Jones" }, gender: "f" } ] }); + c.insert({ persons: [ { name: { first: "Jo" } }, { name: { first: "Ji" } } ] }); + + var query = "FOR doc IN " + cn + " FILTER @value IN doc.persons[*].gender RETURN doc"; + + var tests = [ + [ "m", 7 ], + [ "f", 7 ], + [ "", 0 ], + [ null, 2 ], + [ "Quetzalcoatl", 0 ] + ]; + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length, value); + assertFalse(indexUsed(query, { value: value[0] })); + }); + + // now try again with an index + c.ensureIndex({ type: "hash", fields: [ "persons[*].gender" ] }); + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length, value); + assertTrue(indexUsed(query, { value: value[0] })); + }); + }, + + testNestedSubSubAttribute : function () { + c.insert({ persons: [ { name: { first: "Jane", last: "Doe" }, gender: "f" }, { name: { first: "Joe", last: "Public" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Jack", last: "White" }, gender: "m" }, { name: { first: "John", last: "Black" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Janet", last: "Smith" }, gender: "f" } ] }); + c.insert({ persons: [ { name: { first: "Jill", last: "Jones" }, gender: "f" }, { name: { first: "Jeff", last: "Jackson" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Jimbo", last: "Jonas" }, gender: "m" }, { name: { first: "Jay", last: "Jameson" } } ] }); + c.insert({ persons: [ { name: { first: "Jack", last: "Rabbit" }, gender: "m" }, { name: { first: "Jane", last: "Jackson" }, gender: "f" } ] }); + c.insert({ persons: [ { name: { first: "Janet", last: "Black" }, gender: "f" }, { name: { first: "James", last: "Smith" }, gender: "m" } ] }); + c.insert({ persons: [ { name: { first: "Jill", last: "Doe" }, gender: "f" }, { name: { first: "Jeff", last: "Jones" }, gender: "m" } ] }); + + var query = "FOR doc IN " + cn + " FILTER @value IN doc.persons[*].name.first RETURN doc"; + + var tests = [ + [ "Jane", 2 ], + [ "Joe", 1 ], + [ "Jack", 2 ], + [ "John", 1 ], + [ "Janet", 2 ], + [ "Jill", 2 ], + [ "Jeff", 2 ], + [ "Jimbo", 1 ], + [ "Jay", 1 ], + [ "James", 1 ], + [ "Quetzalcoatl", 0 ] + ]; + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length); + assertFalse(indexUsed(query, { value: value[0] })); + }); + + // now try again with an index + c.ensureIndex({ type: "hash", fields: [ "persons[*].name.first" ] }); + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length); + assertTrue(indexUsed(query, { value: value[0] })); + }); + } + + /* + testNestedNestedSubSubAttribute : function () { + c.insert({ persons: [ { addresses: [ { country: { name: "DE" } }, { country: { name: "IT" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "FR" } }, { country: { name: "GB" } } ] } ] }); + c.insert({ persons: [ ] }); + c.insert({ persons: [ { addresses: [ ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "BG" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "US" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "RU" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "PL" } }, { country: { name: "CH" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "IT" } }, { country: { name: "US" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "DK" } }, { country: { name: "DE" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "NL" } }, { country: { name: "BE" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "GB" } }, { country: { name: "DK" } } ] } ] }); + c.insert({ persons: [ { addresses: [ { country: { name: "DK" } }, { country: { name: "SE" } }, { country: { name: "SE" } } ] } ] }); + + var query = "FOR doc IN " + cn + " FILTER @value IN doc.persons[*].addresses[*].country.name RETURN doc"; + var query = "FOR doc IN " + cn + " FILTER @value != 'hassan' RETURN doc.persons[*].addresses[*].country.name"; + + var tests = [ + [ "DE", 2 ], + [ "IT", 2 ], + [ "FR", 1 ], + [ "GB", 2 ], + [ "BG", 1 ], + [ "US", 2 ], + [ "RU", 1 ], + [ "PL", 1 ], + [ "CH", 1 ], + [ "DK", 3 ], + [ "NL", 1 ], + [ "BE", 1 ], + [ "SE", 1 ], + [ "Quetzalcoatl", 0 ] + ]; + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + require("internal").print(result, value); + assertEqual(value[1], result.length); + assertFalse(indexUsed(query, { value: value[0] })); + }); + + // now try again with an index + c.ensureIndex({ type: "hash", fields: [ "persons[*].addresses[*].country.name" ] }); + + tests.forEach(function(value) { + var result = AQL_EXECUTE(query, { value: value[0] }).json; + assertEqual(value[1], result.length); + assertTrue(indexUsed(query, { value: value[0] })); + }); + } + */ + + }; + +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief executes the test suite +//////////////////////////////////////////////////////////////////////////////// + +jsunity.run(nestedArrayIndexSuite); + +return jsunity.done(); + +// Local Variables: +// mode: outline-minor +// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)" +// End: