1
0
Fork 0

Additional AQL skipSome tests for IndexExecutor (#8938)

* Began work on additional index skipSome tests

* Added more index skip tests
This commit is contained in:
Tobias Gödderz 2019-05-16 11:30:24 +02:00 committed by Michael Hackstein
parent b7cfa24518
commit c9fa5fba11
2 changed files with 295 additions and 45 deletions

View File

@ -527,6 +527,11 @@ std::pair<ExecutionState, IndexStats> IndexExecutor::produceRows(OutputAqlItemRo
}
std::tuple<ExecutionState, IndexExecutor::Stats, size_t> IndexExecutor::skipRows(size_t toSkip) {
// This code does not work correctly with multiple indexes, as it does not
// check for duplicates. Currently, no plan is generated where that can
// happen, because with multiple indexes, the FILTER is not removed and thus
// skipSome is not called on the IndexExecutor.
TRI_ASSERT(_infos.getIndexes().size() <= 1);
TRI_IF_FAILURE("IndexExecutor::skipRows") {
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
}

View File

@ -1,6 +1,7 @@
/*jshint globalstrict:false, strict:false, maxlen: 500 */
/*global assertEqual, assertNotEqual, assertTrue, AQL_EXECUTE, AQL_EXPLAIN */
////////////////////////////////////////////////////////////////////////////////
/// @brief tests for query language, simple queries
///
@ -28,12 +29,12 @@
/// @author Copyright 2019, ArangoDB GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
var jsunity = require("jsunity");
// var errors = require("internal").errors;
var internal = require("internal");
var analyzers = require("@arangodb/analyzers");
var helper = require("@arangodb/aql-helper");
var db = internal.db;
const _ = require('lodash');
const analyzers = require("@arangodb/analyzers");
const internal = require("internal");
const jsunity = require("jsunity");
const db = internal.db;
////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
@ -46,7 +47,7 @@ function aqlSkippingTestsuite () {
/// @brief set up
////////////////////////////////////////////////////////////////////////////////
setUp : function () {
setUpAll : function () {
var c = db._createDocumentCollection('skipCollection', { numberOfShards: 5 });
// c size > 1000 because of internal batchSize of 1000
for (var i = 0; i < 2000; i++) {
@ -58,7 +59,7 @@ function aqlSkippingTestsuite () {
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////
tearDown : function () {
tearDownAll : function () {
db._drop('skipCollection');
},
@ -68,7 +69,7 @@ function aqlSkippingTestsuite () {
var queryOptions = {};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json, [ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ]);
assertEqual([ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ], result.json);
},
testDefaultSkipOffsetWithFullCount: function () {
@ -77,8 +78,8 @@ function aqlSkippingTestsuite () {
var queryOptions = {fullCount: true};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json, [ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ]);
assertEqual(result.stats.fullCount, 100);
assertEqual([ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ], result.json);
assertEqual(100, result.stats.fullCount);
},
testPassSkipOffset: function () {
@ -88,7 +89,7 @@ function aqlSkippingTestsuite () {
var queryOptions = {optimizer: {"rules": ["-move-calculations-down"]}};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json, [ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ]);
assertEqual([ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ], result.json);
},
testPassSkipOffsetWithFullCount: function () {
@ -98,8 +99,8 @@ function aqlSkippingTestsuite () {
var queryOptions = {fullCount: true, optimizer: {"rules": ["-move-calculations-down"]}};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json, [ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ]);
assertEqual(result.stats.fullCount, 100);
assertEqual([ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102 ], result.json);
assertEqual(100, result.stats.fullCount);
},
testPassSkipEnumerateCollection: function () {
@ -108,8 +109,8 @@ function aqlSkippingTestsuite () {
var queryOptions = {};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json.length, 10);
assertEqual(result.stats.scannedFull, 20);
assertEqual(10, result.json.length);
assertEqual(20, result.stats.scannedFull);
},
testPassSkipEnumerateCollectionWithFullCount1: function () {
@ -118,9 +119,9 @@ function aqlSkippingTestsuite () {
var queryOptions = {fullCount: true};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json.length, 20);
assertEqual(result.stats.scannedFull, 2000);
assertEqual(result.stats.fullCount, 2000);
assertEqual(20, result.json.length);
assertEqual(2000, result.stats.scannedFull);
assertEqual(2000, result.stats.fullCount);
},
// FIXME uncomment
@ -156,9 +157,9 @@ function aqlSkippingTestsuite () {
var queryOptions = {fullCount: true};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json.length, 300);
assertEqual(result.stats.scannedFull, 2000);
assertEqual(result.stats.fullCount, 2000);
assertEqual(300, result.json.length);
assertEqual(2000, result.stats.scannedFull);
assertEqual(2000, result.stats.fullCount);
},
testPassSkipEnumerateCollectionWithFullCount3: function () {
@ -168,9 +169,9 @@ function aqlSkippingTestsuite () {
var queryOptions = {fullCount: true};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json.length, 0);
assertEqual(result.stats.scannedFull, 2000);
assertEqual(result.stats.fullCount, 2000);
assertEqual(0, result.json.length);
assertEqual(2000, result.stats.scannedFull);
assertEqual(2000, result.stats.fullCount);
},
testPassSkipEnumerateCollectionWithFullCount4: function () {
@ -180,10 +181,257 @@ function aqlSkippingTestsuite () {
var queryOptions = {fullCount: true};
var result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(result.json.length, 0);
assertEqual(result.stats.scannedFull, 2000);
assertEqual(result.stats.fullCount, 2000);
}
assertEqual(0, result.json.length);
assertEqual(2000, result.stats.scannedFull);
assertEqual(2000, result.stats.fullCount);
},
};
}
function aqlSkippingIndexTestsuite () {
const skipCollection = 'skipCollection';
const explainPlanNodes = (q, b, o) => {
var res = AQL_EXPLAIN(q, b, o);
return res.plan.nodes;
};
return {
////////////////////////////////////////////////////////////////////////////////
/// @brief set up
////////////////////////////////////////////////////////////////////////////////
setUpAll : function () {
const col = db._createDocumentCollection(skipCollection, { numberOfShards: 5 });
col.ensureIndex({ type: "hash", fields: [ "a" ]});
col.ensureIndex({ type: "hash", fields: [ "b" ]});
col.ensureIndex({ type: "hash", fields: [ "c" ]});
const values = _.range(7); // 0..6
// insert a total of 7^4 = 2401 documents:
for (const a of values) {
for (const b of values) {
for (const c of values) {
for (const d of values) {
col.insert({a, b, c, d});
}
}
}
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////
tearDownAll : function () {
db._drop(skipCollection);
},
testOffsetSingleIndex: function () {
// there is a total of 7^3 = 343 documents with a == 1
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1
LIMIT 43, 1000
RETURN doc`;
const bindParams = {};
const queryOptions = {};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(1, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(300, result.json.length);
assertEqual(343, result.stats.scannedIndex);
},
testOffsetLimitSingleIndex: function () {
// there is a total of 7^3 = 343 documents with a == 1
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1
LIMIT 43, 100
RETURN doc`;
const bindParams = {};
const queryOptions = {};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(1, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(100, result.json.length);
assertEqual(143, result.stats.scannedIndex);
},
testOffsetLimitFullCountSingleIndex: function () {
// there is a total of 7^3 = 343 documents with a == 1
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1
LIMIT 43, 100
RETURN doc`;
const bindParams = {};
const queryOptions = {fullCount: true};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(1, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(100, result.json.length);
assertEqual(343, result.stats.scannedIndex);
assertEqual(343, result.stats.fullCount);
},
testOffsetTwoIndexes: function () {
// there is a total of 7^3 = 343 documents with a == 1, same for b == 1,
// and there are 7^2 = 49 documents with a == 1 and b == 1, thus
// 343 + 343 - 49 = 637 documents.
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1 || doc.b == 1
LIMIT 37, 1000
RETURN doc`;
const bindParams = {};
const queryOptions = {};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(2, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
assertEqual(["b"], nodes[1].indexes[1].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(600, result.json.length);
assertEqual(637, result.stats.scannedIndex);
},
testOffsetLimitTwoIndexes: function () {
// there is a total of 7^3 = 343 documents with a == 1, same for b == 1,
// and there are 7^2 = 49 documents with a == 1 and b == 1, thus
// 343 + 343 - 49 = 637 documents.
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1 || doc.b == 1
LIMIT 37, 100
RETURN doc`;
const bindParams = {};
const queryOptions = {};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(2, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
assertEqual(["b"], nodes[1].indexes[1].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(100, result.json.length);
assertEqual(137, result.stats.scannedIndex);
},
testOffsetLimitFullCountTwoIndexes: function () {
// there is a total of 7^3 = 343 documents with a == 1, same for b == 1,
// and there are 7^2 = 49 documents with a == 1 and b == 1, thus
// 343 + 343 - 49 = 637 documents.
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1 || doc.b == 1
LIMIT 37, 100
RETURN doc`;
const bindParams = {};
const queryOptions = {fullCount: true};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(2, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
assertEqual(["b"], nodes[1].indexes[1].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(100, result.json.length);
assertEqual(637, result.stats.scannedIndex);
assertEqual(637, result.stats.fullCount);
},
testOffsetThreeIndexes: function () {
// There are respectively 7^3 = 343 documents with a == 1, b == 1, or c == 1.
// For every two of {a, b, c}, there are 7^2 = 49 documents where both are one.
// And there are 7 documents where all three are one.
// Thus the filter matches 3 * 7^3 - (3 * 7^2 - 7) = 889 documents
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1 || doc.b == 1 || doc.c == 1
LIMIT 89, 1000
RETURN doc`;
const bindParams = {};
const queryOptions = {};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(3, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
assertEqual(["b"], nodes[1].indexes[1].fields);
assertEqual(["c"], nodes[1].indexes[2].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(800, result.json.length);
assertEqual(889, result.stats.scannedIndex);
},
testOffsetLimitThreeIndexes: function () {
// There are respectively 7^3 = 343 documents with a == 1, b == 1, or c == 1.
// For every two of {a, b, c}, there are 7^2 = 49 documents where both are one.
// And there are 7 documents where all three are one.
// Thus the filter matches 3 * 7^3 - (3 * 7^2 - 7) = 889 documents
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1 || doc.b == 1 || doc.c == 1
LIMIT 89, 100
RETURN doc`;
const bindParams = {};
const queryOptions = {};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(3, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
assertEqual(["b"], nodes[1].indexes[1].fields);
assertEqual(["c"], nodes[1].indexes[2].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(100, result.json.length);
assertEqual(189, result.stats.scannedIndex);
},
testOffsetLimitFullCountThreeIndexes: function () {
// There are respectively 7^3 = 343 documents with a == 1, b == 1, or c == 1.
// For every two of {a, b, c}, there are 7^2 = 49 documents where both are one.
// And there are 7 documents where all three are one.
// Thus the filter matches 3 * 7^3 - (3 * 7^2 - 7) = 889 documents
const query = `FOR doc IN ${skipCollection}
FILTER doc.a == 1 || doc.b == 1 || doc.c == 1
LIMIT 89, 100
RETURN doc`;
const bindParams = {};
const queryOptions = {fullCount: true};
const nodes = explainPlanNodes(query, bindParams, queryOptions);
assertEqual('IndexNode', nodes[1].type);
assertEqual(3, nodes[1].indexes.length);
assertEqual(["a"], nodes[1].indexes[0].fields);
assertEqual(["b"], nodes[1].indexes[1].fields);
assertEqual(["c"], nodes[1].indexes[2].fields);
const result = AQL_EXECUTE(query, bindParams, queryOptions);
assertEqual(100, result.json.length);
assertEqual(889, result.stats.scannedIndex);
assertEqual(889, result.stats.fullCount);
},
};
@ -201,7 +449,7 @@ function aqlSkippingIResearchTestsuite () {
/// @brief set up
////////////////////////////////////////////////////////////////////////////////
setUp : function () {
setUpAll : function () {
analyzers.save(db._name() + "::text_en", "text", "{ \"locale\": \"en.UTF-8\", \"ignored_words\": [ ] }", [ "frequency", "norm", "position" ]);
db._drop("UnitTestsCollection");
c = db._create("UnitTestsCollection");
@ -215,8 +463,8 @@ function aqlSkippingIResearchTestsuite () {
db._dropView("UnitTestsView");
v = db._createView("UnitTestsView", "arangosearch", {});
var meta = {
links: {
"UnitTestsCollection": {
links: {
"UnitTestsCollection": {
includeAllFields: true,
storeValues: "id",
fields: {
@ -264,7 +512,7 @@ function aqlSkippingIResearchTestsuite () {
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////
tearDown : function () {
tearDownAll : function () {
var meta = { links : { "UnitTestsCollection": null } };
v.properties(meta);
v.drop();
@ -280,9 +528,9 @@ function aqlSkippingIResearchTestsuite () {
+ "OPTIONS { waitForSync: true, collections : [ 'UnitTestsCollection' ] } "
+ "LIMIT 3,3 RETURN doc");
assertEqual(result.json.length, 3);
assertEqual(3, result.json.length);
result.json.forEach(function(res) {
assertEqual(res.a, "foo");
assertEqual("foo", res.a);
assertTrue(res._id.startsWith('UnitTestsCollection/'));
});
},
@ -294,9 +542,9 @@ function aqlSkippingIResearchTestsuite () {
+ "SORT BM25(doc) "
+ "LIMIT 3,3 RETURN doc");
assertEqual(result.json.length, 3);
assertEqual(3, result.json.length);
result.json.forEach(function(res) {
assertEqual(res.a, "foo");
assertEqual("foo", res.a);
assertTrue(res._id.startsWith('UnitTestsCollection/'));
});
},
@ -308,9 +556,9 @@ function aqlSkippingIResearchTestsuite () {
+ "OPTIONS { waitForSync: true, collections : [ 'UnitTestsCollection' ] } "
+ "LIMIT 3,3 RETURN doc", {}, opts);
assertEqual(result.json.length, 3);
assertEqual(3, result.json.length);
result.json.forEach(function(res) {
assertEqual(res.a, "foo");
assertEqual("foo", res.a);
assertTrue(res._id.startsWith('UnitTestsCollection/'));
});
assertEqual(10, result.stats.fullCount);
@ -356,9 +604,9 @@ function aqlSkippingIResearchTestsuite () {
// skip 3, return 3, out of 10
result = AQL_EXECUTE(query, {}, opts);
assertEqual(result.json.length, 3);
assertEqual(3, result.json.length);
result.json.forEach(function(res) {
assertEqual(res.a, "foo");
assertEqual("foo", res.a);
assertTrue(res._id.startsWith('UnitTestsCollection/'));
});
assertEqual(10, result.stats.fullCount);
@ -373,10 +621,7 @@ function aqlSkippingIResearchTestsuite () {
////////////////////////////////////////////////////////////////////////////////
jsunity.run(aqlSkippingTestsuite);
jsunity.run(aqlSkippingIndexTestsuite);
jsunity.run(aqlSkippingIResearchTestsuite);
// jsunity.run(aqlSkippingIndexTestsuite);
// not needed, tests already in cluded in:
// tests/js/server/aql/aql-skipping.js
return jsunity.done();