Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature-3.4/whiteblacklist (#7259) 

* add blacklist

* added --skipGrey
This commit is contained in:
Frank Celler 2018-11-08 00:00:02 +01:00 committed by Frank Celler
parent a74330250f
commit a6e4c844ad
6 changed files with 232 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
UnitTests/unittest.js
View File

@ -237,6 +237,10 @@ function main (argv) {
options.singleresilient = false;
}
if (options.hasOwnProperty('blacklist')) {
UnitTest.loadBlacklist(options.blacklist);
}
// run the test and store the result
let res = {}; // result
try {

12
js/client/modules/@arangodb/test-utils.js
View File

@ -351,6 +351,11 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
// //////////////////////////////////////////////////////////////////////////////
function filterTestcaseByOptions (testname, options, whichFilter) {
if (options.skipTest(testname, options)) {
whichFilter.filter = 'blacklist';
return false;
}
// These filters require a proper setup, Even if we filter by testcase:
if ((testname.indexOf('-mmfiles') !== -1) && options.storageEngine === 'rocksdb') {
whichFilter.filter = 'skip when running as rocksdb';
@ -418,6 +423,11 @@ function filterTestcaseByOptions (testname, options, whichFilter) {
return false;
}
if (testname.indexOf('-grey') !== -1 && options.skipGrey) {
whichFilter.filter = 'grey';
return false;
}
if (testname.indexOf('-graph') !== -1 && options.skipGraph) {
whichFilter.filter = 'graph';
return false;
@ -504,9 +514,11 @@ function scanTestPaths (paths) {
}
let allTestCases = [];
paths.forEach(function(p) {
allTestCases = allTestCases.concat(doOnePathInner(p));
});
return allTestCases;
}

128
js/client/modules/@arangodb/testing.js
View File

@ -2,28 +2,28 @@
/* global print */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Max Neunhoeffer
// //////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////////////////////////////////////////////////
// DISCLAIMER
//
// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
// Copyright 2014 triagens GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Max Neunhoeffer
// /////////////////////////////////////////////////////////////////////////////
let functionsDocumentation = {
'all': 'run all tests (marked with [x])',
@ -49,6 +49,7 @@ let optionsDocumentation = [
' - `skipRanges`: if set to true the ranges tests are skipped',
' - `skipTimeCritical`: if set to true, time critical tests will be skipped.',
' - `skipNondeterministic`: if set, nondeterministic tests are skipped.',
' - `skipGrey`: if set, grey tests are skipped.',
' - `testBuckets`: split tests in to buckets and execute on, for example',
' 10/2 will split into 10 buckets and execute the third bucket.',
'',
@ -156,6 +157,7 @@ const optionsDefaults = {
'skipMemoryIntense': false,
'skipNightly': true,
'skipNondeterministic': false,
'skipGrey': false,
'skipTimeCritical': false,
'storageEngine': 'rocksdb',
'test': undefined,
@ -195,9 +197,37 @@ const YELLOW = require('internal').COLORS.COLOR_YELLOW;
let failedRuns = {
};
let allTests = [
];
// /////////////////////////////////////////////////////////////////////////////
// blacklisting
// /////////////////////////////////////////////////////////////////////////////
let useBlacklist = false;
let blacklistTests = {};
function skipTest(type, name) {
let ntype = type.toUpperCase();
return useBlacklist && !!blacklistTests[ntype + ":" + name];
}
function loadBlacklist(name) {
let content = fs.read("BLACKLIST");
let a = _.filter(
_.map(content.split("\n"),
function(x) {return x.trim();}),
function(x) {return x.length > 0 && x[0] !== '#';});
for (let i = 0; i < a.length; ++i) {
blacklistTests[a[i]] = true;
}
useBlacklist = true;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief TEST: all
// //////////////////////////////////////////////////////////////////////////////
@ -440,7 +470,7 @@ function findTestCases(options) {
let allTestFiles = {};
for (let testSuiteName in allTestPaths) {
var myList = [];
let files = tu.scanTestPaths(allTestPaths[testSuiteName]);
let files = tu.scanTestPaths(allTestPaths[testSuiteName]);
if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) {
for (let j = 0; j < files.length; j++) {
let foo = {};
@ -452,11 +482,11 @@ function findTestCases(options) {
} else {
myList = myList.concat(files);
}
if (!filterTestcases || (myList.length > 0)) {
allTestFiles[testSuiteName] = myList;
}
}
// print(allTestPaths)
return [found, allTestFiles];
}
@ -539,6 +569,7 @@ function autoTest(options) {
// //////////////////////////////////////////////////////////////////////////////
// / @brief load the available testsuites
// //////////////////////////////////////////////////////////////////////////////
function loadTestSuites () {
let testSuites = _.filter(fs.list(fs.join(__dirname, 'testsuites')),
function (p) {
@ -618,6 +649,7 @@ function iterateTests(cases, options, jsonReply) {
for (let n = 0; n < caselist.length; ++n) {
const currentTest = caselist[n];
var localOptions = _.cloneDeep(options);
localOptions.skipTest = skipTest;
print(BLUE + '================================================================================');
print('Executing test', currentTest);
@ -627,19 +659,33 @@ function iterateTests(cases, options, jsonReply) {
print(CYAN + 'with options:', localOptions, RESET);
}
let result = testFuncs[currentTest](localOptions);
// grrr...normalize structure
delete result.status;
delete result.failed;
delete result.crashed;
let result;
let status = true;
let status = Object.values(result).every(testCase => testCase.status === true);
let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0);
if (!status) {
globalStatus = false;
if (skipTest("SUITE", currentTest)) {
result = {
failed: 0,
status: true,
crashed: false,
};
print(YELLOW + "[SKIPPED] " + currentTest + RESET + "\n");
} else {
result = testFuncs[currentTest](localOptions);
// grrr...normalize structure
delete result.status;
delete result.failed;
delete result.crashed;
let status = Object.values(result).every(testCase => testCase.status === true);
let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0);
if (!status) {
globalStatus = false;
}
result.failed = failed;
result.status = status;
}
result.failed = failed;
result.status = status;
results[currentTest] = result;
if (status && localOptions.cleanup) {
@ -647,6 +693,7 @@ function iterateTests(cases, options, jsonReply) {
} else {
cleanup = false;
}
if (pu.serverCrashed) {
failedRuns[currentTest] = pu.serverFailMessages;
pu.serverFailMessages = "";
@ -661,8 +708,8 @@ function iterateTests(cases, options, jsonReply) {
pu.cleanupDBDirectories(options);
} else {
print('not cleaning up as some tests weren\'t successful:\n' +
pu.getCleanupDBDirectories() +
cleanup + ' - ' + globalStatus + ' - ' + pu.serverCrashed);
pu.getCleanupDBDirectories() + " " +
cleanup + ' - ' + globalStatus + ' - ' + pu.serverCrashed + "\n");
}
} else {
print("not cleaning up since we didn't start the server ourselves\n");
@ -720,12 +767,13 @@ function unitTest (cases, options) {
}
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief exports
// //////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////////////////////////////////////////////////
// exports
// /////////////////////////////////////////////////////////////////////////////
exports.unitTest = unitTest;
exports.internalMembers = internalMembers;
exports.testFuncs = testFuncs;
exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults;
exports.loadBlacklist = loadBlacklist;

158
js/client/modules/@arangodb/testsuites/config.js
View File

@ -2,28 +2,28 @@
/* global print */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Max Neunhoeffer
// //////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////////////////////////////////////////////////
// DISCLAIMER
//
// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
// Copyright 2014 triagens GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Max Neunhoeffer
// /////////////////////////////////////////////////////////////////////////////
const functionsDocumentation = {
'config': 'checks the config file parsing'
@ -41,7 +41,7 @@ const CYAN = require('internal').COLORS.COLOR_CYAN;
// const GREEN = require('internal').COLORS.COLOR_GREEN;
// const RED = require('internal').COLORS.COLOR_RED;
const RESET = require('internal').COLORS.COLOR_RESET;
// const YELLOW = require('internal').COLORS.COLOR_YELLOW;
const YELLOW = require('internal').COLORS.COLOR_YELLOW;
const time = require('internal').time;
const toArgv = require('internal').toArgv;
@ -67,7 +67,7 @@ function config (options) {
let results = {
failed: 0,
absolut: {
absolute: {
failed: 0,
status: true,
total: 0,
@ -99,79 +99,89 @@ function config (options) {
print('absolute config tests');
print('--------------------------------------------------------------------------------');
// we append one cleanup directory for the invoking logic...
let dummyDir = fs.join(fs.getTempPath(), 'configdummy');
fs.makeDirectory(dummyDir);
pu.cleanupDBDirectoriesAppend(dummyDir);
if (options.skipTest('TEST', 'config.absolute')) {
print(YELLOW + "[SKIPPED] config.absolute" + RESET + "\n");
results.absolute.skipped = true;
} else {
// we append one cleanup directory for the invoking logic...
let dummyDir = fs.join(fs.getTempPath(), 'configdummy');
fs.makeDirectory(dummyDir);
pu.cleanupDBDirectoriesAppend(dummyDir);
let startTime = time();
let startTime = time();
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
const args = {
'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const args = {
'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const run = fs.join(pu.BIN_DIR, test);
const run = fs.join(pu.BIN_DIR, test);
results.absolut[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
results.absolute[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
if (!results.absolut[test].status) {
results.absolut.status = false;
results.absolut.failed += 1;
results.failed += 1;
if (!results.absolute[test].status) {
results.absolute.status = false;
results.absolute.failed += 1;
results.failed += 1;
}
results.absolute.total++;
if (options.verbose) {
print('Args for [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.absolute[test].status);
}
}
results.absolut.total++;
if (options.verbose) {
print('Args for [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.absolut[test].status);
}
results.absolute.duration = time() - startTime;
}
results.absolut.duration = time() - startTime;
print('\n--------------------------------------------------------------------------------');
print('relative config tests');
print('--------------------------------------------------------------------------------');
startTime = time();
if (options.skipTest('TEST', 'config.relative')) {
print(YELLOW + "[SKIPPED] config.relative" + RESET + "\n");
results.relative.skipped = true;
} else {
let startTime = time();
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
const args = {
'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const args = {
'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const run = fs.join(pu.BIN_DIR, test);
const run = fs.join(pu.BIN_DIR, test);
results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
if (!results.relative[test].status) {
results.failed += 1;
results.relative.failed += 1;
results.relative.status = false;
if (!results.relative[test].status) {
results.failed += 1;
results.relative.failed += 1;
results.relative.status = false;
}
results.relative.total++;
if (options.verbose) {
print('Args for (relative) [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.relative[test].status);
}
}
results.relative.total++;
if (options.verbose) {
print('Args for (relative) [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.relative[test].status);
}
results.relative.duration = time() - startTime;
}
results.relative.duration = time() - startTime;
print();
return results;

44
js/client/modules/@arangodb/testsuites/fail.js
View File

@ -1,28 +1,28 @@
/* jshint strict: false, sub: true */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Max Neunhoeffer
// //////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////////////////////////////////////////////////
// DISCLAIMER
//
// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
// Copyright 2014 triagens GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Max Neunhoeffer
// /////////////////////////////////////////////////////////////////////////////
const functionsDocumentation = {
'fail' : 'this job will always produce a failed result',

44
js/client/modules/@arangodb/testsuites/resilience.js
View File

@ -2,28 +2,28 @@
/* global */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Max Neunhoeffer
// //////////////////////////////////////////////////////////////////////////////
// /////////////////////////////////////////////////////////////////////////////
// DISCLAIMER
//
// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
// Copyright 2014 triagens GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Max Neunhoeffer
// /////////////////////////////////////////////////////////////////////////////
const functionsDocumentation = {
'resilience': 'resilience tests',