Feature-3.4/whiteblacklist (#7259)

* add blacklist

* added --skipGrey

UnitTests/unittest.js

@@ -237,6 +237,10 @@ function main (argv) {
     options.singleresilient = false;
   }
 
+  if (options.hasOwnProperty('blacklist')) {
+    UnitTest.loadBlacklist(options.blacklist);
+  }
+
   // run the test and store the result
   let res = {}; // result
   try {

js/client/modules/@arangodb/test-utils.js

@@ -351,6 +351,11 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
 // //////////////////////////////////////////////////////////////////////////////
 
 function filterTestcaseByOptions (testname, options, whichFilter) {
+  if (options.skipTest(testname, options)) {
+    whichFilter.filter = 'blacklist';
+    return false;
+  }
+
   // These filters require a proper setup, Even if we filter by testcase:
   if ((testname.indexOf('-mmfiles') !== -1) && options.storageEngine === 'rocksdb') {
     whichFilter.filter = 'skip when running as rocksdb';
@@ -418,6 +423,11 @@ function filterTestcaseByOptions (testname, options, whichFilter) {
     return false;
   }
 
+  if (testname.indexOf('-grey') !== -1 && options.skipGrey) {
+    whichFilter.filter = 'grey';
+    return false;
+  }
+
   if (testname.indexOf('-graph') !== -1 && options.skipGraph) {
     whichFilter.filter = 'graph';
     return false;
@@ -504,9 +514,11 @@ function scanTestPaths (paths) {
   }
 
   let allTestCases = [];
+
   paths.forEach(function(p) {
     allTestCases = allTestCases.concat(doOnePathInner(p));
   });
+
   return allTestCases;
 }
 

js/client/modules/@arangodb/testing.js

@@ -2,28 +2,28 @@
 /* global print */
 'use strict';
 
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
-// / DISCLAIMER
+// DISCLAIMER
-// /
+// 
-// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
-// / Copyright 2014 triagens GmbH, Cologne, Germany
+// Copyright 2014 triagens GmbH, Cologne, Germany
-// /
+// 
-// / Licensed under the Apache License, Version 2.0 (the "License")
+// Licensed under the Apache License, Version 2.0 (the "License")
-// / you may not use this file except in compliance with the License.
+// you may not use this file except in compliance with the License.
-// / You may obtain a copy of the License at
+// You may obtain a copy of the License at
-// /
+// 
-// /     http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
-// /
+//
-// / Unless required by applicable law or agreed to in writing, software
+// Unless required by applicable law or agreed to in writing, software
-// / distributed under the License is distributed on an "AS IS" BASIS,
+// distributed under the License is distributed on an "AS IS" BASIS,
-// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// / See the License for the specific language governing permissions and
+// See the License for the specific language governing permissions and
-// / limitations under the License.
+// limitations under the License.
-// /
+// 
-// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
-// /
+// 
-// / @author Max Neunhoeffer
+// @author Max Neunhoeffer
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
 
 let functionsDocumentation = {
   'all': 'run all tests (marked with [x])',
@@ -49,6 +49,7 @@ let optionsDocumentation = [
   '   - `skipRanges`: if set to true the ranges tests are skipped',
   '   - `skipTimeCritical`: if set to true, time critical tests will be skipped.',
   '   - `skipNondeterministic`: if set, nondeterministic tests are skipped.',
+  '   - `skipGrey`: if set, grey tests are skipped.',
   '   - `testBuckets`: split tests in to buckets and execute on, for example',
   '       10/2 will split into 10 buckets and execute the third bucket.',
   '',
@@ -156,6 +157,7 @@ const optionsDefaults = {
   'skipMemoryIntense': false,
   'skipNightly': true,
   'skipNondeterministic': false,
+  'skipGrey': false,
   'skipTimeCritical': false,
   'storageEngine': 'rocksdb',
   'test': undefined,
@@ -195,9 +197,37 @@ const YELLOW = require('internal').COLORS.COLOR_YELLOW;
 
 let failedRuns = {
 };
+
 let allTests = [
 ];
 
+// /////////////////////////////////////////////////////////////////////////////
+// blacklisting
+// /////////////////////////////////////////////////////////////////////////////
+
+let useBlacklist = false;
+
+let blacklistTests = {};
+
+function skipTest(type, name) {
+  let ntype = type.toUpperCase();
+  return useBlacklist && !!blacklistTests[ntype + ":" + name];
+}
+
+function loadBlacklist(name) {
+  let content = fs.read("BLACKLIST");
+  let a = _.filter(
+    _.map(content.split("\n"),
+          function(x) {return x.trim();}),
+    function(x) {return x.length > 0 && x[0] !== '#';});
+
+  for (let i = 0; i < a.length; ++i) {
+    blacklistTests[a[i]] = true;
+  }
+
+  useBlacklist = true;
+}
+
 // //////////////////////////////////////////////////////////////////////////////
 // / @brief TEST: all
 // //////////////////////////////////////////////////////////////////////////////
@@ -440,7 +470,7 @@ function findTestCases(options) {
   let allTestFiles = {};
   for (let testSuiteName in allTestPaths) {
     var myList = [];
-    let files =  tu.scanTestPaths(allTestPaths[testSuiteName]);
+    let files = tu.scanTestPaths(allTestPaths[testSuiteName]);
     if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) {
       for (let j = 0; j < files.length; j++) {
         let foo = {};
@@ -452,11 +482,11 @@ function findTestCases(options) {
     } else {
       myList = myList.concat(files);
     }
+
     if (!filterTestcases || (myList.length > 0)) {
       allTestFiles[testSuiteName] = myList;
     }
   }
-  // print(allTestPaths)
   return [found, allTestFiles];
 }
 
@@ -539,6 +569,7 @@ function autoTest(options) {
 // //////////////////////////////////////////////////////////////////////////////
 // / @brief load the available testsuites
 // //////////////////////////////////////////////////////////////////////////////
+
 function loadTestSuites () {
   let testSuites = _.filter(fs.list(fs.join(__dirname, 'testsuites')),
                             function (p) {
@@ -618,6 +649,7 @@ function iterateTests(cases, options, jsonReply) {
   for (let n = 0; n < caselist.length; ++n) {
     const currentTest = caselist[n];
     var localOptions = _.cloneDeep(options);
+    localOptions.skipTest = skipTest;
 
     print(BLUE + '================================================================================');
     print('Executing test', currentTest);
@@ -627,19 +659,33 @@ function iterateTests(cases, options, jsonReply) {
       print(CYAN + 'with options:', localOptions, RESET);
     }
 
-    let result = testFuncs[currentTest](localOptions);
+    let result;
-    // grrr...normalize structure
+    let status = true;
-    delete result.status;
-    delete result.failed;
-    delete result.crashed;
 
-    let status = Object.values(result).every(testCase => testCase.status === true);
+    if (skipTest("SUITE", currentTest)) {
-    let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0);
+      result = {
-    if (!status) {
+        failed: 0,
-      globalStatus = false;
+        status: true,
+        crashed: false,
+      };
+
+      print(YELLOW + "[SKIPPED] " + currentTest + RESET + "\n");
+    } else {
+      result = testFuncs[currentTest](localOptions);
+      // grrr...normalize structure
+      delete result.status;
+      delete result.failed;
+      delete result.crashed;
+
+      let status = Object.values(result).every(testCase => testCase.status === true);
+      let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0);
+      if (!status) {
+        globalStatus = false;
+      }
+      result.failed = failed;
+      result.status = status;
     }
-    result.failed = failed;
+
-    result.status = status;
     results[currentTest] = result;
 
     if (status && localOptions.cleanup) {
@@ -647,6 +693,7 @@ function iterateTests(cases, options, jsonReply) {
     } else {
       cleanup = false;
     }
+
     if (pu.serverCrashed) {
       failedRuns[currentTest] = pu.serverFailMessages;
       pu.serverFailMessages = "";
@@ -661,8 +708,8 @@ function iterateTests(cases, options, jsonReply) {
       pu.cleanupDBDirectories(options);
     } else {
       print('not cleaning up as some tests weren\'t successful:\n' +
-            pu.getCleanupDBDirectories() +
+            pu.getCleanupDBDirectories() + " " +
-           cleanup + ' - ' + globalStatus + ' - ' + pu.serverCrashed);
+            cleanup + ' - ' + globalStatus + ' - ' + pu.serverCrashed + "\n");
     }
   } else {
     print("not cleaning up since we didn't start the server ourselves\n");
@@ -720,12 +767,13 @@ function unitTest (cases, options) {
   }
 }
 
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
-// / @brief exports
+// exports
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
 
 exports.unitTest = unitTest;
 
 exports.internalMembers = internalMembers;
 exports.testFuncs = testFuncs;
 exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults;
+exports.loadBlacklist = loadBlacklist;

js/client/modules/@arangodb/testsuites/config.js

@@ -2,28 +2,28 @@
 /* global print */
 'use strict';
 
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
-// / DISCLAIMER
+// DISCLAIMER
-// /
+// 
-// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
-// / Copyright 2014 triagens GmbH, Cologne, Germany
+// Copyright 2014 triagens GmbH, Cologne, Germany
-// /
+// 
-// / Licensed under the Apache License, Version 2.0 (the "License")
+// Licensed under the Apache License, Version 2.0 (the "License")
-// / you may not use this file except in compliance with the License.
+// you may not use this file except in compliance with the License.
-// / You may obtain a copy of the License at
+// You may obtain a copy of the License at
-// /
+// 
-// /     http://www.apache.org/licenses/LICENSE-2.0
+//      http://www.apache.org/licenses/LICENSE-2.0
-// /
+// 
-// / Unless required by applicable law or agreed to in writing, software
+// Unless required by applicable law or agreed to in writing, software
-// / distributed under the License is distributed on an "AS IS" BASIS,
+// distributed under the License is distributed on an "AS IS" BASIS,
-// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// / See the License for the specific language governing permissions and
+// See the License for the specific language governing permissions and
-// / limitations under the License.
+// limitations under the License.
-// /
+// 
-// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
-// /
+// 
-// / @author Max Neunhoeffer
+// @author Max Neunhoeffer
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
 
 const functionsDocumentation = {
   'config': 'checks the config file parsing'
@@ -41,7 +41,7 @@ const CYAN = require('internal').COLORS.COLOR_CYAN;
 // const GREEN = require('internal').COLORS.COLOR_GREEN;
 // const RED = require('internal').COLORS.COLOR_RED;
 const RESET = require('internal').COLORS.COLOR_RESET;
-// const YELLOW = require('internal').COLORS.COLOR_YELLOW;
+const YELLOW = require('internal').COLORS.COLOR_YELLOW;
 
 const time = require('internal').time;
 const toArgv = require('internal').toArgv;
@@ -67,7 +67,7 @@ function config (options) {
 
   let results = {
     failed: 0,
-    absolut: {
+    absolute: {
       failed: 0,
       status: true,
       total: 0,
@@ -99,79 +99,89 @@ function config (options) {
   print('absolute config tests');
   print('--------------------------------------------------------------------------------');
 
-  // we append one cleanup directory for the invoking logic...
+  if (options.skipTest('TEST', 'config.absolute')) {
-  let dummyDir = fs.join(fs.getTempPath(), 'configdummy');
+    print(YELLOW + "[SKIPPED] config.absolute" + RESET + "\n");
-  fs.makeDirectory(dummyDir);
+    results.absolute.skipped = true;
-  pu.cleanupDBDirectoriesAppend(dummyDir);
+  } else {
+    // we append one cleanup directory for the invoking logic...
+    let dummyDir = fs.join(fs.getTempPath(), 'configdummy');
+    fs.makeDirectory(dummyDir);
+    pu.cleanupDBDirectoriesAppend(dummyDir);
 
-  let startTime = time();
+    let startTime = time();
 
-  for (let i = 0; i < ts.length; i++) {
+    for (let i = 0; i < ts.length; i++) {
-    const test = ts[i];
+      const test = ts[i];
-    print(CYAN + 'checking "' + test + '"' + RESET);
+      print(CYAN + 'checking "' + test + '"' + RESET);
 
-    const args = {
+      const args = {
-      'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'),
+        'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'),
-      'flatCommands': ['--check-configuration']
+        'flatCommands': ['--check-configuration']
-    };
+      };
 
-    const run = fs.join(pu.BIN_DIR, test);
+      const run = fs.join(pu.BIN_DIR, test);
 
-    results.absolut[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
+      results.absolute[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
 
-    if (!results.absolut[test].status) {
+      if (!results.absolute[test].status) {
-      results.absolut.status = false;
+        results.absolute.status = false;
-      results.absolut.failed += 1;
+        results.absolute.failed += 1;
-      results.failed += 1;
+        results.failed += 1;
+      }
+
+      results.absolute.total++;
+
+      if (options.verbose) {
+        print('Args for [' + test + ']:');
+        print(yaml.safeDump(args));
+        print('Result: ' + results.absolute[test].status);
+      }
     }
 
-    results.absolut.total++;
+    results.absolute.duration = time() - startTime;
-
-    if (options.verbose) {
-      print('Args for [' + test + ']:');
-      print(yaml.safeDump(args));
-      print('Result: ' + results.absolut[test].status);
-    }
   }
 
-  results.absolut.duration = time() - startTime;
-
   print('\n--------------------------------------------------------------------------------');
   print('relative config tests');
   print('--------------------------------------------------------------------------------');
 
-  startTime = time();
+  if (options.skipTest('TEST', 'config.relative')) {
+    print(YELLOW + "[SKIPPED] config.relative" + RESET + "\n");
+    results.relative.skipped = true;
+  } else {
+    let startTime = time();
 
-  for (let i = 0; i < ts.length; i++) {
+    for (let i = 0; i < ts.length; i++) {
-    const test = ts[i];
+      const test = ts[i];
-    print(CYAN + 'checking "' + test + '"' + RESET);
+      print(CYAN + 'checking "' + test + '"' + RESET);
 
-    const args = {
+      const args = {
-      'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'),
+        'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'),
-      'flatCommands': ['--check-configuration']
+        'flatCommands': ['--check-configuration']
-    };
+      };
 
-    const run = fs.join(pu.BIN_DIR, test);
+      const run = fs.join(pu.BIN_DIR, test);
 
-    results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
+      results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, false, options.coreCheck);
 
-    if (!results.relative[test].status) {
+      if (!results.relative[test].status) {
-      results.failed += 1;
+        results.failed += 1;
-      results.relative.failed += 1;
+        results.relative.failed += 1;
-      results.relative.status = false;
+        results.relative.status = false;
+      }
+
+      results.relative.total++;
+
+      if (options.verbose) {
+        print('Args for (relative) [' + test + ']:');
+        print(yaml.safeDump(args));
+        print('Result: ' + results.relative[test].status);
+      }
     }
 
-    results.relative.total++;
+    results.relative.duration = time() - startTime;
-
-    if (options.verbose) {
-      print('Args for (relative) [' + test + ']:');
-      print(yaml.safeDump(args));
-      print('Result: ' + results.relative[test].status);
-    }
   }
 
-  results.relative.duration = time() - startTime;
-
   print();
 
   return results;

js/client/modules/@arangodb/testsuites/fail.js

@@ -1,28 +1,28 @@
 /* jshint strict: false, sub: true */
 'use strict';
 
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
-// / DISCLAIMER
+// DISCLAIMER
-// /
+// 
-// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
-// / Copyright 2014 triagens GmbH, Cologne, Germany
+// Copyright 2014 triagens GmbH, Cologne, Germany
-// /
+// 
-// / Licensed under the Apache License, Version 2.0 (the "License")
+// Licensed under the Apache License, Version 2.0 (the "License")
-// / you may not use this file except in compliance with the License.
+// you may not use this file except in compliance with the License.
-// / You may obtain a copy of the License at
+// You may obtain a copy of the License at
-// /
+// 
-// /     http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
-// /
+// 
-// / Unless required by applicable law or agreed to in writing, software
+// Unless required by applicable law or agreed to in writing, software
-// / distributed under the License is distributed on an "AS IS" BASIS,
+// distributed under the License is distributed on an "AS IS" BASIS,
-// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// / See the License for the specific language governing permissions and
+// See the License for the specific language governing permissions and
-// / limitations under the License.
+// limitations under the License.
-// /
+// 
-// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
-// /
+// 
-// / @author Max Neunhoeffer
+// @author Max Neunhoeffer
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
 
 const functionsDocumentation = {
   'fail'   : 'this job will always produce a failed result',

js/client/modules/@arangodb/testsuites/resilience.js

@@ -2,28 +2,28 @@
 /* global */
 'use strict';
 
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
-// / DISCLAIMER
+// DISCLAIMER
-// /
+// 
-// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2018 ArangoDB GmbH, Cologne, Germany
-// / Copyright 2014 triagens GmbH, Cologne, Germany
+// Copyright 2014 triagens GmbH, Cologne, Germany
-// /
+// 
-// / Licensed under the Apache License, Version 2.0 (the "License")
+// Licensed under the Apache License, Version 2.0 (the "License")
-// / you may not use this file except in compliance with the License.
+// you may not use this file except in compliance with the License.
-// / You may obtain a copy of the License at
+// You may obtain a copy of the License at
-// /
+// 
-// /     http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
-// /
+// 
-// / Unless required by applicable law or agreed to in writing, software
+// Unless required by applicable law or agreed to in writing, software
-// / distributed under the License is distributed on an "AS IS" BASIS,
+// distributed under the License is distributed on an "AS IS" BASIS,
-// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// / See the License for the specific language governing permissions and
+// See the License for the specific language governing permissions and
-// / limitations under the License.
+// limitations under the License.
-// /
+// 
-// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
-// /
+// 
-// / @author Max Neunhoeffer
+// @author Max Neunhoeffer
-// //////////////////////////////////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////////////////
 
 const functionsDocumentation = {
   'resilience': 'resilience tests',