Bug fix 3.5/connect mocha results to testing js (#9493)

* connect the mocha testresults if we run it in an extra arangosh instance

* fix status handling, lint

* sometimes we have the string undefined to ignore

* fix filter forwarding for spawned arangoshs - add mocha version

* migrate mochaGrep into --testCase parametrizing

README_maintainers.md

@@ -418,9 +418,13 @@ Testing a single test with the framework directly on a server:
 
     scripts/unittest single_server --test js/server/tests/aql/aql-escaping.js
 
-You can also only execute a single test case in a jsunity test suite (in this case `testTokens`):
+You can also only execute a filtered test case in a jsunity/mocha/gtest test suite (in this case `testTokens`):
 
     scripts/unittest single_server --test js/server/tests/aql/aql-escaping.js --testCase testTokens
+    
+    scripts/unittest shell_client --test shell-util-spec.js --testCase zip
+    
+    scripts/unittest gtest --testCase IResearchDocumentTest.*
 
 Testing a single test with the framework via arangosh:
 

js/client/modules/@arangodb/test-utils.js

@@ -617,7 +617,6 @@ function scanTestPaths (paths, options) {
 function runThere (options, instanceInfo, file) {
   try {
     let testCode;
-    let mochaGrep = options.mochaGrep ? ', ' + JSON.stringify(options.mochaGrep) : '';
     if (file.indexOf('-spec') === -1) {
       let testCase = JSON.stringify(options.testCase);
       if (options.testCase === undefined) {
@@ -626,6 +625,7 @@ function runThere (options, instanceInfo, file) {
       testCode = 'const runTest = require("jsunity").runTest; ' +
         'return runTest(' + JSON.stringify(file) + ', true, ' + testCase + ');';
     } else {
+      let mochaGrep = options.testCase ? ', ' + JSON.stringify(options.testCase) : '';
       testCode = 'const runTest = require("@arangodb/mocha-runner"); ' +
         'return runTest(' + JSON.stringify(file) + ', true' + mochaGrep + ');';
     }
@@ -755,7 +755,7 @@ function runInArangosh (options, instanceInfo, file, addArgs) {
 
   args['javascript.unit-tests'] = fs.join(pu.TOP_DIR, file);
 
-  args['javascript.unit-test-filter'] = options.testFilter;
+  args['javascript.unit-test-filter'] = options.testCase;
 
   if (!options.verbose) {
     args['log.level'] = 'warning';
@@ -791,7 +791,7 @@ function runInLocalArangosh (options, instanceInfo, file, addArgs) {
     testCode = 'const runTest = require("jsunity").runTest;\n ' +
       'return runTest(' + JSON.stringify(file) + ', true, ' + testCase + ');\n';
   } else {
-    let mochaGrep = options.mochaGrep ? ', ' + JSON.stringify(options.mochaGrep) : '';
+    let mochaGrep = options.testCase ? ', ' + JSON.stringify(options.testCase) : '';
     testCode = 'const runTest = require("@arangodb/mocha-runner"); ' +
       'return runTest(' + JSON.stringify(file) + ', true' + mochaGrep + ');\n';
   }

js/client/modules/@arangodb/testing.js

@@ -156,7 +156,6 @@ const optionsDefaults = {
   'loopSleepWhen': 1,
   'minPort': 1024,
   'maxPort': 32768,
-  'mochaGrep': undefined,
   'onlyNightly': false,
   'password': '',
   'protocol': 'tcp',

js/common/modules/@arangodb/mocha-runner.js

@@ -112,7 +112,9 @@ function buildJson (results) {
     duration: results.stats.duration,
     status: !results.stats.failures,
     failed: results.stats.failures,
-    total: results.stats.tests
+    total: results.stats.tests,
+    totalSetUp: 0, // we can't get setup time spent in mocha
+    totalTearDown: 0
   };
   for (const test of findTests(results)) {
     const name = test[0];

js/common/modules/@arangodb/testrunner.js

@@ -7,28 +7,36 @@ var runTest = require('jsunity').runTest,
   _ = require('lodash'),
   internal = require('internal');
 
+const deflRes = {
+  failed: 0,
+  status: true,
+  duration: 0,
+  total: 0,
+  totalSetUp: 0,
+  totalTearDown: 0
+};
+const statusKeys = [
+  'failed',
+  'status',
+  'duration',
+  'total',
+  'totalSetUp',
+  'totalTearDown'
+];
+
 // //////////////////////////////////////////////////////////////////////////////
   // / @brief runs all jsunity tests
   // //////////////////////////////////////////////////////////////////////////////
 
-function runJSUnityTests (tests) {
-  let env = require('internal').env;
-  let instanceinfo;
-  if (!env.hasOwnProperty('INSTANCEINFO')) {
-    throw new Error('env.INSTANCEINFO was not set by caller!');
-  }
-  instanceinfo = JSON.parse(env.INSTANCEINFO);
-  if (!instanceinfo) {
-    throw new Error('env.INSTANCEINFO was not set by caller!');
-  }
-  var result = true;
-  var allResults = [];
+function runJSUnityTests (tests, instanceinfo) {
+  let env = internal.env;
+  var allResults = _.clone(deflRes);
   var failed = [];
   var res;
 
   // find out whether we're on server or client...
   var runenvironment = 'arangod';
-  if (typeof (require('internal').arango) === 'object') {
+  if (typeof (internal.arango) === 'object') {
     runenvironment = 'arangosh';
   }
 
@@ -38,17 +46,32 @@ function runJSUnityTests (tests) {
   }
 
   _.each(tests, function (file) {
-    if (result) {
+    if (allResults.status) {
       print('\n' + Date() + ' ' + runenvironment + ": Running JSUnity test from file '" + file + "'");
     } else {
       print('\n' + Date() + ' ' + runenvironment +
-        ": Skipping JSUnity test from file '" + file + "' due to previous errors");
+            ": Skipping JSUnity test from file '" + file + "' due to previous errors");
+      if (!allResults.hasOwnProperty('ALLTESTS')) {
+        allResults['ALLTESTS'] = {
+          status: false,
+          skipped: true,
+          message: 'test failed, skipping others: '
+        };
+      }
+      allResults.ALLTESTS.message += file + ', ';
+      return;
     }
 
     try {
       res = runTest(file, true, unitTestFilter);
-      allResults.push(res);
-      result = result && res.status;
+      allResults[file] = res;
+      allResults.duration += res.duration;
+      allResults.total +=1;
+      allResults.totalSetUp += res.totalSetup;
+      allResults.totalTearDown += res.totalTearDown;
+
+      allResults.status = allResults.status && res.status;
+
       if (!res.status) {
         failed.push(file);
       }
@@ -63,17 +86,16 @@ function runJSUnityTests (tests) {
         );
         err = err.cause;
       }
-      result = false;
+      allResults.status = false;
     }
 
     internal.wait(0); // force GC
   });
-  fs.write(fs.join(instanceinfo.rootDir, 'testresult.json'), JSON.stringify(allResults));
 
   if (failed.length > 1) {
     print('The following ' + failed.length + ' test files produced errors: ', failed.join(', '));
   }
-  return result;
+  return allResults;
 }
 
 // //////////////////////////////////////////////////////////////////////////////
@@ -81,11 +103,16 @@ function runJSUnityTests (tests) {
 // //////////////////////////////////////////////////////////////////////////////
 
 function runMochaTests (testFiles) {
-  var result = true;
+  var result = _.clone(deflRes);
 
   if (testFiles.length > 0) {
+    var unitTestFilter = internal.unitTestFilter();
+    if ((unitTestFilter === "") || (unitTestFilter === "undefined")) {
+      unitTestFilter = undefined;
+    }
+
     print('\nRunning Mocha Tests: ' + testFiles.join(', '));
-    result = require('@arangodb/mocha-runner')(testFiles);
+    result = require('@arangodb/mocha-runner')(testFiles, true, unitTestFilter);
   }
 
   return result;
@@ -96,7 +123,16 @@ function runMochaTests (testFiles) {
 // //////////////////////////////////////////////////////////////////////////////
 
 function runCommandLineTests () {
-  var result = true,
+  let instanceinfo;
+  let env = internal.env;
+  if (!env.hasOwnProperty('INSTANCEINFO')) {
+    throw new Error('env.INSTANCEINFO was not set by caller!');
+  }
+  instanceinfo = JSON.parse(env.INSTANCEINFO);
+  if (!instanceinfo) {
+    throw new Error('env.INSTANCEINFO was not set by caller!');
+  }
+  var result,
     unitTests = internal.unitTests(),
     isSpecRegEx = /.+-spec.*\.js/,
     isSpec = function (unitTest) {
@@ -105,12 +141,35 @@ function runCommandLineTests () {
     jsUnity = _.reject(unitTests, isSpec),
     mocha = _.filter(unitTests, isSpec);
 
-  result = (
-    runJSUnityTests(jsUnity)
-    && runMochaTests(mocha)
-  );
+  let resultJ = runJSUnityTests(jsUnity, instanceinfo);
+  let resultM = runMochaTests(mocha);
 
-  internal.setUnitTestsResult(result);
+  result = {
+    failed: resultJ.failed + resultM.failed,
+    total: resultJ.total + resultM.total,
+    duration: resultJ.duration + resultM.duration,
+    status: resultJ.status & resultM.status
+  };
+
+  let addResults = function(which) {
+    for (let key in which) {
+      if (which.hasOwnProperty(key)) {
+        if (! statusKeys.includes(key)) {
+          if (result.hasOwnProperty(key)) {
+            print('Duplicate test in "' + key + '" - \n"' + JSON.stringify(which) + "\n" + JSON.stringify(result));
+            throw new Error();
+          }
+        }
+        result[key] = which[key];
+      }
+    }
+  };
+
+  addResults(resultJ);
+  addResults(resultM);
+  fs.write(fs.join(instanceinfo.rootDir, 'testresult.json'), JSON.stringify(result));
+
+  internal.setUnitTestsResult(result.status);
 }
 
 exports.runCommandLineTests = runCommandLineTests;