diff --git a/README_maintainers.md b/README_maintainers.md index 4f7b7548b6..fcee3bcaab 100644 --- a/README_maintainers.md +++ b/README_maintainers.md @@ -823,3 +823,21 @@ Choose the `Npcap Loopback Adapter` number - 1: --sniffProgram c:/Programm Files/wireshark/tshark.exe You can later on use Wireshark to inpsect the capture files. + + +### Evaluating json test reports from previous testruns + +All test results of testruns are dumped to a json file named `UNITTEST_RESULT.json` which can be used +for later analyzing of timings etc. + +Currently available analyzers are: + + - unitTestPrettyPrintResults - Prints a pretty summary and writes an ASCII representation into `out/testfailures.txt` (if any errors) + - saveToJunitXML - saves jUnit compatible XML files + - locateLongRunning - searches the 10 longest running tests from a testsuite + - locateShortServerLife - whether the servers lifetime for the tests isn't at least 10 times as long as startup/shutdown + - locateLongSetupTeardown - locate tests that may use a lot of time in setup/teardown + - yaml - dumps the json file as a yaml file + + + ./scripts/examine_results.js -- 'yaml,locateLongRunning' --readFile out/UNITTEST_RESULT.json diff --git a/UnitTests/unittest.js b/UnitTests/unittest.js index d597a93d19..235a3087bc 100644 --- a/UnitTests/unittest.js +++ b/UnitTests/unittest.js @@ -3,251 +3,72 @@ 'use strict'; const _ = require('lodash'); +const internal = require('internal'); +const rp = require('@arangodb/result-processing'); -const UnitTest = require('@arangodb/testing'); - -const internalMembers = UnitTest.internalMembers; -const fs = require('fs'); -const internal = require('internal'); // js/common/bootstrap/modules/internal.js +const unitTest = require('@arangodb/testing').unitTest; +const optionsDefaults = require('@arangodb/testing').optionsDefaults; +const makeDirectoryRecursive = require('fs').makeDirectoryRecursive; +const killRemainingProcesses = require('@arangodb/process-utils').killRemainingProcesses; const inspect = internal.inspect; -const abortSignal = 6; - -let testOutputDirectory; - -function makePathGeneric (path) { - return path.split(fs.pathSeparator); -} - -function xmlEscape (s) { - return s.replace(/[<>&"]/g, function (c) { - return '&' + { - '<': 'lt', - '>': 'gt', - '&': 'amp', - '"': 'quot' - }[c] + ';'; - }); -} - -function buildXml () { - let xml = ['\n']; - - xml.text = function (s) { - Array.prototype.push.call(this, s); - return this; - }; - - xml.elem = function (tagName, attrs, close) { - this.text('<').text(tagName); - attrs = attrs || {}; - - for (let a in attrs) { - if (attrs.hasOwnProperty(a)) { - this.text(' ').text(a).text('="') - .text(xmlEscape(String(attrs[a]))).text('"'); - } - } - - if (close) { - this.text('/'); - } - - this.text('>\n'); - - return this; - }; - - return xml; -} // ////////////////////////////////////////////////////////////////////////////// -// @brief converts results to XML representation -// ////////////////////////////////////////////////////////////////////////////// - -function resultsToXml (results, baseName, cluster, isRocksDb) { - let clprefix = ''; - - if (cluster) { - clprefix = 'CL_'; - } - - if (isRocksDb) { - clprefix += 'RX_'; - } else { - clprefix += 'MM_'; - } - - const isSignificant = function (a, b) { - return (internalMembers.indexOf(b) === -1) && a.hasOwnProperty(b); - }; - - for (let resultName in results) { - if (isSignificant(results, resultName)) { - let run = results[resultName]; - - for (let runName in run) { - if (isSignificant(run, runName)) { - const xmlName = clprefix + resultName + '_' + runName; - const current = run[runName]; - - if (current.skipped) { - continue; - } - - let xml = buildXml(); - let total = 0; - - if (current.hasOwnProperty('total')) { - total = current.total; - } - - let failuresFound = current.failed; - xml.elem('testsuite', { - errors: 0, - failures: failuresFound, - tests: total, - name: xmlName, - time: 0 + current.duration - }); - - let seen = false; - - for (let oneTestName in current) { - if (isSignificant(current, oneTestName)) { - const oneTest = current[oneTestName]; - const success = (oneTest.status === true); - - seen = true; - - xml.elem('testcase', { - name: clprefix + oneTestName, - time: 0 + oneTest.duration - }, success); - - if (!success) { - xml.elem('failure'); - xml.text('\n'); - xml.elem('/failure'); - xml.elem('/testcase'); - } - } - } - - if (!seen) { - if (failuresFound === 0) { - xml.elem('testcase', { - name: 'all_tests_in_' + xmlName, - time: 0 + current.duration - }, true); - } else { - xml.elem('testcase', { - name: 'all_tests_in_' + xmlName, - failures: failuresFound, - time: 0 + current.duration - }, true); - } - } - - xml.elem('/testsuite'); - - const fn = makePathGeneric(baseName + xmlName + '.xml').join('_'); - - fs.write(testOutputDirectory + fn, xml.join('')); - } - } - } - } -} - -// ////////////////////////////////////////////////////////////////////////////// -// @brief runs the test using testing.js +// @brief runs the test using the test facilities in js/client/modules/@arangodb // ////////////////////////////////////////////////////////////////////////////// function main (argv) { start_pretty_print(); - // parse arguments - let testSuits = []; // e.g all, http_server, recovery, ... + let testSuites = []; // e.g all, http_server, recovery, ... let options = {}; while (argv.length >= 1) { - if (argv[0].slice(0, 1) === '{') { // stop parsing if there is a json document - break; - } - if (argv[0].slice(0, 1) === '-') { // break parsing if we hit some -option break; } - testSuits.push(argv[0]); // append first arg to test suits + testSuites.push(argv[0]); // append first arg to test suits argv = argv.slice(1); // and remove first arg (c++:pop_front/bash:shift) } - // convert arguments if (argv.length >= 1) { try { - if (argv[0].slice(0, 1) === '{') { - options = JSON.parse(argv[0]); // parse options form json - } else { - options = internal.parseArgv(argv, 0); // parse option with parseArgv function - } + options = internal.parseArgv(argv, 0); } catch (x) { - print('failed to parse the json options: ' + x.message + '\n' + String(x.stack)); + print('failed to parse the options: ' + x.message + '\n' + String(x.stack)); print('argv: ', argv); - return -1; + throw x; } } if (options.hasOwnProperty('testOutput')) { - testOutputDirectory = options.testOutput + '/'; - } else { - testOutputDirectory = 'out/'; + options.testOutputDirectory = options.testOutput + '/'; } - - options.testOutputDirectory = testOutputDirectory; - - // force json reply - options.jsonReply = true; + _.defaults(options, optionsDefaults); // create output directory try { - fs.makeDirectoryRecursive(testOutputDirectory); + makeDirectoryRecursive(options.testOutputDirectory); } catch (x) { print("failed to create test directory - " + x.message); throw x; } - // by default we set this to error, so we always have a proper result for the caller try { - fs.write(testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', "false", true); - fs.write(testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', "true", true); - let testFailureText = 'testfailures.txt'; - if (options.hasOwnProperty('testFailureText')) { - testFailureText = options.testFailureText; - } - fs.write(fs.join(testOutputDirectory, testFailureText), - "Incomplete testrun with these testsuites: '" + testSuits + - "'\nand these options: " + JSON.stringify(options) + "\n"); + // safeguard: mark test as failed if we die for some reason + rp.writeDefaultReports(options, testSuites); } catch (x) { print('failed to write default test result: ' + x.message); - throw(x); + throw x; } - if (options.hasOwnProperty('cluster') && options.cluster) { - // cluster beats resilient single server - options.singleresilient = false; - } - - if (options.hasOwnProperty('blacklist')) { - UnitTest.loadBlacklist(options.blacklist); - } - - // run the test and store the result - let res = {}; // result + let result = {}; try { - // run tests - res = UnitTest.unitTest(testSuits, options, testOutputDirectory) || {}; + result = unitTest(testSuites, options); } catch (x) { + if (x.message === "USAGE ERROR") { + throw x; + } print('caught exception during test execution!'); if (x.message !== undefined) { @@ -260,74 +81,38 @@ function main (argv) { print(x); } - print(JSON.stringify(res)); + print(JSON.stringify(result)); + throw x; } - _.defaults(res, { + _.defaults(result, { status: false, crashed: true }); - let running = require("internal").getExternalSpawned(); - let i = 0; - for (i = 0; i < running.length; i++) { - let status = require("internal").statusExternal(running[i].pid, false); - if (status.status === "TERMINATED") { - print("process exited without us joining it (marking crashy): " + JSON.stringify(running[i]) + JSON.stringify(status)); - } - else { - print("Killing remaining process & marking crashy: " + JSON.stringify(running[i])); - print(require("internal").killExternal(running[i].pid, abortSignal)); - } - res.crashed = true; - }; + killRemainingProcesses(result); - // whether or not there was an error try { - fs.write(testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', String(res.status), true); - fs.write(testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', String(res.crashed), true); + rp.writeReports(options, result); } catch (x) { print('failed to write test result: ' + x.message); } + rp.dumpAllResults(options, result); if (options.writeXmlReport) { - let j; - try { - j = JSON.stringify(res); - } catch (err) { - j = inspect(res); - } - - fs.write(testOutputDirectory + '/UNITTEST_RESULT.json', j, true); - - try { - let isCluster = false; - let isRocksDb = false; - let prefix = ''; - if (options.hasOwnProperty('prefix')) { - prefix = options.prefix; - } - if (options.hasOwnProperty('cluster') && options.cluster) { - isCluster = true; - } - if (options.hasOwnProperty('storageEngine')) { - isRocksDb = (options.storageEngine === 'rocksdb'); - } - - resultsToXml(res, 'UNITTEST_RESULT_' + prefix, isCluster, isRocksDb); + rp.analyze.saveToJunitXML(options, result); } catch (x) { print('exception while serializing status xml!'); print(x.message); print(x.stack); - print(inspect(res)); + print(inspect(result)); } } - // creates yaml like dump at the end - UnitTest.unitTestPrettyPrintResults(res, testOutputDirectory, options); + rp.analyze.unitTestPrettyPrintResults(options, result); - return res.status && running.length === 0; + return result.status; } let result = main(ARGUMENTS); diff --git a/arangod/RestHandler/RestStatusHandler.cpp b/arangod/RestHandler/RestStatusHandler.cpp index c25c298097..49e96665c1 100644 --- a/arangod/RestHandler/RestStatusHandler.cpp +++ b/arangod/RestHandler/RestStatusHandler.cpp @@ -66,7 +66,7 @@ RestStatus RestStatusHandler::execute() { result.add("server", VPackValue("arango")); result.add("version", VPackValue(ARANGODB_VERSION)); - result.add("pid", VPackValue(Thread::currentProcessId())); + result.add("pid", VPackValue(static_cast(Thread::currentProcessId()))); #ifdef USE_ENTERPRISE result.add("license", VPackValue("enterprise")); diff --git a/js/client/modules/@arangodb/clusterstats.js b/js/client/modules/@arangodb/clusterstats.js new file mode 100644 index 0000000000..716287eea4 --- /dev/null +++ b/js/client/modules/@arangodb/clusterstats.js @@ -0,0 +1,116 @@ +/* jshint strict: false, sub: true */ +/* global print, arango */ +'use strict'; + +// ////////////////////////////////////////////////////////////////////////////// +// / DISCLAIMER +// / +// / Copyright 2016 ArangoDB GmbH, Cologne, Germany +// / Copyright 2014 triagens GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License") +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is ArangoDB GmbH, Cologne, Germany +// / +// / @author Wilfried Goesgens +// ////////////////////////////////////////////////////////////////////////////// + +const internal = require('internal'); +const fs = require('fs'); + +const download = internal.download; +const time = internal.time; +const sleep = internal.sleep; + +const pu = require('@arangodb/process-utils'); + +const instanceInfo = JSON.parse(internal.env.INSTANCEINFO); +const options = JSON.parse(internal.env.OPTIONS); +const outfn = fs.join(instanceInfo.rootDir, 'stats.jsonl'); +const opts = Object.assign(pu.makeAuthorizationHeaders(options), + { method: 'GET' }); + +while(true) { + let state = { + state: true, + before: time(), + delta: [], + fails: [] + }; + let results = []; + for (let i = 0; i < 60; i++) { + const before = time(); + let oneSet = { state: true }; + results.push(oneSet); + instanceInfo.arangods.forEach(arangod => { + let serverId = arangod.role + '_' + arangod.port; + let beforeCall = time(); + let procStats = pu.getProcessStats(arangod.pid); + if (arangod.role === "agent") { + let reply = download(arangod.url + '/_api/version', '', opts); + if (reply.hasOwnProperty('error') || reply.code !== 200) { + print("fail: " + JSON.stringify(reply) + + " - ps before: " + JSON.stringify(procStats) + + " - ps now: " + JSON.stringify(pu.getProcessStats(arangod.pid))); + state.state = false; + oneSet.state = false; + oneSet[serverId] = { + error: true, + start: beforeCall, + delta: time() - beforeCall + }; + } else { + let statisticsReply = JSON.parse(reply.body); + oneSet[serverId] = { + error: false, + start: beforeCall, + delta: time() - beforeCall + }; + } + } else { + let reply = download(arangod.url + '/_admin/statistics', '', opts); + if (reply.hasOwnProperty('error') || reply.code !== 200) { + print("fail: " + JSON.stringify(reply) + + " - ps before: " + JSON.stringify(procStats) + + " - ps now: " + JSON.stringify(pu.getProcessStats(arangod.pid))); + state.state = false; + oneSet.state = false; + oneSet[serverId] = { + error: true, + start: beforeCall, + delta: time() - beforeCall + }; + } else { + let statisticsReply = JSON.parse(reply.body); + oneSet[serverId] = { + error: false, + start: beforeCall, + delta: time() - beforeCall, + uptime: statisticsReply.server.uptime + }; + } + } + }); + state['delta'].push(time() - before); + if (state.delta > 1000) { + print("marking FAIL since it took to long"); + state.state = false; + } + if (!oneSet.state) { + state.fails.push(oneSet); + } + sleep(1); + } + fs.append(outfn, JSON.stringify(state) + "\n"); + +} diff --git a/js/client/modules/@arangodb/process-utils.js b/js/client/modules/@arangodb/process-utils.js index f7ba11ed81..f587c8b09d 100755 --- a/js/client/modules/@arangodb/process-utils.js +++ b/js/client/modules/@arangodb/process-utils.js @@ -28,6 +28,7 @@ /* Modules: */ const _ = require('lodash'); const fs = require('fs'); +const rp = require('@arangodb/result-processing'); const yaml = require('js-yaml'); const internal = require('internal'); const crashUtils = require('@arangodb/crash-utils'); @@ -39,6 +40,7 @@ const executeExternal = internal.executeExternal; const executeExternalAndWait = internal.executeExternalAndWait; const killExternal = internal.killExternal; const statusExternal = internal.statusExternal; +const statisticsExternal = internal.statisticsExternal; const base64Encode = internal.base64Encode; const testPort = internal.testPort; const download = internal.download; @@ -234,14 +236,22 @@ function setupBinaries (builddir, buildType, configDir) { BIN_DIR = fs.join(TOP_DIR, BIN_DIR); } + if (!fs.exists(BIN_DIR)) { + BIN_DIR = fs.join(TOP_DIR, 'bin'); + } + UNITTESTS_DIR = fs.join(fs.join(builddir, 'tests')); if (!fs.exists(UNITTESTS_DIR)) { UNITTESTS_DIR = fs.join(TOP_DIR, UNITTESTS_DIR); } if (buildType !== '') { - BIN_DIR = fs.join(BIN_DIR, buildType); - UNITTESTS_DIR = fs.join(UNITTESTS_DIR, buildType); + if (fs.exists(fs.join(BIN_DIR, buildType))) { + BIN_DIR = fs.join(BIN_DIR, buildType); + } + if (fs.exists(fs.join(UNITTESTS_DIR, buildType))) { + UNITTESTS_DIR = fs.join(UNITTESTS_DIR, buildType); + } } ARANGOBACKUP_BIN = fs.join(BIN_DIR, 'arangobackup' + executableExt); @@ -587,6 +597,88 @@ function killWithCoreDump (options, instanceInfo) { instanceInfo.exitStatus = killExternal(instanceInfo.pid, abortSignal); } +// ////////////////////////////////////////////////////////////////////////////// +// / @brief aggregates information from /proc about the SUT +// ////////////////////////////////////////////////////////////////////////////// +function getProcessStats(pid) { + let processStats = statisticsExternal(pid); + if (platform === 'linux') { + let pidStr = "" + pid; + let ioraw; + try { + ioraw = fs.readBuffer(fs.join('/', 'proc', pidStr, 'io')); + } catch (x) { + print(x.stack); + throw x; + } + let lineStart = 0; + let maxBuffer = ioraw.length; + for (let j = 0; j < maxBuffer; j++) { + if (ioraw[j] === 10) { // \n + const line = ioraw.asciiSlice(lineStart, j); + lineStart = j + 1; + let x = line.split(":"); + processStats[x[0]] = parseInt(x[1]); + } + } + } + return processStats; +} + +function initProcessStats(instanceInfo) { + instanceInfo.arangods.forEach((arangod) => { + arangod.stats = getProcessStats(arangod.pid); + }); +} + +function getDeltaProcessStats(instanceInfo) { + let deltaStats = {}; + instanceInfo.arangods.forEach((arangod) => { + let newStats = getProcessStats(arangod.pid); + let myDeltaStats = {}; + for (let key in arangod.stats) { + myDeltaStats[key] = newStats[key] - arangod.stats[key]; + } + deltaStats[arangod.pid + '_' + arangod.role] = myDeltaStats; + arangod.stats = newStats; + }); + return deltaStats; +} + +function summarizeStats(deltaStats) { + let sumStats = {}; + for (let instance in deltaStats) { + for (let key in deltaStats[instance]) { + if (!sumStats.hasOwnProperty(key)) { + sumStats[key] = 0; + } + sumStats[key] += deltaStats[instance][key]; + } + } + return sumStats; +} + +// ////////////////////////////////////////////////////////////////////////////// +// / @brief if we forgot about processes, this safe guard will clean up and mark failed +// ////////////////////////////////////////////////////////////////////////////// + +function killRemainingProcesses(results) { + let running = internal.getExternalSpawned(); + results.status = results.status && (running.length === 0); + let i = 0; + for (i = 0; i < running.length; i++) { + let status = require("internal").statusExternal(running[i].pid, false); + if (status.status === "TERMINATED") { + print("process exited without us joining it (marking crashy): " + JSON.stringify(running[i]) + JSON.stringify(status)); + } + else { + print("Killing remaining process & marking crashy: " + JSON.stringify(running[i])); + print(killExternal(running[i].pid, abortSignal)); + } + results.crashed = true; + }; +} + // ////////////////////////////////////////////////////////////////////////////// // / @brief executes a command and waits for result // ////////////////////////////////////////////////////////////////////////////// @@ -1039,7 +1131,12 @@ function abortSurvivors(arangod, options) { } function checkInstanceAlive (instanceInfo, options) { - if (options.activefailover && instanceInfo.hasOwnProperty('authOpts')) { + if (options.activefailover && + instanceInfo.hasOwnProperty('authOpts') && + (instanceInfo.url !== instanceInfo.agencyUrl) + ) { + // only detect a leader after we actually know one has been started. + // the agency won't tell us anything about leaders. let d = detectCurrentLeader(instanceInfo); if (instanceInfo.endpoint !== d.endpoint) { print(Date() + ' failover has happened, leader is no more! Marking Crashy!'); @@ -1270,6 +1367,16 @@ function shutdownInstance (instanceInfo, options, forceTerminate) { } } + if (options.cluster && instanceInfo.hasOwnProperty('clusterHealthMonitor')) { + try { + instanceInfo.clusterHealthMonitor['kill'] = killExternal(instanceInfo.clusterHealthMonitor.pid); + instanceInfo.clusterHealthMonitor['statusExternal'] = statusExternal(instanceInfo.clusterHealthMonitor.pid, true); + } + catch (x) { + print(x); + } + } + // Shut down all non-agency servers: const n = instanceInfo.arangods.length; @@ -1431,7 +1538,6 @@ function detectCurrentLeader(instanceInfo) { throw "Leader is not selected"; } } - opts['method'] = 'GET'; reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', opts); let res; @@ -1452,9 +1558,6 @@ function detectCurrentLeader(instanceInfo) { } function checkClusterAlive(options, instanceInfo, addArgs) { - // disabled because not in use (jslint) - // let coordinatorUrl = instanceInfo.url - // let response let httpOptions = makeAuthorizationHeaders(options); httpOptions.method = 'POST'; httpOptions.returnBodyOnError = true; @@ -1512,6 +1615,7 @@ function checkClusterAlive(options, instanceInfo, addArgs) { throw new Error('cluster startup timed out after 10 minutes!'); } } + print("Determining server IDs"); instanceInfo.arangods.forEach(arangod => { // agents don't support the ID call... @@ -1524,6 +1628,19 @@ function checkClusterAlive(options, instanceInfo, addArgs) { arangod.id = res['id']; } }); + + if ((options.cluster || options.agency) && + !instanceInfo.hasOwnProperty('clusterHealthMonitor') && + !options.disableClusterMonitor) { + print("spawning cluster health inspector"); + internal.env.INSTANCEINFO = JSON.stringify(instanceInfo); + internal.env.OPTIONS = JSON.stringify(options); + let args = makeArgsArangosh(options); + args['javascript.execute'] = fs.join('js', 'client', 'modules', '@arangodb', 'clusterstats.js'); + const argv = toArgv(args); + instanceInfo.clusterHealthMonitor = executeExternal(ARANGOSH_BIN, argv); + instanceInfo.clusterHealthMonitorFile = fs.join(instanceInfo.rootDir, 'stats.jsonl'); + } } // ////////////////////////////////////////////////////////////////////////////// // / @brief starts an instance @@ -1710,6 +1827,7 @@ function launchFinalize(options, instanceInfo, startTime) { } print(processInfo.join('\n') + '\n'); internal.sleep(options.sleepBeforeStart); + initProcessStats(instanceInfo); } // ////////////////////////////////////////////////////////////////////////////// @@ -1848,6 +1966,9 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir) instanceInfo.role = 'agent'; print(Date() + ' Agency Endpoint: ' + instanceInfo.endpoint); } + + checkClusterAlive(options, instanceInfo, addArgs); + return instanceInfo; } @@ -1988,6 +2109,12 @@ function reStartInstance(options, instanceInfo, moreArgs) { launchFinalize(options, instanceInfo, startTime); } +function aggregateFatalErrors(currentTest) { + if (serverCrashedLocal) { + rp.addFailRunsMessage(currentTest, serverFailMessagesLocal); + serverFailMessagesLocal = ""; + } +} // exports.analyzeServerCrash = analyzeServerCrash; exports.makeArgs = { arangod: makeArgsArangod, @@ -2008,6 +2135,7 @@ exports.coverageEnvironment = coverageEnvironment; exports.executeArangod = executeArangod; exports.executeAndWait = executeAndWait; +exports.killRemainingProcesses = killRemainingProcesses; exports.stopProcdump = stopProcdump; exports.createBaseConfig = createBaseConfigBuilder; @@ -2021,14 +2149,17 @@ exports.run = { }; exports.shutdownInstance = shutdownInstance; +exports.getProcessStats = getProcessStats; +exports.getDeltaProcessStats = getDeltaProcessStats; +exports.summarizeStats = summarizeStats; exports.startArango = startArango; exports.startInstance = startInstance; exports.reStartInstance = reStartInstance; exports.setupBinaries = setupBinaries; exports.executableExt = executableExt; exports.serverCrashed = serverCrashedLocal; -exports.serverFailMessages = serverFailMessagesLocal; +exports.aggregateFatalErrors = aggregateFatalErrors; exports.cleanupDBDirectoriesAppend = cleanupDBDirectoriesAppend; exports.cleanupDBDirectories = cleanupDBDirectories; exports.cleanupLastDirectory = cleanupLastDirectory; diff --git a/js/client/modules/@arangodb/result-processing.js b/js/client/modules/@arangodb/result-processing.js new file mode 100644 index 0000000000..19b9d6f0d4 --- /dev/null +++ b/js/client/modules/@arangodb/result-processing.js @@ -0,0 +1,769 @@ +/* jshint strict: false, sub: true */ +/* global print */ +'use strict'; + +// ////////////////////////////////////////////////////////////////////////////// +// / DISCLAIMER +// / +// / Copyright 2016 ArangoDB GmbH, Cologne, Germany +// / Copyright 2014 triagens GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License") +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is ArangoDB GmbH, Cologne, Germany +// / +// / @author Wilfried Goesgens +// ////////////////////////////////////////////////////////////////////////////// + +// ////////////////////////////////////////////////////////////////////////////// +// / @brief internal members of the results +// ////////////////////////////////////////////////////////////////////////////// + +const internal = require('internal'); +const inspect = internal.inspect; +const fs = require('fs'); +const pu = require('@arangodb/process-utils'); +const cu = require('@arangodb/crash-utils'); +const yaml = require('js-yaml'); + +/* Constants: */ +const BLUE = internal.COLORS.COLOR_BLUE; +const CYAN = internal.COLORS.COLOR_CYAN; +const GREEN = internal.COLORS.COLOR_GREEN; +const RED = internal.COLORS.COLOR_RED; +const RESET = internal.COLORS.COLOR_RESET; +const YELLOW = internal.COLORS.COLOR_YELLOW; + +const internalMembers = [ + 'code', + 'error', + 'status', + 'duration', + 'failed', + 'total', + 'crashed', + 'ok', + 'message', + 'suiteName', + 'processStats', + 'startupTime', + 'testDuration', + 'shutdownTime', + 'totalSetUp', + 'totalTearDown', + 'setUpDuration', + 'setUpAllDuration', + 'teardownAllDuration' +]; + +function skipInternalMember (r, a) { + return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1; +} + +function makePathGeneric (path) { + return path.split(fs.pathSeparator); +} + +let failedRuns = { +}; + +function gatherFailed(result) { + return Object.values(result).reduce(function(prev, testCase) { + if (testCase instanceof Object) { + return prev + !testCase.status; + } else { + return prev; + } + }, 0 + ); +} + +function gatherStatus(result) { + return Object.values(result).reduce(function(prev, testCase) { + if (testCase instanceof Object) { + return prev && testCase.status; + } else { + return prev; + } + }, true + ); +} + +// ////////////////////////////////////////////////////////////////////////////// +// / @brief pretty prints the result +// ////////////////////////////////////////////////////////////////////////////// + +function fancyTimeFormat(time) +{ + // Hours, minutes and seconds + var hrs = ~~(time / 3600); + var mins = ~~((time % 3600) / 60); + var secs = ~~time % 60; + + // Output like "1:01" or "4:03:59" or "123:03:59" + var ret = ""; + + if (hrs > 0) { + ret += "" + hrs + ":" + (mins < 10 ? "0" : ""); + } + + ret += "" + mins + ":" + (secs < 10 ? "0" : ""); + ret += "" + secs; + return ret; +} + + +function iterateTestResults(options, results, state, handlers) { + let bucketName = ""; + if (options.testBuckets) { + let n = options.testBuckets.split('/'); + bucketName = "_" + n[1]; + } + try { + /* jshint forin: false */ + // i.e. shell_server_aql + for (let testRunName in results) { + if (skipInternalMember(results, testRunName)) { + continue; + } + let testRun = results[testRunName]; + + if (handlers.hasOwnProperty('testRun')) { + handlers.testRun(options, state, testRun, testRunName); + } + + // i.e. tests/js/server/aql/aql-cross.js + for (let testSuiteName in testRun) { + if (skipInternalMember(testRun, testSuiteName)) { + continue; + } + let testSuite = testRun[testSuiteName]; + + if (handlers.hasOwnProperty('testSuite')) { + handlers.testSuite(options, state, testSuite, testSuiteName); + } + + // i.e. testDocument1 + for (let testName in testSuite) { + if (skipInternalMember(testSuite, testName)) { + continue; + } + let testCase = testSuite[testName]; + + if (handlers.hasOwnProperty('testCase')) { + handlers.testCase(options, state, testCase, testName); + } + } + if (handlers.hasOwnProperty('endTestSuite')) { + handlers.endTestSuite(options, state, testSuite, testSuiteName); + } + } + if (handlers.hasOwnProperty('endTestRun')) { + handlers.endTestRun(options, state, testRun, testRunName); + } + } + } catch (x) { + print("Exception occured in running tests: " + x.message + "\n" + x.stack); + } +} + +function locateFailState(options, results) { + let failedStates = { + state: true, + failCount: 0, + thisFailedTestCount: 0, + runFailedTestCount: 0, + currentSuccess: true, + failedTests: {}, + thisFailedTests: [] + }; + iterateTestResults(options, results, failedStates, { + testRun: function(options, state, testRun, testRunName) { + if (!testRun.state) { + state.state = false; + state.thisFailedTestCount = testRun.failed; + } + }, + testCase: function(options, state, testCase, testCaseName) { + if (!testCase.state) { + state.state = false; + state.runFailedTestCount ++; + state.thisFailedTests.push(testCaseName); + } + }, + endTestRun: function(options, state, testRun, testRunName) { + if (state.thisFailedTestCount !== 0) { + if (state.thisFailedTestCount !== state.runFailedTestCount) { + // todo error message + } + state.failedTests[testRunName] = state.thisFailedTests; + state.failCount += state.runFailedTestCount; + + state.thisFailedTests = []; + state.thisFailedTestCount = 0; + } + } + }); +} + +// ////////////////////////////////////////////////////////////////////////////// +// @brief converts results to XML representation +// ////////////////////////////////////////////////////////////////////////////// + +function saveToJunitXML(options, results) { + function xmlEscape (s) { + return s.replace(/[<>&"]/g, function (c) { + return '&' + { + '<': 'lt', + '>': 'gt', + '&': 'amp', + '"': 'quot' + }[c] + ';'; + }); + } + + function buildXml () { + let xml = ['\n']; + + xml.text = function (s) { + Array.prototype.push.call(this, s); + return this; + }; + + xml.elem = function (tagName, attrs, close) { + this.text('<').text(tagName); + attrs = attrs || {}; + + for (let a in attrs) { + if (attrs.hasOwnProperty(a)) { + this.text(' ').text(a).text('="') + .text(xmlEscape(String(attrs[a]))).text('"'); + } + } + + if (close) { + this.text('/'); + } + + this.text('>\n'); + + return this; + }; + + return xml; + } + let xmlState = { + xml: undefined, + xmlName: '', + testRunName: '', + seenTestCases: false, + }; + let prefix = options.cluster ? 'CL_' : '' + + (options.storageEngine === 'rocksdb') ? 'RX_': 'MM_'; + iterateTestResults(options, results, xmlState, { + testRun: function(options, state, testRun, testRunName) {state.testRunName = testRunName;}, + testSuite: function(options, state, testSuite, testSuiteName) { + let total = 0; + state.seenTestCases = false; + state.xml = buildXml(); + state.xmlName = prefix + state.testRunName + '_' + makePathGeneric(testSuiteName).join('_'); + if (testSuite.hasOwnProperty('total')) { + total = testSuite.total; + } + + state.xml.elem('testsuite', { + errors: 0, + failures: testSuite.failed, + tests: total, + name: state.xmlName, + time: 0 + testSuite.duration + }); + + }, + testCase: function(options, state, testCase, testCaseName) { + const success = (testCase.status === true); + + state.xml.elem('testcase', { + name: prefix + testCaseName, + time: 0 + testCase.duration + }, success); + + state.seenTestCases = true; + if (!success) { + state.xml.elem('failure'); + state.xml.text('\n'); + state.xml.elem('/failure'); + state.xml.elem('/testcase'); + } + }, + endTestSuite: function(options, state, testSuite, testSuiteName) { + if (!state.seenTestCases) { + if (testSuite.failed === 0) { + state.xml.elem('testcase', { + name: 'all_tests_in_' + state.xmlName, + time: 0 + testSuite.duration + }, true); + } else { + state.xml.elem('testcase', { + name: 'all_tests_in_' + state.xmlName, + failures: testSuite.failuresFound, + time: 0 + testSuite.duration + }, true); + } + } + state.xml.elem('/testsuite'); + fs.write(fs.join(options.testOutputDirectory, + 'UNITTEST_RESULT_' + state.xmlName + '.xml'), + state.xml.join('')); + + }, + endTestRun: function(options, state, testRun, testRunName) {} + }); +} + +function unitTestPrettyPrintResults (options, results) { + let onlyFailedMessages = ''; + let failedMessages = ''; + let SuccessMessages = ''; + let failedSuiteCount = 0; + let failedTestsCount = 0; + let successCases = {}; + let failedCases = {}; + let failsOfOneSuite = {}; + let failsOfOneSuiteCount = 0; + let bucketName = ""; + let testRunStatistics = ""; + let isSuccess = true; + let suiteSuccess = true; + + if (options.testBuckets) { + let n = options.testBuckets.split('/'); + bucketName = "_" + n[1]; + } + function testCaseMessage (test) { + if (typeof test.message === 'object' && test.message.hasOwnProperty('body')) { + return test.message.body; + } else { + return test.message; + } + } + + let failedStates = { + state: true, + failCount: 0, + thisFailedTestCount: 0, + runFailedTestCount: 0, + currentSuccess: true, + failedTests: {}, + thisFailedTests: [] + }; + + iterateTestResults(options, results, failedStates, { + testRun: function(options, state, testRun, testRunName) { + }, + testSuite: function(options, state, testSuite, testSuiteName) { + if (testSuite.status) { + successCases[testSuiteName] = testSuite; + } else { + ++failedSuiteCount; + isSuccess = false; + if (testSuite.hasOwnProperty('message')) { + failedCases[testSuiteName] = { + test: testCaseMessage(testSuite) + }; + } + } + }, + testCase: function(options, state, testCase, testCaseName) { + if (!testCase.status) { + failsOfOneSuite[testCaseName] = testCaseMessage(testCase); + failsOfOneSuiteCount ++; + + } + }, + endTestSuite: function(options, state, testSuite, testSuiteName) { + failedTestsCount++; + if (failsOfOneSuiteCount !== 0) { + failedCases[testSuiteName] = failsOfOneSuite; + failsOfOneSuite = {}; + failsOfOneSuiteCount = 0; + } + }, + endTestRun: function(options, state, testRun, testRunName) { + if (isSuccess) { + SuccessMessages += '* Test "' + testRunName + bucketName + '"\n'; + + for (let name in successCases) { + if (!successCases.hasOwnProperty(name)) { + continue; + } + + let details = successCases[name]; + + if (details.skipped) { + SuccessMessages += YELLOW + ' [SKIPPED] ' + name + RESET + '\n'; + } else { + SuccessMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n'; + } + } + } else { + let m = '* Test "' + testRunName + bucketName + '"\n'; + onlyFailedMessages += m; + failedMessages += m; + + for (let name in successCases) { + if (!successCases.hasOwnProperty(name)) { + continue; + } + + let details = successCases[name]; + + if (details.skipped) { + m = ' [SKIPPED] ' + name; + failedMessages += YELLOW + m + RESET + '\n'; + onlyFailedMessages += m + '\n'; + } else { + failedMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n'; + } + } + for (let name in failedCases) { + if (!failedCases.hasOwnProperty(name)) { + continue; + } + + m = ' [FAILED] ' + name; + failedMessages += RED + m + RESET + '\n\n'; + onlyFailedMessages += m + '\n\n'; + + let details = failedCases[name]; + + let count = 0; + for (let one in details) { + if (!details.hasOwnProperty(one)) { + continue; + } + + if (count > 0) { + failedMessages += '\n'; + onlyFailedMessages += '\n'; + } + m = ' "' + one + '" failed: ' + details[one]; + failedMessages += RED + m + RESET + '\n\n'; + onlyFailedMessages += m + '\n\n'; + count++; + } + } + } + isSuccess = true; + successCases = {}; + failedCases = {}; + } + }); + let color,statusMessage; + let crashedText = ''; + let crashText = ''; + let failText = ''; + if (results.status === true) { + color = GREEN; + statusMessage = 'Success'; + } else { + color = RED; + statusMessage = 'Fail'; + } + if (results.crashed === true) { + color = RED; + for (let failed in failedRuns) { + crashedText += ' [' + failed + '] : ' + failedRuns[failed].replace(/^/mg, ' '); + } + crashedText += "\nMarking crashy!"; + crashText = color + crashedText + RESET; + } + if (results.status !== true) { + failText = '\n Suites failed: ' + failedSuiteCount + ' Tests Failed: ' + failedTestsCount; + onlyFailedMessages += failText + '\n'; + failText = RED + failText + RESET; + } + if (cu.GDB_OUTPUT !== '') { + // write more verbose failures to the testFailureText file + onlyFailedMessages += '\n\n' + cu.GDB_OUTPUT; + } + print(` +${YELLOW}================================================================================' +TEST RESULTS +================================================================================${RESET} +${SuccessMessages} +${failedMessages}${color} * Overall state: ${statusMessage}${RESET}${crashText}${failText}`); + + onlyFailedMessages; + if (crashedText !== '') { + onlyFailedMessages += '\n' + crashedText; + } + fs.write(options.testOutputDirectory + options.testFailureText, onlyFailedMessages); +} + +function locateLongRunning(options, results) { + let testRunStatistics = ""; + let sortedByDuration = []; + + let failedStates = { + state: true, + failCount: 0, + thisFailedTestCount: 0, + runFailedTestCount: 0, + currentSuccess: true, + failedTests: {}, + thisFailedTests: [] + }; + iterateTestResults(options, results, failedStates, { + testRun: function(options, state, testRun, testRunName) { + let startup = testRun['startupTime']; + let testDuration = testRun['testDuration']; + let shutdown = testRun['shutdownTime']; + let color = GREEN; + if (((startup + shutdown) * 10) > testDuration) { + color = RED; + } + + testRunStatistics += `${color}${testRunName} - startup [${startup}] => run [${testDuration}] => shutdown [${shutdown}]${RESET} +`; + }, + testSuite: function(options, state, testSuite, testSuiteName) { + if (testSuite.hasOwnProperty('duration') && testSuite.duration !== 0) { + sortedByDuration.push( + { + testName: testSuiteName, + duration: testSuite.duration, + test: testSuite, + count: Object.keys(testSuite).filter(testCase => skipInternalMember(testSuite, testCase)).length + }); + } else { + print(RED + "This test doesn't have a duration: " + testSuiteName + "\n" + JSON.stringify(testSuite) + RESET); + } + }, + testCase: function(options, state, testCase, testCaseName) { + }, + endTestSuite: function(options, state, testSuite, testSuiteName) { + }, + endTestRun: function(options, state, testRun, testRunName) { + sortedByDuration.sort(function(a, b) { + return a.duration - b.duration; + }); + let results = {}; + for (let i = sortedByDuration.length - 1; (i >= 0) && (i > sortedByDuration.length - 11); i --) { + let key = " - " + fancyTimeFormat(sortedByDuration[i].duration / 1000) + " - " + + sortedByDuration[i].count + " - " + + sortedByDuration[i].testName.replace('/\\/g', '/'); + let testCases = []; + let thisTestSuite = sortedByDuration[i]; + for (let testName in thisTestSuite.test) { + if (skipInternalMember(thisTestSuite.test, testName)) { + continue; + } + let test = thisTestSuite.test[testName]; + let duration = 0; + if (test.hasOwnProperty('duration')) { + duration += test.duration; + } + if (test.hasOwnProperty('setUpDuration')) { + duration += test.setUpDuration; + } + if (test.hasOwnProperty('tearDownDuration')) { + duration += test.tearDownDuration; + } + testCases.push({ + testName: testName, + duration: duration + }); + } + testCases.sort(function(a, b) { + return a.duration - b.duration; + }); + + let statistics = []; + for (let j = Object.keys(testCases).length - 1; (j >= 0) && (j > Object.keys(testCases).length - 11); j --) { + statistics.push(fancyTimeFormat(testCases[j].duration / 1000) + " - " + testCases[j].testName); + } + results[key] = { + 'processStatistics': pu.sumarizeStats(thisTestSuite.test['processStats']), + 'stats': statistics + }; + + } + testRunStatistics += yaml.safeDump(results); + sortedByDuration = []; + } + }); + print(testRunStatistics); +} + +function locateLongSetupTeardown(options, results) { + let testRunStatistics = " Setup | Run | tests | setupAll | suite name\n"; + let sortedByDuration = []; + let failedStates = {}; + + iterateTestResults(options, results, failedStates, { + testRun: function(options, state, testRun, testRunName) { + }, + testSuite: function(options, state, testSuite, testSuiteName) { + let setupAllDuration = 0; + if (testSuite.hasOwnProperty('setUpAllDuration')) { + Object.keys(testSuite.setUpAllDuration).forEach(testName => { + setupAllDuration += testSuite.setUpAllDuration[testName]; + }); + } + let hasSetupAll = setupAllDuration !== 0; + + if (testSuite.hasOwnProperty('totalSetUp') && + testSuite.hasOwnProperty('totalTearDown')) { + sortedByDuration.push({ + testName: testSuiteName, + setupTearDown: testSuite.totalSetUp + testSuite.totalTearDown, + duration: testSuite.duration, + hasSetupAll: hasSetupAll, + count: Object.keys(testSuite).filter(testCase => ! skipInternalMember(testSuite, testCase)).length, + }); + } else { + print(RED + "This test doesn't have a duration: " + testSuiteName + "\n" + JSON.stringify(testSuite) + RESET); + } + }, + testCase: function(options, state, testCase, testCaseName) { + }, + endTestSuite: function(options, state, testSuite, testSuiteName) { + }, + endTestRun: function(options, state, testRun, testRunName) { + sortedByDuration.sort(function(a, b) { + return a.setupTearDown - b.setupTearDown; + }); + let results = []; + for (let i = sortedByDuration.length - 1; (i >= 0) && (i > sortedByDuration.length - 11); i --) { + let key = " " + + fancyTimeFormat(sortedByDuration[i].setupTearDown / 1000) + " | " + + fancyTimeFormat(sortedByDuration[i].duration / 1000) + " | " + + sortedByDuration[i].count + " | " + + sortedByDuration[i].hasSetupAll + " | " + + sortedByDuration[i].testName.replace('/\\/g', '/'); + results.push(key); + } + testRunStatistics += yaml.safeDump(results); + sortedByDuration = []; + } + }); + print(testRunStatistics); +} + + + +function locateShortServerLife(options, results) { + let rc = true; + let testRunStatistics = ""; + let sortedByDuration = []; + + let failedStates = { + state: true, + failCount: 0, + thisFailedTestCount: 0, + runFailedTestCount: 0, + currentSuccess: true, + failedTests: {}, + thisFailedTests: [] + }; + iterateTestResults(options, results, failedStates, { + testRun: function(options, state, testRun, testRunName) { + if (testRun.hasOwnProperty('startupTime') && + testRun.hasOwnProperty('testDuration') && + testRun.hasOwnProperty('shutdownTime')) { + let startup = testRun['startupTime']; + let testDuration = testRun['testDuration']; + let shutdown = testRun['shutdownTime']; + let color = GREEN; + let OK = "YES"; + if (((startup + shutdown) * 10) > testDuration) { + color = RED; + OK = "NOP"; + rc = false; + } + + testRunStatistics += `${color}[${OK}] ${testRunName} - startup [${startup}] => run [${testDuration}] => shutdown [${shutdown}]${RESET}`; + } else { + testRunStatistics += `${YELLOW}${testRunName} doesn't have server start statistics${RESET}`; + } + } + }); + print(testRunStatistics); + return rc; +} + +// ////////////////////////////////////////////////////////////////////////////// +// @brief simple status representations +// ////////////////////////////////////////////////////////////////////////////// + +function writeDefaultReports(options, testSuites) { + fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', "false", true); + fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', "true", true); + let testFailureText = 'testfailures.txt'; + if (options.hasOwnProperty('testFailureText')) { + testFailureText = options.testFailureText; + } + fs.write(fs.join(options.testOutputDirectory, testFailureText), + "Incomplete testrun with these testsuites: '" + testSuites + + "'\nand these options: " + JSON.stringify(options) + "\n"); + +} + +function writeReports(options, results) { + fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', String(results.status), true); + fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', String(results.crashed), true); +} + +function dumpAllResults(options, results) { + let j; + + try { + j = JSON.stringify(results); + } catch (err) { + j = inspect(results); + } + + fs.write(options.testOutputDirectory + '/UNITTEST_RESULT.json', j, true); +} + + +function addFailRunsMessage(testcase, message) { + failedRuns[testcase] = message; +} + +function yamlDumpResults(options, results) { + try { + print(yaml.safeDump(JSON.parse(JSON.stringify(results)))); + } catch (err) { + print(RED + 'cannot dump results: ' + String(err) + RESET); + print(RED + require('internal').inspect(results) + RESET); + } +} + +exports.gatherStatus = gatherStatus; +exports.gatherFailed = gatherFailed; +exports.yamlDumpResults = yamlDumpResults; +exports.addFailRunsMessage = addFailRunsMessage; +exports.dumpAllResults = dumpAllResults; +exports.writeDefaultReports = writeDefaultReports; +exports.writeReports = writeReports; + +exports.analyze = { + unitTestPrettyPrintResults: unitTestPrettyPrintResults, + saveToJunitXML: saveToJunitXML, + locateLongRunning: locateLongRunning, + locateShortServerLife: locateShortServerLife, + locateLongSetupTeardown: locateLongSetupTeardown, + yaml: yamlDumpResults +}; diff --git a/js/client/modules/@arangodb/test-utils.js b/js/client/modules/@arangodb/test-utils.js index b2e6df7a99..dce321d07c 100755 --- a/js/client/modules/@arangodb/test-utils.js +++ b/js/client/modules/@arangodb/test-utils.js @@ -103,7 +103,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS let env = {}; let customInstanceInfos = {}; let healthCheck = function () {return true;}; - + let beforeStart = time(); if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('healthCheck')) { healthCheck = startStopHandlers.healthCheck; } @@ -162,7 +162,11 @@ function performTests (options, testList, testname, runFn, serverOptions, startS _.defaults(env, customInstanceInfos.postStart.env); } - let results = { shutdown: true }; + let testrunStart = time(); + let results = { + shutdown: true, + startupTime: testrunStart - beforeStart + }; let continueTesting = true; let serverDead = false; let count = 0; @@ -234,6 +238,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS continue; } else if (reply.hasOwnProperty('status')) { results[te] = reply; + results[te]['processStats'] = pu.getDeltaProcessStats(instanceInfo); if (results[te].status === false) { options.cleanup = false; @@ -399,7 +404,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS results.status = true; print(RED + 'No testcase matched the filter.' + RESET); } - + let shutDownStart = time(); + results['testDuration'] = shutDownStart - testrunStart; print(Date() + ' Shutting down...'); if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('preStop')) { customInstanceInfos['preStop'] = startStopHandlers.preStop(options, @@ -426,6 +432,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS } results.shutdown = results.shutdown && pu.shutdownInstance(instanceInfo, clonedOpts, forceTerminate); + loadClusterTestStabilityInfo(results, instanceInfo); + if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('postStop')) { customInstanceInfos['postStop'] = startStopHandlers.postStop(options, serverOptions, @@ -437,6 +445,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS results.setup.message = 'custom postStop failed!'; } } + results['shutdownTime'] = time() - shutDownStart; + print('done.'); return results; @@ -447,11 +457,6 @@ function performTests (options, testList, testname, runFn, serverOptions, startS // ////////////////////////////////////////////////////////////////////////////// function filterTestcaseByOptions (testname, options, whichFilter) { - if (options.skipTest(testname, options)) { - whichFilter.filter = 'blacklist'; - return false; - } - // These filters require a proper setup, Even if we filter by testcase: if ((testname.indexOf('-mmfiles') !== -1) && options.storageEngine === 'rocksdb') { whichFilter.filter = 'skip when running as rocksdb'; @@ -621,7 +626,7 @@ function scanTestPaths (paths, options) { } return rc; }); - if (filteredTestCases.length === 0) { + if ((filteredTestCases.length === 0) && (options.extremeVerbosity !== 'silence')) { print("No testcase matched the filter: " + JSON.stringify(allFiltered)); return []; } @@ -629,6 +634,41 @@ function scanTestPaths (paths, options) { return allTestCases; } + +function loadClusterTestStabilityInfo(results, instanceInfo){ + try { + if (instanceInfo.hasOwnProperty('clusterHealthMonitorFile')) { + let status = true; + let slow = []; + let buf = fs.readBuffer(instanceInfo.clusterHealthMonitorFile); + let lineStart = 0; + let maxBuffer = buf.length; + + for (let j = 0; j < maxBuffer; j++) { + if (buf[j] === 10) { // \n + const line = buf.asciiSlice(lineStart, j); + lineStart = j + 1; + let val = JSON.parse(line); + if (val.state === false) { + slow.push(val); + status = false; + } + } + } + if (!status) { + print('found ' + slow.length + ' slow lines!'); + results.status = false; + results.message += 'found ' + slow.length + ' slow lines!'; + } else { + print('everything fast!'); + } + } + } + catch(x) { + print(x); + } +} + // ////////////////////////////////////////////////////////////////////////////// // / @brief runs a remote unittest file using /_admin/execute // ////////////////////////////////////////////////////////////////////////////// @@ -704,7 +744,7 @@ function runThere (options, instanceInfo, file) { }; } } -runThere.info = 'runThere'; +runThere.info = 'runInArangod'; function readTestResult(path, rc, testCase) { const jsonFN = fs.join(path, 'testresult.json'); @@ -791,7 +831,7 @@ function runInArangosh (options, instanceInfo, file, addArgs) { let rc = pu.executeAndWait(pu.ARANGOSH_BIN, toArgv(args), options, 'arangosh', instanceInfo.rootDir, options.coreCheck); return readTestResult(instanceInfo.rootDir, rc, args['javascript.unit-tests']); } -runInArangosh.info = 'arangosh'; +runInArangosh.info = 'runInExternalArangosh'; // ////////////////////////////////////////////////////////////////////////////// // / @brief runs a local unittest file in the current arangosh @@ -834,7 +874,7 @@ function runInLocalArangosh (options, instanceInfo, file, addArgs) { }; } } -runInLocalArangosh.info = 'localarangosh'; +runInLocalArangosh.info = 'runInLocalArangosh'; // ////////////////////////////////////////////////////////////////////////////// // / @brief runs a unittest file using rspec @@ -977,7 +1017,7 @@ function runInRSpec (options, instanceInfo, file, addArgs) { fs.remove(tmpname); return result; } -runInRSpec.info = 'runInRSpec'; +runInRSpec.info = 'runInLocalRSpec'; exports.runThere = runThere; exports.runInArangosh = runInArangosh; diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 13be1f8850..68abe7fa10 100755 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -23,8 +23,26 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // @author Max Neunhoeffer +// @author Wilfried Goesgnes // ///////////////////////////////////////////////////////////////////////////// +const _ = require('lodash'); +const fs = require('fs'); + +const pu = require('@arangodb/process-utils'); +const rp = require('@arangodb/result-processing'); +const cu = require('@arangodb/crash-utils'); +const tu = require('@arangodb/test-utils'); +const internal = require('internal'); +const platform = internal.platform; + +const BLUE = internal.COLORS.COLOR_BLUE; +const CYAN = internal.COLORS.COLOR_CYAN; +const GREEN = internal.COLORS.COLOR_GREEN; +const RED = internal.COLORS.COLOR_RED; +const RESET = internal.COLORS.COLOR_RESET; +const YELLOW = internal.COLORS.COLOR_YELLOW; + let functionsDocumentation = { 'all': 'run all tests (marked with [x])', 'find': 'searches all testcases, and eventually filters them by `--test`, ' + @@ -39,8 +57,6 @@ let optionsDocumentation = [ ' The following properties of `options` are defined:', '', ' - `testOutput`: set the output directory for testresults, defaults to `out`', - ' - `jsonReply`: if set a json is returned which the caller has to ', - ' present the user', ' - `force`: if set to true the tests are continued even if one fails', '', " - `skipLogAnalysis`: don't try to crawl the server logs", @@ -77,15 +93,17 @@ let optionsDocumentation = [ ' - `agencySupervision`: run supervision in agency', ' - `oneTestTimeout`: how long a single testsuite (.js, .rb) should run', ' - `isAsan`: doubles oneTestTimeot value if set to true (for ASAN-related builds)', - ' - `test`: path to single test to execute for "single" test target', - ' - `cleanup`: if set to true (the default), the cluster data files', - ' and logs are removed after termination of the test.', + ' - `test`: path to single test to execute for "single" test target, ', + ' or pattern to filter for other suites', + ' - `cleanup`: if set to false the data files', + ' and logs are not removed after termination of the test.', '', ' - `protocol`: the protocol to talk to the server - [tcp (default), ssl, unix]', ' - `sniff`: if we should try to launch tcpdump / windump for a testrun', ' false / true / sudo', ' - `sniffDevice`: the device tcpdump / tshark should use', ' - `sniffProgram`: specify your own programm', + '', ' - `build`: the directory containing the binaries', ' - `buildType`: Windows build type (Debug, Release), leave empty on linux', ' - `configDir`: the directory containing the config files, defaults to', @@ -94,6 +112,8 @@ let optionsDocumentation = [ ' - `dumpAgencyOnError`: if we should create an agency dump if an error occurs', ' - `prefix`: prefix for the tests in the xml reports', '', + ' - `disableClusterMonitor`: if set to false, an arangosh is started that will send', + ' keepalive requests to all cluster instances, and report on error', ' - `disableMonitor`: if set to true on windows, procdump will not be attached.', ' - `rr`: if set to true arangod instances are run with rr', ' - `exceptionFilter`: on windows you can use this to abort tests on specific exceptions', @@ -135,7 +155,7 @@ const optionsDefaults = { 'agencyWaitForSync': false, 'agencySupervision': true, 'build': '', - 'buildType': '', + 'buildType': (platform.substr(0, 3) === 'win') ? 'RelWithDebInfo':'', 'cleanup': true, 'cluster': false, 'concurrency': 3, @@ -150,7 +170,6 @@ const optionsDefaults = { 'force': true, 'getSockStat': false, 'arangosearch':true, - 'jsonReply': false, 'loopEternal': false, 'loopSleepSec': 1, 'loopSleepWhen': 1, @@ -181,6 +200,7 @@ const optionsDefaults = { 'storageEngine': 'rocksdb', 'test': undefined, 'testBuckets': undefined, + 'testOutputDirectory': 'out/', 'useReconnect': true, 'username': 'root', 'valgrind': false, @@ -193,266 +213,14 @@ const optionsDefaults = { 'testFailureText': 'testfailures.txt', 'testCase': undefined, 'disableMonitor': false, + 'disableClusterMonitor': true, 'sleepBeforeStart' : 0, }; -const _ = require('lodash'); -const fs = require('fs'); -const yaml = require('js-yaml'); +let globalStatus = true; -const pu = require('@arangodb/process-utils'); -const cu = require('@arangodb/crash-utils'); -const tu = require('@arangodb/test-utils'); - -const BLUE = require('internal').COLORS.COLOR_BLUE; -const CYAN = require('internal').COLORS.COLOR_CYAN; -const GREEN = require('internal').COLORS.COLOR_GREEN; -const RED = require('internal').COLORS.COLOR_RED; -const RESET = require('internal').COLORS.COLOR_RESET; -const YELLOW = require('internal').COLORS.COLOR_YELLOW; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief test functions for all -// ////////////////////////////////////////////////////////////////////////////// - -let failedRuns = { -}; - -let allTests = [ -]; - -// ///////////////////////////////////////////////////////////////////////////// -// blacklisting -// ///////////////////////////////////////////////////////////////////////////// - -let useBlacklist = false; - -let blacklistTests = {}; - -function skipTest(type, name) { - let ntype = type.toUpperCase(); - return useBlacklist && !!blacklistTests[ntype + ":" + name]; -} - -function loadBlacklist(name) { - let content = fs.read("BLACKLIST"); - let a = _.filter( - _.map(content.split("\n"), - function(x) {return x.trim();}), - function(x) {return x.length > 0 && x[0] !== '#';}); - - for (let i = 0; i < a.length; ++i) { - blacklistTests[a[i]] = true; - } - - useBlacklist = true; -} - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief TEST: all -// ////////////////////////////////////////////////////////////////////////////// - -let testFuncs = { - 'all': function () {} -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief internal members of the results -// ////////////////////////////////////////////////////////////////////////////// - -const internalMembers = [ - 'code', - 'error', - 'status', - 'duration', - 'failed', - 'total', - 'crashed', - 'ok', - 'message', - 'suiteName' -]; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief pretty prints the result -// ////////////////////////////////////////////////////////////////////////////// - -function testCaseMessage (test) { - if (typeof test.message === 'object' && test.message.hasOwnProperty('body')) { - return test.message.body; - } else { - return test.message; - } -} - -function unitTestPrettyPrintResults (res, testOutputDirectory, options) { - function skipInternalMember (r, a) { - return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1; - } - print(YELLOW + '================================================================================'); - print('TEST RESULTS'); - print('================================================================================\n' + RESET); - - let failedSuite = 0; - let failedTests = 0; - - let onlyFailedMessages = ''; - let failedMessages = ''; - let SuccessMessages = ''; - let bucketName = ""; - if (options.testBuckets) { - let n = options.testBuckets.split('/'); - bucketName = "_" + n[1]; - } - try { - /* jshint forin: false */ - for (let testrunName in res) { - if (skipInternalMember(res, testrunName)) { - continue; - } - - let testrun = res[testrunName]; - - let successCases = {}; - let failedCases = {}; - let isSuccess = true; - - for (let testName in testrun) { - if (skipInternalMember(testrun, testName)) { - continue; - } - - let test = testrun[testName]; - - if (test.status) { - successCases[testName] = test; - } else { - isSuccess = false; - ++failedSuite; - - if (test.hasOwnProperty('message')) { - ++failedTests; - failedCases[testName] = { - test: testCaseMessage(test) - }; - } else { - let fails = failedCases[testName] = {}; - - for (let oneName in test) { - if (skipInternalMember(test, oneName)) { - continue; - } - - let oneTest = test[oneName]; - - if (!oneTest.status) { - ++failedTests; - fails[oneName] = testCaseMessage(oneTest); - } - } - } - } - } - - if (isSuccess) { - SuccessMessages += '* Test "' + testrunName + bucketName + '"\n'; - - for (let name in successCases) { - if (!successCases.hasOwnProperty(name)) { - continue; - } - - let details = successCases[name]; - - if (details.skipped) { - SuccessMessages += YELLOW + ' [SKIPPED] ' + name + RESET + '\n'; - } else { - SuccessMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n'; - } - } - } else { - let m = '* Test "' + testrunName + bucketName + '"\n'; - onlyFailedMessages += m; - failedMessages += m; - - for (let name in successCases) { - if (!successCases.hasOwnProperty(name)) { - continue; - } - - let details = successCases[name]; - - if (details.skipped) { - failedMessages += YELLOW + ' [SKIPPED] ' + name + RESET + '\n'; - onlyFailedMessages += ' [SKIPPED] ' + name + '\n'; - } else { - failedMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n'; - } - } - - for (let name in failedCases) { - if (!failedCases.hasOwnProperty(name)) { - continue; - } - - failedMessages += RED + ' [FAILED] ' + name + RESET + '\n\n'; - onlyFailedMessages += ' [FAILED] ' + name + '\n\n'; - - let details = failedCases[name]; - - let count = 0; - for (let one in details) { - if (!details.hasOwnProperty(one)) { - continue; - } - - if (count > 0) { - failedMessages += '\n'; - onlyFailedMessages += '\n'; - } - failedMessages += RED + ' "' + one + '" failed: ' + details[one] + RESET + '\n\n'; - onlyFailedMessages += ' "' + one + '" failed: ' + details[one] + '\n\n'; - count++; - } - } - } - } - print(SuccessMessages); - print(failedMessages); - /* jshint forin: true */ - - let color = (!res.crashed && res.status === true) ? GREEN : RED; - let crashText = ''; - let crashedText = ''; - if (res.crashed === true) { - for (let failed in failedRuns) { - crashedText += ' [' + failed + '] : ' + failedRuns[failed].replace(/^/mg, ' '); - } - crashedText += "\nMarking crashy!"; - crashText = RED + crashedText + RESET; - } - print('\n' + color + '* Overall state: ' + ((res.status === true) ? 'Success' : 'Fail') + RESET + crashText); - - let failText = ''; - if (res.status !== true) { - failText = ' Suites failed: ' + failedSuite + ' Tests Failed: ' + failedTests; - print(color + failText + RESET); - } - - failedMessages = onlyFailedMessages; - if (crashedText !== '') { - failedMessages += '\n' + crashedText; - } - if (cu.GDB_OUTPUT !== '' || failText !== '') { - failedMessages += '\n\n' + cu.GDB_OUTPUT + failText + '\n'; - } - fs.write(testOutputDirectory + options.testFailureText, failedMessages); - } catch (x) { - print('exception caught while pretty printing result: '); - print(x.message); - print(JSON.stringify(res)); - } -} +let allTests = []; +let testFuncs = {}; // ////////////////////////////////////////////////////////////////////////////// // / @brief print usage information @@ -501,7 +269,9 @@ function findTestCases(options) { let allTestFiles = {}; for (let testSuiteName in allTestPaths) { var myList = []; - let files = tu.scanTestPaths(allTestPaths[testSuiteName], options); + var _opts = _.clone(options); + _opts.extremeVerbosity = 'silence'; + let files = tu.scanTestPaths(allTestPaths[testSuiteName], _opts); if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) { for (let j = 0; j < files.length; j++) { let foo = {}; @@ -557,7 +327,6 @@ function findTest(options) { } } - function autoTest(options) { if (!options.hasOwnProperty('test') || (typeof (options.test) === 'undefined')) { return { @@ -620,16 +389,13 @@ function loadTestSuites () { throw x; } } + testFuncs['all'] = allTests; testFuncs['find'] = findTest; testFuncs['auto'] = autoTest; } -let globalStatus = true; - -function iterateTests(cases, options, jsonReply) { - // tests to run +function translateTestList(cases) { let caselist = []; - const expandWildcard = ( name ) => { if (!name.endsWith('*')) { return name; @@ -640,29 +406,18 @@ function iterateTests(cases, options, jsonReply) { for (let n = 0; n < cases.length; ++n) { let splitted = expandWildcard(cases[n]).split(/[,;|]/); - for (let m = 0; m < splitted.length; ++m) { let which = splitted[m]; - if (which === 'all') { - caselist = caselist.concat(allTests); - } else if (testFuncs.hasOwnProperty(which)) { + if (testFuncs.hasOwnProperty(which)) { caselist.push(which); } else { print('Unknown test "' + which + '"\nKnown tests are: ' + Object.keys(testFuncs).sort().join(', ')); - - return { - status: false - }; + throw new Error("USAGE ERROR"); } } } - - let results = {}; - let cleanup = true; - - // real ugly hack. there are some suites which are just placeholders - // for other suites + // Expand meta tests like ldap, all caselist = (function() { let flattened = []; for (let n = 0; n < caselist.length; ++n) { @@ -675,12 +430,27 @@ function iterateTests(cases, options, jsonReply) { } return flattened; })(); + if (cases === undefined || cases.length === 0) { + printUsage(); + + print('\nFATAL: "which" is undefined\n'); + throw new Error("USAGE ERROR"); + } + return caselist; +} +function iterateTests(cases, options) { + // tests to run + let caselist = []; + + let results = {}; + let cleanup = true; + + caselist = translateTestList(cases); // running all tests for (let n = 0; n < caselist.length; ++n) { const currentTest = caselist[n]; var localOptions = _.cloneDeep(options); - localOptions.skipTest = skipTest; let printTestName = currentTest; if (options.testBuckets) { printTestName += " - " + options.testBuckets; @@ -697,34 +467,23 @@ function iterateTests(cases, options, jsonReply) { let status = true; let shutdownSuccess = true; - if (skipTest("SUITE", currentTest)) { - result = { - failed: 0, - status: true, - crashed: false, - }; - - print(YELLOW + "[SKIPPED] " + currentTest + RESET + "\n"); - } else { - result = testFuncs[currentTest](localOptions); - // grrr...normalize structure - delete result.status; - delete result.failed; - delete result.crashed; - if (result.hasOwnProperty('shutdown')) { - shutdownSuccess = result['shutdown']; - delete result.shutdown; - } - - status = Object.values(result).every(testCase => testCase.status === true); - let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0); - if (!status) { - globalStatus = false; - } - result.failed = failed; - result.status = status; + result = testFuncs[currentTest](localOptions); + // grrr...normalize structure + delete result.status; + delete result.failed; + delete result.crashed; + if (result.hasOwnProperty('shutdown')) { + shutdownSuccess = result['shutdown']; + delete result.shutdown; } + status = rp.gatherStatus(result); + let failed = rp.gatherFailed(result); + if (!status) { + globalStatus = false; + } + result.failed = failed; + result.status = status; results[currentTest] = result; if (status && localOptions.cleanup && shutdownSuccess ) { @@ -732,11 +491,7 @@ function iterateTests(cases, options, jsonReply) { } else { cleanup = false; } - - if (pu.serverCrashed) { - failedRuns[currentTest] = pu.serverFailMessages; - pu.serverFailMessages = ""; - } + pu.aggregateFatalErrors(currentTest); } results.status = globalStatus; @@ -755,14 +510,8 @@ function iterateTests(cases, options, jsonReply) { } if (options.extremeVerbosity === true) { - try { - print(yaml.safeDump(JSON.parse(JSON.stringify(results)))); - } catch (err) { - print(RED + 'cannot dump results: ' + String(err) + RESET); - print(RED + require('internal').inspect(results) + RESET); - } + rp.yamlDumpResults(options, results); } - return results; } @@ -779,20 +528,10 @@ function unitTest (cases, options) { if (typeof options !== 'object') { options = {}; } - loadTestSuites(); + loadTestSuites(options); + // testsuites may register more defaults... _.defaults(options, optionsDefaults); - if (cases === undefined || cases.length === 0) { - printUsage(); - - print('FATAL: "which" is undefined\n'); - - return { - status: false, - crashed: false - }; - } - try { pu.setupBinaries(options.build, options.buildType, options.configDir); } @@ -808,25 +547,18 @@ function unitTest (cases, options) { }] }; } - const jsonReply = options.jsonReply; - delete options.jsonReply; - let results = iterateTests(cases, options, jsonReply); - - if (jsonReply === true) { - return results; + if ((cases.length === 1) && cases[0] === 'auto') { + return autoTest(options); } else { - return globalStatus; + return iterateTests(cases, options); } } // ///////////////////////////////////////////////////////////////////////////// // exports // ///////////////////////////////////////////////////////////////////////////// - +exports.optionsDefaults = optionsDefaults; exports.unitTest = unitTest; -exports.internalMembers = internalMembers; exports.testFuncs = testFuncs; -exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults; -exports.loadBlacklist = loadBlacklist; diff --git a/js/client/modules/@arangodb/testsuites/agency.js b/js/client/modules/@arangodb/testsuites/agency.js index ca1177cff0..28401560f5 100644 --- a/js/client/modules/@arangodb/testsuites/agency.js +++ b/js/client/modules/@arangodb/testsuites/agency.js @@ -50,7 +50,6 @@ function agency (options) { options.agency = true; options.cluster = false; - let results = tu.performTests(options, testCases, 'agency', tu.runInArangosh); options.agency = saveAgency; diff --git a/js/client/modules/@arangodb/testsuites/audit.js b/js/client/modules/@arangodb/testsuites/audit.js index ad33a59a59..ddc929f810 100644 --- a/js/client/modules/@arangodb/testsuites/audit.js +++ b/js/client/modules/@arangodb/testsuites/audit.js @@ -74,6 +74,7 @@ function auditLog(onServer) { 'server.authentication': 'true', 'server.jwt-secret': 'haxxmann', 'log.level': 'audit-authentication=info', + 'log.force-direct': true }; print(CYAN + 'Audit log server tests...' + RESET); diff --git a/js/client/modules/@arangodb/testsuites/config.js b/js/client/modules/@arangodb/testsuites/config.js index fd7a51a842..27aea556af 100644 --- a/js/client/modules/@arangodb/testsuites/config.js +++ b/js/client/modules/@arangodb/testsuites/config.js @@ -99,89 +99,79 @@ function config (options) { print('absolute config tests'); print('--------------------------------------------------------------------------------'); - if (options.skipTest('TEST', 'config.absolute')) { - print(YELLOW + "[SKIPPED] config.absolute" + RESET + "\n"); - results.absolute.skipped = true; - } else { - // we append one cleanup directory for the invoking logic... - let dummyDir = fs.join(fs.getTempPath(), 'configdummy'); - fs.makeDirectory(dummyDir); - pu.cleanupDBDirectoriesAppend(dummyDir); + // we append one cleanup directory for the invoking logic... + let dummyDir = fs.join(fs.getTempPath(), 'configdummy'); + fs.makeDirectory(dummyDir); + pu.cleanupDBDirectoriesAppend(dummyDir); - let startTime = time(); + let startTime = time(); - for (let i = 0; i < ts.length; i++) { - const test = ts[i]; - print(CYAN + 'checking "' + test + '"' + RESET); + for (let i = 0; i < ts.length; i++) { + const test = ts[i]; + print(CYAN + 'checking "' + test + '"' + RESET); - const args = { - 'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'), - 'flatCommands': ['--check-configuration'] - }; + const args = { + 'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'), + 'flatCommands': ['--check-configuration'] + }; - const run = fs.join(pu.BIN_DIR, test); + const run = fs.join(pu.BIN_DIR, test); - results.absolute[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck); + results.absolute[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck); - if (!results.absolute[test].status) { - results.absolute.status = false; - results.absolute.failed += 1; - results.failed += 1; - } - - results.absolute.total++; - - if (options.verbose) { - print('Args for [' + test + ']:'); - print(yaml.safeDump(args)); - print('Result: ' + results.absolute[test].status); - } + if (!results.absolute[test].status) { + results.absolute.status = false; + results.absolute.failed += 1; + results.failed += 1; } - results.absolute.duration = time() - startTime; + results.absolute.total++; + + if (options.verbose) { + print('Args for [' + test + ']:'); + print(yaml.safeDump(args)); + print('Result: ' + results.absolute[test].status); + } } + results.absolute.duration = time() - startTime; + print('\n--------------------------------------------------------------------------------'); print('relative config tests'); print('--------------------------------------------------------------------------------'); - if (options.skipTest('TEST', 'config.relative')) { - print(YELLOW + "[SKIPPED] config.relative" + RESET + "\n"); - results.relative.skipped = true; - } else { - let startTime = time(); + startTime = time(); - for (let i = 0; i < ts.length; i++) { - const test = ts[i]; - print(CYAN + 'checking "' + test + '"' + RESET); + for (let i = 0; i < ts.length; i++) { + const test = ts[i]; + print(CYAN + 'checking "' + test + '"' + RESET); - const args = { - 'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'), - 'flatCommands': ['--check-configuration'] - }; + const args = { + 'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'), + 'flatCommands': ['--check-configuration'] + }; - const run = fs.join(pu.BIN_DIR, test); + const run = fs.join(pu.BIN_DIR, test); - results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck); + results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck); - if (!results.relative[test].status) { - results.failed += 1; - results.relative.failed += 1; - results.relative.status = false; - } - - results.relative.total++; - - if (options.verbose) { - print('Args for (relative) [' + test + ']:'); - print(yaml.safeDump(args)); - print('Result: ' + results.relative[test].status); - } + if (!results.relative[test].status) { + results.failed += 1; + results.relative.failed += 1; + results.relative.status = false; } - results.relative.duration = time() - startTime; + results.relative.total++; + + if (options.verbose) { + print('Args for (relative) [' + test + ']:'); + print(yaml.safeDump(args)); + print('Result: ' + results.relative[test].status); + } } + results.relative.duration = time() - startTime; + print(); return results; diff --git a/js/client/modules/@arangodb/testsuites/fail.js b/js/client/modules/@arangodb/testsuites/fail.js index 91fa612324..744533f352 100644 --- a/js/client/modules/@arangodb/testsuites/fail.js +++ b/js/client/modules/@arangodb/testsuites/fail.js @@ -134,10 +134,10 @@ function success (options) { duration: 2, failed: 1, failTest: { - status: false, + status: true, total: 1, duration: 1, - message: 'this testcase will always success.' + message: 'this testcase will always succeed.' }, failSuccessTest: { status: true, diff --git a/js/common/bootstrap/modules/internal.js b/js/common/bootstrap/modules/internal.js index 0b3788a89d..31a1b2af53 100644 --- a/js/common/bootstrap/modules/internal.js +++ b/js/common/bootstrap/modules/internal.js @@ -560,6 +560,14 @@ global.DEFINE_MODULE('internal', (function () { delete global.SYS_PROCESS_JSON_FILE; } + // ////////////////////////////////////////////////////////////////////////////// + // / @brief statisticsExternal + // ////////////////////////////////////////////////////////////////////////////// + + if (global.SYS_PROCESS_STATISTICS_EXTERNAL) { + exports.statisticsExternal = global.SYS_PROCESS_STATISTICS_EXTERNAL; + delete global.SYS_PROCESS_STATISTICS_EXTERNAL; + } // ////////////////////////////////////////////////////////////////////////////// // / @brief executeExternal // ////////////////////////////////////////////////////////////////////////////// @@ -756,7 +764,18 @@ global.DEFINE_MODULE('internal', (function () { } else if (!isNaN(argv[i + 1])) { ret[option] = parseInt(argv[i + 1]); } else { - ret[option] = argv[i + 1]; + if (ret.hasOwnProperty(option)) { + if (Array.isArray(ret[option])) { + ret[option].push(argv[i + 1]); + } else { + ret[option] = [ + ret[option], + argv[i + 1] + ]; + } + } else { + ret[option] = argv[i + 1]; + } } } diff --git a/js/common/modules/jsunity.js b/js/common/modules/jsunity.js index ee46bf1294..5a69f59ca5 100644 --- a/js/common/modules/jsunity.js +++ b/js/common/modules/jsunity.js @@ -44,6 +44,8 @@ var TEARDOWNS = 0; var TOTALSETUPS = 0; var TOTALTEARDOWNS = 0; + + var jsUnity = require('./jsunity/jsunity').jsUnity; var STARTTEST = 0.0; var ENDTEST = 0.0; @@ -141,6 +143,15 @@ jsUnity.results.end = function (passed, failed, duration) { } }; +jsUnity.results.beginSetUpAll = function(index) { + SETUPS = jsUnity.env.getDate(); +}; + +jsUnity.results.endSetUpAll = function(index) { + RESULTS.setUpAllDuration = jsUnity.env.getDate() - SETUPS; + TOTALSETUPS += RESULTS.setUpAllDuration; +}; + jsUnity.results.beginSetUp = function(index, testName) { if (testCount === 0) { @@ -171,6 +182,15 @@ jsUnity.results.endTeardown = function(index, testName) { TOTALTEARDOWNS += RESULTS[testName].tearDownDuration; }; +jsUnity.results.beginTeardownAll = function(index) { + TEARDOWNS = jsUnity.env.getDate(); +}; + +jsUnity.results.endTeardownAll = function(index) { + RESULTS.teardownAllDuration = jsUnity.env.getDate() - TEARDOWNS; + TOTALTEARDOWNS += RESULTS.teardownAllDuration; +}; + // ////////////////////////////////////////////////////////////////////////////// // / @brief runs a test with context // ////////////////////////////////////////////////////////////////////////////// @@ -255,11 +275,15 @@ function Run (testsuite) { PASSED += result.passed; FAILED += result.failed; DURATION += result.duration; - - + let duplicates = []; for (var attrname in RESULTS) { - if (RESULTS.hasOwnProperty(attrname)) { + if (typeof(RESULTS[attrname]) === 'number') { + if (!COMPLETE.hasOwnProperty(attrname)) { + COMPLETE[attrname] = { }; + } + COMPLETE[attrname][suite.suiteName] = RESULTS[attrname]; + } else if (RESULTS.hasOwnProperty(attrname)) { if (COMPLETE.hasOwnProperty(attrname)) { print("Duplicate testsuite '" + attrname + "' - already have: " + JSON.stringify(COMPLETE[attrname]) + ""); duplicates.push(attrname); diff --git a/js/common/modules/jsunity/jsunity.js b/js/common/modules/jsunity/jsunity.js index e4c26536f5..487765c14d 100644 --- a/js/common/modules/jsunity/jsunity.js +++ b/js/common/modules/jsunity/jsunity.js @@ -378,6 +378,10 @@ var jsUnity = exports.jsUnity = (function () { jsUnity.log.info(plural(total, "test") + " found"); }, + beginSetUpAll: function(index, testName) {}, + + endSetUpAll: function(index, testName) {}, + beginSetUp: function(index, testName) {}, endSetUp: function(index, testName) {}, @@ -399,6 +403,10 @@ var jsUnity = exports.jsUnity = (function () { endTeardown: function(index, testName) {}, + beginTeardownAll: function(index, testName) {}, + + endTeardownAll: function(index, testName) {}, + end: function (passed, failed, duration) { jsUnity.log.info(plural(passed, "test") + " passed"); jsUnity.log.info(plural(failed, "test") + " failed"); @@ -507,7 +515,9 @@ var jsUnity = exports.jsUnity = (function () { var runSuite; try { + this.results.beginSetUpAll(suite.scope); setUpAll(suite.suiteName); + this.results.endSetUpAll(suite.scope); runSuite = true; } catch (setUpAllError) { runSuite = false; @@ -583,7 +593,9 @@ var jsUnity = exports.jsUnity = (function () { } try { + this.results.beginTeardownAll(suite.scope); tearDownAll(suite.suiteName); + this.results.endTeardownAll(suite.scope); } catch (tearDownAllError) { if (tearDownAllError.stack !== undefined) { this.results.fail(0, suite.suiteName, diff --git a/lib/Basics/process-utils.cpp b/lib/Basics/process-utils.cpp index 61019a8f38..0552a6bd63 100644 --- a/lib/Basics/process-utils.cpp +++ b/lib/Basics/process-utils.cpp @@ -173,6 +173,18 @@ ExternalProcess::~ExternalProcess() { ExternalProcessStatus::ExternalProcessStatus() : _status(TRI_EXT_NOT_STARTED), _exitStatus(0), _errorMessage() {} +static ExternalProcess* TRI_LookupSpawnedProcess(TRI_pid_t pid) { + { + MUTEX_LOCKER(mutexLocker, ExternalProcessesLock); + auto found = std::find_if(ExternalProcesses.begin(), ExternalProcesses.end(), + [pid](const ExternalProcess * m) -> bool { return m->_pid == pid; }); + if (found != ExternalProcesses.end()) { + return *found; + } + } + return nullptr; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief creates pipe pair //////////////////////////////////////////////////////////////////////////////// @@ -655,13 +667,14 @@ static time_t _FileTime_to_POSIX(FILETIME* ft) { return (ts - 116444736000000000) / 10000000; } -ProcessInfo TRI_ProcessInfoSelf() { +ProcessInfo TRI_ProcessInfoH(HANDLE processHandle, TRI_pid_t pid) { ProcessInfo result; + PROCESS_MEMORY_COUNTERS_EX pmc; pmc.cb = sizeof(PROCESS_MEMORY_COUNTERS_EX); // compiler warning wird in kauf genommen!c // http://msdn.microsoft.com/en-us/library/windows/desktop/ms684874(v=vs.85).aspx - if (GetProcessMemoryInfo(GetCurrentProcess(), (PPROCESS_MEMORY_COUNTERS)&pmc, pmc.cb)) { + if (GetProcessMemoryInfo(processHandle, (PPROCESS_MEMORY_COUNTERS)&pmc, pmc.cb)) { result._majorPageFaults = pmc.PageFaultCount; // there is not any corresponce to minflt in linux result._minorPageFaults = 0; @@ -686,7 +699,7 @@ ProcessInfo TRI_ProcessInfoSelf() { } /// computing times FILETIME creationTime, exitTime, kernelTime, userTime; - if (GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime, &kernelTime, &userTime)) { + if (GetProcessTimes(processHandle, &creationTime, &exitTime, &kernelTime, &userTime)) { // see remarks in // http://msdn.microsoft.com/en-us/library/windows/desktop/ms683223(v=vs.85).aspx // value in seconds @@ -697,8 +710,7 @@ ProcessInfo TRI_ProcessInfoSelf() { // the function '_FileTime_to_POSIX' should be called } /// computing number of threads - DWORD myPID = GetCurrentProcessId(); - HANDLE snapShot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, myPID); + HANDLE snapShot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, pid); if (snapShot != INVALID_HANDLE_VALUE) { THREADENTRY32 te32; @@ -706,16 +718,35 @@ ProcessInfo TRI_ProcessInfoSelf() { if (Thread32First(snapShot, &te32)) { result._numberThreads++; while (Thread32Next(snapShot, &te32)) { - if (te32.th32OwnerProcessID == myPID) { + if (te32.th32OwnerProcessID == pid) { result._numberThreads++; } } } + else { + LOG_TOPIC("66667", ERR, arangodb::Logger::FIXME) << "failed to acquire thread from snapshot - " << GetLastError(); + } CloseHandle(snapShot); } + else { + LOG_TOPIC("66668", ERR, arangodb::Logger::FIXME) << "failed to acquire process threads count - " << GetLastError(); + } return result; } + +ProcessInfo TRI_ProcessInfoSelf() { + return TRI_ProcessInfoH(GetCurrentProcess(), GetCurrentProcessId()); +} + +ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) { + auto external = TRI_LookupSpawnedProcess(pid); + if (external != nullptr) { + return TRI_ProcessInfoH(external->_process, pid); + } + return {}; +} + #endif //////////////////////////////////////////////////////////////////////////////// @@ -882,7 +913,7 @@ ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) { } #else - +#ifndef _WIN32 ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) { ProcessInfo result; @@ -890,7 +921,7 @@ ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) { return result; } - +#endif #endif //////////////////////////////////////////////////////////////////////////////// @@ -983,17 +1014,7 @@ ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait, uint32 status._status = TRI_EXT_NOT_FOUND; status._exitStatus = 0; - ExternalProcess* external = nullptr; - { - MUTEX_LOCKER(mutexLocker, ExternalProcessesLock); - - for (auto& it : ExternalProcesses) { - if (it->_pid == pid._pid) { - external = it; - break; - } - } - } + auto external = TRI_LookupSpawnedProcess(pid._pid); if (external == nullptr) { status._errorMessage = diff --git a/lib/Basics/threads-posix.h b/lib/Basics/threads-posix.h index 6a01c5bbb8..5808403b0c 100644 --- a/lib/Basics/threads-posix.h +++ b/lib/Basics/threads-posix.h @@ -38,6 +38,8 @@ #define TRI_pid_t pid_t +#define TRI_vpack_pid_t int + //////////////////////////////////////////////////////////////////////////////// /// @brief thread library identifier //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/Basics/threads-win32.h b/lib/Basics/threads-win32.h index a79eeb2ae2..bde5b11e13 100644 --- a/lib/Basics/threads-win32.h +++ b/lib/Basics/threads-win32.h @@ -31,7 +31,9 @@ /// @brief process identifier //////////////////////////////////////////////////////////////////////////////// -#define TRI_pid_t int +#define TRI_pid_t DWORD + +#define TRI_vpack_pid_t int //////////////////////////////////////////////////////////////////////////////// /// @brief thread identifier diff --git a/lib/V8/v8-utils.cpp b/lib/V8/v8-utils.cpp index 116ae57b6e..94c6cb66d9 100644 --- a/lib/V8/v8-utils.cpp +++ b/lib/V8/v8-utils.cpp @@ -2821,13 +2821,12 @@ static void JS_Output(v8::FunctionCallbackInfo const& args) { /// @END_EXAMPLE_ARANGOSH_OUTPUT //////////////////////////////////////////////////////////////////////////////// -static void JS_ProcessStatistics(v8::FunctionCallbackInfo const& args) { +static void ProcessStatisticsToV8(v8::FunctionCallbackInfo const& args, ProcessInfo const& info) { TRI_V8_TRY_CATCH_BEGIN(isolate) v8::HandleScope scope(isolate); v8::Handle result = v8::Object::New(isolate); - ProcessInfo info = TRI_ProcessInfoSelf(); double rss = (double)info._residentSize; double rssp = 0; @@ -2855,6 +2854,36 @@ static void JS_ProcessStatistics(v8::FunctionCallbackInfo const& args TRI_V8_TRY_CATCH_END } +static void JS_ProcessStatisticsSelf(v8::FunctionCallbackInfo const& args) { + ProcessStatisticsToV8(args, TRI_ProcessInfoSelf()); +} + +static void JS_ProcessStatisticsExternal(v8::FunctionCallbackInfo const& args) { + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + if (args.Length() != 1) { + TRI_V8_THROW_EXCEPTION_USAGE( + "statisticsExternal()"); + } + + auto& server = application_features::ApplicationServer::server(); + V8SecurityFeature& v8security = server.getFeature(); + + if (v8security.isInternalModuleHardened(isolate)) { + TRI_V8_THROW_EXCEPTION_MESSAGE( + TRI_ERROR_FORBIDDEN, + "not allowed to execute or modify state of external processes"); + } + + ExternalId pid; + + pid._pid = static_cast(TRI_ObjectToUInt64(isolate, args[0], true)); + + ProcessStatisticsToV8(args, TRI_ProcessInfo(pid._pid)); + TRI_V8_TRY_CATCH_END +} + static void JS_GetPid(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_BEGIN(isolate) v8::HandleScope scope(isolate); @@ -4355,11 +4384,7 @@ static void JS_StatusExternal(v8::FunctionCallbackInfo const& args) { ExternalId pid; -#ifndef _WIN32 pid._pid = static_cast(TRI_ObjectToUInt64(isolate, args[0], true)); -#else - pid._pid = static_cast(TRI_ObjectToUInt64(isolate, args[0], true)); -#endif bool wait = false; if (args.Length() >= 2) { wait = TRI_ObjectToBoolean(isolate, args[1]); @@ -4550,6 +4575,11 @@ static void JS_KillExternal(v8::FunctionCallbackInfo const& args) { #else pid._pid = static_cast(TRI_ObjectToUInt64(isolate, args[0], true)); #endif + if (pid._pid == 0) { + TRI_V8_THROW_EXCEPTION_MESSAGE( + TRI_ERROR_FORBIDDEN, + "not allowed to kill 0"); + } // return the result v8::Handle result = v8::Object::New(isolate); @@ -5540,6 +5570,11 @@ void TRI_InitV8Utils(v8::Isolate* isolate, v8::Handle context, TRI_AddGlobalFunctionVocbase( isolate, TRI_V8_ASCII_STRING(isolate, "SYS_SPLIT_WORDS_ICU"), JS_SplitWordlist); + TRI_AddGlobalFunctionVocbase(isolate, + TRI_V8_ASCII_STRING(isolate, + "SYS_PROCESS_STATISTICS_EXTERNAL"), + JS_ProcessStatisticsExternal); + TRI_AddGlobalFunctionVocbase(isolate, TRI_V8_ASCII_STRING(isolate, "SYS_GET_EXTERNAL_SPAWNED"), @@ -5578,7 +5613,7 @@ void TRI_InitV8Utils(v8::Isolate* isolate, v8::Handle context, TRI_AddGlobalFunctionVocbase(isolate, TRI_V8_ASCII_STRING(isolate, "SYS_PROCESS_STATISTICS"), - JS_ProcessStatistics); + JS_ProcessStatisticsSelf); TRI_AddGlobalFunctionVocbase(isolate, TRI_V8_ASCII_STRING(isolate, "SYS_GET_PID"), JS_GetPid); TRI_AddGlobalFunctionVocbase(isolate, TRI_V8_ASCII_STRING(isolate, "SYS_RAND"), JS_Rand); diff --git a/scripts/examine_results.js b/scripts/examine_results.js new file mode 100755 index 0000000000..93bc0dcab3 --- /dev/null +++ b/scripts/examine_results.js @@ -0,0 +1,92 @@ +#!bin/arangosh --javascript.execute +/* jshint globalstrict:false, unused:false */ +/* global print, start_pretty_print, ARGUMENTS */ +'use strict'; +const _ = require('lodash'); +const fs = require('fs'); +const internal = require('internal'); +const rp = require('@arangodb/result-processing'); + +const optionsDefaults = require('@arangodb/testing').optionsDefaults; + +/* Constants: */ +const BLUE = internal.COLORS.COLOR_BLUE; +const CYAN = internal.COLORS.COLOR_CYAN; +const GREEN = internal.COLORS.COLOR_GREEN; +const RED = internal.COLORS.COLOR_RED; +const RESET = internal.COLORS.COLOR_RESET; +const YELLOW = internal.COLORS.COLOR_YELLOW; + +function main (argv) { + start_pretty_print(); + + let analyzers = []; // e.g all, http_server, recovery, ... + let options = {}; + + while (argv.length >= 1) { + if (argv[0].slice(0, 1) === '-') { // break parsing if we hit some -option + break; + } + analyzers = _.merge(analyzers, argv[0].split(',')); + + argv = argv.slice(1); // and remove first arg (c++:pop_front/bash:shift) + } + + if (argv.length >= 1) { + try { + options = internal.parseArgv(argv, 0); + } catch (x) { + print('failed to parse the options: ' + x.message + '\n' + String(x.stack)); + print('argv: ', argv); + throw x; + } + } + + if (!options.hasOwnProperty('readFile')) { + print("need a file to load!"); + process.exit(1); + } + let readFile; + if (Array.isArray(options.readFile)) { + readFile = options.readFile; + } else { + readFile = [ options.readFile ]; + } + let rc = readFile.forEach(file => { + let ret = true; + let resultsDump; + try { + resultsDump = fs.read(file); + } catch (ex) { + print(RED + "File not found [" + file + "]: " + ex.message + RESET + "\n"); + return false; + } + + let results; + try { + results = JSON.parse(resultsDump); + } + catch (ex) { + print(RED + "Failed to parse " + options.readFile + " - " + ex.message + RESET + "\n"); + return false; + } + _.defaults(options, optionsDefaults); + try { + print(YELLOW + "Analyzing: " + file + RESET); + ret = ret && analyzers.forEach(function(which) { + rp.analyze[which](options, results); + }); + } catch (ex) { + print("Failed to analyze [" + file + "]: " + ex.message + RESET + "\n"); + return false; + } + return ret; + }); + return rc; +} + +let result = main(ARGUMENTS); + +if (!result) { + process.exit(1); +}