Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fixed version of arangosh result evaluation refactoring - properly count objects (#10080)

This commit is contained in:
Wilfried Goesgens 2019-09-26 12:07:52 +02:00 committed by Jan
parent e89d72d02c
commit 636b2e5d5f
20 changed files with 1498 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README_maintainers.md
View File

@ -823,3 +823,21 @@ Choose the `Npcap Loopback Adapter` number - 1:
--sniffProgram c:/Programm Files/wireshark/tshark.exe
You can later on use Wireshark to inpsect the capture files.
### Evaluating json test reports from previous testruns
All test results of testruns are dumped to a json file named `UNITTEST_RESULT.json` which can be used
for later analyzing of timings etc.
Currently available analyzers are:
- unitTestPrettyPrintResults - Prints a pretty summary and writes an ASCII representation into `out/testfailures.txt` (if any errors)
- saveToJunitXML - saves jUnit compatible XML files
- locateLongRunning - searches the 10 longest running tests from a testsuite
- locateShortServerLife - whether the servers lifetime for the tests isn't at least 10 times as long as startup/shutdown
- locateLongSetupTeardown - locate tests that may use a lot of time in setup/teardown
- yaml - dumps the json file as a yaml file
./scripts/examine_results.js -- 'yaml,locateLongRunning' --readFile out/UNITTEST_RESULT.json

281
UnitTests/unittest.js
View File

@ -3,251 +3,72 @@
'use strict';
const _ = require('lodash');
const internal = require('internal');
const rp = require('@arangodb/result-processing');
const UnitTest = require('@arangodb/testing');
const internalMembers = UnitTest.internalMembers;
const fs = require('fs');
const internal = require('internal'); // js/common/bootstrap/modules/internal.js
const unitTest = require('@arangodb/testing').unitTest;
const optionsDefaults = require('@arangodb/testing').optionsDefaults;
const makeDirectoryRecursive = require('fs').makeDirectoryRecursive;
const killRemainingProcesses = require('@arangodb/process-utils').killRemainingProcesses;
const inspect = internal.inspect;
const abortSignal = 6;
let testOutputDirectory;
function makePathGeneric (path) {
return path.split(fs.pathSeparator);
}
function xmlEscape (s) {
return s.replace(/[<>&"]/g, function (c) {
return '&' + {
'<': 'lt',
'>': 'gt',
'&': 'amp',
'"': 'quot'
}[c] + ';';
});
}
function buildXml () {
let xml = ['<?xml version="1.0" encoding="UTF-8"?>\n'];
xml.text = function (s) {
Array.prototype.push.call(this, s);
return this;
};
xml.elem = function (tagName, attrs, close) {
this.text('<').text(tagName);
attrs = attrs || {};
for (let a in attrs) {
if (attrs.hasOwnProperty(a)) {
this.text(' ').text(a).text('="')
.text(xmlEscape(String(attrs[a]))).text('"');
}
}
if (close) {
this.text('/');
}
this.text('>\n');
return this;
};
return xml;
}
// //////////////////////////////////////////////////////////////////////////////
// @brief converts results to XML representation
// //////////////////////////////////////////////////////////////////////////////
function resultsToXml (results, baseName, cluster, isRocksDb) {
let clprefix = '';
if (cluster) {
clprefix = 'CL_';
}
if (isRocksDb) {
clprefix += 'RX_';
} else {
clprefix += 'MM_';
}
const isSignificant = function (a, b) {
return (internalMembers.indexOf(b) === -1) && a.hasOwnProperty(b);
};
for (let resultName in results) {
if (isSignificant(results, resultName)) {
let run = results[resultName];
for (let runName in run) {
if (isSignificant(run, runName)) {
const xmlName = clprefix + resultName + '_' + runName;
const current = run[runName];
if (current.skipped) {
continue;
}
let xml = buildXml();
let total = 0;
if (current.hasOwnProperty('total')) {
total = current.total;
}
let failuresFound = current.failed;
xml.elem('testsuite', {
errors: 0,
failures: failuresFound,
tests: total,
name: xmlName,
time: 0 + current.duration
});
let seen = false;
for (let oneTestName in current) {
if (isSignificant(current, oneTestName)) {
const oneTest = current[oneTestName];
const success = (oneTest.status === true);
seen = true;
xml.elem('testcase', {
name: clprefix + oneTestName,
time: 0 + oneTest.duration
}, success);
if (!success) {
xml.elem('failure');
xml.text('<![CDATA[' + oneTest.message + ']]>\n');
xml.elem('/failure');
xml.elem('/testcase');
}
}
}
if (!seen) {
if (failuresFound === 0) {
xml.elem('testcase', {
name: 'all_tests_in_' + xmlName,
time: 0 + current.duration
}, true);
} else {
xml.elem('testcase', {
name: 'all_tests_in_' + xmlName,
failures: failuresFound,
time: 0 + current.duration
}, true);
}
}
xml.elem('/testsuite');
const fn = makePathGeneric(baseName + xmlName + '.xml').join('_');
fs.write(testOutputDirectory + fn, xml.join(''));
}
}
}
}
}
// //////////////////////////////////////////////////////////////////////////////
// @brief runs the test using testing.js
// @brief runs the test using the test facilities in js/client/modules/@arangodb
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
start_pretty_print();
// parse arguments
let testSuits = []; // e.g all, http_server, recovery, ...
let testSuites = []; // e.g all, http_server, recovery, ...
let options = {};
while (argv.length >= 1) {
if (argv[0].slice(0, 1) === '{') { // stop parsing if there is a json document
break;
}
if (argv[0].slice(0, 1) === '-') { // break parsing if we hit some -option
break;
}
testSuits.push(argv[0]); // append first arg to test suits
testSuites.push(argv[0]); // append first arg to test suits
argv = argv.slice(1); // and remove first arg (c++:pop_front/bash:shift)
}
// convert arguments
if (argv.length >= 1) {
try {
if (argv[0].slice(0, 1) === '{') {
options = JSON.parse(argv[0]); // parse options form json
} else {
options = internal.parseArgv(argv, 0); // parse option with parseArgv function
}
options = internal.parseArgv(argv, 0);
} catch (x) {
print('failed to parse the json options: ' + x.message + '\n' + String(x.stack));
print('failed to parse the options: ' + x.message + '\n' + String(x.stack));
print('argv: ', argv);
return -1;
throw x;
}
}
if (options.hasOwnProperty('testOutput')) {
testOutputDirectory = options.testOutput + '/';
} else {
testOutputDirectory = 'out/';
options.testOutputDirectory = options.testOutput + '/';
}
options.testOutputDirectory = testOutputDirectory;
// force json reply
options.jsonReply = true;
_.defaults(options, optionsDefaults);
// create output directory
try {
fs.makeDirectoryRecursive(testOutputDirectory);
makeDirectoryRecursive(options.testOutputDirectory);
} catch (x) {
print("failed to create test directory - " + x.message);
throw x;
}
// by default we set this to error, so we always have a proper result for the caller
try {
fs.write(testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', "false", true);
fs.write(testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', "true", true);
let testFailureText = 'testfailures.txt';
if (options.hasOwnProperty('testFailureText')) {
testFailureText = options.testFailureText;
}
fs.write(fs.join(testOutputDirectory, testFailureText),
"Incomplete testrun with these testsuites: '" + testSuits +
"'\nand these options: " + JSON.stringify(options) + "\n");
// safeguard: mark test as failed if we die for some reason
rp.writeDefaultReports(options, testSuites);
} catch (x) {
print('failed to write default test result: ' + x.message);
throw(x);
throw x;
}
if (options.hasOwnProperty('cluster') && options.cluster) {
// cluster beats resilient single server
options.singleresilient = false;
}
if (options.hasOwnProperty('blacklist')) {
UnitTest.loadBlacklist(options.blacklist);
}
// run the test and store the result
let res = {}; // result
let result = {};
try {
// run tests
res = UnitTest.unitTest(testSuits, options, testOutputDirectory) || {};
result = unitTest(testSuites, options);
} catch (x) {
if (x.message === "USAGE ERROR") {
throw x;
}
print('caught exception during test execution!');
if (x.message !== undefined) {
@ -260,74 +81,38 @@ function main (argv) {
print(x);
}
print(JSON.stringify(res));
print(JSON.stringify(result));
throw x;
}
_.defaults(res, {
_.defaults(result, {
status: false,
crashed: true
});
let running = require("internal").getExternalSpawned();
let i = 0;
for (i = 0; i < running.length; i++) {
let status = require("internal").statusExternal(running[i].pid, false);
if (status.status === "TERMINATED") {
print("process exited without us joining it (marking crashy): " + JSON.stringify(running[i]) + JSON.stringify(status));
}
else {
print("Killing remaining process & marking crashy: " + JSON.stringify(running[i]));
print(require("internal").killExternal(running[i].pid, abortSignal));
}
res.crashed = true;
};
killRemainingProcesses(result);
// whether or not there was an error
try {
fs.write(testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', String(res.status), true);
fs.write(testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', String(res.crashed), true);
rp.writeReports(options, result);
} catch (x) {
print('failed to write test result: ' + x.message);
}
rp.dumpAllResults(options, result);
if (options.writeXmlReport) {
let j;
try {
j = JSON.stringify(res);
} catch (err) {
j = inspect(res);
}
fs.write(testOutputDirectory + '/UNITTEST_RESULT.json', j, true);
try {
let isCluster = false;
let isRocksDb = false;
let prefix = '';
if (options.hasOwnProperty('prefix')) {
prefix = options.prefix;
}
if (options.hasOwnProperty('cluster') && options.cluster) {
isCluster = true;
}
if (options.hasOwnProperty('storageEngine')) {
isRocksDb = (options.storageEngine === 'rocksdb');
}
resultsToXml(res, 'UNITTEST_RESULT_' + prefix, isCluster, isRocksDb);
rp.analyze.saveToJunitXML(options, result);
} catch (x) {
print('exception while serializing status xml!');
print(x.message);
print(x.stack);
print(inspect(res));
print(inspect(result));
}
}
// creates yaml like dump at the end
UnitTest.unitTestPrettyPrintResults(res, testOutputDirectory, options);
rp.analyze.unitTestPrettyPrintResults(options, result);
return res.status && running.length === 0;
return result.status;
}
let result = main(ARGUMENTS);

2
arangod/RestHandler/RestStatusHandler.cpp
View File

@ -66,7 +66,7 @@ RestStatus RestStatusHandler::execute() {
result.add("server", VPackValue("arango"));
result.add("version", VPackValue(ARANGODB_VERSION));
result.add("pid", VPackValue(Thread::currentProcessId()));
result.add("pid", VPackValue(static_cast<TRI_vpack_pid_t>(Thread::currentProcessId())));
#ifdef USE_ENTERPRISE
result.add("license", VPackValue("enterprise"));

116
js/client/modules/@arangodb/clusterstats.js Normal file
View File

@ -0,0 +1,116 @@
/* jshint strict: false, sub: true */
/* global print, arango */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Wilfried Goesgens
// //////////////////////////////////////////////////////////////////////////////
const internal = require('internal');
const fs = require('fs');
const download = internal.download;
const time = internal.time;
const sleep = internal.sleep;
const pu = require('@arangodb/process-utils');
const instanceInfo = JSON.parse(internal.env.INSTANCEINFO);
const options = JSON.parse(internal.env.OPTIONS);
const outfn = fs.join(instanceInfo.rootDir, 'stats.jsonl');
const opts = Object.assign(pu.makeAuthorizationHeaders(options),
{ method: 'GET' });
while(true) {
let state = {
state: true,
before: time(),
delta: [],
fails: []
};
let results = [];
for (let i = 0; i < 60; i++) {
const before = time();
let oneSet = { state: true };
results.push(oneSet);
instanceInfo.arangods.forEach(arangod => {
let serverId = arangod.role + '_' + arangod.port;
let beforeCall = time();
let procStats = pu.getProcessStats(arangod.pid);
if (arangod.role === "agent") {
let reply = download(arangod.url + '/_api/version', '', opts);
if (reply.hasOwnProperty('error') || reply.code !== 200) {
print("fail: " + JSON.stringify(reply) +
" - ps before: " + JSON.stringify(procStats) +
" - ps now: " + JSON.stringify(pu.getProcessStats(arangod.pid)));
state.state = false;
oneSet.state = false;
oneSet[serverId] = {
error: true,
start: beforeCall,
delta: time() - beforeCall
};
} else {
let statisticsReply = JSON.parse(reply.body);
oneSet[serverId] = {
error: false,
start: beforeCall,
delta: time() - beforeCall
};
}
} else {
let reply = download(arangod.url + '/_admin/statistics', '', opts);
if (reply.hasOwnProperty('error') || reply.code !== 200) {
print("fail: " + JSON.stringify(reply) +
" - ps before: " + JSON.stringify(procStats) +
" - ps now: " + JSON.stringify(pu.getProcessStats(arangod.pid)));
state.state = false;
oneSet.state = false;
oneSet[serverId] = {
error: true,
start: beforeCall,
delta: time() - beforeCall
};
} else {
let statisticsReply = JSON.parse(reply.body);
oneSet[serverId] = {
error: false,
start: beforeCall,
delta: time() - beforeCall,
uptime: statisticsReply.server.uptime
};
}
}
});
state['delta'].push(time() - before);
if (state.delta > 1000) {
print("marking FAIL since it took to long");
state.state = false;
}
if (!oneSet.state) {
state.fails.push(oneSet);
}
sleep(1);
}
fs.append(outfn, JSON.stringify(state) + "\n");
}

147
js/client/modules/@arangodb/process-utils.js
View File

@ -28,6 +28,7 @@
/* Modules: */
const _ = require('lodash');
const fs = require('fs');
const rp = require('@arangodb/result-processing');
const yaml = require('js-yaml');
const internal = require('internal');
const crashUtils = require('@arangodb/crash-utils');
@ -39,6 +40,7 @@ const executeExternal = internal.executeExternal;
const executeExternalAndWait = internal.executeExternalAndWait;
const killExternal = internal.killExternal;
const statusExternal = internal.statusExternal;
const statisticsExternal = internal.statisticsExternal;
const base64Encode = internal.base64Encode;
const testPort = internal.testPort;
const download = internal.download;
@ -234,14 +236,22 @@ function setupBinaries (builddir, buildType, configDir) {
BIN_DIR = fs.join(TOP_DIR, BIN_DIR);
}
if (!fs.exists(BIN_DIR)) {
BIN_DIR = fs.join(TOP_DIR, 'bin');
}
UNITTESTS_DIR = fs.join(fs.join(builddir, 'tests'));
if (!fs.exists(UNITTESTS_DIR)) {
UNITTESTS_DIR = fs.join(TOP_DIR, UNITTESTS_DIR);
}
if (buildType !== '') {
BIN_DIR = fs.join(BIN_DIR, buildType);
UNITTESTS_DIR = fs.join(UNITTESTS_DIR, buildType);
if (fs.exists(fs.join(BIN_DIR, buildType))) {
BIN_DIR = fs.join(BIN_DIR, buildType);
}
if (fs.exists(fs.join(UNITTESTS_DIR, buildType))) {
UNITTESTS_DIR = fs.join(UNITTESTS_DIR, buildType);
}
}
ARANGOBACKUP_BIN = fs.join(BIN_DIR, 'arangobackup' + executableExt);
@ -587,6 +597,88 @@ function killWithCoreDump (options, instanceInfo) {
instanceInfo.exitStatus = killExternal(instanceInfo.pid, abortSignal);
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief aggregates information from /proc about the SUT
// //////////////////////////////////////////////////////////////////////////////
function getProcessStats(pid) {
let processStats = statisticsExternal(pid);
if (platform === 'linux') {
let pidStr = "" + pid;
let ioraw;
try {
ioraw = fs.readBuffer(fs.join('/', 'proc', pidStr, 'io'));
} catch (x) {
print(x.stack);
throw x;
}
let lineStart = 0;
let maxBuffer = ioraw.length;
for (let j = 0; j < maxBuffer; j++) {
if (ioraw[j] === 10) { // \n
const line = ioraw.asciiSlice(lineStart, j);
lineStart = j + 1;
let x = line.split(":");
processStats[x[0]] = parseInt(x[1]);
}
}
}
return processStats;
}
function initProcessStats(instanceInfo) {
instanceInfo.arangods.forEach((arangod) => {
arangod.stats = getProcessStats(arangod.pid);
});
}
function getDeltaProcessStats(instanceInfo) {
let deltaStats = {};
instanceInfo.arangods.forEach((arangod) => {
let newStats = getProcessStats(arangod.pid);
let myDeltaStats = {};
for (let key in arangod.stats) {
myDeltaStats[key] = newStats[key] - arangod.stats[key];
}
deltaStats[arangod.pid + '_' + arangod.role] = myDeltaStats;
arangod.stats = newStats;
});
return deltaStats;
}
function summarizeStats(deltaStats) {
let sumStats = {};
for (let instance in deltaStats) {
for (let key in deltaStats[instance]) {
if (!sumStats.hasOwnProperty(key)) {
sumStats[key] = 0;
}
sumStats[key] += deltaStats[instance][key];
}
}
return sumStats;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief if we forgot about processes, this safe guard will clean up and mark failed
// //////////////////////////////////////////////////////////////////////////////
function killRemainingProcesses(results) {
let running = internal.getExternalSpawned();
results.status = results.status && (running.length === 0);
let i = 0;
for (i = 0; i < running.length; i++) {
let status = require("internal").statusExternal(running[i].pid, false);
if (status.status === "TERMINATED") {
print("process exited without us joining it (marking crashy): " + JSON.stringify(running[i]) + JSON.stringify(status));
}
else {
print("Killing remaining process & marking crashy: " + JSON.stringify(running[i]));
print(killExternal(running[i].pid, abortSignal));
}
results.crashed = true;
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes a command and waits for result
// //////////////////////////////////////////////////////////////////////////////
@ -1039,7 +1131,12 @@ function abortSurvivors(arangod, options) {
}
function checkInstanceAlive (instanceInfo, options) {
if (options.activefailover && instanceInfo.hasOwnProperty('authOpts')) {
if (options.activefailover &&
instanceInfo.hasOwnProperty('authOpts') &&
(instanceInfo.url !== instanceInfo.agencyUrl)
) {
// only detect a leader after we actually know one has been started.
// the agency won't tell us anything about leaders.
let d = detectCurrentLeader(instanceInfo);
if (instanceInfo.endpoint !== d.endpoint) {
print(Date() + ' failover has happened, leader is no more! Marking Crashy!');
@ -1270,6 +1367,16 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
}
}
if (options.cluster && instanceInfo.hasOwnProperty('clusterHealthMonitor')) {
try {
instanceInfo.clusterHealthMonitor['kill'] = killExternal(instanceInfo.clusterHealthMonitor.pid);
instanceInfo.clusterHealthMonitor['statusExternal'] = statusExternal(instanceInfo.clusterHealthMonitor.pid, true);
}
catch (x) {
print(x);
}
}
// Shut down all non-agency servers:
const n = instanceInfo.arangods.length;
@ -1431,7 +1538,6 @@ function detectCurrentLeader(instanceInfo) {
throw "Leader is not selected";
}
}
opts['method'] = 'GET';
reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', opts);
let res;
@ -1452,9 +1558,6 @@ function detectCurrentLeader(instanceInfo) {
}
function checkClusterAlive(options, instanceInfo, addArgs) {
// disabled because not in use (jslint)
// let coordinatorUrl = instanceInfo.url
// let response
let httpOptions = makeAuthorizationHeaders(options);
httpOptions.method = 'POST';
httpOptions.returnBodyOnError = true;
@ -1512,6 +1615,7 @@ function checkClusterAlive(options, instanceInfo, addArgs) {
throw new Error('cluster startup timed out after 10 minutes!');
}
}
print("Determining server IDs");
instanceInfo.arangods.forEach(arangod => {
// agents don't support the ID call...
@ -1524,6 +1628,19 @@ function checkClusterAlive(options, instanceInfo, addArgs) {
arangod.id = res['id'];
}
});
if ((options.cluster || options.agency) &&
!instanceInfo.hasOwnProperty('clusterHealthMonitor') &&
!options.disableClusterMonitor) {
print("spawning cluster health inspector");
internal.env.INSTANCEINFO = JSON.stringify(instanceInfo);
internal.env.OPTIONS = JSON.stringify(options);
let args = makeArgsArangosh(options);
args['javascript.execute'] = fs.join('js', 'client', 'modules', '@arangodb', 'clusterstats.js');
const argv = toArgv(args);
instanceInfo.clusterHealthMonitor = executeExternal(ARANGOSH_BIN, argv);
instanceInfo.clusterHealthMonitorFile = fs.join(instanceInfo.rootDir, 'stats.jsonl');
}
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief starts an instance
@ -1710,6 +1827,7 @@ function launchFinalize(options, instanceInfo, startTime) {
}
print(processInfo.join('\n') + '\n');
internal.sleep(options.sleepBeforeStart);
initProcessStats(instanceInfo);
}
// //////////////////////////////////////////////////////////////////////////////
@ -1848,6 +1966,9 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir)
instanceInfo.role = 'agent';
print(Date() + ' Agency Endpoint: ' + instanceInfo.endpoint);
}
checkClusterAlive(options, instanceInfo, addArgs);
return instanceInfo;
}
@ -1988,6 +2109,12 @@ function reStartInstance(options, instanceInfo, moreArgs) {
launchFinalize(options, instanceInfo, startTime);
}
function aggregateFatalErrors(currentTest) {
if (serverCrashedLocal) {
rp.addFailRunsMessage(currentTest, serverFailMessagesLocal);
serverFailMessagesLocal = "";
}
}
// exports.analyzeServerCrash = analyzeServerCrash;
exports.makeArgs = {
arangod: makeArgsArangod,
@ -2008,6 +2135,7 @@ exports.coverageEnvironment = coverageEnvironment;
exports.executeArangod = executeArangod;
exports.executeAndWait = executeAndWait;
exports.killRemainingProcesses = killRemainingProcesses;
exports.stopProcdump = stopProcdump;
exports.createBaseConfig = createBaseConfigBuilder;
@ -2021,14 +2149,17 @@ exports.run = {
};
exports.shutdownInstance = shutdownInstance;
exports.getProcessStats = getProcessStats;
exports.getDeltaProcessStats = getDeltaProcessStats;
exports.summarizeStats = summarizeStats;
exports.startArango = startArango;
exports.startInstance = startInstance;
exports.reStartInstance = reStartInstance;
exports.setupBinaries = setupBinaries;
exports.executableExt = executableExt;
exports.serverCrashed = serverCrashedLocal;
exports.serverFailMessages = serverFailMessagesLocal;
exports.aggregateFatalErrors = aggregateFatalErrors;
exports.cleanupDBDirectoriesAppend = cleanupDBDirectoriesAppend;
exports.cleanupDBDirectories = cleanupDBDirectories;
exports.cleanupLastDirectory = cleanupLastDirectory;

769
js/client/modules/@arangodb/result-processing.js Normal file
View File

@ -0,0 +1,769 @@
/* jshint strict: false, sub: true */
/* global print */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Wilfried Goesgens
// //////////////////////////////////////////////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////
// / @brief internal members of the results
// //////////////////////////////////////////////////////////////////////////////
const internal = require('internal');
const inspect = internal.inspect;
const fs = require('fs');
const pu = require('@arangodb/process-utils');
const cu = require('@arangodb/crash-utils');
const yaml = require('js-yaml');
/* Constants: */
const BLUE = internal.COLORS.COLOR_BLUE;
const CYAN = internal.COLORS.COLOR_CYAN;
const GREEN = internal.COLORS.COLOR_GREEN;
const RED = internal.COLORS.COLOR_RED;
const RESET = internal.COLORS.COLOR_RESET;
const YELLOW = internal.COLORS.COLOR_YELLOW;
const internalMembers = [
'code',
'error',
'status',
'duration',
'failed',
'total',
'crashed',
'ok',
'message',
'suiteName',
'processStats',
'startupTime',
'testDuration',
'shutdownTime',
'totalSetUp',
'totalTearDown',
'setUpDuration',
'setUpAllDuration',
'teardownAllDuration'
];
function skipInternalMember (r, a) {
return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1;
}
function makePathGeneric (path) {
return path.split(fs.pathSeparator);
}
let failedRuns = {
};
function gatherFailed(result) {
return Object.values(result).reduce(function(prev, testCase) {
if (testCase instanceof Object) {
return prev + !testCase.status;
} else {
return prev;
}
}, 0
);
}
function gatherStatus(result) {
return Object.values(result).reduce(function(prev, testCase) {
if (testCase instanceof Object) {
return prev && testCase.status;
} else {
return prev;
}
}, true
);
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief pretty prints the result
// //////////////////////////////////////////////////////////////////////////////
function fancyTimeFormat(time)
{
// Hours, minutes and seconds
var hrs = ~~(time / 3600);
var mins = ~~((time % 3600) / 60);
var secs = ~~time % 60;
// Output like "1:01" or "4:03:59" or "123:03:59"
var ret = "";
if (hrs > 0) {
ret += "" + hrs + ":" + (mins < 10 ? "0" : "");
}
ret += "" + mins + ":" + (secs < 10 ? "0" : "");
ret += "" + secs;
return ret;
}
function iterateTestResults(options, results, state, handlers) {
let bucketName = "";
if (options.testBuckets) {
let n = options.testBuckets.split('/');
bucketName = "_" + n[1];
}
try {
/* jshint forin: false */
// i.e. shell_server_aql
for (let testRunName in results) {
if (skipInternalMember(results, testRunName)) {
continue;
}
let testRun = results[testRunName];
if (handlers.hasOwnProperty('testRun')) {
handlers.testRun(options, state, testRun, testRunName);
}
// i.e. tests/js/server/aql/aql-cross.js
for (let testSuiteName in testRun) {
if (skipInternalMember(testRun, testSuiteName)) {
continue;
}
let testSuite = testRun[testSuiteName];
if (handlers.hasOwnProperty('testSuite')) {
handlers.testSuite(options, state, testSuite, testSuiteName);
}
// i.e. testDocument1
for (let testName in testSuite) {
if (skipInternalMember(testSuite, testName)) {
continue;
}
let testCase = testSuite[testName];
if (handlers.hasOwnProperty('testCase')) {
handlers.testCase(options, state, testCase, testName);
}
}
if (handlers.hasOwnProperty('endTestSuite')) {
handlers.endTestSuite(options, state, testSuite, testSuiteName);
}
}
if (handlers.hasOwnProperty('endTestRun')) {
handlers.endTestRun(options, state, testRun, testRunName);
}
}
} catch (x) {
print("Exception occured in running tests: " + x.message + "\n" + x.stack);
}
}
function locateFailState(options, results) {
let failedStates = {
state: true,
failCount: 0,
thisFailedTestCount: 0,
runFailedTestCount: 0,
currentSuccess: true,
failedTests: {},
thisFailedTests: []
};
iterateTestResults(options, results, failedStates, {
testRun: function(options, state, testRun, testRunName) {
if (!testRun.state) {
state.state = false;
state.thisFailedTestCount = testRun.failed;
}
},
testCase: function(options, state, testCase, testCaseName) {
if (!testCase.state) {
state.state = false;
state.runFailedTestCount ++;
state.thisFailedTests.push(testCaseName);
}
},
endTestRun: function(options, state, testRun, testRunName) {
if (state.thisFailedTestCount !== 0) {
if (state.thisFailedTestCount !== state.runFailedTestCount) {
// todo error message
}
state.failedTests[testRunName] = state.thisFailedTests;
state.failCount += state.runFailedTestCount;
state.thisFailedTests = [];
state.thisFailedTestCount = 0;
}
}
});
}
// //////////////////////////////////////////////////////////////////////////////
// @brief converts results to XML representation
// //////////////////////////////////////////////////////////////////////////////
function saveToJunitXML(options, results) {
function xmlEscape (s) {
return s.replace(/[<>&"]/g, function (c) {
return '&' + {
'<': 'lt',
'>': 'gt',
'&': 'amp',
'"': 'quot'
}[c] + ';';
});
}
function buildXml () {
let xml = ['<?xml version="1.0" encoding="UTF-8"?>\n'];
xml.text = function (s) {
Array.prototype.push.call(this, s);
return this;
};
xml.elem = function (tagName, attrs, close) {
this.text('<').text(tagName);
attrs = attrs || {};
for (let a in attrs) {
if (attrs.hasOwnProperty(a)) {
this.text(' ').text(a).text('="')
.text(xmlEscape(String(attrs[a]))).text('"');
}
}
if (close) {
this.text('/');
}
this.text('>\n');
return this;
};
return xml;
}
let xmlState = {
xml: undefined,
xmlName: '',
testRunName: '',
seenTestCases: false,
};
let prefix = options.cluster ? 'CL_' : '' +
(options.storageEngine === 'rocksdb') ? 'RX_': 'MM_';
iterateTestResults(options, results, xmlState, {
testRun: function(options, state, testRun, testRunName) {state.testRunName = testRunName;},
testSuite: function(options, state, testSuite, testSuiteName) {
let total = 0;
state.seenTestCases = false;
state.xml = buildXml();
state.xmlName = prefix + state.testRunName + '_' + makePathGeneric(testSuiteName).join('_');
if (testSuite.hasOwnProperty('total')) {
total = testSuite.total;
}
state.xml.elem('testsuite', {
errors: 0,
failures: testSuite.failed,
tests: total,
name: state.xmlName,
time: 0 + testSuite.duration
});
},
testCase: function(options, state, testCase, testCaseName) {
const success = (testCase.status === true);
state.xml.elem('testcase', {
name: prefix + testCaseName,
time: 0 + testCase.duration
}, success);
state.seenTestCases = true;
if (!success) {
state.xml.elem('failure');
state.xml.text('<![CDATA[' + testCase.message + ']]>\n');
state.xml.elem('/failure');
state.xml.elem('/testcase');
}
},
endTestSuite: function(options, state, testSuite, testSuiteName) {
if (!state.seenTestCases) {
if (testSuite.failed === 0) {
state.xml.elem('testcase', {
name: 'all_tests_in_' + state.xmlName,
time: 0 + testSuite.duration
}, true);
} else {
state.xml.elem('testcase', {
name: 'all_tests_in_' + state.xmlName,
failures: testSuite.failuresFound,
time: 0 + testSuite.duration
}, true);
}
}
state.xml.elem('/testsuite');
fs.write(fs.join(options.testOutputDirectory,
'UNITTEST_RESULT_' + state.xmlName + '.xml'),
state.xml.join(''));
},
endTestRun: function(options, state, testRun, testRunName) {}
});
}
function unitTestPrettyPrintResults (options, results) {
let onlyFailedMessages = '';
let failedMessages = '';
let SuccessMessages = '';
let failedSuiteCount = 0;
let failedTestsCount = 0;
let successCases = {};
let failedCases = {};
let failsOfOneSuite = {};
let failsOfOneSuiteCount = 0;
let bucketName = "";
let testRunStatistics = "";
let isSuccess = true;
let suiteSuccess = true;
if (options.testBuckets) {
let n = options.testBuckets.split('/');
bucketName = "_" + n[1];
}
function testCaseMessage (test) {
if (typeof test.message === 'object' && test.message.hasOwnProperty('body')) {
return test.message.body;
} else {
return test.message;
}
}
let failedStates = {
state: true,
failCount: 0,
thisFailedTestCount: 0,
runFailedTestCount: 0,
currentSuccess: true,
failedTests: {},
thisFailedTests: []
};
iterateTestResults(options, results, failedStates, {
testRun: function(options, state, testRun, testRunName) {
},
testSuite: function(options, state, testSuite, testSuiteName) {
if (testSuite.status) {
successCases[testSuiteName] = testSuite;
} else {
++failedSuiteCount;
isSuccess = false;
if (testSuite.hasOwnProperty('message')) {
failedCases[testSuiteName] = {
test: testCaseMessage(testSuite)
};
}
}
},
testCase: function(options, state, testCase, testCaseName) {
if (!testCase.status) {
failsOfOneSuite[testCaseName] = testCaseMessage(testCase);
failsOfOneSuiteCount ++;
}
},
endTestSuite: function(options, state, testSuite, testSuiteName) {
failedTestsCount++;
if (failsOfOneSuiteCount !== 0) {
failedCases[testSuiteName] = failsOfOneSuite;
failsOfOneSuite = {};
failsOfOneSuiteCount = 0;
}
},
endTestRun: function(options, state, testRun, testRunName) {
if (isSuccess) {
SuccessMessages += '* Test "' + testRunName + bucketName + '"\n';
for (let name in successCases) {
if (!successCases.hasOwnProperty(name)) {
continue;
}
let details = successCases[name];
if (details.skipped) {
SuccessMessages += YELLOW + ' [SKIPPED] ' + name + RESET + '\n';
} else {
SuccessMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n';
}
}
} else {
let m = '* Test "' + testRunName + bucketName + '"\n';
onlyFailedMessages += m;
failedMessages += m;
for (let name in successCases) {
if (!successCases.hasOwnProperty(name)) {
continue;
}
let details = successCases[name];
if (details.skipped) {
m = ' [SKIPPED] ' + name;
failedMessages += YELLOW + m + RESET + '\n';
onlyFailedMessages += m + '\n';
} else {
failedMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n';
}
}
for (let name in failedCases) {
if (!failedCases.hasOwnProperty(name)) {
continue;
}
m = ' [FAILED] ' + name;
failedMessages += RED + m + RESET + '\n\n';
onlyFailedMessages += m + '\n\n';
let details = failedCases[name];
let count = 0;
for (let one in details) {
if (!details.hasOwnProperty(one)) {
continue;
}
if (count > 0) {
failedMessages += '\n';
onlyFailedMessages += '\n';
}
m = ' "' + one + '" failed: ' + details[one];
failedMessages += RED + m + RESET + '\n\n';
onlyFailedMessages += m + '\n\n';
count++;
}
}
}
isSuccess = true;
successCases = {};
failedCases = {};
}
});
let color,statusMessage;
let crashedText = '';
let crashText = '';
let failText = '';
if (results.status === true) {
color = GREEN;
statusMessage = 'Success';
} else {
color = RED;
statusMessage = 'Fail';
}
if (results.crashed === true) {
color = RED;
for (let failed in failedRuns) {
crashedText += ' [' + failed + '] : ' + failedRuns[failed].replace(/^/mg, ' ');
}
crashedText += "\nMarking crashy!";
crashText = color + crashedText + RESET;
}
if (results.status !== true) {
failText = '\n Suites failed: ' + failedSuiteCount + ' Tests Failed: ' + failedTestsCount;
onlyFailedMessages += failText + '\n';
failText = RED + failText + RESET;
}
if (cu.GDB_OUTPUT !== '') {
// write more verbose failures to the testFailureText file
onlyFailedMessages += '\n\n' + cu.GDB_OUTPUT;
}
print(`
${YELLOW}================================================================================'
TEST RESULTS
================================================================================${RESET}
${SuccessMessages}
${failedMessages}${color} * Overall state: ${statusMessage}${RESET}${crashText}${failText}`);
onlyFailedMessages;
if (crashedText !== '') {
onlyFailedMessages += '\n' + crashedText;
}
fs.write(options.testOutputDirectory + options.testFailureText, onlyFailedMessages);
}
function locateLongRunning(options, results) {
let testRunStatistics = "";
let sortedByDuration = [];
let failedStates = {
state: true,
failCount: 0,
thisFailedTestCount: 0,
runFailedTestCount: 0,
currentSuccess: true,
failedTests: {},
thisFailedTests: []
};
iterateTestResults(options, results, failedStates, {
testRun: function(options, state, testRun, testRunName) {
let startup = testRun['startupTime'];
let testDuration = testRun['testDuration'];
let shutdown = testRun['shutdownTime'];
let color = GREEN;
if (((startup + shutdown) * 10) > testDuration) {
color = RED;
}
testRunStatistics += `${color}${testRunName} - startup [${startup}] => run [${testDuration}] => shutdown [${shutdown}]${RESET}
`;
},
testSuite: function(options, state, testSuite, testSuiteName) {
if (testSuite.hasOwnProperty('duration') && testSuite.duration !== 0) {
sortedByDuration.push(
{
testName: testSuiteName,
duration: testSuite.duration,
test: testSuite,
count: Object.keys(testSuite).filter(testCase => skipInternalMember(testSuite, testCase)).length
});
} else {
print(RED + "This test doesn't have a duration: " + testSuiteName + "\n" + JSON.stringify(testSuite) + RESET);
}
},
testCase: function(options, state, testCase, testCaseName) {
},
endTestSuite: function(options, state, testSuite, testSuiteName) {
},
endTestRun: function(options, state, testRun, testRunName) {
sortedByDuration.sort(function(a, b) {
return a.duration - b.duration;
});
let results = {};
for (let i = sortedByDuration.length - 1; (i >= 0) && (i > sortedByDuration.length - 11); i --) {
let key = " - " + fancyTimeFormat(sortedByDuration[i].duration / 1000) + " - " +
sortedByDuration[i].count + " - " +
sortedByDuration[i].testName.replace('/\\/g', '/');
let testCases = [];
let thisTestSuite = sortedByDuration[i];
for (let testName in thisTestSuite.test) {
if (skipInternalMember(thisTestSuite.test, testName)) {
continue;
}
let test = thisTestSuite.test[testName];
let duration = 0;
if (test.hasOwnProperty('duration')) {
duration += test.duration;
}
if (test.hasOwnProperty('setUpDuration')) {
duration += test.setUpDuration;
}
if (test.hasOwnProperty('tearDownDuration')) {
duration += test.tearDownDuration;
}
testCases.push({
testName: testName,
duration: duration
});
}
testCases.sort(function(a, b) {
return a.duration - b.duration;
});
let statistics = [];
for (let j = Object.keys(testCases).length - 1; (j >= 0) && (j > Object.keys(testCases).length - 11); j --) {
statistics.push(fancyTimeFormat(testCases[j].duration / 1000) + " - " + testCases[j].testName);
}
results[key] = {
'processStatistics': pu.sumarizeStats(thisTestSuite.test['processStats']),
'stats': statistics
};
}
testRunStatistics += yaml.safeDump(results);
sortedByDuration = [];
}
});
print(testRunStatistics);
}
function locateLongSetupTeardown(options, results) {
let testRunStatistics = " Setup | Run | tests | setupAll | suite name\n";
let sortedByDuration = [];
let failedStates = {};
iterateTestResults(options, results, failedStates, {
testRun: function(options, state, testRun, testRunName) {
},
testSuite: function(options, state, testSuite, testSuiteName) {
let setupAllDuration = 0;
if (testSuite.hasOwnProperty('setUpAllDuration')) {
Object.keys(testSuite.setUpAllDuration).forEach(testName => {
setupAllDuration += testSuite.setUpAllDuration[testName];
});
}
let hasSetupAll = setupAllDuration !== 0;
if (testSuite.hasOwnProperty('totalSetUp') &&
testSuite.hasOwnProperty('totalTearDown')) {
sortedByDuration.push({
testName: testSuiteName,
setupTearDown: testSuite.totalSetUp + testSuite.totalTearDown,
duration: testSuite.duration,
hasSetupAll: hasSetupAll,
count: Object.keys(testSuite).filter(testCase => ! skipInternalMember(testSuite, testCase)).length,
});
} else {
print(RED + "This test doesn't have a duration: " + testSuiteName + "\n" + JSON.stringify(testSuite) + RESET);
}
},
testCase: function(options, state, testCase, testCaseName) {
},
endTestSuite: function(options, state, testSuite, testSuiteName) {
},
endTestRun: function(options, state, testRun, testRunName) {
sortedByDuration.sort(function(a, b) {
return a.setupTearDown - b.setupTearDown;
});
let results = [];
for (let i = sortedByDuration.length - 1; (i >= 0) && (i > sortedByDuration.length - 11); i --) {
let key = " " +
fancyTimeFormat(sortedByDuration[i].setupTearDown / 1000) + " | " +
fancyTimeFormat(sortedByDuration[i].duration / 1000) + " | " +
sortedByDuration[i].count + " | " +
sortedByDuration[i].hasSetupAll + " | " +
sortedByDuration[i].testName.replace('/\\/g', '/');
results.push(key);
}
testRunStatistics += yaml.safeDump(results);
sortedByDuration = [];
}
});
print(testRunStatistics);
}
function locateShortServerLife(options, results) {
let rc = true;
let testRunStatistics = "";
let sortedByDuration = [];
let failedStates = {
state: true,
failCount: 0,
thisFailedTestCount: 0,
runFailedTestCount: 0,
currentSuccess: true,
failedTests: {},
thisFailedTests: []
};
iterateTestResults(options, results, failedStates, {
testRun: function(options, state, testRun, testRunName) {
if (testRun.hasOwnProperty('startupTime') &&
testRun.hasOwnProperty('testDuration') &&
testRun.hasOwnProperty('shutdownTime')) {
let startup = testRun['startupTime'];
let testDuration = testRun['testDuration'];
let shutdown = testRun['shutdownTime'];
let color = GREEN;
let OK = "YES";
if (((startup + shutdown) * 10) > testDuration) {
color = RED;
OK = "NOP";
rc = false;
}
testRunStatistics += `${color}[${OK}] ${testRunName} - startup [${startup}] => run [${testDuration}] => shutdown [${shutdown}]${RESET}`;
} else {
testRunStatistics += `${YELLOW}${testRunName} doesn't have server start statistics${RESET}`;
}
}
});
print(testRunStatistics);
return rc;
}
// //////////////////////////////////////////////////////////////////////////////
// @brief simple status representations
// //////////////////////////////////////////////////////////////////////////////
function writeDefaultReports(options, testSuites) {
fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', "false", true);
fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', "true", true);
let testFailureText = 'testfailures.txt';
if (options.hasOwnProperty('testFailureText')) {
testFailureText = options.testFailureText;
}
fs.write(fs.join(options.testOutputDirectory, testFailureText),
"Incomplete testrun with these testsuites: '" + testSuites +
"'\nand these options: " + JSON.stringify(options) + "\n");
}
function writeReports(options, results) {
fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json', String(results.status), true);
fs.write(options.testOutputDirectory + '/UNITTEST_RESULT_CRASHED.json', String(results.crashed), true);
}
function dumpAllResults(options, results) {
let j;
try {
j = JSON.stringify(results);
} catch (err) {
j = inspect(results);
}
fs.write(options.testOutputDirectory + '/UNITTEST_RESULT.json', j, true);
}
function addFailRunsMessage(testcase, message) {
failedRuns[testcase] = message;
}
function yamlDumpResults(options, results) {
try {
print(yaml.safeDump(JSON.parse(JSON.stringify(results))));
} catch (err) {
print(RED + 'cannot dump results: ' + String(err) + RESET);
print(RED + require('internal').inspect(results) + RESET);
}
}
exports.gatherStatus = gatherStatus;
exports.gatherFailed = gatherFailed;
exports.yamlDumpResults = yamlDumpResults;
exports.addFailRunsMessage = addFailRunsMessage;
exports.dumpAllResults = dumpAllResults;
exports.writeDefaultReports = writeDefaultReports;
exports.writeReports = writeReports;
exports.analyze = {
unitTestPrettyPrintResults: unitTestPrettyPrintResults,
saveToJunitXML: saveToJunitXML,
locateLongRunning: locateLongRunning,
locateShortServerLife: locateShortServerLife,
locateLongSetupTeardown: locateLongSetupTeardown,
yaml: yamlDumpResults
};

66
js/client/modules/@arangodb/test-utils.js
View File

@ -103,7 +103,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
let env = {};
let customInstanceInfos = {};
let healthCheck = function () {return true;};
let beforeStart = time();
if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('healthCheck')) {
healthCheck = startStopHandlers.healthCheck;
}
@ -162,7 +162,11 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
_.defaults(env, customInstanceInfos.postStart.env);
}
let results = { shutdown: true };
let testrunStart = time();
let results = {
shutdown: true,
startupTime: testrunStart - beforeStart
};
let continueTesting = true;
let serverDead = false;
let count = 0;
@ -234,6 +238,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
continue;
} else if (reply.hasOwnProperty('status')) {
results[te] = reply;
results[te]['processStats'] = pu.getDeltaProcessStats(instanceInfo);
if (results[te].status === false) {
options.cleanup = false;
@ -399,7 +404,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
results.status = true;
print(RED + 'No testcase matched the filter.' + RESET);
}
let shutDownStart = time();
results['testDuration'] = shutDownStart - testrunStart;
print(Date() + ' Shutting down...');
if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('preStop')) {
customInstanceInfos['preStop'] = startStopHandlers.preStop(options,
@ -426,6 +432,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
}
results.shutdown = results.shutdown && pu.shutdownInstance(instanceInfo, clonedOpts, forceTerminate);
loadClusterTestStabilityInfo(results, instanceInfo);
if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('postStop')) {
customInstanceInfos['postStop'] = startStopHandlers.postStop(options,
serverOptions,
@ -437,6 +445,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
results.setup.message = 'custom postStop failed!';
}
}
results['shutdownTime'] = time() - shutDownStart;
print('done.');
return results;
@ -447,11 +457,6 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
// //////////////////////////////////////////////////////////////////////////////
function filterTestcaseByOptions (testname, options, whichFilter) {
if (options.skipTest(testname, options)) {
whichFilter.filter = 'blacklist';
return false;
}
// These filters require a proper setup, Even if we filter by testcase:
if ((testname.indexOf('-mmfiles') !== -1) && options.storageEngine === 'rocksdb') {
whichFilter.filter = 'skip when running as rocksdb';
@ -621,7 +626,7 @@ function scanTestPaths (paths, options) {
}
return rc;
});
if (filteredTestCases.length === 0) {
if ((filteredTestCases.length === 0) && (options.extremeVerbosity !== 'silence')) {
print("No testcase matched the filter: " + JSON.stringify(allFiltered));
return [];
}
@ -629,6 +634,41 @@ function scanTestPaths (paths, options) {
return allTestCases;
}
function loadClusterTestStabilityInfo(results, instanceInfo){
try {
if (instanceInfo.hasOwnProperty('clusterHealthMonitorFile')) {
let status = true;
let slow = [];
let buf = fs.readBuffer(instanceInfo.clusterHealthMonitorFile);
let lineStart = 0;
let maxBuffer = buf.length;
for (let j = 0; j < maxBuffer; j++) {
if (buf[j] === 10) { // \n
const line = buf.asciiSlice(lineStart, j);
lineStart = j + 1;
let val = JSON.parse(line);
if (val.state === false) {
slow.push(val);
status = false;
}
}
}
if (!status) {
print('found ' + slow.length + ' slow lines!');
results.status = false;
results.message += 'found ' + slow.length + ' slow lines!';
} else {
print('everything fast!');
}
}
}
catch(x) {
print(x);
}
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief runs a remote unittest file using /_admin/execute
// //////////////////////////////////////////////////////////////////////////////
@ -704,7 +744,7 @@ function runThere (options, instanceInfo, file) {
};
}
}
runThere.info = 'runThere';
runThere.info = 'runInArangod';
function readTestResult(path, rc, testCase) {
const jsonFN = fs.join(path, 'testresult.json');
@ -791,7 +831,7 @@ function runInArangosh (options, instanceInfo, file, addArgs) {
let rc = pu.executeAndWait(pu.ARANGOSH_BIN, toArgv(args), options, 'arangosh', instanceInfo.rootDir, options.coreCheck);
return readTestResult(instanceInfo.rootDir, rc, args['javascript.unit-tests']);
}
runInArangosh.info = 'arangosh';
runInArangosh.info = 'runInExternalArangosh';
// //////////////////////////////////////////////////////////////////////////////
// / @brief runs a local unittest file in the current arangosh
@ -834,7 +874,7 @@ function runInLocalArangosh (options, instanceInfo, file, addArgs) {
};
}
}
runInLocalArangosh.info = 'localarangosh';
runInLocalArangosh.info = 'runInLocalArangosh';
// //////////////////////////////////////////////////////////////////////////////
// / @brief runs a unittest file using rspec
@ -977,7 +1017,7 @@ function runInRSpec (options, instanceInfo, file, addArgs) {
fs.remove(tmpname);
return result;
}
runInRSpec.info = 'runInRSpec';
runInRSpec.info = 'runInLocalRSpec';
exports.runThere = runThere;
exports.runInArangosh = runInArangosh;

424
js/client/modules/@arangodb/testing.js
View File

@ -23,8 +23,26 @@
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Max Neunhoeffer
// @author Wilfried Goesgnes
// /////////////////////////////////////////////////////////////////////////////
const _ = require('lodash');
const fs = require('fs');
const pu = require('@arangodb/process-utils');
const rp = require('@arangodb/result-processing');
const cu = require('@arangodb/crash-utils');
const tu = require('@arangodb/test-utils');
const internal = require('internal');
const platform = internal.platform;
const BLUE = internal.COLORS.COLOR_BLUE;
const CYAN = internal.COLORS.COLOR_CYAN;
const GREEN = internal.COLORS.COLOR_GREEN;
const RED = internal.COLORS.COLOR_RED;
const RESET = internal.COLORS.COLOR_RESET;
const YELLOW = internal.COLORS.COLOR_YELLOW;
let functionsDocumentation = {
'all': 'run all tests (marked with [x])',
'find': 'searches all testcases, and eventually filters them by `--test`, ' +
@ -39,8 +57,6 @@ let optionsDocumentation = [
' The following properties of `options` are defined:',
'',
' - `testOutput`: set the output directory for testresults, defaults to `out`',
' - `jsonReply`: if set a json is returned which the caller has to ',
' present the user',
' - `force`: if set to true the tests are continued even if one fails',
'',
" - `skipLogAnalysis`: don't try to crawl the server logs",
@ -77,15 +93,17 @@ let optionsDocumentation = [
' - `agencySupervision`: run supervision in agency',
' - `oneTestTimeout`: how long a single testsuite (.js, .rb) should run',
' - `isAsan`: doubles oneTestTimeot value if set to true (for ASAN-related builds)',
' - `test`: path to single test to execute for "single" test target',
' - `cleanup`: if set to true (the default), the cluster data files',
' and logs are removed after termination of the test.',
' - `test`: path to single test to execute for "single" test target, ',
' or pattern to filter for other suites',
' - `cleanup`: if set to false the data files',
' and logs are not removed after termination of the test.',
'',
' - `protocol`: the protocol to talk to the server - [tcp (default), ssl, unix]',
' - `sniff`: if we should try to launch tcpdump / windump for a testrun',
' false / true / sudo',
' - `sniffDevice`: the device tcpdump / tshark should use',
' - `sniffProgram`: specify your own programm',
'',
' - `build`: the directory containing the binaries',
' - `buildType`: Windows build type (Debug, Release), leave empty on linux',
' - `configDir`: the directory containing the config files, defaults to',
@ -94,6 +112,8 @@ let optionsDocumentation = [
' - `dumpAgencyOnError`: if we should create an agency dump if an error occurs',
' - `prefix`: prefix for the tests in the xml reports',
'',
' - `disableClusterMonitor`: if set to false, an arangosh is started that will send',
' keepalive requests to all cluster instances, and report on error',
' - `disableMonitor`: if set to true on windows, procdump will not be attached.',
' - `rr`: if set to true arangod instances are run with rr',
' - `exceptionFilter`: on windows you can use this to abort tests on specific exceptions',
@ -135,7 +155,7 @@ const optionsDefaults = {
'agencyWaitForSync': false,
'agencySupervision': true,
'build': '',
'buildType': '',
'buildType': (platform.substr(0, 3) === 'win') ? 'RelWithDebInfo':'',
'cleanup': true,
'cluster': false,
'concurrency': 3,
@ -150,7 +170,6 @@ const optionsDefaults = {
'force': true,
'getSockStat': false,
'arangosearch':true,
'jsonReply': false,
'loopEternal': false,
'loopSleepSec': 1,
'loopSleepWhen': 1,
@ -181,6 +200,7 @@ const optionsDefaults = {
'storageEngine': 'rocksdb',
'test': undefined,
'testBuckets': undefined,
'testOutputDirectory': 'out/',
'useReconnect': true,
'username': 'root',
'valgrind': false,
@ -193,266 +213,14 @@ const optionsDefaults = {
'testFailureText': 'testfailures.txt',
'testCase': undefined,
'disableMonitor': false,
'disableClusterMonitor': true,
'sleepBeforeStart' : 0,
};
const _ = require('lodash');
const fs = require('fs');
const yaml = require('js-yaml');
let globalStatus = true;
const pu = require('@arangodb/process-utils');
const cu = require('@arangodb/crash-utils');
const tu = require('@arangodb/test-utils');
const BLUE = require('internal').COLORS.COLOR_BLUE;
const CYAN = require('internal').COLORS.COLOR_CYAN;
const GREEN = require('internal').COLORS.COLOR_GREEN;
const RED = require('internal').COLORS.COLOR_RED;
const RESET = require('internal').COLORS.COLOR_RESET;
const YELLOW = require('internal').COLORS.COLOR_YELLOW;
// //////////////////////////////////////////////////////////////////////////////
// / @brief test functions for all
// //////////////////////////////////////////////////////////////////////////////
let failedRuns = {
};
let allTests = [
];
// /////////////////////////////////////////////////////////////////////////////
// blacklisting
// /////////////////////////////////////////////////////////////////////////////
let useBlacklist = false;
let blacklistTests = {};
function skipTest(type, name) {
let ntype = type.toUpperCase();
return useBlacklist && !!blacklistTests[ntype + ":" + name];
}
function loadBlacklist(name) {
let content = fs.read("BLACKLIST");
let a = _.filter(
_.map(content.split("\n"),
function(x) {return x.trim();}),
function(x) {return x.length > 0 && x[0] !== '#';});
for (let i = 0; i < a.length; ++i) {
blacklistTests[a[i]] = true;
}
useBlacklist = true;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief TEST: all
// //////////////////////////////////////////////////////////////////////////////
let testFuncs = {
'all': function () {}
};
// //////////////////////////////////////////////////////////////////////////////
// / @brief internal members of the results
// //////////////////////////////////////////////////////////////////////////////
const internalMembers = [
'code',
'error',
'status',
'duration',
'failed',
'total',
'crashed',
'ok',
'message',
'suiteName'
];
// //////////////////////////////////////////////////////////////////////////////
// / @brief pretty prints the result
// //////////////////////////////////////////////////////////////////////////////
function testCaseMessage (test) {
if (typeof test.message === 'object' && test.message.hasOwnProperty('body')) {
return test.message.body;
} else {
return test.message;
}
}
function unitTestPrettyPrintResults (res, testOutputDirectory, options) {
function skipInternalMember (r, a) {
return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1;
}
print(YELLOW + '================================================================================');
print('TEST RESULTS');
print('================================================================================\n' + RESET);
let failedSuite = 0;
let failedTests = 0;
let onlyFailedMessages = '';
let failedMessages = '';
let SuccessMessages = '';
let bucketName = "";
if (options.testBuckets) {
let n = options.testBuckets.split('/');
bucketName = "_" + n[1];
}
try {
/* jshint forin: false */
for (let testrunName in res) {
if (skipInternalMember(res, testrunName)) {
continue;
}
let testrun = res[testrunName];
let successCases = {};
let failedCases = {};
let isSuccess = true;
for (let testName in testrun) {
if (skipInternalMember(testrun, testName)) {
continue;
}
let test = testrun[testName];
if (test.status) {
successCases[testName] = test;
} else {
isSuccess = false;
++failedSuite;
if (test.hasOwnProperty('message')) {
++failedTests;
failedCases[testName] = {
test: testCaseMessage(test)
};
} else {
let fails = failedCases[testName] = {};
for (let oneName in test) {
if (skipInternalMember(test, oneName)) {
continue;
}
let oneTest = test[oneName];
if (!oneTest.status) {
++failedTests;
fails[oneName] = testCaseMessage(oneTest);
}
}
}
}
}
if (isSuccess) {
SuccessMessages += '* Test "' + testrunName + bucketName + '"\n';
for (let name in successCases) {
if (!successCases.hasOwnProperty(name)) {
continue;
}
let details = successCases[name];
if (details.skipped) {
SuccessMessages += YELLOW + ' [SKIPPED] ' + name + RESET + '\n';
} else {
SuccessMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n';
}
}
} else {
let m = '* Test "' + testrunName + bucketName + '"\n';
onlyFailedMessages += m;
failedMessages += m;
for (let name in successCases) {
if (!successCases.hasOwnProperty(name)) {
continue;
}
let details = successCases[name];
if (details.skipped) {
failedMessages += YELLOW + ' [SKIPPED] ' + name + RESET + '\n';
onlyFailedMessages += ' [SKIPPED] ' + name + '\n';
} else {
failedMessages += GREEN + ' [SUCCESS] ' + name + RESET + '\n';
}
}
for (let name in failedCases) {
if (!failedCases.hasOwnProperty(name)) {
continue;
}
failedMessages += RED + ' [FAILED] ' + name + RESET + '\n\n';
onlyFailedMessages += ' [FAILED] ' + name + '\n\n';
let details = failedCases[name];
let count = 0;
for (let one in details) {
if (!details.hasOwnProperty(one)) {
continue;
}
if (count > 0) {
failedMessages += '\n';
onlyFailedMessages += '\n';
}
failedMessages += RED + ' "' + one + '" failed: ' + details[one] + RESET + '\n\n';
onlyFailedMessages += ' "' + one + '" failed: ' + details[one] + '\n\n';
count++;
}
}
}
}
print(SuccessMessages);
print(failedMessages);
/* jshint forin: true */
let color = (!res.crashed && res.status === true) ? GREEN : RED;
let crashText = '';
let crashedText = '';
if (res.crashed === true) {
for (let failed in failedRuns) {
crashedText += ' [' + failed + '] : ' + failedRuns[failed].replace(/^/mg, ' ');
}
crashedText += "\nMarking crashy!";
crashText = RED + crashedText + RESET;
}
print('\n' + color + '* Overall state: ' + ((res.status === true) ? 'Success' : 'Fail') + RESET + crashText);
let failText = '';
if (res.status !== true) {
failText = ' Suites failed: ' + failedSuite + ' Tests Failed: ' + failedTests;
print(color + failText + RESET);
}
failedMessages = onlyFailedMessages;
if (crashedText !== '') {
failedMessages += '\n' + crashedText;
}
if (cu.GDB_OUTPUT !== '' || failText !== '') {
failedMessages += '\n\n' + cu.GDB_OUTPUT + failText + '\n';
}
fs.write(testOutputDirectory + options.testFailureText, failedMessages);
} catch (x) {
print('exception caught while pretty printing result: ');
print(x.message);
print(JSON.stringify(res));
}
}
let allTests = [];
let testFuncs = {};
// //////////////////////////////////////////////////////////////////////////////
// / @brief print usage information
@ -501,7 +269,9 @@ function findTestCases(options) {
let allTestFiles = {};
for (let testSuiteName in allTestPaths) {
var myList = [];
let files = tu.scanTestPaths(allTestPaths[testSuiteName], options);
var _opts = _.clone(options);
_opts.extremeVerbosity = 'silence';
let files = tu.scanTestPaths(allTestPaths[testSuiteName], _opts);
if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) {
for (let j = 0; j < files.length; j++) {
let foo = {};
@ -557,7 +327,6 @@ function findTest(options) {
}
}
function autoTest(options) {
if (!options.hasOwnProperty('test') || (typeof (options.test) === 'undefined')) {
return {
@ -620,16 +389,13 @@ function loadTestSuites () {
throw x;
}
}
testFuncs['all'] = allTests;
testFuncs['find'] = findTest;
testFuncs['auto'] = autoTest;
}
let globalStatus = true;
function iterateTests(cases, options, jsonReply) {
// tests to run
function translateTestList(cases) {
let caselist = [];
const expandWildcard = ( name ) => {
if (!name.endsWith('*')) {
return name;
@ -640,29 +406,18 @@ function iterateTests(cases, options, jsonReply) {
for (let n = 0; n < cases.length; ++n) {
let splitted = expandWildcard(cases[n]).split(/[,;|]/);
for (let m = 0; m < splitted.length; ++m) {
let which = splitted[m];
if (which === 'all') {
caselist = caselist.concat(allTests);
} else if (testFuncs.hasOwnProperty(which)) {
if (testFuncs.hasOwnProperty(which)) {
caselist.push(which);
} else {
print('Unknown test "' + which + '"\nKnown tests are: ' + Object.keys(testFuncs).sort().join(', '));
return {
status: false
};
throw new Error("USAGE ERROR");
}
}
}
let results = {};
let cleanup = true;
// real ugly hack. there are some suites which are just placeholders
// for other suites
// Expand meta tests like ldap, all
caselist = (function() {
let flattened = [];
for (let n = 0; n < caselist.length; ++n) {
@ -675,12 +430,27 @@ function iterateTests(cases, options, jsonReply) {
}
return flattened;
})();
if (cases === undefined || cases.length === 0) {
printUsage();
print('\nFATAL: "which" is undefined\n');
throw new Error("USAGE ERROR");
}
return caselist;
}
function iterateTests(cases, options) {
// tests to run
let caselist = [];
let results = {};
let cleanup = true;
caselist = translateTestList(cases);
// running all tests
for (let n = 0; n < caselist.length; ++n) {
const currentTest = caselist[n];
var localOptions = _.cloneDeep(options);
localOptions.skipTest = skipTest;
let printTestName = currentTest;
if (options.testBuckets) {
printTestName += " - " + options.testBuckets;
@ -697,34 +467,23 @@ function iterateTests(cases, options, jsonReply) {
let status = true;
let shutdownSuccess = true;
if (skipTest("SUITE", currentTest)) {
result = {
failed: 0,
status: true,
crashed: false,
};
print(YELLOW + "[SKIPPED] " + currentTest + RESET + "\n");
} else {
result = testFuncs[currentTest](localOptions);
// grrr...normalize structure
delete result.status;
delete result.failed;
delete result.crashed;
if (result.hasOwnProperty('shutdown')) {
shutdownSuccess = result['shutdown'];
delete result.shutdown;
}
status = Object.values(result).every(testCase => testCase.status === true);
let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0);
if (!status) {
globalStatus = false;
}
result.failed = failed;
result.status = status;
result = testFuncs[currentTest](localOptions);
// grrr...normalize structure
delete result.status;
delete result.failed;
delete result.crashed;
if (result.hasOwnProperty('shutdown')) {
shutdownSuccess = result['shutdown'];
delete result.shutdown;
}
status = rp.gatherStatus(result);
let failed = rp.gatherFailed(result);
if (!status) {
globalStatus = false;
}
result.failed = failed;
result.status = status;
results[currentTest] = result;
if (status && localOptions.cleanup && shutdownSuccess ) {
@ -732,11 +491,7 @@ function iterateTests(cases, options, jsonReply) {
} else {
cleanup = false;
}
if (pu.serverCrashed) {
failedRuns[currentTest] = pu.serverFailMessages;
pu.serverFailMessages = "";
}
pu.aggregateFatalErrors(currentTest);
}
results.status = globalStatus;
@ -755,14 +510,8 @@ function iterateTests(cases, options, jsonReply) {
}
if (options.extremeVerbosity === true) {
try {
print(yaml.safeDump(JSON.parse(JSON.stringify(results))));
} catch (err) {
print(RED + 'cannot dump results: ' + String(err) + RESET);
print(RED + require('internal').inspect(results) + RESET);
}
rp.yamlDumpResults(options, results);
}
return results;
}
@ -779,20 +528,10 @@ function unitTest (cases, options) {
if (typeof options !== 'object') {
options = {};
}
loadTestSuites();
loadTestSuites(options);
// testsuites may register more defaults...
_.defaults(options, optionsDefaults);
if (cases === undefined || cases.length === 0) {
printUsage();
print('FATAL: "which" is undefined\n');
return {
status: false,
crashed: false
};
}
try {
pu.setupBinaries(options.build, options.buildType, options.configDir);
}
@ -808,25 +547,18 @@ function unitTest (cases, options) {
}]
};
}
const jsonReply = options.jsonReply;
delete options.jsonReply;
let results = iterateTests(cases, options, jsonReply);
if (jsonReply === true) {
return results;
if ((cases.length === 1) && cases[0] === 'auto') {
return autoTest(options);
} else {
return globalStatus;
return iterateTests(cases, options);
}
}
// /////////////////////////////////////////////////////////////////////////////
// exports
// /////////////////////////////////////////////////////////////////////////////
exports.optionsDefaults = optionsDefaults;
exports.unitTest = unitTest;
exports.internalMembers = internalMembers;
exports.testFuncs = testFuncs;
exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults;
exports.loadBlacklist = loadBlacklist;

1
js/client/modules/@arangodb/testsuites/agency.js
View File

@ -50,7 +50,6 @@ function agency (options) {
options.agency = true;
options.cluster = false;
let results = tu.performTests(options, testCases, 'agency', tu.runInArangosh);
options.agency = saveAgency;

1
js/client/modules/@arangodb/testsuites/audit.js
View File

@ -74,6 +74,7 @@ function auditLog(onServer) {
'server.authentication': 'true',
'server.jwt-secret': 'haxxmann',
'log.level': 'audit-authentication=info',
'log.force-direct': true
};
print(CYAN + 'Audit log server tests...' + RESET);

110
js/client/modules/@arangodb/testsuites/config.js
View File

@ -99,89 +99,79 @@ function config (options) {
print('absolute config tests');
print('--------------------------------------------------------------------------------');
if (options.skipTest('TEST', 'config.absolute')) {
print(YELLOW + "[SKIPPED] config.absolute" + RESET + "\n");
results.absolute.skipped = true;
} else {
// we append one cleanup directory for the invoking logic...
let dummyDir = fs.join(fs.getTempPath(), 'configdummy');
fs.makeDirectory(dummyDir);
pu.cleanupDBDirectoriesAppend(dummyDir);
// we append one cleanup directory for the invoking logic...
let dummyDir = fs.join(fs.getTempPath(), 'configdummy');
fs.makeDirectory(dummyDir);
pu.cleanupDBDirectoriesAppend(dummyDir);
let startTime = time();
let startTime = time();
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
const args = {
'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const args = {
'configuration': fs.join(pu.CONFIG_ARANGODB_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const run = fs.join(pu.BIN_DIR, test);
const run = fs.join(pu.BIN_DIR, test);
results.absolute[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck);
results.absolute[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck);
if (!results.absolute[test].status) {
results.absolute.status = false;
results.absolute.failed += 1;
results.failed += 1;
}
results.absolute.total++;
if (options.verbose) {
print('Args for [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.absolute[test].status);
}
if (!results.absolute[test].status) {
results.absolute.status = false;
results.absolute.failed += 1;
results.failed += 1;
}
results.absolute.duration = time() - startTime;
results.absolute.total++;
if (options.verbose) {
print('Args for [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.absolute[test].status);
}
}
results.absolute.duration = time() - startTime;
print('\n--------------------------------------------------------------------------------');
print('relative config tests');
print('--------------------------------------------------------------------------------');
if (options.skipTest('TEST', 'config.relative')) {
print(YELLOW + "[SKIPPED] config.relative" + RESET + "\n");
results.relative.skipped = true;
} else {
let startTime = time();
startTime = time();
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
for (let i = 0; i < ts.length; i++) {
const test = ts[i];
print(CYAN + 'checking "' + test + '"' + RESET);
const args = {
'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const args = {
'configuration': fs.join(pu.CONFIG_RELATIVE_DIR, test + '.conf'),
'flatCommands': ['--check-configuration']
};
const run = fs.join(pu.BIN_DIR, test);
const run = fs.join(pu.BIN_DIR, test);
results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck);
results.relative[test] = pu.executeAndWait(run, toArgv(args), options, test, rootDir, options.coreCheck);
if (!results.relative[test].status) {
results.failed += 1;
results.relative.failed += 1;
results.relative.status = false;
}
results.relative.total++;
if (options.verbose) {
print('Args for (relative) [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.relative[test].status);
}
if (!results.relative[test].status) {
results.failed += 1;
results.relative.failed += 1;
results.relative.status = false;
}
results.relative.duration = time() - startTime;
results.relative.total++;
if (options.verbose) {
print('Args for (relative) [' + test + ']:');
print(yaml.safeDump(args));
print('Result: ' + results.relative[test].status);
}
}
results.relative.duration = time() - startTime;
print();
return results;

4
js/client/modules/@arangodb/testsuites/fail.js
View File

@ -134,10 +134,10 @@ function success (options) {
duration: 2,
failed: 1,
failTest: {
status: false,
status: true,
total: 1,
duration: 1,
message: 'this testcase will always success.'
message: 'this testcase will always succeed.'
},
failSuccessTest: {
status: true,

21
js/common/bootstrap/modules/internal.js
View File

@ -560,6 +560,14 @@ global.DEFINE_MODULE('internal', (function () {
delete global.SYS_PROCESS_JSON_FILE;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief statisticsExternal
// //////////////////////////////////////////////////////////////////////////////
if (global.SYS_PROCESS_STATISTICS_EXTERNAL) {
exports.statisticsExternal = global.SYS_PROCESS_STATISTICS_EXTERNAL;
delete global.SYS_PROCESS_STATISTICS_EXTERNAL;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executeExternal
// //////////////////////////////////////////////////////////////////////////////
@ -756,7 +764,18 @@ global.DEFINE_MODULE('internal', (function () {
} else if (!isNaN(argv[i + 1])) {
ret[option] = parseInt(argv[i + 1]);
} else {
ret[option] = argv[i + 1];
if (ret.hasOwnProperty(option)) {
if (Array.isArray(ret[option])) {
ret[option].push(argv[i + 1]);
} else {
ret[option] = [
ret[option],
argv[i + 1]
];
}
} else {
ret[option] = argv[i + 1];
}
}
}

30
js/common/modules/jsunity.js
View File

@ -44,6 +44,8 @@ var TEARDOWNS = 0;
var TOTALSETUPS = 0;
var TOTALTEARDOWNS = 0;
var jsUnity = require('./jsunity/jsunity').jsUnity;
var STARTTEST = 0.0;
var ENDTEST = 0.0;
@ -141,6 +143,15 @@ jsUnity.results.end = function (passed, failed, duration) {
}
};
jsUnity.results.beginSetUpAll = function(index) {
SETUPS = jsUnity.env.getDate();
};
jsUnity.results.endSetUpAll = function(index) {
RESULTS.setUpAllDuration = jsUnity.env.getDate() - SETUPS;
TOTALSETUPS += RESULTS.setUpAllDuration;
};
jsUnity.results.beginSetUp = function(index, testName) {
if (testCount === 0)
{
@ -171,6 +182,15 @@ jsUnity.results.endTeardown = function(index, testName) {
TOTALTEARDOWNS += RESULTS[testName].tearDownDuration;
};
jsUnity.results.beginTeardownAll = function(index) {
TEARDOWNS = jsUnity.env.getDate();
};
jsUnity.results.endTeardownAll = function(index) {
RESULTS.teardownAllDuration = jsUnity.env.getDate() - TEARDOWNS;
TOTALTEARDOWNS += RESULTS.teardownAllDuration;
};
// //////////////////////////////////////////////////////////////////////////////
// / @brief runs a test with context
// //////////////////////////////////////////////////////////////////////////////
@ -255,11 +275,15 @@ function Run (testsuite) {
PASSED += result.passed;
FAILED += result.failed;
DURATION += result.duration;
let duplicates = [];
for (var attrname in RESULTS) {
if (RESULTS.hasOwnProperty(attrname)) {
if (typeof(RESULTS[attrname]) === 'number') {
if (!COMPLETE.hasOwnProperty(attrname)) {
COMPLETE[attrname] = { };
}
COMPLETE[attrname][suite.suiteName] = RESULTS[attrname];
} else if (RESULTS.hasOwnProperty(attrname)) {
if (COMPLETE.hasOwnProperty(attrname)) {
print("Duplicate testsuite '" + attrname + "' - already have: " + JSON.stringify(COMPLETE[attrname]) + "");
duplicates.push(attrname);

12
js/common/modules/jsunity/jsunity.js
View File

@ -378,6 +378,10 @@ var jsUnity = exports.jsUnity = (function () {
jsUnity.log.info(plural(total, "test") + " found");
},
beginSetUpAll: function(index, testName) {},
endSetUpAll: function(index, testName) {},
beginSetUp: function(index, testName) {},
endSetUp: function(index, testName) {},
@ -399,6 +403,10 @@ var jsUnity = exports.jsUnity = (function () {
endTeardown: function(index, testName) {},
beginTeardownAll: function(index, testName) {},
endTeardownAll: function(index, testName) {},
end: function (passed, failed, duration) {
jsUnity.log.info(plural(passed, "test") + " passed");
jsUnity.log.info(plural(failed, "test") + " failed");
@ -507,7 +515,9 @@ var jsUnity = exports.jsUnity = (function () {
var runSuite;
try {
this.results.beginSetUpAll(suite.scope);
setUpAll(suite.suiteName);
this.results.endSetUpAll(suite.scope);
runSuite = true;
} catch (setUpAllError) {
runSuite = false;
@ -583,7 +593,9 @@ var jsUnity = exports.jsUnity = (function () {
}
try {
this.results.beginTeardownAll(suite.scope);
tearDownAll(suite.suiteName);
this.results.endTeardownAll(suite.scope);
} catch (tearDownAllError) {
if (tearDownAllError.stack !== undefined) {
this.results.fail(0, suite.suiteName,

59
lib/Basics/process-utils.cpp
View File

@ -173,6 +173,18 @@ ExternalProcess::~ExternalProcess() {
ExternalProcessStatus::ExternalProcessStatus()
: _status(TRI_EXT_NOT_STARTED), _exitStatus(0), _errorMessage() {}
static ExternalProcess* TRI_LookupSpawnedProcess(TRI_pid_t pid) {
{
MUTEX_LOCKER(mutexLocker, ExternalProcessesLock);
auto found = std::find_if(ExternalProcesses.begin(), ExternalProcesses.end(),
[pid](const ExternalProcess * m) -> bool { return m->_pid == pid; });
if (found != ExternalProcesses.end()) {
return *found;
}
}
return nullptr;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief creates pipe pair
////////////////////////////////////////////////////////////////////////////////
@ -655,13 +667,14 @@ static time_t _FileTime_to_POSIX(FILETIME* ft) {
return (ts - 116444736000000000) / 10000000;
}
ProcessInfo TRI_ProcessInfoSelf() {
ProcessInfo TRI_ProcessInfoH(HANDLE processHandle, TRI_pid_t pid) {
ProcessInfo result;
PROCESS_MEMORY_COUNTERS_EX pmc;
pmc.cb = sizeof(PROCESS_MEMORY_COUNTERS_EX);
// compiler warning wird in kauf genommen!c
// http://msdn.microsoft.com/en-us/library/windows/desktop/ms684874(v=vs.85).aspx
if (GetProcessMemoryInfo(GetCurrentProcess(), (PPROCESS_MEMORY_COUNTERS)&pmc, pmc.cb)) {
if (GetProcessMemoryInfo(processHandle, (PPROCESS_MEMORY_COUNTERS)&pmc, pmc.cb)) {
result._majorPageFaults = pmc.PageFaultCount;
// there is not any corresponce to minflt in linux
result._minorPageFaults = 0;
@ -686,7 +699,7 @@ ProcessInfo TRI_ProcessInfoSelf() {
}
/// computing times
FILETIME creationTime, exitTime, kernelTime, userTime;
if (GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime, &kernelTime, &userTime)) {
if (GetProcessTimes(processHandle, &creationTime, &exitTime, &kernelTime, &userTime)) {
// see remarks in
// http://msdn.microsoft.com/en-us/library/windows/desktop/ms683223(v=vs.85).aspx
// value in seconds
@ -697,8 +710,7 @@ ProcessInfo TRI_ProcessInfoSelf() {
// the function '_FileTime_to_POSIX' should be called
}
/// computing number of threads
DWORD myPID = GetCurrentProcessId();
HANDLE snapShot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, myPID);
HANDLE snapShot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, pid);
if (snapShot != INVALID_HANDLE_VALUE) {
THREADENTRY32 te32;
@ -706,16 +718,35 @@ ProcessInfo TRI_ProcessInfoSelf() {
if (Thread32First(snapShot, &te32)) {
result._numberThreads++;
while (Thread32Next(snapShot, &te32)) {
if (te32.th32OwnerProcessID == myPID) {
if (te32.th32OwnerProcessID == pid) {
result._numberThreads++;
}
}
}
else {
LOG_TOPIC("66667", ERR, arangodb::Logger::FIXME) << "failed to acquire thread from snapshot - " << GetLastError();
}
CloseHandle(snapShot);
}
else {
LOG_TOPIC("66668", ERR, arangodb::Logger::FIXME) << "failed to acquire process threads count - " << GetLastError();
}
return result;
}
ProcessInfo TRI_ProcessInfoSelf() {
return TRI_ProcessInfoH(GetCurrentProcess(), GetCurrentProcessId());
}
ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) {
auto external = TRI_LookupSpawnedProcess(pid);
if (external != nullptr) {
return TRI_ProcessInfoH(external->_process, pid);
}
return {};
}
#endif
////////////////////////////////////////////////////////////////////////////////
@ -882,7 +913,7 @@ ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) {
}
#else
#ifndef _WIN32
ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) {
ProcessInfo result;
@ -890,7 +921,7 @@ ProcessInfo TRI_ProcessInfo(TRI_pid_t pid) {
return result;
}
#endif
#endif
////////////////////////////////////////////////////////////////////////////////
@ -983,17 +1014,7 @@ ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait, uint32
status._status = TRI_EXT_NOT_FOUND;
status._exitStatus = 0;
ExternalProcess* external = nullptr;
{
MUTEX_LOCKER(mutexLocker, ExternalProcessesLock);
for (auto& it : ExternalProcesses) {
if (it->_pid == pid._pid) {
external = it;
break;
}
}
}
auto external = TRI_LookupSpawnedProcess(pid._pid);
if (external == nullptr) {
status._errorMessage =

2
lib/Basics/threads-posix.h
View File

@ -38,6 +38,8 @@
#define TRI_pid_t pid_t
#define TRI_vpack_pid_t int
////////////////////////////////////////////////////////////////////////////////
/// @brief thread library identifier
////////////////////////////////////////////////////////////////////////////////

4
lib/Basics/threads-win32.h
View File

@ -31,7 +31,9 @@
/// @brief process identifier
////////////////////////////////////////////////////////////////////////////////
#define TRI_pid_t int
#define TRI_pid_t DWORD
#define TRI_vpack_pid_t int
////////////////////////////////////////////////////////////////////////////////
/// @brief thread identifier

49
lib/V8/v8-utils.cpp
View File

@ -2821,13 +2821,12 @@ static void JS_Output(v8::FunctionCallbackInfo<v8::Value> const& args) {
/// @END_EXAMPLE_ARANGOSH_OUTPUT
////////////////////////////////////////////////////////////////////////////////
static void JS_ProcessStatistics(v8::FunctionCallbackInfo<v8::Value> const& args) {
static void ProcessStatisticsToV8(v8::FunctionCallbackInfo<v8::Value> const& args, ProcessInfo const& info) {
TRI_V8_TRY_CATCH_BEGIN(isolate)
v8::HandleScope scope(isolate);
v8::Handle<v8::Object> result = v8::Object::New(isolate);
ProcessInfo info = TRI_ProcessInfoSelf();
double rss = (double)info._residentSize;
double rssp = 0;
@ -2855,6 +2854,36 @@ static void JS_ProcessStatistics(v8::FunctionCallbackInfo<v8::Value> const& args
TRI_V8_TRY_CATCH_END
}
static void JS_ProcessStatisticsSelf(v8::FunctionCallbackInfo<v8::Value> const& args) {
ProcessStatisticsToV8(args, TRI_ProcessInfoSelf());
}
static void JS_ProcessStatisticsExternal(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_TRY_CATCH_BEGIN(isolate);
v8::HandleScope scope(isolate);
if (args.Length() != 1) {
TRI_V8_THROW_EXCEPTION_USAGE(
"statisticsExternal(<external-identifier>)");
}
auto& server = application_features::ApplicationServer::server();
V8SecurityFeature& v8security = server.getFeature<V8SecurityFeature>();
if (v8security.isInternalModuleHardened(isolate)) {
TRI_V8_THROW_EXCEPTION_MESSAGE(
TRI_ERROR_FORBIDDEN,
"not allowed to execute or modify state of external processes");
}
ExternalId pid;
pid._pid = static_cast<TRI_pid_t>(TRI_ObjectToUInt64(isolate, args[0], true));
ProcessStatisticsToV8(args, TRI_ProcessInfo(pid._pid));
TRI_V8_TRY_CATCH_END
}
static void JS_GetPid(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_TRY_CATCH_BEGIN(isolate)
v8::HandleScope scope(isolate);
@ -4355,11 +4384,7 @@ static void JS_StatusExternal(v8::FunctionCallbackInfo<v8::Value> const& args) {
ExternalId pid;
#ifndef _WIN32
pid._pid = static_cast<TRI_pid_t>(TRI_ObjectToUInt64(isolate, args[0], true));
#else
pid._pid = static_cast<DWORD>(TRI_ObjectToUInt64(isolate, args[0], true));
#endif
bool wait = false;
if (args.Length() >= 2) {
wait = TRI_ObjectToBoolean(isolate, args[1]);
@ -4550,6 +4575,11 @@ static void JS_KillExternal(v8::FunctionCallbackInfo<v8::Value> const& args) {
#else
pid._pid = static_cast<DWORD>(TRI_ObjectToUInt64(isolate, args[0], true));
#endif
if (pid._pid == 0) {
TRI_V8_THROW_EXCEPTION_MESSAGE(
TRI_ERROR_FORBIDDEN,
"not allowed to kill 0");
}
// return the result
v8::Handle<v8::Object> result = v8::Object::New(isolate);
@ -5540,6 +5570,11 @@ void TRI_InitV8Utils(v8::Isolate* isolate, v8::Handle<v8::Context> context,
TRI_AddGlobalFunctionVocbase(
isolate, TRI_V8_ASCII_STRING(isolate, "SYS_SPLIT_WORDS_ICU"), JS_SplitWordlist);
TRI_AddGlobalFunctionVocbase(isolate,
TRI_V8_ASCII_STRING(isolate,
"SYS_PROCESS_STATISTICS_EXTERNAL"),
JS_ProcessStatisticsExternal);
TRI_AddGlobalFunctionVocbase(isolate,
TRI_V8_ASCII_STRING(isolate,
"SYS_GET_EXTERNAL_SPAWNED"),
@ -5578,7 +5613,7 @@ void TRI_InitV8Utils(v8::Isolate* isolate, v8::Handle<v8::Context> context,
TRI_AddGlobalFunctionVocbase(isolate,
TRI_V8_ASCII_STRING(isolate,
"SYS_PROCESS_STATISTICS"),
JS_ProcessStatistics);
JS_ProcessStatisticsSelf);
TRI_AddGlobalFunctionVocbase(isolate,
TRI_V8_ASCII_STRING(isolate, "SYS_GET_PID"), JS_GetPid);
TRI_AddGlobalFunctionVocbase(isolate, TRI_V8_ASCII_STRING(isolate, "SYS_RAND"), JS_Rand);

92
scripts/examine_results.js Executable file
View File

@ -0,0 +1,92 @@
#!bin/arangosh --javascript.execute
/* jshint globalstrict:false, unused:false */
/* global print, start_pretty_print, ARGUMENTS */
'use strict';
const _ = require('lodash');
const fs = require('fs');
const internal = require('internal');
const rp = require('@arangodb/result-processing');
const optionsDefaults = require('@arangodb/testing').optionsDefaults;
/* Constants: */
const BLUE = internal.COLORS.COLOR_BLUE;
const CYAN = internal.COLORS.COLOR_CYAN;
const GREEN = internal.COLORS.COLOR_GREEN;
const RED = internal.COLORS.COLOR_RED;
const RESET = internal.COLORS.COLOR_RESET;
const YELLOW = internal.COLORS.COLOR_YELLOW;
function main (argv) {
start_pretty_print();
let analyzers = []; // e.g all, http_server, recovery, ...
let options = {};
while (argv.length >= 1) {
if (argv[0].slice(0, 1) === '-') { // break parsing if we hit some -option
break;
}
analyzers = _.merge(analyzers, argv[0].split(','));
argv = argv.slice(1); // and remove first arg (c++:pop_front/bash:shift)
}
if (argv.length >= 1) {
try {
options = internal.parseArgv(argv, 0);
} catch (x) {
print('failed to parse the options: ' + x.message + '\n' + String(x.stack));
print('argv: ', argv);
throw x;
}
}
if (!options.hasOwnProperty('readFile')) {
print("need a file to load!");
process.exit(1);
}
let readFile;
if (Array.isArray(options.readFile)) {
readFile = options.readFile;
} else {
readFile = [ options.readFile ];
}
let rc = readFile.forEach(file => {
let ret = true;
let resultsDump;
try {
resultsDump = fs.read(file);
} catch (ex) {
print(RED + "File not found [" + file + "]: " + ex.message + RESET + "\n");
return false;
}
let results;
try {
results = JSON.parse(resultsDump);
}
catch (ex) {
print(RED + "Failed to parse " + options.readFile + " - " + ex.message + RESET + "\n");
return false;
}
_.defaults(options, optionsDefaults);
try {
print(YELLOW + "Analyzing: " + file + RESET);
ret = ret && analyzers.forEach(function(which) {
rp.analyze[which](options, results);
});
} catch (ex) {
print("Failed to analyze [" + file + "]: " + ex.message + RESET + "\n");
return false;
}
return ret;
});
return rc;
}
let result = main(ARGUMENTS);
if (!result) {
process.exit(1);
}