Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Implement structures in the same way for all other tests, and move the pretty printer over, so users calling testing.js directly can also enjoy human readable test results.

This commit is contained in:
Willi Goesgens 2014-09-23 16:56:24 +02:00
parent ede622ea73
commit 357764ac57
3 changed files with 209 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
js/common/modules/reportgenerator.js
View File

@ -28,15 +28,15 @@ var generatePerfReport = function (reportName, testdata, longdesc) {
var j = 0;
for (i = 0; i < measurements.length; i++) {
if (oneResultSet.hasOwnProperty([measurements[i]])) {
// for (i = 0; i < measurements.length; i++) {
// if (oneResultSet.hasOwnProperty([measurements[i]])) {
var s = oneResultSet[measurements[i]].sum;
var t = oneResultSet[measurements[i]].avg;
x = x + '\t<test name="/' + testname + '/' + testrunner + '/' + measurements[i] + '" executed="yes">\n';
x = x + '\t\t<description><![CDATA[This is the description of the test ' + testname + ' executed by the testrunner ' + testrunner + ' run component ' + measurements[i] + ']]></description>\n';
x = x + '\t\t<targets>\n\t\t\t<target threaded="false">AQL</target>\n\t\t</targets>\n';
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + s + '">Sum ' + measurements[i] + '</commandline>\n';
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + s + '">Sum time</commandline>\n';
j += 1;
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + t + '">Average ' + measurements[i] + '</commandline>\n';
@ -45,13 +45,13 @@ var generatePerfReport = function (reportName, testdata, longdesc) {
x = x + '\t\t<result>\n';
x += '\t\t\t<success passed="yes" state="100" hasTimedOut="false" />\n';
x += '\t\t\t<compiletime unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<performance unit="GFLOPs" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<performance unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<executiontime unit="s" mesure="'+ oneResultSet[measurements[0]].avg + '" isRelevant="true" />\n';
x += '\t\t</result>\n';
x = x + '\t</test>\n';
}
}
// }
// }
}
}
}

148
js/server/modules/org/arangodb/testing.js
View File

@ -57,6 +57,8 @@
///
/// The following properties of `options` are defined:
///
/// - `jsonReply`: if set a json is returned which the caller has to
/// present the user
/// - `force`: if set to true the tests are continued even if one fails
/// - `skipBoost`: if set to true the boost unittests are skipped
/// - `skipGeo`: if set to true the geo index tests are skipped
@ -110,7 +112,8 @@ var optionsDefaults = { "cluster": false,
"test": undefined,
"skipServer": false,
"skipClient": false,
"cleanup": true };
"cleanup": true,
"jsonReply": false};
function findTopDir () {
var topDir = fs.normalize(fs.makeAbsolute("."));
@ -397,9 +400,11 @@ function executeAndWait (cmd, args) {
var pid = executeExternal(cmd, args);
var res = statusExternal(pid, true);
if (res.status === "TERMINATED") {
return res.exit;
return { status: (res.exit === 0), message: ""};
}
else {
return { status: (res === 0), message: res.exit};
}
return res;
}
function runInArangosh (options, instanceInfo, file, addArgs) {
@ -543,14 +548,14 @@ testFuncs.config = function () {
results[t] = executeAndWait(fs.join(topDir,"bin",t),
["--configuration", fs.join(topDir,"etc","arangodb",t+".conf"),
"--help"]);
print("Config test "+t+"...",results[t]);
print("Config test "+t+"...",results[t].status);
}
for (i = 0; i < ts.length; i++) {
t = ts[i];
results[t+"_rel"] = executeAndWait(fs.join(topDir,"bin",t),
["--configuration", fs.join(topDir,"etc","relative",
t+".conf"), "--help"]);
print("Config test "+t+" (relative)...",results[t+"_rel"]);
print("Config test "+t+" (relative)...",results[t+"_rel"].status);
}
return results;
@ -642,7 +647,13 @@ function rubyTests (options, ssl) {
fs.join("UnitTests","HttpInterface",n)];
var pid = executeExternal("rspec", args);
var r = statusExternal(pid, true);
result[n] = r.exit;
if (r.status === "TERMINATED") {
result[n] = { status: (r.exit === 0), message: r.exit};
}
else {
result[n] = { status: (r === 0), message: r.exit};
}
if (r.exit !== 0 && !options.force) {
break;
}
@ -954,17 +965,24 @@ testFuncs.authentication_parameters = function (options) {
"authparams");
var r;
var i;
var re = [];
var expectAuthFullRC = [401, 401, 401, 401, 401, 401, 401];
var all_ok = true;
results["auth_full"] = {};
for (i = 0; i < urlsTodo.length; i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code);
}
if (_.isEqual(re,[401, 401, 401, 401, 401, 401, 401])) {
results.auth_full = 0;
if (r.code === expectAuthFullRC[i]) {
results.auth_full[urlsTodo[i]] = { status: true, message: ""};
}
else {
results.auth_full = re;
results.auth_full[urlsTodo[i]] = {
status: false,
message: "we expected " + expectAuthFullRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
}
results.auth_full.status = all_ok;
print("Shutting down...");
shutdownInstance(instanceInfo,options);
print("done.");
@ -973,17 +991,24 @@ testFuncs.authentication_parameters = function (options) {
["--server.disable-authentication", "false",
"--server.authenticate-system-only", "true"],
"authparams2");
re = [];
var expectAuthSystemRC = [401, 401, 401, 401, 401, 404, 404];
all_ok = true;
results["auth_system"] = {};
for (i = 0;i < urlsTodo.length;i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code);
}
if (_.isEqual(re, [401, 401, 401, 401, 401, 404, 404])) {
results.auth_system = 0;
if (r.code === expectAuthSystemRC[i]) {
results.auth_system[urlsTodo[i]] = { status: true, message: ""};
}
else {
results.auth_system = re;
results.auth_system[urlsTodo[i]] = {
status: false,
message: "we expected " + expectAuthSystemRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
}
results.auth_system.status = all_ok;
print("Shutting down...");
shutdownInstance(instanceInfo,options);
print("done.");
@ -992,23 +1017,59 @@ testFuncs.authentication_parameters = function (options) {
["--server.disable-authentication", "true",
"--server.authenticate-system-only", "true"],
"authparams3");
re = [];
var expectAuthNoneRC = [404, 404, 200, 301, 301, 404, 404];
results["auth_none"] = {};
all_ok = true;
for (i = 0;i < urlsTodo.length;i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code);
}
if (_.isEqual(re, [404, 404, 200, 301, 301, 404, 404])) {
results.auth_none = 0;
if (r.code === expectAuthNoneRC[i]) {
results.auth_none[urlsTodo[i]] = { status: true, message: ""};
}
else {
results.auth_none = re;
results.auth_none[urlsTodo[i]] = {
status: false,
message: "we expected " + expectAuthNoneRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
}
results.auth_none.status = all_ok;
print("Shutting down...");
shutdownInstance(instanceInfo,options);
print("done.");
return results;
};
var internalMembers = ["code", "error", "status", "duration", "failed", "total"];
function unitTestPrettyPrintResults(r) {
for (var testrun in r) {
if (r.hasOwnProperty(testrun) && (testrun !== 'all_ok')) {
print("Testrun: " + testrun);
for (var test in r[testrun]) {
if (r[testrun].hasOwnProperty(test) && (test !== 'ok')) {
if (r[testrun][test].status) {
print(" " + test + ": Success");
}
else {
print(" " + test + ": Fail");
for (var oneTest in r[testrun][test]) {
if ((r[testrun][test].hasOwnProperty(oneTest)) &&
(internalMembers.indexOf(oneTest) === -1) &&
(!r[testrun][test][oneTest].status)) {
print(" -> " + oneTest + " Failed; Verbose message:");
print(r[testrun][test][oneTest].message);
}
}
}
}
}
}
}
print("Overal state: " + ((r.all_ok === true) ? "Success" : "Fail"));
}
var allTests =
[
@ -1055,6 +1116,8 @@ function printUsage () {
}
function UnitTest (which, options) {
var allok = true;
var results = {};
if (typeof options !== "object") {
options = {};
}
@ -1063,20 +1126,19 @@ function UnitTest (which, options) {
printUsage();
return;
}
var jsonReply = options.jsonReply
delete(options.jsonReply);
var i;
var ok;
var r;
if (which === "all") {
var n;
var results = {};
var allok = true;
for (n = 0;n < allTests.length;n++) {
print("Doing test",allTests[n],"with options",options);
results[allTests[n]] = r = testFuncs[allTests[n]](options);
ok = true;
for (i in r) {
if (r.hasOwnProperty(i)) {
if (r[i] !== 0 && r[i].status !== true) {
if (r[i].status !== true) {
ok = false;
}
}
@ -1085,32 +1147,48 @@ function UnitTest (which, options) {
if (!ok) {
allok = false;
}
results.all_ok = allok;
}
results.all_ok = allok;
if (jsonReply === true ) {
return results;
}
if (!testFuncs.hasOwnProperty(which)) {
else {
unitTestPrettyPrintResults(results);
return allok;
}
}
else if (!testFuncs.hasOwnProperty(which)) {
printUsage();
throw 'Unknown test "' + which + '"';
}
var rr = {};
rr[which] = r = testFuncs[which](options);
else {
var r;
results[which] = r = testFuncs[which](options);
ok = true;
for (i in r) {
if (r.hasOwnProperty(i) &&
(which !== "single" || i !== "test")) {
if (r[i] !== 0 && r[i].status !== true) {
if (r[i].status !== true) {
ok = false;
allok = false;
}
}
}
r.ok = ok;
rr.all_ok = ok;
return rr;
results.all_ok = ok;
if (jsonReply === true ) {
return results;
}
else {
unitTestPrettyPrintResults(results);
return allok;
}
}
}
exports.UnitTest = UnitTest;
exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults;
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

29
scripts/unittest.js
View File

@ -13,7 +13,9 @@ function resultsToXml(results, baseName) {
});
}
for (var test in results.shell_server_aql) {
for (var testrun in results) {
for (var test in results[testrun]) {
var xml = [ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ];
xml.text = function (s) {
@ -37,25 +39,25 @@ function resultsToXml(results, baseName) {
xml.elem("testsuite", {
errors: 0,
failures: results.shell_server_aql[test].failed,
failures: results[testrun][test].failed,
name: test,
tests: results.shell_server_aql[test].total,
time: results.shell_server_aql[test].duration
tests: results[testrun][test].total,
time: results[testrun][test].duration
});
for (var oneTest in results.shell_server_aql[test]) {
for (var oneTest in results[testrun][test]) {
if (internalMembers.indexOf(oneTest) === -1) {
var result = results.shell_server_aql[test][oneTest].status;
var result = results[testrun][test][oneTest].status;
var success = (typeof(result) === 'boolean')? result : false;
xml.elem("testcase", {
name: oneTest,
time: results.shell_server_aql[test][oneTest].duration
time: results[testrun][test][oneTest].duration
}, success);
if (!success) {
xml.elem("failure");
xml.text('<![CDATA[' + results.shell_server_aql[test][oneTest].message + ']]>\n');
xml.text('<![CDATA[' + results[testrun][test][oneTest].message + ']]>\n');
xml.elem("/failure");
xml.elem("/testcase");
}
@ -63,12 +65,13 @@ function resultsToXml(results, baseName) {
}
xml.elem("/testsuite");
var fn = baseName + test.replace(/\//g, '_') + ".xml";
var fn = baseName + testrun.replace(/\//g, '_') + '_' + test.replace(/\//g, '_') + ".xml";
//print('Writing: '+ fn);
fs.write(fn, xml.join(""));
}
}
}
function main (argv) {
@ -81,17 +84,19 @@ function main (argv) {
if (argv.length >= 3) {
options = JSON.parse(argv[2]);
}
var UnitTest = require("org/arangodb/testing").UnitTest;
options.jsonReply = true;
var UnitTest = require("org/arangodb/testing");
start_pretty_print();
var r = UnitTest(test,options);
var r = UnitTest.UnitTest(test,options);
fs.write("UNITTEST_RESULT.json",JSON.stringify(r));
fs.write("UNITTEST_RESULT_SUMMARY.txt",JSON.stringify(r.all_ok));
try {
resultsToXml(r, "UNITTEST_RESULT_");
}
catch (x) {
print("Exception while serializing status xml!");
print(x.message);
}
print(r);
UnitTest.unitTestPrettyPrintResults(r);
}