Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Implement structures in the same way for all other tests, and move the pretty printer over, so users calling testing.js directly can also enjoy human readable test results.

This commit is contained in:
Willi Goesgens 2014-09-23 16:56:24 +02:00
parent ede622ea73
commit 357764ac57
3 changed files with 209 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
js/common/modules/reportgenerator.js
View File

@ -28,30 +28,30 @@ var generatePerfReport = function (reportName, testdata, longdesc) {
var j = 0; var j = 0;
for (i = 0; i < measurements.length; i++) { // for (i = 0; i < measurements.length; i++) {
if (oneResultSet.hasOwnProperty([measurements[i]])) { // if (oneResultSet.hasOwnProperty([measurements[i]])) {
var s = oneResultSet[measurements[i]].sum; var s = oneResultSet[measurements[i]].sum;
var t = oneResultSet[measurements[i]].avg; var t = oneResultSet[measurements[i]].avg;
x = x + '\t<test name="/' + testname + '/' + testrunner + '/' + measurements[i] + '" executed="yes">\n'; x = x + '\t<test name="/' + testname + '/' + testrunner + '/' + measurements[i] + '" executed="yes">\n';
x = x + '\t\t<description><![CDATA[This is the description of the test ' + testname + ' executed by the testrunner ' + testrunner + ' run component ' + measurements[i] + ']]></description>\n'; x = x + '\t\t<description><![CDATA[This is the description of the test ' + testname + ' executed by the testrunner ' + testrunner + ' run component ' + measurements[i] + ']]></description>\n';
x = x + '\t\t<targets>\n\t\t\t<target threaded="false">AQL</target>\n\t\t</targets>\n'; x = x + '\t\t<targets>\n\t\t\t<target threaded="false">AQL</target>\n\t\t</targets>\n';
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + s + '">Sum ' + measurements[i] + '</commandline>\n'; x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + s + '">Sum time</commandline>\n';
j += 1; j += 1;
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + t + '">Average ' + measurements[i] + '</commandline>\n'; x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + t + '">Average ' + measurements[i] + '</commandline>\n';
j += 1; j += 1;
x = x + '\t\t<result>\n'; x = x + '\t\t<result>\n';
x += '\t\t\t<success passed="yes" state="100" hasTimedOut="false" />\n'; x += '\t\t\t<success passed="yes" state="100" hasTimedOut="false" />\n';
x += '\t\t\t<compiletime unit="s" mesure="1.0" isRelevant="false" />\n'; x += '\t\t\t<compiletime unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<performance unit="GFLOPs" mesure="1.0" isRelevant="false" />\n'; x += '\t\t\t<performance unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<executiontime unit="s" mesure="'+ oneResultSet[measurements[0]].avg + '" isRelevant="true" />\n'; x += '\t\t\t<executiontime unit="s" mesure="'+ oneResultSet[measurements[0]].avg + '" isRelevant="true" />\n';
x += '\t\t</result>\n'; x += '\t\t</result>\n';
x = x + '\t</test>\n'; x = x + '\t</test>\n';
} // }
} // }
} }
} }
} }

186
js/server/modules/org/arangodb/testing.js
View File

@ -57,6 +57,8 @@
/// ///
/// The following properties of `options` are defined: /// The following properties of `options` are defined:
/// ///
/// - `jsonReply`: if set a json is returned which the caller has to
/// present the user
/// - `force`: if set to true the tests are continued even if one fails /// - `force`: if set to true the tests are continued even if one fails
/// - `skipBoost`: if set to true the boost unittests are skipped /// - `skipBoost`: if set to true the boost unittests are skipped
/// - `skipGeo`: if set to true the geo index tests are skipped /// - `skipGeo`: if set to true the geo index tests are skipped
@ -110,7 +112,8 @@ var optionsDefaults = { "cluster": false,
"test": undefined, "test": undefined,
"skipServer": false, "skipServer": false,
"skipClient": false, "skipClient": false,
"cleanup": true }; "cleanup": true,
"jsonReply": false};
function findTopDir () { function findTopDir () {
var topDir = fs.normalize(fs.makeAbsolute(".")); var topDir = fs.normalize(fs.makeAbsolute("."));
@ -397,9 +400,11 @@ function executeAndWait (cmd, args) {
var pid = executeExternal(cmd, args); var pid = executeExternal(cmd, args);
var res = statusExternal(pid, true); var res = statusExternal(pid, true);
if (res.status === "TERMINATED") { if (res.status === "TERMINATED") {
return res.exit; return { status: (res.exit === 0), message: ""};
}
else {
return { status: (res === 0), message: res.exit};
} }
return res;
} }
function runInArangosh (options, instanceInfo, file, addArgs) { function runInArangosh (options, instanceInfo, file, addArgs) {
@ -543,14 +548,14 @@ testFuncs.config = function () {
results[t] = executeAndWait(fs.join(topDir,"bin",t), results[t] = executeAndWait(fs.join(topDir,"bin",t),
["--configuration", fs.join(topDir,"etc","arangodb",t+".conf"), ["--configuration", fs.join(topDir,"etc","arangodb",t+".conf"),
"--help"]); "--help"]);
print("Config test "+t+"...",results[t]); print("Config test "+t+"...",results[t].status);
} }
for (i = 0; i < ts.length; i++) { for (i = 0; i < ts.length; i++) {
t = ts[i]; t = ts[i];
results[t+"_rel"] = executeAndWait(fs.join(topDir,"bin",t), results[t+"_rel"] = executeAndWait(fs.join(topDir,"bin",t),
["--configuration", fs.join(topDir,"etc","relative", ["--configuration", fs.join(topDir,"etc","relative",
t+".conf"), "--help"]); t+".conf"), "--help"]);
print("Config test "+t+" (relative)...",results[t+"_rel"]); print("Config test "+t+" (relative)...",results[t+"_rel"].status);
} }
return results; return results;
@ -642,7 +647,13 @@ function rubyTests (options, ssl) {
fs.join("UnitTests","HttpInterface",n)]; fs.join("UnitTests","HttpInterface",n)];
var pid = executeExternal("rspec", args); var pid = executeExternal("rspec", args);
var r = statusExternal(pid, true); var r = statusExternal(pid, true);
result[n] = r.exit;
if (r.status === "TERMINATED") {
result[n] = { status: (r.exit === 0), message: r.exit};
}
else {
result[n] = { status: (r === 0), message: r.exit};
}
if (r.exit !== 0 && !options.force) { if (r.exit !== 0 && !options.force) {
break; break;
} }
@ -954,17 +965,24 @@ testFuncs.authentication_parameters = function (options) {
"authparams"); "authparams");
var r; var r;
var i; var i;
var re = []; var expectAuthFullRC = [401, 401, 401, 401, 401, 401, 401];
for (i = 0;i < urlsTodo.length;i++) { var all_ok = true;
results["auth_full"] = {};
for (i = 0; i < urlsTodo.length; i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true}); r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code); if (r.code === expectAuthFullRC[i]) {
} results.auth_full[urlsTodo[i]] = { status: true, message: ""};
if (_.isEqual(re,[401, 401, 401, 401, 401, 401, 401])) { }
results.auth_full = 0; else {
} results.auth_full[urlsTodo[i]] = {
else { status: false,
results.auth_full = re; message: "we expected " + expectAuthFullRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
} }
results.auth_full.status = all_ok;
print("Shutting down..."); print("Shutting down...");
shutdownInstance(instanceInfo,options); shutdownInstance(instanceInfo,options);
print("done."); print("done.");
@ -973,17 +991,24 @@ testFuncs.authentication_parameters = function (options) {
["--server.disable-authentication", "false", ["--server.disable-authentication", "false",
"--server.authenticate-system-only", "true"], "--server.authenticate-system-only", "true"],
"authparams2"); "authparams2");
re = []; var expectAuthSystemRC = [401, 401, 401, 401, 401, 404, 404];
all_ok = true;
results["auth_system"] = {};
for (i = 0;i < urlsTodo.length;i++) { for (i = 0;i < urlsTodo.length;i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true}); r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code); if (r.code === expectAuthSystemRC[i]) {
} results.auth_system[urlsTodo[i]] = { status: true, message: ""};
if (_.isEqual(re, [401, 401, 401, 401, 401, 404, 404])) { }
results.auth_system = 0; else {
} results.auth_system[urlsTodo[i]] = {
else { status: false,
results.auth_system = re; message: "we expected " + expectAuthSystemRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
} }
results.auth_system.status = all_ok;
print("Shutting down..."); print("Shutting down...");
shutdownInstance(instanceInfo,options); shutdownInstance(instanceInfo,options);
print("done."); print("done.");
@ -992,23 +1017,59 @@ testFuncs.authentication_parameters = function (options) {
["--server.disable-authentication", "true", ["--server.disable-authentication", "true",
"--server.authenticate-system-only", "true"], "--server.authenticate-system-only", "true"],
"authparams3"); "authparams3");
re = []; var expectAuthNoneRC = [404, 404, 200, 301, 301, 404, 404];
results["auth_none"] = {};
all_ok = true;
for (i = 0;i < urlsTodo.length;i++) { for (i = 0;i < urlsTodo.length;i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true}); r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code); if (r.code === expectAuthNoneRC[i]) {
} results.auth_none[urlsTodo[i]] = { status: true, message: ""};
if (_.isEqual(re, [404, 404, 200, 301, 301, 404, 404])) { }
results.auth_none = 0; else {
} results.auth_none[urlsTodo[i]] = {
else { status: false,
results.auth_none = re; message: "we expected " + expectAuthNoneRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
} }
results.auth_none.status = all_ok;
print("Shutting down..."); print("Shutting down...");
shutdownInstance(instanceInfo,options); shutdownInstance(instanceInfo,options);
print("done."); print("done.");
return results; return results;
}; };
var internalMembers = ["code", "error", "status", "duration", "failed", "total"];
function unitTestPrettyPrintResults(r) {
for (var testrun in r) {
if (r.hasOwnProperty(testrun) && (testrun !== 'all_ok')) {
print("Testrun: " + testrun);
for (var test in r[testrun]) {
if (r[testrun].hasOwnProperty(test) && (test !== 'ok')) {
if (r[testrun][test].status) {
print(" " + test + ": Success");
}
else {
print(" " + test + ": Fail");
for (var oneTest in r[testrun][test]) {
if ((r[testrun][test].hasOwnProperty(oneTest)) &&
(internalMembers.indexOf(oneTest) === -1) &&
(!r[testrun][test][oneTest].status)) {
print(" -> " + oneTest + " Failed; Verbose message:");
print(r[testrun][test][oneTest].message);
}
}
}
}
}
}
}
print("Overal state: " + ((r.all_ok === true) ? "Success" : "Fail"));
}
var allTests = var allTests =
[ [
@ -1055,6 +1116,8 @@ function printUsage () {
} }
function UnitTest (which, options) { function UnitTest (which, options) {
var allok = true;
var results = {};
if (typeof options !== "object") { if (typeof options !== "object") {
options = {}; options = {};
} }
@ -1063,20 +1126,19 @@ function UnitTest (which, options) {
printUsage(); printUsage();
return; return;
} }
var jsonReply = options.jsonReply
delete(options.jsonReply);
var i; var i;
var ok; var ok;
var r;
if (which === "all") { if (which === "all") {
var n; var n;
var results = {};
var allok = true;
for (n = 0;n < allTests.length;n++) { for (n = 0;n < allTests.length;n++) {
print("Doing test",allTests[n],"with options",options); print("Doing test",allTests[n],"with options",options);
results[allTests[n]] = r = testFuncs[allTests[n]](options); results[allTests[n]] = r = testFuncs[allTests[n]](options);
ok = true; ok = true;
for (i in r) { for (i in r) {
if (r.hasOwnProperty(i)) { if (r.hasOwnProperty(i)) {
if (r[i] !== 0 && r[i].status !== true) { if (r[i].status !== true) {
ok = false; ok = false;
} }
} }
@ -1085,32 +1147,48 @@ function UnitTest (which, options) {
if (!ok) { if (!ok) {
allok = false; allok = false;
} }
results.all_ok = allok;
} }
results.all_ok = allok; results.all_ok = allok;
return results; if (jsonReply === true ) {
} return results;
if (!testFuncs.hasOwnProperty(which)) { }
printUsage(); else {
throw 'Unknown test "'+which+'"'; unitTestPrettyPrintResults(results);
} return allok;
var rr = {}; }
rr[which] = r = testFuncs[which](options); }
ok = true; else if (!testFuncs.hasOwnProperty(which)) {
for (i in r) { printUsage();
if (r.hasOwnProperty(i) && throw 'Unknown test "' + which + '"';
(which !== "single" || i !== "test")) { }
if (r[i] !== 0 && r[i].status !== true) { else {
ok = false; var r;
} results[which] = r = testFuncs[which](options);
ok = true;
for (i in r) {
if (r.hasOwnProperty(i) &&
(which !== "single" || i !== "test")) {
if (r[i].status !== true) {
ok = false;
allok = false;
}
}
}
r.ok = ok;
results.all_ok = ok;
if (jsonReply === true ) {
return results;
}
else {
unitTestPrettyPrintResults(results);
return allok;
} }
} }
r.ok = ok;
rr.all_ok = ok;
return rr;
} }
exports.UnitTest = UnitTest; exports.UnitTest = UnitTest;
exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults;
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE // --SECTION-- END-OF-FILE
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------

109
scripts/unittest.js
View File

@ -13,60 +13,63 @@ function resultsToXml(results, baseName) {
}); });
} }
for (var test in results.shell_server_aql) { for (var testrun in results) {
var xml = [ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ];
for (var test in results[testrun]) {
var xml = [ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ];
xml.text = function (s) { xml.text = function (s) {
Array.prototype.push.call(this, s); Array.prototype.push.call(this, s);
return this; return this;
}; };
xml.elem = function (tagName, attrs, close) {
this.text("<").text(tagName);
for (var a in attrs || {}) { xml.elem = function (tagName, attrs, close) {
this.text(" ").text(a).text("=\"") this.text("<").text(tagName);
.text(xmlEscape(String(attrs[a]))).text("\"");
}
close && this.text("/");
this.text(">\n");
return this;
};
xml.elem("testsuite", {
errors: 0,
failures: results.shell_server_aql[test].failed,
name: test,
tests: results.shell_server_aql[test].total,
time: results.shell_server_aql[test].duration
});
for (var oneTest in results.shell_server_aql[test]) {
if (internalMembers.indexOf(oneTest) === -1) {
var result = results.shell_server_aql[test][oneTest].status;
var success = (typeof(result) === 'boolean')? result : false;
xml.elem("testcase", { for (var a in attrs || {}) {
name: oneTest, this.text(" ").text(a).text("=\"")
time: results.shell_server_aql[test][oneTest].duration .text(xmlEscape(String(attrs[a]))).text("\"");
}, success); }
if (!success) { close && this.text("/");
xml.elem("failure"); this.text(">\n");
xml.text('<![CDATA[' + results.shell_server_aql[test][oneTest].message + ']]>\n');
xml.elem("/failure"); return this;
xml.elem("/testcase"); };
xml.elem("testsuite", {
errors: 0,
failures: results[testrun][test].failed,
name: test,
tests: results[testrun][test].total,
time: results[testrun][test].duration
});
for (var oneTest in results[testrun][test]) {
if (internalMembers.indexOf(oneTest) === -1) {
var result = results[testrun][test][oneTest].status;
var success = (typeof(result) === 'boolean')? result : false;
xml.elem("testcase", {
name: oneTest,
time: results[testrun][test][oneTest].duration
}, success);
if (!success) {
xml.elem("failure");
xml.text('<![CDATA[' + results[testrun][test][oneTest].message + ']]>\n');
xml.elem("/failure");
xml.elem("/testcase");
}
} }
} }
xml.elem("/testsuite");
var fn = baseName + testrun.replace(/\//g, '_') + '_' + test.replace(/\//g, '_') + ".xml";
//print('Writing: '+ fn);
fs.write(fn, xml.join(""));
} }
xml.elem("/testsuite");
var fn = baseName + test.replace(/\//g, '_') + ".xml";
//print('Writing: '+ fn);
fs.write(fn, xml.join(""));
} }
} }
@ -81,17 +84,19 @@ function main (argv) {
if (argv.length >= 3) { if (argv.length >= 3) {
options = JSON.parse(argv[2]); options = JSON.parse(argv[2]);
} }
var UnitTest = require("org/arangodb/testing").UnitTest; options.jsonReply = true;
var UnitTest = require("org/arangodb/testing");
start_pretty_print(); start_pretty_print();
var r = UnitTest(test,options); var r = UnitTest.UnitTest(test,options);
fs.write("UNITTEST_RESULT.json",JSON.stringify(r)); fs.write("UNITTEST_RESULT.json",JSON.stringify(r));
fs.write("UNITTEST_RESULT_SUMMARY.txt",JSON.stringify(r.all_ok)); fs.write("UNITTEST_RESULT_SUMMARY.txt",JSON.stringify(r.all_ok));
try { try {
resultsToXml(r, "UNITTEST_RESULT_"); resultsToXml(r, "UNITTEST_RESULT_");
} }
catch (x) { catch (x) {
print("Exception while serializing status xml!");
print(x.message); print(x.message);
} }
print(r); UnitTest.unitTestPrettyPrintResults(r);
} }