Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Implement structures in the same way for all other tests, and move the pretty printer over, so users calling testing.js directly can also enjoy human readable test results.

This commit is contained in:
Willi Goesgens 2014-09-23 16:56:24 +02:00
parent ede622ea73
commit 357764ac57
3 changed files with 209 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
js/common/modules/reportgenerator.js
View File

@ -28,30 +28,30 @@ var generatePerfReport = function (reportName, testdata, longdesc) {
var j = 0;
for (i = 0; i < measurements.length; i++) {
if (oneResultSet.hasOwnProperty([measurements[i]])) {
// for (i = 0; i < measurements.length; i++) {
// if (oneResultSet.hasOwnProperty([measurements[i]])) {
var s = oneResultSet[measurements[i]].sum;
var t = oneResultSet[measurements[i]].avg;
x = x + '\t<test name="/' + testname + '/' + testrunner + '/' + measurements[i] + '" executed="yes">\n';
x = x + '\t\t<description><![CDATA[This is the description of the test ' + testname + ' executed by the testrunner ' + testrunner + ' run component ' + measurements[i] + ']]></description>\n';
x = x + '\t\t<targets>\n\t\t\t<target threaded="false">AQL</target>\n\t\t</targets>\n';
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + s + '">Sum ' + measurements[i] + '</commandline>\n';
j += 1;
x = x + '\t<test name="/' + testname + '/' + testrunner + '/' + measurements[i] + '" executed="yes">\n';
x = x + '\t\t<description><![CDATA[This is the description of the test ' + testname + ' executed by the testrunner ' + testrunner + ' run component ' + measurements[i] + ']]></description>\n';
x = x + '\t\t<targets>\n\t\t\t<target threaded="false">AQL</target>\n\t\t</targets>\n';
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + s + '">Sum time</commandline>\n';
j += 1;
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + t + '">Average ' + measurements[i] + '</commandline>\n';
j += 1;
x += '\t\t<commandline rank="' + j + '" time="20100128-195406.590832" duration="' + t + '">Average ' + measurements[i] + '</commandline>\n';
j += 1;
x = x + '\t\t<result>\n';
x += '\t\t\t<success passed="yes" state="100" hasTimedOut="false" />\n';
x += '\t\t\t<compiletime unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<performance unit="GFLOPs" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<executiontime unit="s" mesure="'+ oneResultSet[measurements[0]].avg + '" isRelevant="true" />\n';
x += '\t\t</result>\n';
x = x + '\t</test>\n';
}
}
x = x + '\t\t<result>\n';
x += '\t\t\t<success passed="yes" state="100" hasTimedOut="false" />\n';
x += '\t\t\t<compiletime unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<performance unit="s" mesure="1.0" isRelevant="false" />\n';
x += '\t\t\t<executiontime unit="s" mesure="'+ oneResultSet[measurements[0]].avg + '" isRelevant="true" />\n';
x += '\t\t</result>\n';
x = x + '\t</test>\n';
// }
// }
}
}
}

186
js/server/modules/org/arangodb/testing.js
View File

@ -57,6 +57,8 @@
///
/// The following properties of `options` are defined:
///
/// - `jsonReply`: if set a json is returned which the caller has to
/// present the user
/// - `force`: if set to true the tests are continued even if one fails
/// - `skipBoost`: if set to true the boost unittests are skipped
/// - `skipGeo`: if set to true the geo index tests are skipped
@ -110,7 +112,8 @@ var optionsDefaults = { "cluster": false,
"test": undefined,
"skipServer": false,
"skipClient": false,
"cleanup": true };
"cleanup": true,
"jsonReply": false};
function findTopDir () {
var topDir = fs.normalize(fs.makeAbsolute("."));
@ -397,9 +400,11 @@ function executeAndWait (cmd, args) {
var pid = executeExternal(cmd, args);
var res = statusExternal(pid, true);
if (res.status === "TERMINATED") {
return res.exit;
return { status: (res.exit === 0), message: ""};
}
else {
return { status: (res === 0), message: res.exit};
}
return res;
}
function runInArangosh (options, instanceInfo, file, addArgs) {
@ -543,14 +548,14 @@ testFuncs.config = function () {
results[t] = executeAndWait(fs.join(topDir,"bin",t),
["--configuration", fs.join(topDir,"etc","arangodb",t+".conf"),
"--help"]);
print("Config test "+t+"...",results[t]);
print("Config test "+t+"...",results[t].status);
}
for (i = 0; i < ts.length; i++) {
t = ts[i];
results[t+"_rel"] = executeAndWait(fs.join(topDir,"bin",t),
["--configuration", fs.join(topDir,"etc","relative",
t+".conf"), "--help"]);
print("Config test "+t+" (relative)...",results[t+"_rel"]);
print("Config test "+t+" (relative)...",results[t+"_rel"].status);
}
return results;
@ -642,7 +647,13 @@ function rubyTests (options, ssl) {
fs.join("UnitTests","HttpInterface",n)];
var pid = executeExternal("rspec", args);
var r = statusExternal(pid, true);
result[n] = r.exit;
if (r.status === "TERMINATED") {
result[n] = { status: (r.exit === 0), message: r.exit};
}
else {
result[n] = { status: (r === 0), message: r.exit};
}
if (r.exit !== 0 && !options.force) {
break;
}
@ -954,17 +965,24 @@ testFuncs.authentication_parameters = function (options) {
"authparams");
var r;
var i;
var re = [];
for (i = 0;i < urlsTodo.length;i++) {
var expectAuthFullRC = [401, 401, 401, 401, 401, 401, 401];
var all_ok = true;
results["auth_full"] = {};
for (i = 0; i < urlsTodo.length; i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code);
}
if (_.isEqual(re,[401, 401, 401, 401, 401, 401, 401])) {
results.auth_full = 0;
}
else {
results.auth_full = re;
if (r.code === expectAuthFullRC[i]) {
results.auth_full[urlsTodo[i]] = { status: true, message: ""};
}
else {
results.auth_full[urlsTodo[i]] = {
status: false,
message: "we expected " + expectAuthFullRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
}
results.auth_full.status = all_ok;
print("Shutting down...");
shutdownInstance(instanceInfo,options);
print("done.");
@ -973,17 +991,24 @@ testFuncs.authentication_parameters = function (options) {
["--server.disable-authentication", "false",
"--server.authenticate-system-only", "true"],
"authparams2");
re = [];
var expectAuthSystemRC = [401, 401, 401, 401, 401, 404, 404];
all_ok = true;
results["auth_system"] = {};
for (i = 0;i < urlsTodo.length;i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code);
}
if (_.isEqual(re, [401, 401, 401, 401, 401, 404, 404])) {
results.auth_system = 0;
}
else {
results.auth_system = re;
if (r.code === expectAuthSystemRC[i]) {
results.auth_system[urlsTodo[i]] = { status: true, message: ""};
}
else {
results.auth_system[urlsTodo[i]] = {
status: false,
message: "we expected " + expectAuthSystemRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
}
results.auth_system.status = all_ok;
print("Shutting down...");
shutdownInstance(instanceInfo,options);
print("done.");
@ -992,23 +1017,59 @@ testFuncs.authentication_parameters = function (options) {
["--server.disable-authentication", "true",
"--server.authenticate-system-only", "true"],
"authparams3");
re = [];
var expectAuthNoneRC = [404, 404, 200, 301, 301, 404, 404];
results["auth_none"] = {};
all_ok = true;
for (i = 0;i < urlsTodo.length;i++) {
r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true});
re.push(r.code);
}
if (_.isEqual(re, [404, 404, 200, 301, 301, 404, 404])) {
results.auth_none = 0;
}
else {
results.auth_none = re;
if (r.code === expectAuthNoneRC[i]) {
results.auth_none[urlsTodo[i]] = { status: true, message: ""};
}
else {
results.auth_none[urlsTodo[i]] = {
status: false,
message: "we expected " + expectAuthNoneRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r)
};
all_ok = false;
}
}
results.auth_none.status = all_ok;
print("Shutting down...");
shutdownInstance(instanceInfo,options);
print("done.");
return results;
};
var internalMembers = ["code", "error", "status", "duration", "failed", "total"];
function unitTestPrettyPrintResults(r) {
for (var testrun in r) {
if (r.hasOwnProperty(testrun) && (testrun !== 'all_ok')) {
print("Testrun: " + testrun);
for (var test in r[testrun]) {
if (r[testrun].hasOwnProperty(test) && (test !== 'ok')) {
if (r[testrun][test].status) {
print(" " + test + ": Success");
}
else {
print(" " + test + ": Fail");
for (var oneTest in r[testrun][test]) {
if ((r[testrun][test].hasOwnProperty(oneTest)) &&
(internalMembers.indexOf(oneTest) === -1) &&
(!r[testrun][test][oneTest].status)) {
print(" -> " + oneTest + " Failed; Verbose message:");
print(r[testrun][test][oneTest].message);
}
}
}
}
}
}
}
print("Overal state: " + ((r.all_ok === true) ? "Success" : "Fail"));
}
var allTests =
[
@ -1055,6 +1116,8 @@ function printUsage () {
}
function UnitTest (which, options) {
var allok = true;
var results = {};
if (typeof options !== "object") {
options = {};
}
@ -1063,20 +1126,19 @@ function UnitTest (which, options) {
printUsage();
return;
}
var jsonReply = options.jsonReply
delete(options.jsonReply);
var i;
var ok;
var r;
if (which === "all") {
var n;
var results = {};
var allok = true;
for (n = 0;n < allTests.length;n++) {
print("Doing test",allTests[n],"with options",options);
results[allTests[n]] = r = testFuncs[allTests[n]](options);
ok = true;
for (i in r) {
if (r.hasOwnProperty(i)) {
if (r[i] !== 0 && r[i].status !== true) {
if (r[i].status !== true) {
ok = false;
}
}
@ -1085,32 +1147,48 @@ function UnitTest (which, options) {
if (!ok) {
allok = false;
}
results.all_ok = allok;
}
results.all_ok = allok;
return results;
}
if (!testFuncs.hasOwnProperty(which)) {
printUsage();
throw 'Unknown test "'+which+'"';
}
var rr = {};
rr[which] = r = testFuncs[which](options);
ok = true;
for (i in r) {
if (r.hasOwnProperty(i) &&
(which !== "single" || i !== "test")) {
if (r[i] !== 0 && r[i].status !== true) {
ok = false;
}
if (jsonReply === true ) {
return results;
}
else {
unitTestPrettyPrintResults(results);
return allok;
}
}
else if (!testFuncs.hasOwnProperty(which)) {
printUsage();
throw 'Unknown test "' + which + '"';
}
else {
var r;
results[which] = r = testFuncs[which](options);
ok = true;
for (i in r) {
if (r.hasOwnProperty(i) &&
(which !== "single" || i !== "test")) {
if (r[i].status !== true) {
ok = false;
allok = false;
}
}
}
r.ok = ok;
results.all_ok = ok;
if (jsonReply === true ) {
return results;
}
else {
unitTestPrettyPrintResults(results);
return allok;
}
}
r.ok = ok;
rr.all_ok = ok;
return rr;
}
exports.UnitTest = UnitTest;
exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults;
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

109
scripts/unittest.js
View File

@ -13,60 +13,63 @@ function resultsToXml(results, baseName) {
});
}
for (var test in results.shell_server_aql) {
var xml = [ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ];
for (var testrun in results) {
for (var test in results[testrun]) {
var xml = [ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ];
xml.text = function (s) {
Array.prototype.push.call(this, s);
return this;
};
xml.elem = function (tagName, attrs, close) {
this.text("<").text(tagName);
xml.text = function (s) {
Array.prototype.push.call(this, s);
return this;
};
for (var a in attrs || {}) {
this.text(" ").text(a).text("=\"")
.text(xmlEscape(String(attrs[a]))).text("\"");
}
close && this.text("/");
this.text(">\n");
return this;
};
xml.elem("testsuite", {
errors: 0,
failures: results.shell_server_aql[test].failed,
name: test,
tests: results.shell_server_aql[test].total,
time: results.shell_server_aql[test].duration
});
for (var oneTest in results.shell_server_aql[test]) {
if (internalMembers.indexOf(oneTest) === -1) {
var result = results.shell_server_aql[test][oneTest].status;
var success = (typeof(result) === 'boolean')? result : false;
xml.elem = function (tagName, attrs, close) {
this.text("<").text(tagName);
xml.elem("testcase", {
name: oneTest,
time: results.shell_server_aql[test][oneTest].duration
}, success);
if (!success) {
xml.elem("failure");
xml.text('<![CDATA[' + results.shell_server_aql[test][oneTest].message + ']]>\n');
xml.elem("/failure");
xml.elem("/testcase");
for (var a in attrs || {}) {
this.text(" ").text(a).text("=\"")
.text(xmlEscape(String(attrs[a]))).text("\"");
}
close && this.text("/");
this.text(">\n");
return this;
};
xml.elem("testsuite", {
errors: 0,
failures: results[testrun][test].failed,
name: test,
tests: results[testrun][test].total,
time: results[testrun][test].duration
});
for (var oneTest in results[testrun][test]) {
if (internalMembers.indexOf(oneTest) === -1) {
var result = results[testrun][test][oneTest].status;
var success = (typeof(result) === 'boolean')? result : false;
xml.elem("testcase", {
name: oneTest,
time: results[testrun][test][oneTest].duration
}, success);
if (!success) {
xml.elem("failure");
xml.text('<![CDATA[' + results[testrun][test][oneTest].message + ']]>\n');
xml.elem("/failure");
xml.elem("/testcase");
}
}
}
xml.elem("/testsuite");
var fn = baseName + testrun.replace(/\//g, '_') + '_' + test.replace(/\//g, '_') + ".xml";
//print('Writing: '+ fn);
fs.write(fn, xml.join(""));
}
xml.elem("/testsuite");
var fn = baseName + test.replace(/\//g, '_') + ".xml";
//print('Writing: '+ fn);
fs.write(fn, xml.join(""));
}
}
@ -81,17 +84,19 @@ function main (argv) {
if (argv.length >= 3) {
options = JSON.parse(argv[2]);
}
var UnitTest = require("org/arangodb/testing").UnitTest;
options.jsonReply = true;
var UnitTest = require("org/arangodb/testing");
start_pretty_print();
var r = UnitTest(test,options);
var r = UnitTest.UnitTest(test,options);
fs.write("UNITTEST_RESULT.json",JSON.stringify(r));
fs.write("UNITTEST_RESULT_SUMMARY.txt",JSON.stringify(r.all_ok));
try {
resultsToXml(r, "UNITTEST_RESULT_");
}
catch (x) {
print("Exception while serializing status xml!");
print(x.message);
}
print(r);
UnitTest.unitTestPrettyPrintResults(r);
}