From 357764ac57ab68cea95b96d44f5883c0043cde24 Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Tue, 23 Sep 2014 16:56:24 +0200 Subject: [PATCH] Implement structures in the same way for all other tests, and move the pretty printer over, so users calling testing.js directly can also enjoy human readable test results. --- js/common/modules/reportgenerator.js | 40 ++--- js/server/modules/org/arangodb/testing.js | 186 +++++++++++++++------- scripts/unittest.js | 109 +++++++------ 3 files changed, 209 insertions(+), 126 deletions(-) diff --git a/js/common/modules/reportgenerator.js b/js/common/modules/reportgenerator.js index 133d02e840..051a0df1e8 100644 --- a/js/common/modules/reportgenerator.js +++ b/js/common/modules/reportgenerator.js @@ -28,30 +28,30 @@ var generatePerfReport = function (reportName, testdata, longdesc) { var j = 0; - for (i = 0; i < measurements.length; i++) { - if (oneResultSet.hasOwnProperty([measurements[i]])) { +// for (i = 0; i < measurements.length; i++) { +// if (oneResultSet.hasOwnProperty([measurements[i]])) { var s = oneResultSet[measurements[i]].sum; var t = oneResultSet[measurements[i]].avg; - x = x + '\t\n'; - x = x + '\t\t\n'; - x = x + '\t\t\n\t\t\tAQL\n\t\t\n'; - - x += '\t\tSum ' + measurements[i] + '\n'; - j += 1; + x = x + '\t\n'; + x = x + '\t\t\n'; + x = x + '\t\t\n\t\t\tAQL\n\t\t\n'; + + x += '\t\tSum time\n'; + j += 1; - x += '\t\tAverage ' + measurements[i] + '\n'; - j += 1; + x += '\t\tAverage ' + measurements[i] + '\n'; + j += 1; - x = x + '\t\t\n'; - x += '\t\t\t\n'; - x += '\t\t\t\n'; - x += '\t\t\t\n'; - x += '\t\t\t\n'; - - x += '\t\t\n'; - x = x + '\t\n'; - } - } + x = x + '\t\t\n'; + x += '\t\t\t\n'; + x += '\t\t\t\n'; + x += '\t\t\t\n'; + x += '\t\t\t\n'; + + x += '\t\t\n'; + x = x + '\t\n'; +// } + // } } } } diff --git a/js/server/modules/org/arangodb/testing.js b/js/server/modules/org/arangodb/testing.js index bca9bf8a96..b428ed7011 100644 --- a/js/server/modules/org/arangodb/testing.js +++ b/js/server/modules/org/arangodb/testing.js @@ -57,6 +57,8 @@ /// /// The following properties of `options` are defined: /// +/// - `jsonReply`: if set a json is returned which the caller has to +/// present the user /// - `force`: if set to true the tests are continued even if one fails /// - `skipBoost`: if set to true the boost unittests are skipped /// - `skipGeo`: if set to true the geo index tests are skipped @@ -110,7 +112,8 @@ var optionsDefaults = { "cluster": false, "test": undefined, "skipServer": false, "skipClient": false, - "cleanup": true }; + "cleanup": true, + "jsonReply": false}; function findTopDir () { var topDir = fs.normalize(fs.makeAbsolute(".")); @@ -397,9 +400,11 @@ function executeAndWait (cmd, args) { var pid = executeExternal(cmd, args); var res = statusExternal(pid, true); if (res.status === "TERMINATED") { - return res.exit; + return { status: (res.exit === 0), message: ""}; + } + else { + return { status: (res === 0), message: res.exit}; } - return res; } function runInArangosh (options, instanceInfo, file, addArgs) { @@ -543,14 +548,14 @@ testFuncs.config = function () { results[t] = executeAndWait(fs.join(topDir,"bin",t), ["--configuration", fs.join(topDir,"etc","arangodb",t+".conf"), "--help"]); - print("Config test "+t+"...",results[t]); + print("Config test "+t+"...",results[t].status); } for (i = 0; i < ts.length; i++) { t = ts[i]; results[t+"_rel"] = executeAndWait(fs.join(topDir,"bin",t), ["--configuration", fs.join(topDir,"etc","relative", t+".conf"), "--help"]); - print("Config test "+t+" (relative)...",results[t+"_rel"]); + print("Config test "+t+" (relative)...",results[t+"_rel"].status); } return results; @@ -642,7 +647,13 @@ function rubyTests (options, ssl) { fs.join("UnitTests","HttpInterface",n)]; var pid = executeExternal("rspec", args); var r = statusExternal(pid, true); - result[n] = r.exit; + + if (r.status === "TERMINATED") { + result[n] = { status: (r.exit === 0), message: r.exit}; + } + else { + result[n] = { status: (r === 0), message: r.exit}; + } if (r.exit !== 0 && !options.force) { break; } @@ -954,17 +965,24 @@ testFuncs.authentication_parameters = function (options) { "authparams"); var r; var i; - var re = []; - for (i = 0;i < urlsTodo.length;i++) { + var expectAuthFullRC = [401, 401, 401, 401, 401, 401, 401]; + var all_ok = true; + results["auth_full"] = {}; + for (i = 0; i < urlsTodo.length; i++) { r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true}); - re.push(r.code); - } - if (_.isEqual(re,[401, 401, 401, 401, 401, 401, 401])) { - results.auth_full = 0; - } - else { - results.auth_full = re; + if (r.code === expectAuthFullRC[i]) { + results.auth_full[urlsTodo[i]] = { status: true, message: ""}; + } + else { + results.auth_full[urlsTodo[i]] = { + status: false, + message: "we expected " + expectAuthFullRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r) + }; + all_ok = false; + } } + results.auth_full.status = all_ok; + print("Shutting down..."); shutdownInstance(instanceInfo,options); print("done."); @@ -973,17 +991,24 @@ testFuncs.authentication_parameters = function (options) { ["--server.disable-authentication", "false", "--server.authenticate-system-only", "true"], "authparams2"); - re = []; + var expectAuthSystemRC = [401, 401, 401, 401, 401, 404, 404]; + all_ok = true; + results["auth_system"] = {}; for (i = 0;i < urlsTodo.length;i++) { r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true}); - re.push(r.code); - } - if (_.isEqual(re, [401, 401, 401, 401, 401, 404, 404])) { - results.auth_system = 0; - } - else { - results.auth_system = re; + if (r.code === expectAuthSystemRC[i]) { + results.auth_system[urlsTodo[i]] = { status: true, message: ""}; + } + else { + results.auth_system[urlsTodo[i]] = { + status: false, + message: "we expected " + expectAuthSystemRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r) + }; + all_ok = false; + } } + results.auth_system.status = all_ok; + print("Shutting down..."); shutdownInstance(instanceInfo,options); print("done."); @@ -992,23 +1017,59 @@ testFuncs.authentication_parameters = function (options) { ["--server.disable-authentication", "true", "--server.authenticate-system-only", "true"], "authparams3"); - re = []; + var expectAuthNoneRC = [404, 404, 200, 301, 301, 404, 404]; + results["auth_none"] = {}; + all_ok = true; for (i = 0;i < urlsTodo.length;i++) { r = download(instanceInfo.url+urlsTodo[i],"",{followRedirects:false,returnBodyOnError:true}); - re.push(r.code); - } - if (_.isEqual(re, [404, 404, 200, 301, 301, 404, 404])) { - results.auth_none = 0; - } - else { - results.auth_none = re; + if (r.code === expectAuthNoneRC[i]) { + results.auth_none[urlsTodo[i]] = { status: true, message: ""}; + } + else { + results.auth_none[urlsTodo[i]] = { + status: false, + message: "we expected " + expectAuthNoneRC[i] + " and we got " + r.code + " Full Status: " + JSON.stringify(r) + }; + all_ok = false; + } } + results.auth_none.status = all_ok; + print("Shutting down..."); shutdownInstance(instanceInfo,options); print("done."); return results; }; +var internalMembers = ["code", "error", "status", "duration", "failed", "total"]; + +function unitTestPrettyPrintResults(r) { + for (var testrun in r) { + if (r.hasOwnProperty(testrun) && (testrun !== 'all_ok')) { + print("Testrun: " + testrun); + for (var test in r[testrun]) { + if (r[testrun].hasOwnProperty(test) && (test !== 'ok')) { + if (r[testrun][test].status) { + print(" " + test + ": Success"); + } + else { + print(" " + test + ": Fail"); + for (var oneTest in r[testrun][test]) { + if ((r[testrun][test].hasOwnProperty(oneTest)) && + (internalMembers.indexOf(oneTest) === -1) && + (!r[testrun][test][oneTest].status)) { + print(" -> " + oneTest + " Failed; Verbose message:"); + print(r[testrun][test][oneTest].message); + } + } + } + } + } + } + } + print("Overal state: " + ((r.all_ok === true) ? "Success" : "Fail")); +} + var allTests = [ @@ -1055,6 +1116,8 @@ function printUsage () { } function UnitTest (which, options) { + var allok = true; + var results = {}; if (typeof options !== "object") { options = {}; } @@ -1063,20 +1126,19 @@ function UnitTest (which, options) { printUsage(); return; } + var jsonReply = options.jsonReply + delete(options.jsonReply); var i; var ok; - var r; if (which === "all") { var n; - var results = {}; - var allok = true; for (n = 0;n < allTests.length;n++) { print("Doing test",allTests[n],"with options",options); results[allTests[n]] = r = testFuncs[allTests[n]](options); ok = true; for (i in r) { if (r.hasOwnProperty(i)) { - if (r[i] !== 0 && r[i].status !== true) { + if (r[i].status !== true) { ok = false; } } @@ -1085,32 +1147,48 @@ function UnitTest (which, options) { if (!ok) { allok = false; } + results.all_ok = allok; } results.all_ok = allok; - return results; - } - if (!testFuncs.hasOwnProperty(which)) { - printUsage(); - throw 'Unknown test "'+which+'"'; - } - var rr = {}; - rr[which] = r = testFuncs[which](options); - ok = true; - for (i in r) { - if (r.hasOwnProperty(i) && - (which !== "single" || i !== "test")) { - if (r[i] !== 0 && r[i].status !== true) { - ok = false; - } + if (jsonReply === true ) { + return results; + } + else { + unitTestPrettyPrintResults(results); + return allok; + } + } + else if (!testFuncs.hasOwnProperty(which)) { + printUsage(); + throw 'Unknown test "' + which + '"'; + } + else { + var r; + results[which] = r = testFuncs[which](options); + ok = true; + for (i in r) { + if (r.hasOwnProperty(i) && + (which !== "single" || i !== "test")) { + if (r[i].status !== true) { + ok = false; + allok = false; + } + } + } + r.ok = ok; + results.all_ok = ok; + if (jsonReply === true ) { + return results; + } + else { + unitTestPrettyPrintResults(results); + return allok; } } - r.ok = ok; - rr.all_ok = ok; - return rr; } exports.UnitTest = UnitTest; - +exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults; // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // ----------------------------------------------------------------------------- diff --git a/scripts/unittest.js b/scripts/unittest.js index 2e815ecbac..f12aa6e478 100644 --- a/scripts/unittest.js +++ b/scripts/unittest.js @@ -13,60 +13,63 @@ function resultsToXml(results, baseName) { }); } - for (var test in results.shell_server_aql) { - var xml = [ "\n" ]; + for (var testrun in results) { + + for (var test in results[testrun]) { + var xml = [ "\n" ]; - xml.text = function (s) { - Array.prototype.push.call(this, s); - return this; - }; - - xml.elem = function (tagName, attrs, close) { - this.text("<").text(tagName); + xml.text = function (s) { + Array.prototype.push.call(this, s); + return this; + }; - for (var a in attrs || {}) { - this.text(" ").text(a).text("=\"") - .text(xmlEscape(String(attrs[a]))).text("\""); - } - - close && this.text("/"); - this.text(">\n"); - - return this; - }; - - xml.elem("testsuite", { - errors: 0, - failures: results.shell_server_aql[test].failed, - name: test, - tests: results.shell_server_aql[test].total, - time: results.shell_server_aql[test].duration - }); - - for (var oneTest in results.shell_server_aql[test]) { - if (internalMembers.indexOf(oneTest) === -1) { - var result = results.shell_server_aql[test][oneTest].status; - var success = (typeof(result) === 'boolean')? result : false; + xml.elem = function (tagName, attrs, close) { + this.text("<").text(tagName); - xml.elem("testcase", { - name: oneTest, - time: results.shell_server_aql[test][oneTest].duration - }, success); - - if (!success) { - xml.elem("failure"); - xml.text('\n'); - xml.elem("/failure"); - xml.elem("/testcase"); + for (var a in attrs || {}) { + this.text(" ").text(a).text("=\"") + .text(xmlEscape(String(attrs[a]))).text("\""); + } + + close && this.text("/"); + this.text(">\n"); + + return this; + }; + + xml.elem("testsuite", { + errors: 0, + failures: results[testrun][test].failed, + name: test, + tests: results[testrun][test].total, + time: results[testrun][test].duration + }); + + for (var oneTest in results[testrun][test]) { + if (internalMembers.indexOf(oneTest) === -1) { + var result = results[testrun][test][oneTest].status; + var success = (typeof(result) === 'boolean')? result : false; + + xml.elem("testcase", { + name: oneTest, + time: results[testrun][test][oneTest].duration + }, success); + + if (!success) { + xml.elem("failure"); + xml.text('\n'); + xml.elem("/failure"); + xml.elem("/testcase"); + } } } + + xml.elem("/testsuite"); + var fn = baseName + testrun.replace(/\//g, '_') + '_' + test.replace(/\//g, '_') + ".xml"; + //print('Writing: '+ fn); + fs.write(fn, xml.join("")); + } - - xml.elem("/testsuite"); - var fn = baseName + test.replace(/\//g, '_') + ".xml"; - //print('Writing: '+ fn); - fs.write(fn, xml.join("")); - } } @@ -81,17 +84,19 @@ function main (argv) { if (argv.length >= 3) { options = JSON.parse(argv[2]); } - var UnitTest = require("org/arangodb/testing").UnitTest; + options.jsonReply = true; + var UnitTest = require("org/arangodb/testing"); start_pretty_print(); - var r = UnitTest(test,options); + var r = UnitTest.UnitTest(test,options); fs.write("UNITTEST_RESULT.json",JSON.stringify(r)); fs.write("UNITTEST_RESULT_SUMMARY.txt",JSON.stringify(r.all_ok)); try { resultsToXml(r, "UNITTEST_RESULT_"); } catch (x) { + print("Exception while serializing status xml!"); print(x.message); } - - print(r); + + UnitTest.unitTestPrettyPrintResults(r); }