Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/cleanup rspec integration (#5517)

This commit is contained in:
Wilfried Goesgens 2018-06-07 11:47:46 +02:00 committed by Jan
parent 25f77ceeb1
commit eb9bf93c98
14 changed files with 174 additions and 522 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README_maintainers.md
View File

@ -155,6 +155,22 @@ Dependencies
`cd UnitTests/HttpInterface; bundler`
* catch (compile time, shipped in the 3rdParty directory)
Invoking
--------
Since several testing technoligies are utilized, and different arangodb startup options may be required
(even different compilation options may be required) the framework is split into testsuites.
Get a list of the available testsuites and options by invoking:
./scripts/unittest
To locate the suite(s) associated with a specific test file use:
./scripts/unittest find --test js/common/tests/shell/shell-aqlfunctions.js
or to run all of them:
./scripts/unittest auto --test js/common/tests/shell/shell-aqlfunctions.js
Filename conventions
====================

12
UnitTests/HttpInterface/README
View File

@ -1,12 +0,0 @@
Conventions for testing framework:
==================================
All files in this directory whose name starts with "api-"
and ends with ".js" automatically take part in http
or https server tests, with the exception of
"api-replication-spec.js".
If the filename contains the string "-cluster-", then it is
only executed when testing in cluster mode. If the filename
contains the string "-noncluster-", then it is only executed
when testing in single instance mode.

4
UnitTests/HttpInterface/run-tests
View File

@ -1,4 +0,0 @@
#!/bin/sh
test -d logs || mkdir logs
rspec -I . --color --format d `find . -name "api-*.rb" -a \! -name "*-cluster-*"`

4
UnitTests/HttpInterface/run-tests-replication
View File

@ -1,4 +0,0 @@
#!/bin/sh
test -d logs || mkdir logs
rspec -I . --color --format d `find . -name "api-replication-*.rb" -a \! -name "*-cluster-*"`

253
UnitTests/HttpInterface/stress-test.rb
View File

@ -1,253 +0,0 @@
#! ruby -rubygems
# coding: utf-8
require 'arangodb'
# configuration
$number_reader = 30
$number_writer = 20
$number_deleter = 10
$maximal_size = 1000
$verbose = false
################################################################################
## list of all active documents
################################################################################
$documents = []
$doc_mutex = Mutex.new
$doc_cv = ConditionVariable.new
################################################################################
## print version number of the server
################################################################################
doc = ArangoDB.get("/_admin/version")
puts "starting stress test, ArangoDB #{doc.parsed_response['version']}"
################################################################################
## create a collection for testing
################################################################################
cn = "StressTest#{Time.now.to_i}"
ArangoDB.delete("/_api/collection/#{cn}")
body = "{ \"name\" : \"#{cn}\" }"
doc = ArangoDB.post("/_api/collection", :body => body)
$cid = doc.parsed_response['id']
puts "create collection \"#{cn}\": #{$cid}"
################################################################################
## function to a read a document
################################################################################
def read_document (i, pos)
did = nil
$doc_mutex.synchronize {
if $documents.length == 0
return pos
end
pos = (pos * 13 + 1) % $documents.length
did = $documents[pos]
if pos + 1 == $documents.length
$documents.pop()
else
$documents[pos] = $documents.pop()
end
}
res = ArangoDB.get("/_api/document/#{did}")
if res.code == 200
if $verbose
puts "read document #{pos} in thread #{i}: #{did}"
end
else
puts "FAILED READ:"
puts "thread: #{i}"
puts "document identifier: #{did}"
puts JSON.pretty_generate(res.parsed_response)
puts "--------------"
end
$doc_mutex.synchronize {
$documents << did
}
return pos
end
################################################################################
## function to a create a document
################################################################################
def create_document (i)
len = 0
body = "{ \"Hallo\" : #{i} }"
res = ArangoDB.post("/_api/document?collection=#{$cid}", :body => body)
if res.code == 201 or res.code == 202
did = res.parsed_response['_id']
$doc_mutex.synchronize {
$documents << did
len = $documents.length
}
if $verbose
puts "created document #{len} in thread #{i}: #{did}"
end
else
puts "FAILED CREATE:"
puts JSON.pretty_generate(res.parsed_response)
puts "--------------"
end
end
################################################################################
## function to a delete a document
################################################################################
def delete_document (i, pos)
did = nil
$doc_mutex.synchronize {
if $documents.length == 0
return pos
end
pos = (pos * 13 + 1) % $documents.length
did = $documents[pos]
if pos + 1 == $documents.length
$documents.pop()
else
$documents[pos] = $documents.pop()
end
}
res = ArangoDB.delete("/_api/document/#{did}")
if res.code == 200
if $verbose
puts "deleted document #{pos} in thread #{i}: #{did}"
end
else
puts "FAILED DELETE:"
puts "thread: #{i}"
puts "document identifier: #{did}"
puts JSON.pretty_generate(res.parsed_response)
puts "--------------"
end
return pos
end
################################################################################
## reader
################################################################################
def reader (i)
pos = i % $maximal_size
while true
$doc_mutex.synchronize {
while $documents.length <= 10
$doc_cv.wait($doc_mutex)
end
}
pos = read_document(i, pos)
$doc_mutex.synchronize {
$doc_cv.broadcast
}
end
end
################################################################################
## writer
################################################################################
def writer (i)
while true
$doc_mutex.synchronize {
while $maximal_size <= $documents.length
$doc_cv.wait($doc_mutex)
end
}
create_document(i)
$doc_mutex.synchronize {
$doc_cv.broadcast
}
end
end
################################################################################
## deleter
################################################################################
def deleter (i)
pos = i % $maximal_size
while true
$doc_mutex.synchronize {
while $documents.length < $maximal_size
$doc_cv.wait($doc_mutex)
end
}
pos = delete_document(i, pos)
$doc_mutex.synchronize {
$doc_cv.broadcast
}
end
end
################################################################################
## main
################################################################################
threads = []
$number_reader.times { |i|
threads << Thread.new(i) {
reader(i)
}
}
$number_writer.times { |i|
threads << Thread.new(i) {
writer(i)
}
}
$number_deleter.times { |i|
threads << Thread.new(i) {
deleter(i)
}
}
if $verbose
sleep
else
while true
$doc_mutex.synchronize {
puts "number of documents: #{$documents.length}"
}
sleep(10)
end
end

0
UnitTests/HttpInterface/api-replication-global-spec.rb → UnitTests/HttpReplication/api-replication-global-spec.rb
View File

0
UnitTests/HttpInterface/api-replication-mmfiles-spec.rb → UnitTests/HttpReplication/api-replication-mmfiles-spec.rb
View File

0
UnitTests/HttpInterface/api-replication-rocksdb-spec.rb → UnitTests/HttpReplication/api-replication-rocksdb-spec.rb
View File

0
UnitTests/HttpInterface/arangodb.rb → UnitTests/arangodbRspecLib/arangodb.rb
View File

0
UnitTests/HttpInterface/arangomultipartbody.rb → UnitTests/arangodbRspecLib/arangomultipartbody.rb
View File

3
js/client/modules/@arangodb/process-utils.js
View File

@ -1249,7 +1249,8 @@ function startInstance (protocol, options, addArgs, testname, tmpDir) {
let instanceInfo = {
rootDir,
arangods: []
arangods: [],
protocol: protocol
};
const startTime = time();

146
js/client/modules/@arangodb/test-utils.js
View File

@ -117,7 +117,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
_.defaults(env, customInstanceInfos.preStart.env);
}
let instanceInfo = pu.startInstance('tcp', options, serverOptions, testname);
let instanceInfo = pu.startInstance(options.protocol, options, serverOptions, testname);
if (instanceInfo === false) {
if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('startFailed')) {
@ -223,7 +223,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
let delta = diffArray(collectionsBefore, collectionsAfter).filter(function(name) {
return (name[0] !== '_'); // exclude system collections from the comparison
});
print(delta);
if (delta.length !== 0) {
results[te] = {
status: false,
@ -345,7 +345,6 @@ function filterTestcaseByOptions (testname, options, whichFilter) {
if (options.replication) {
whichFilter.filter = 'replication';
if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) {
whichFilter.filter = 'testcase';
return ((testname.search(options.test) >= 0) &&
@ -354,6 +353,7 @@ function filterTestcaseByOptions (testname, options, whichFilter) {
return testname.indexOf('replication') !== -1;
}
} else if (testname.indexOf('replication') !== -1) {
whichFilter.filter = 'replication';
return false;
}
@ -571,15 +571,16 @@ function runInArangosh (options, instanceInfo, file, addArgs) {
args = Object.assign(args, addArgs);
}
require('internal').env.INSTANCEINFO = JSON.stringify(instanceInfo);
const jsonFN = fs.join(instanceInfo.rootDir, 'testresult.json');
let rc = pu.executeAndWait(pu.ARANGOSH_BIN, toArgv(args), options, 'arangosh', instanceInfo.rootDir, options.coreCheck);
let result;
try {
result = JSON.parse(fs.read(instanceInfo.rootDir + '/testresult.json'));
result = JSON.parse(fs.read(jsonFN));
fs.remove(instanceInfo.rootDir + '/testresult.json');
fs.remove(jsonFN);
} catch (x) {
if (options.extremeVerbosity) {
print('failed to read ' + instanceInfo.rootDir + '/testresult.json');
print('failed to read ' + jsonFN);
}
return rc;
}
@ -594,6 +595,138 @@ function runInArangosh (options, instanceInfo, file, addArgs) {
}
runInArangosh.info = 'arangosh';
// //////////////////////////////////////////////////////////////////////////////
// / @brief runs a unittest file using rspec
// //////////////////////////////////////////////////////////////////////////////
function camelize (str) {
return str.replace(/(?:^\w|[A-Z]|\b\w)/g, function (letter, index) {
return index === 0 ? letter.toLowerCase() : letter.toUpperCase();
}).replace(/\s+/g, '');
}
const parseRspecJson = function (testCase, res, totalDuration) {
let tName = camelize(testCase.description);
let status = (testCase.status === 'passed');
res[tName] = {
status: status,
message: testCase.full_description,
duration: totalDuration // RSpec doesn't offer per testcase time...
};
res.total++;
if (!status) {
const msg = yaml.safeDump(testCase)
.replace(/.*rspec\/core.*\n/gm, '')
.replace(/.*rspec\\core.*\n/gm, '')
.replace(/.*lib\/ruby.*\n/, '')
.replace(/.*- >-.*\n/gm, '')
.replace(/\n *`/gm, ' `');
print('RSpec test case falied: \n' + msg);
res[tName].message += '\n' + msg;
}
return status ? 0 : 1;
};
function runInRSpec (options, instanceInfo, file, addArgs) {
const tmpname = fs.join(instanceInfo.rootDir, 'testconfig.rb');
const jsonFN = fs.join(instanceInfo.rootDir, 'testresult.json');
let command;
let args;
let rspec;
let ssl = "0";
if (instanceInfo.protocol === 'ssl') {
ssl = "1";
}
let rspecConfig = 'RSpec.configure do |c|\n' +
' c.add_setting :ARANGO_SERVER\n' +
' c.ARANGO_SERVER = "' +
instanceInfo.endpoint.substr(6) + '"\n' +
' c.add_setting :ARANGO_SSL\n' +
' c.ARANGO_SSL = "' + ssl + '"\n' +
' c.add_setting :ARANGO_USER\n' +
' c.ARANGO_USER = "' + options.username + '"\n' +
' c.add_setting :ARANGO_PASSWORD\n' +
' c.ARANGO_PASSWORD = "' + options.password + '"\n' +
' c.add_setting :SKIP_TIMECRITICAL\n' +
' c.SKIP_TIMECRITICAL = ' + JSON.stringify(options.skipTimeCritical) + '\n' +
'end\n';
fs.write(tmpname, rspecConfig);
if (options.extremeVerbosity === true) {
print('rspecConfig: \n' + rspecConfig);
}
try {
fs.makeDirectory(pu.LOGS_DIR);
} catch (err) {}
if (options.ruby === '') {
command = options.rspec;
rspec = undefined;
} else {
command = options.ruby;
rspec = options.rspec;
}
args = ['--color',
'-I', fs.join('UnitTests', 'arangodbRspecLib'),
'--format', 'd',
'--format', 'j',
'--out', jsonFN,
'--require', tmpname,
file
];
if (rspec !== undefined) {
args = [rspec].concat(args);
}
const res = pu.executeAndWait(command, args, options, 'arangosh', instanceInfo.rootDir);
let result = {
total: 0,
failed: 0,
status: res.status
};
try {
const jsonResult = JSON.parse(fs.read(jsonFN));
if (options.extremeVerbosity) {
print(yaml.safeDump(jsonResult));
}
for (let j = 0; j < jsonResult.examples.length; ++j) {
result.failed += parseRspecJson(
jsonResult.examples[j], result,
jsonResult.summary.duration);
}
result.duration = jsonResult.summary.duration;
} catch (x) {
result.failed = 1;
result.message = 'Failed to parse rspec results for: ' + file;
if (res.status === false) {
options.cleanup = false;
}
}
fs.remove(jsonFN);
fs.remove(tmpname);
return result;
}
runInRSpec.info = 'runInRSpec';
// //////////////////////////////////////////////////////////////////////////////
// / @brief
// //////////////////////////////////////////////////////////////////////////////
function makeResults (testname, instanceInfo) {
const startTime = time();
@ -640,6 +773,7 @@ function makeResults (testname, instanceInfo) {
exports.runThere = runThere;
exports.runInArangosh = runInArangosh;
exports.runInRSpec = runInRSpec;
exports.makePathUnix = makePathUnix;
exports.makePathGeneric = makePathGeneric;

2
js/client/modules/@arangodb/testing.js
View File

@ -74,6 +74,7 @@ let optionsDocumentation = [
' - `cleanup`: if set to true (the default), the cluster data files',
' and logs are removed after termination of the test.',
'',
' - `protocol`: the protocol to talk to the server - [tcp (default), ssl, unix]',
' - `build`: the directory containing the binaries',
' - `buildType`: Windows build type (Debug, Release), leave empty on linux',
' - `configDir`: the directory containing the config files, defaults to',
@ -135,6 +136,7 @@ const optionsDefaults = {
'mochaGrep': undefined,
'onlyNightly': false,
'password': '',
'protocol': 'tcp',
'replication': false,
'rr': false,
'sanitizer': false,

256
js/client/modules/@arangodb/testsuites/rspec.js
View File

@ -53,7 +53,7 @@ const RESET = require('internal').COLORS.COLOR_RESET;
// const YELLOW = require('internal').COLORS.COLOR_YELLOW;
const testPaths = {
'http_replication': [fs.join('UnitTests', 'HttpInterface')],
'http_replication': [fs.join('UnitTests', 'HttpReplication')],
'http_server': [fs.join('UnitTests', 'HttpInterface')],
'server_http': ['js/common/tests/http'],
'ssl_server': [fs.join('UnitTests', 'HttpInterface')]
@ -70,243 +70,6 @@ function serverHttp (options) {
return tu.performTests(options, testCases, 'server_http', tu.runThere);
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief runs ruby tests using RSPEC
// //////////////////////////////////////////////////////////////////////////////
function camelize (str) {
return str.replace(/(?:^\w|[A-Z]|\b\w)/g, function (letter, index) {
return index === 0 ? letter.toLowerCase() : letter.toUpperCase();
}).replace(/\s+/g, '');
}
function rubyTests (options, ssl) {
let instanceInfo;
if (ssl) {
instanceInfo = pu.startInstance('ssl', options, {}, 'ssl_server');
} else {
instanceInfo = pu.startInstance('tcp', options, {}, 'http_server');
}
if (instanceInfo === false) {
return {
status: false,
message: 'failed to start server!'
};
}
const tmpname = fs.getTempFile() + '.rb';
let rspecConfig = 'RSpec.configure do |c|\n' +
' c.add_setting :ARANGO_SERVER\n' +
' c.ARANGO_SERVER = "' +
instanceInfo.endpoint.substr(6) + '"\n' +
' c.add_setting :ARANGO_SSL\n' +
' c.ARANGO_SSL = "' + (ssl ? '1' : '0') + '"\n' +
' c.add_setting :ARANGO_USER\n' +
' c.ARANGO_USER = "' + options.username + '"\n' +
' c.add_setting :ARANGO_PASSWORD\n' +
' c.ARANGO_PASSWORD = "' + options.password + '"\n' +
' c.add_setting :SKIP_TIMECRITICAL\n' +
' c.SKIP_TIMECRITICAL = ' + JSON.stringify(options.skipTimeCritical) + '\n' +
'end\n';
fs.write(tmpname, rspecConfig);
if (options.extremeVerbosity === true) {
print('rspecConfig: \n' + rspecConfig);
}
try {
fs.makeDirectory(pu.LOGS_DIR);
} catch (err) {}
let files = [];
if (ssl) {
files = tu.scanTestPaths(testPaths.ssl_server);
} else {
files = tu.scanTestPaths(testPaths.http_server);
}
let continueTesting = true;
let filtered = {};
let results = {};
let args;
let command;
let rspec;
if (options.ruby === '') {
command = options.rspec;
rspec = undefined;
} else {
command = options.ruby;
rspec = options.rspec;
}
const parseRspecJson = function (testCase, res, totalDuration) {
let tName = camelize(testCase.description);
let status = (testCase.status === 'passed');
res[tName] = {
status: status,
message: testCase.full_description,
duration: totalDuration // RSpec doesn't offer per testcase time...
};
res.total++;
if (!status) {
const msg = yaml.safeDump(testCase)
.replace(/.*rspec\/core.*\n/gm, '')
.replace(/.*rspec\\core.*\n/gm, '')
.replace(/.*lib\/ruby.*\n/, '')
.replace(/.*- >-.*\n/gm, '')
.replace(/\n *`/gm, ' `');
print('RSpec test case falied: \n' + msg);
res[tName].message += '\n' + msg;
}
return status ? 0 : 1;
};
let count = 0;
let graphCount = 0;
files = tu.splitBuckets(options, files);
for (let i = 0; i < files.length; i++) {
const te = files[i];
if ((te.search('api-') !== -1) && te.substr(-3) === '.rb') {
let tfn = te;
if (tu.filterTestcaseByOptions(tfn, options, filtered)) {
count += 1;
if (!continueTesting) {
print('Skipping ' + te + ' server is gone.');
results[te] = {
status: false,
message: instanceInfo.exitStatus
};
instanceInfo.exitStatus = 'server is gone.';
break;
}
let collectionsBefore = [];
db._collections().forEach(collection => {
collectionsBefore.push(collection._name);
});
const subFolder = ssl ? 'ssl_server' : 'http_server';
const resultfn = fs.join(options.testOutputDirectory, subFolder, te + '.json');
args = ['--color',
'-I', fs.join('UnitTests', 'HttpInterface'),
'--format', 'd',
'--format', 'j',
'--out', resultfn,
'--require', tmpname,
tfn
];
if (rspec !== undefined) {
args = [rspec].concat(args);
}
print('\n' + Date() + ' rspec trying', tfn, '...');
const res = pu.executeAndWait(command, args, options, 'arangosh', instanceInfo.rootDir);
results[te] = {
total: 0,
failed: 0,
status: res.status
};
try {
const jsonResult = JSON.parse(fs.read(resultfn));
if (options.extremeVerbosity) {
print(yaml.safeDump(jsonResult));
}
for (let j = 0; j < jsonResult.examples.length; ++j) {
results[te].failed += parseRspecJson(
jsonResult.examples[j], results[te],
jsonResult.summary.duration);
}
results[te].duration = jsonResult.summary.duration;
} catch (x) {
print('Failed to parse rspec result: ' + x);
results[te]['complete_' + te] = res;
if (res.status === false) {
options.cleanup = false;
if (!options.force) {
break;
}
}
}
continueTesting = pu.arangod.check.instanceAlive(instanceInfo, options);
if (continueTesting) {
// Check whether some collections were left behind, and if mark test as failed.
let collectionsAfter = [];
db._collections().forEach(collection => {
collectionsAfter.push(collection._name);
});
let delta = tu.diffArray(collectionsBefore, collectionsAfter, _.isEqual).filter(function (name) {
return (name[0] !== '_'); // exclude system collections from the comparison
});
if (delta.length !== 0) {
results[te] = {
status: false,
message: 'Cleanup missing - test left over collections! [' + delta + '] - Original test status: ' + JSON.stringify(results[te])
};
collectionsBefore = [];
db._collections().forEach(collection => {
collectionsBefore.push(collection._name);
});
}
if (db._graphs.count() !== graphCount) {
results[te] = {
status: false,
message: 'Cleanup of graphs missing - found graph definitions: [ ' +
JSON.stringify(db._graphs.toArray()) +
' ] - Original test status: ' +
JSON.stringify(results[te])
};
graphCount = db._graphs.count();
}
}
} else {
if (options.extremeVerbosity) {
print('Skipped ' + te + ' because of ' + filtered.filter);
}
}
}
}
print('Shutting down...');
if (count === 0) {
results['ALLTESTS'] = {
status: false,
skipped: true
};
results.status = false;
print(RED + 'No testcase matched the filter.' + RESET);
}
fs.remove(tmpname);
pu.shutdownInstance(instanceInfo, options);
print('done.');
return results;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief TEST: http_replication
// //////////////////////////////////////////////////////////////////////////////
@ -316,8 +79,10 @@ function httpReplication (options) {
'replication': true
};
_.defaults(opts, options);
let testCases = tu.scanTestPaths(testPaths.http_replication);
return rubyTests(opts, false);
return tu.performTests(opts, testCases, 'http_replication', tu.runInRSpec);
}
// //////////////////////////////////////////////////////////////////////////////
@ -329,7 +94,10 @@ function httpServer (options) {
'httpTrustedOrigin': 'http://was-erlauben-strunz.it'
};
_.defaults(opts, options);
return rubyTests(opts, false);
let testCases = tu.scanTestPaths(testPaths.http_server);
return tu.performTests(opts, testCases, 'http_server', tu.runInRSpec);
}
// //////////////////////////////////////////////////////////////////////////////
@ -346,10 +114,14 @@ function sslServer (options) {
};
}
var opts = {
'httpTrustedOrigin': 'http://was-erlauben-strunz.it'
'httpTrustedOrigin': 'http://was-erlauben-strunz.it',
'protocol': 'ssl'
};
_.defaults(opts, options);
return rubyTests(opts, true);
let testCases = tu.scanTestPaths(testPaths.ssl_server);
return tu.performTests(opts, testCases, 'ssl_server', tu.runInRSpec);
}
exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {