Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

preserve the instance shutdown state, and if its bad, don't clean up the test directory (#9122) 

* preserve the instance shutdown state, and if its bad, don't clean up the test directory

* Update js/client/modules/@arangodb/testing.js

Co-Authored-By: Tobias Gödderz <github@tobias.goedderz.info>

* Update js/client/modules/@arangodb/testsuites/permissions_server.js

Co-Authored-By: Tobias Gödderz <github@tobias.goedderz.info>

* Update js/client/modules/@arangodb/testsuites/permissions_server.js

Co-Authored-By: Tobias Gödderz <github@tobias.goedderz.info>

* Update js/client/modules/@arangodb/testsuites/single.js

Co-Authored-By: Tobias Gödderz <github@tobias.goedderz.info>

* fix handling in custom start/stop handlers

* remove shutdown state from reporting structure

* fix shutdown of second instance

* one more to adjust

* only add jwt if we run the agency with authentification

* lint
This commit is contained in:
Wilfried Goesgens 2019-05-29 14:59:34 +02:00 committed by KVS85
parent a5a72ef6d4
commit 065b87f9a8
17 changed files with 187 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
js/client/modules/@arangodb/process-utils.js
View File

@ -831,9 +831,11 @@ function runArangoBenchmark (options, instanceInfo, cmds, rootDir, coreCheck = f
function dumpAgency(instanceInfo, options) {
function dumpAgent(arangod, path, method, fn) {
let opts = {
method: method,
jwt: crypto.jwtEncode(instanceInfo.authOpts['server.jwt-secret'], {'server_id': 'none', 'iss': 'arangodb'}, 'HS256')
method: method
};
if (instanceInfo.hasOwnProperty('authOpts')) {
opts['jwt'] = crypto.jwtEncode(instanceInfo.authOpts['server.jwt-secret'], {'server_id': 'none', 'iss': 'arangodb'}, 'HS256');
}
print('--------------------------------- '+ fn + ' -----------------------------------------------');
let agencyReply = download(arangod.url + path, method === 'POST' ? '[["/"]]' : '', opts);
if (agencyReply.code === 200) {
@ -1088,6 +1090,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
if (forceTerminate === undefined) {
forceTerminate = false;
}
let shutdownSuccess = !forceTerminate;
// we need to find the leading server
if (options.activefailover) {
@ -1096,6 +1099,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
print(Date() + ' failover has happened, leader is no more! Marking Crashy!');
serverCrashedLocal = true;
forceTerminate = true;
shutdownSuccess = false;
}
}
@ -1119,6 +1123,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
}
} catch (err) {
print(Date() + " error while setting cluster maintenance mode:", err);
shutdownSuccess = false;
}
}
@ -1190,6 +1195,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
print(Date() + ' forcefully terminating ' + yaml.safeDump(arangod) +
' after ' + timeout + 's grace period; marking crashy.');
serverCrashedLocal = true;
shutdownSuccess = false;
arangod.exitStatus = killExternal(arangod.pid, abortSignal);
analyzeServerCrash(arangod,
options,
@ -1212,6 +1218,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
analyzeServerCrash(arangod, options, 'instance "' + arangod.role + '" Shutdown - ' + arangod.exitStatus.signal);
print(Date() + " shutdownInstance: Marking crashy - " + JSON.stringify(arangod));
serverCrashedLocal = true;
shutdownSuccess = false;
}
stopProcdump(options, arangod);
} else {
@ -1252,6 +1259,7 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
}
cleanupDirectories.unshift(instanceInfo.rootDir);
return shutdownSuccess;
}
function detectCurrentLeader(instanceInfo) {

25
js/client/modules/@arangodb/test-utils.js
View File

@ -117,7 +117,8 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
setup: {
status: false,
message: 'custom preStart failed!' + customInstanceInfos.preStart.message
}
},
shutdown: customInstanceInfos.preStart.shutdown
};
}
_.defaults(env, customInstanceInfos.preStart.env);
@ -147,18 +148,20 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
customInstanceInfos,
startStopHandlers);
if (customInstanceInfos.postStart.state === false) {
pu.shutdownInstance(instanceInfo, options);
let shutdownStatus = customInstanceInfos.postStart.shutdown;
shutdownStatus = shutdownStatus && pu.shutdownInstance(instanceInfo, options);
return {
setup: {
status: false,
message: 'custom postStart failed: ' + customInstanceInfos.postStart.message
message: 'custom postStart failed: ' + customInstanceInfos.postStart.message,
shutdown: shutdownStatus
}
};
}
_.defaults(env, customInstanceInfos.postStart.env);
}
let results = {};
let results = { shutdown: true };
let continueTesting = true;
let serverDead = false;
let count = 0;
@ -318,7 +321,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
startStopHandlers);
if (customInstanceInfos.alive.state === false) {
continueTesting = false;
results.setup.message = 'custom preStop failed!';
results.setup.message = 'custom alive check failed!';
}
collectionsBefore = [];
try {
@ -373,8 +376,14 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
customInstanceInfos,
startStopHandlers);
if (customInstanceInfos.preStop.state === false) {
results.setup.status = false;
results.setup.message = 'custom preStop failed!';
if (!results.hasOwnProperty('setup')) {
results['setup'] = {};
}
results.setup['status'] = false;
results.setup['message'] = 'custom preStop failed!' + customInstanceInfos.preStop.message;
if (customInstanceInfos.preStop.hasOwnProperty('shutdown')) {
results.shutdown = results.shutdown && customInstanceInfos.preStop.shutdown;
}
}
}
@ -383,7 +392,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
if (serverOptions['server.jwt-secret'] && !clonedOpts['server.jwt-secret']) {
clonedOpts['server.jwt-secret'] = serverOptions['server.jwt-secret'];
}
pu.shutdownInstance(instanceInfo, clonedOpts, forceTerminate);
results.shutdown = results.shutdown && pu.shutdownInstance(instanceInfo, clonedOpts, forceTerminate);
if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('postStop')) {
customInstanceInfos['postStop'] = startStopHandlers.postStop(options,

7
js/client/modules/@arangodb/testing.js
View File

@ -683,6 +683,7 @@ function iterateTests(cases, options, jsonReply) {
let result;
let status = true;
let shutdownSuccess = true;
if (skipTest("SUITE", currentTest)) {
result = {
@ -698,6 +699,10 @@ function iterateTests(cases, options, jsonReply) {
delete result.status;
delete result.failed;
delete result.crashed;
if (result.hasOwnProperty('shutdown')) {
shutdownSuccess = result['shutdown'];
delete result.shutdown;
}
status = Object.values(result).every(testCase => testCase.status === true);
let failed = Object.values(result).reduce((prev, testCase) => prev + !testCase.status, 0);
@ -710,7 +715,7 @@ function iterateTests(cases, options, jsonReply) {
results[currentTest] = result;
if (status && localOptions.cleanup) {
if (status && localOptions.cleanup && shutdownSuccess ) {
pu.cleanupLastDirectory(localOptions);
} else {
cleanup = false;

2
js/client/modules/@arangodb/testsuites/arangobench.js
View File

@ -237,7 +237,7 @@ function arangobench (options) {
}
print(CYAN + 'Shutting down...' + RESET);
pu.shutdownInstance(instanceInfo, options);
results['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print(CYAN + 'done.' + RESET);
return results;

2
js/client/modules/@arangodb/testsuites/authentication.js
View File

@ -260,7 +260,7 @@ function authenticationParameters (options) {
results[testName].status = results[testName].failed === 0;
print(CYAN + 'Shutting down ' + authTestNames[test] + ' test...' + RESET);
pu.shutdownInstance(instanceInfo, options);
results['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print(CYAN + 'done with ' + authTestNames[test] + ' test.' + RESET);
if (cleanup) {

17
js/client/modules/@arangodb/testsuites/backup.js
View File

@ -69,6 +69,7 @@ const testPaths = {
const failPreStartMessage = (msg) => {
return {
state: false,
shutdown: true,
message: msg
};
};
@ -135,7 +136,14 @@ const generateDumpData = (options) => {
if (isAlive(instanceInfo, options)) {
options['server.jwt-secret'] = 'haxxmann';
pu.shutdownInstance(instanceInfo, options);
if (!pu.shutdownInstance(instanceInfo, options)) {
path = {
state: false,
failed: 1,
shutdown: false,
message: "shutdown of dump server failed"
};
}
}
log('done.');
print();
@ -166,7 +174,8 @@ const setServerOptions = (options, serverOptions, customInstanceInfos, startStop
serverOptions['server.authentication'] = 'false';
}
return {
state: true
state: true,
shutdown: true
};
};
@ -186,11 +195,13 @@ const setupBackupTest = (options, serverOptions, instanceInfo, customInstanceInf
restore.failed = 1;
return {
state: false,
message: restore.message
message: restore.message,
shutdown: true
};
}
return {
shutdown: true,
state: true
};
};

2
js/client/modules/@arangodb/testsuites/dump.js
View File

@ -122,7 +122,7 @@ class DumpRestoreHelper {
fs.remove(this.fn);
}
print(CYAN + 'Shutting down...' + RESET);
pu.shutdownInstance(this.instanceInfo, this.options);
this.results['shutdown'] = pu.shutdownInstance(this.instanceInfo, this.options);
print(CYAN + 'done.' + RESET);
print();

6
js/client/modules/@arangodb/testsuites/endpoints.js
View File

@ -80,6 +80,7 @@ function endpoints (options) {
return Object.keys(endpoints).reduce((results, endpointName) => {
results.failed = 0;
results.shutdown = true;
let testName = 'endpoint-' + endpointName;
results[testName] = (function () {
let endpoint = endpoints[endpointName]();
@ -107,11 +108,12 @@ function endpoints (options) {
print(CYAN + 'Shutting down...' + RESET);
// mop: mehhh...when launched with a socket we can't use download :S
pu.shutdownInstance(instanceInfo, Object.assign(options, {useKillExternal: true}));
oneTestResult['shutdown'] = pu.shutdownInstance(instanceInfo, Object.assign(options, {useKillExternal: true}));
print(CYAN + 'done.' + RESET);
if (!oneTestResult.status) {
if (!oneTestResult.status || !oneTestResult.shutdown) {
results.failed += 1;
results.shutdown = false;
} else {
pu.cleanupLastDirectory(options);
}

2
js/client/modules/@arangodb/testsuites/export.js
View File

@ -103,7 +103,7 @@ function exportTest (options) {
function shutdown () {
print(CYAN + 'Shutting down...' + RESET);
pu.shutdownInstance(instanceInfo, options);
results['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print(CYAN + 'done.' + RESET);
print();
return results;

2
js/client/modules/@arangodb/testsuites/foxxmanager.js
View File

@ -70,7 +70,7 @@ function foxxManager (options) {
}
print('Shutting down...');
pu.shutdownInstance(instanceInfo, options);
results['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print('done.');
return results;

2
js/client/modules/@arangodb/testsuites/importing.js
View File

@ -260,7 +260,7 @@ function importing (options) {
}
print('Shutting down...');
pu.shutdownInstance(instanceInfo, options);
result['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print('done.');
return result;

29
js/client/modules/@arangodb/testsuites/permissions_server.js
View File

@ -56,7 +56,7 @@ const testPaths = {
function permissions_server(options) {
let count = 0;
let results = {};
let results = { shutdown: true };
let filtered = {};
const tests = tu.scanTestPaths(testPaths.permissions_server);
@ -84,22 +84,35 @@ function permissions_server(options) {
paramsSecondRun = executeScript(content, true, testFile);
} catch (ex) {
let shutdownStatus = pu.shutdownInstance(instanceInfo, clonedOpts, false); // stop
results[testFile] = {
status: false,
messages: 'Warmup of system failed: ' + ex
messages: 'Warmup of system failed: ' + ex,
shutdown: shutdownStatus
};
pu.shutdownInstance(instanceInfo, clonedOpts, false); // stop
results['shutdown'] = results['shutdown'] && shutdownStatus;
return;
}
if (paramsSecondRun.hasOwnProperty('server.jwt-secret')) {
clonedOpts['server.jwt-secret'] = paramsSecondRun['server.jwt-secret'];
}
pu.shutdownInstance(instanceInfo, clonedOpts, false); // stop
pu.reStartInstance(clonedOpts, instanceInfo, paramsSecondRun); // restart with restricted permissions
results[testFile] = tu.runInLocalArangosh(options, instanceInfo, testFile, {});
pu.shutdownInstance(instanceInfo, clonedOpts, false);
if (!results[testFile].status) {
let shutdownStatus = pu.shutdownInstance(instanceInfo, clonedOpts, false); // stop
if (shutdownStatus) {
pu.reStartInstance(clonedOpts, instanceInfo, paramsSecondRun); // restart with restricted permissions
results[testFile] = tu.runInLocalArangosh(options, instanceInfo, testFile, {});
shutdownStatus = pu.shutdownInstance(instanceInfo, clonedOpts, false);
}
else {
results[testFile] = {
status: false,
message: "failed to stop instance",
shutdown: false
};
}
results['shutdown'] = results['shutdown'] && shutdownStatus;
if (!results[testFile].status || !shutdownStatus) {
print("Not cleaning up " + instanceInfo.rootDir);
results.status = false;
}

2
js/client/modules/@arangodb/testsuites/queryCacheAuthorization.js
View File

@ -148,7 +148,7 @@ function queryCacheAuthorization (options) {
]);
run(requests);
pu.shutdownInstance(adbInstance, options);
results['shutdown'] = pu.shutdownInstance(adbInstance, options);
return results;
}

5
js/client/modules/@arangodb/testsuites/readOnly.js
View File

@ -190,7 +190,7 @@ function readOnly (options) {
print(res); */`
]);
if (res.status !== true) {
pu.shutdownInstance(adbInstance, options);
let shutdownStatus = pu.shutdownInstance(adbInstance, options);
return {
readOnly : {
status: false,
@ -198,6 +198,7 @@ function readOnly (options) {
message: 'the readonly suite failed to setup the environment.',
duration: 2,
failed: 1,
shutdown: shutdownStatus,
failTest: {
status: false,
total: 1,
@ -211,7 +212,7 @@ function readOnly (options) {
requests[0][2] += bodies.pop().indexes.filter(idx => idx.type === 'hash')[0].id;
run(requests);
pu.shutdownInstance(adbInstance, options);
results['shutdown'] = pu.shutdownInstance(adbInstance, options);
return results;
}

106
js/client/modules/@arangodb/testsuites/replication.js
View File

@ -122,9 +122,21 @@ function replicationFuzz (options) {
customInstanceInfos,
startStopHandlers) {
if (pu.arangod.check.instanceAlive(customInstanceInfos.postStart.instanceInfo, options)) {
pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options);
if (!pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options)) {
return {
state: false,
shutdown: false,
message: " failed to shutdown other instance"
};
}
else { return {}; }
} else {
return {
state: false,
shutdown: false,
message: " alive check of other instance failed"
};
}
return {};
},
postStop: function (options,
@ -191,9 +203,21 @@ function replicationRandom (options) {
customInstanceInfos,
startStopHandlers) {
if (pu.arangod.check.instanceAlive(customInstanceInfos.postStart.instanceInfo, options)) {
pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options);
if (!pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options)) {
return {
state: false,
shutdown: false,
message: " failed to shutdown other instance"
};
}
else { return {}; }
} else {
return {
state: false,
shutdown: false,
message: " alive check of other instance failed"
};
}
return {};
},
postStop: function (options,
@ -259,9 +283,21 @@ function replicationAql (options) {
customInstanceInfos,
startStopHandlers) {
if (pu.arangod.check.instanceAlive(customInstanceInfos.postStart.instanceInfo, options)) {
pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options);
if (!pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options)) {
return {
state: false,
shutdown: false,
message: " failed to shutdown other instance"
};
}
else { return {}; }
} else {
return {
state: false,
shutdown: false,
message: " alive check of other instance failed"
};
}
return {};
},
postStop: function (options,
@ -287,7 +323,7 @@ function replicationAql (options) {
var _replicationOngoing = function(path) {
this.func = function replicationOngoing (options) {
let testCases = tu.scanTestPaths(testPaths[path]);
let startStopHandlers = {
postStart: function (options,
serverOptions,
@ -328,9 +364,21 @@ var _replicationOngoing = function(path) {
customInstanceInfos,
startStopHandlers) {
if (pu.arangod.check.instanceAlive(customInstanceInfos.postStart.instanceInfo, options)) {
pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options);
if (!pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options)) {
return {
state: false,
shutdown: false,
message: " failed to shutdown other instance"
};
}
else { return {}; }
} else {
return {
state: false,
shutdown: false,
message: " alive check of other instance failed"
};
}
return {};
},
postStop: function (options,
@ -371,6 +419,7 @@ function replicationStatic (options) {
customInstanceInfos,
startStopHandlers) {
let message;
let shutdownState = true;
let res = true;
print("starting replication slave: ");
let slave = pu.startInstance('tcp', options, {}, 'slave_static');
@ -391,7 +440,7 @@ function replicationStatic (options) {
state = res.status;
if (!state) {
message = 'failed to setup slave connection' + res.message;
pu.shutdownInstance(slave, options);
shutdownState = pu.shutdownInstance(slave, options);
}
slave['isSlaveInstance'] = true;
} else {
@ -401,6 +450,7 @@ function replicationStatic (options) {
instanceInfo: slave,
message: message,
state: state,
shutdown: shutdownState,
env: {
'flatCommands': slave.endpoint
}
@ -421,9 +471,21 @@ function replicationStatic (options) {
customInstanceInfos,
startStopHandlers) {
if (pu.arangod.check.instanceAlive(customInstanceInfos.postStart.instanceInfo, options)) {
pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options);
if (!pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options)) {
return {
state: false,
shutdown: false,
message: " failed to shutdown other instance"
};
}
else { return {}; }
} else {
return {
state: false,
shutdown: false,
message: " alive check of other instance failed"
};
}
return {};
},
postStop: function (options,
@ -463,6 +525,7 @@ function replicationSync (options) {
customInstanceInfos,
startStopHandlers) {
let message;
let shutdownState = true;
let res = true;
print("starting replication slave: ");
let slave = pu.startInstance('tcp', options, {"log.level" : "replication=trace", "--log.level": "replication=trace"}, 'slave_sync');
@ -481,7 +544,7 @@ function replicationSync (options) {
state = res.status;
if (!state) {
message = 'failed to setup slave connection' + res.message;
pu.shutdownInstance(slave, options);
shutdownState = pu.shutdownInstance(slave, options);
}
slave['isSlaveInstance'] = true;
} else {
@ -492,6 +555,7 @@ function replicationSync (options) {
instanceInfo: slave,
message: message,
state: state,
shutdown: shutdownState,
env: {
'flatCommands': slave.endpoint
}
@ -512,9 +576,21 @@ function replicationSync (options) {
customInstanceInfos,
startStopHandlers) {
if (pu.arangod.check.instanceAlive(customInstanceInfos.postStart.instanceInfo, options)) {
pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options);
if (!pu.shutdownInstance(customInstanceInfos.postStart.instanceInfo, options)) {
return {
state: false,
shutdown: false,
message: " failed to shutdown other instance"
};
}
else { return {}; }
} else {
return {
state: false,
shutdown: false,
message: " alive check of other instance failed"
};
}
return {};
},
postStop: function (options,

4
js/client/modules/@arangodb/testsuites/single.js
View File

@ -97,7 +97,7 @@ function singleClient (options) {
options.cleanup = false;
}
pu.shutdownInstance(instanceInfo, options);
result['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print('done.');
@ -152,7 +152,7 @@ function singleServer (options) {
options.cleanup = false;
}
pu.shutdownInstance(instanceInfo, options);
result['shutdown'] = pu.shutdownInstance(instanceInfo, options);
print('done.');

18
js/client/modules/@arangodb/testsuites/stress.js
View File

@ -83,11 +83,12 @@ function runStressTest (options, command, testname) {
print('cannot execute command: (' +
reply.code + ') ' + reply.message);
pu.shutdownInstance(instanceInfo, options);
let shutdownStatus = pu.shutdownInstance(instanceInfo, options);
return {
status: false,
message: reply.hasOwnProperty('body') ? reply.body : yaml.safeDump(reply)
message: reply.hasOwnProperty('body') ? reply.body : yaml.safeDump(reply),
shutdown: shutdownStatus
};
}
@ -117,19 +118,24 @@ function runStressTest (options, command, testname) {
print(yaml.safeDump(check));
pu.shutdownInstance(instanceInfo, options);
let shutdownStatus = pu.shutdownInstance(instanceInfo, options);
return {
status: false,
message: check.hasOwnProperty('body') ? check.body : yaml.safeDump(check)
message: check.hasOwnProperty('body') ? check.body : yaml.safeDump(check),
shutdown: shutdownStatus
};
}
print('Shutting down...');
pu.shutdownInstance(instanceInfo, options);
let shutdownStatus = pu.shutdownInstance(instanceInfo, options);
print('done.');
return {};
return {
status: true,
message: "",
shutdown: shutdownStatus
};
}
// //////////////////////////////////////////////////////////////////////////////