mirror of https://gitee.com/bigwinds/arangodb
1832 lines
61 KiB
JavaScript
Executable File
1832 lines
61 KiB
JavaScript
Executable File
/* jshint strict: false, sub: true */
|
|
/* global print, arango */
|
|
'use strict';
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / DISCLAIMER
|
|
// /
|
|
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
|
|
// / Copyright 2014 triagens GmbH, Cologne, Germany
|
|
// /
|
|
// / Licensed under the Apache License, Version 2.0 (the "License")
|
|
// / you may not use this file except in compliance with the License.
|
|
// / You may obtain a copy of the License at
|
|
// /
|
|
// / http://www.apache.org/licenses/LICENSE-2.0
|
|
// /
|
|
// / Unless required by applicable law or agreed to in writing, software
|
|
// / distributed under the License is distributed on an "AS IS" BASIS,
|
|
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// / See the License for the specific language governing permissions and
|
|
// / limitations under the License.
|
|
// /
|
|
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
|
|
// /
|
|
// / @author Max Neunhoeffer
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
/* Modules: */
|
|
const _ = require('lodash');
|
|
const fs = require('fs');
|
|
const yaml = require('js-yaml');
|
|
const internal = require('internal');
|
|
const toArgv = internal.toArgv;
|
|
const crashUtils = require('@arangodb/crash-utils');
|
|
const crypto = require('@arangodb/crypto');
|
|
|
|
/* Functions: */
|
|
const executeExternal = internal.executeExternal;
|
|
const executeExternalAndWait = internal.executeExternalAndWait;
|
|
const killExternal = internal.killExternal;
|
|
const statusExternal = internal.statusExternal;
|
|
const base64Encode = internal.base64Encode;
|
|
const testPort = internal.testPort;
|
|
const download = internal.download;
|
|
const time = internal.time;
|
|
const wait = internal.wait;
|
|
const sleep = internal.sleep;
|
|
|
|
/* Constants: */
|
|
// const BLUE = internal.COLORS.COLOR_BLUE;
|
|
const CYAN = internal.COLORS.COLOR_CYAN;
|
|
const GREEN = internal.COLORS.COLOR_GREEN;
|
|
const RED = internal.COLORS.COLOR_RED;
|
|
const RESET = internal.COLORS.COLOR_RESET;
|
|
// const YELLOW = internal.COLORS.COLOR_YELLOW;
|
|
|
|
const platform = internal.platform;
|
|
|
|
const abortSignal = 6;
|
|
|
|
class ConfigBuilder {
|
|
constructor(type) {
|
|
this.config = {
|
|
'log.foreground-tty': 'true'
|
|
};
|
|
this.type = type;
|
|
switch (type) {
|
|
case 'restore':
|
|
this.config.configuration = fs.join(CONFIG_DIR, 'arangorestore.conf');
|
|
this.executable = ARANGORESTORE_BIN;
|
|
break;
|
|
case 'dump':
|
|
this.config.configuration = fs.join(CONFIG_DIR, 'arangodump.conf');
|
|
this.executable = ARANGODUMP_BIN;
|
|
break;
|
|
default:
|
|
throw 'Sorry this type of Arango-Binary is not yet implemented: ' + type;
|
|
}
|
|
}
|
|
|
|
setAuth(username, password) {
|
|
this.config['server.username'] = username;
|
|
this.config['server.password'] = password;
|
|
}
|
|
setEndpoint(endpoint) { this.config['server.endpoint'] = endpoint; }
|
|
setDatabase(database) {
|
|
if (this.haveSetAllDatabases()) {
|
|
throw new Error("must not specify all-databases and database");
|
|
}
|
|
this.config['server.database'] = database;
|
|
}
|
|
setIncludeSystem(active) {
|
|
if (this.type !== 'restore' && this.type !== 'dump') {
|
|
throw '"include-system-collections" is not supported for binary: ' + this.type;
|
|
}
|
|
this.config['include-system-collections'] = active ? 'true' : 'false';
|
|
}
|
|
setOutputDirectory(dir) {
|
|
if (this.type !== 'dump') {
|
|
throw '"output-directory" is not supported for binary: ' + this.type;
|
|
}
|
|
this.config['output-directory'] = fs.join(this.rootDir, dir);
|
|
}
|
|
setInputDirectory(dir, createDatabase) {
|
|
if (this.type !== 'restore') {
|
|
throw '"input-directory" is not supported for binary: ' + this.type;
|
|
}
|
|
this.config['input-directory'] = fs.join(this.rootDir, dir);
|
|
if (createDatabase) {
|
|
this.config['create-database'] = 'true';
|
|
} else {
|
|
this.config['create-database'] = 'false';
|
|
}
|
|
}
|
|
setMaskings(dir) {
|
|
if (this.type !== 'dump') {
|
|
throw '"maskings" is not supported for binary: ' + this.type;
|
|
}
|
|
this.config['maskings'] = fs.join(TOP_DIR, "tests/js/common/test-data/maskings", dir);
|
|
}
|
|
activateEncryption() { this.config['encription.keyfile'] = fs.join(this.rootDir, 'secret-key'); }
|
|
setRootDir(dir) { this.rootDir = dir; }
|
|
restrictToCollection(collection) {
|
|
if (this.type !== 'restore' && this.type !== 'dump') {
|
|
throw '"collection" is not supported for binary: ' + this.type;
|
|
}
|
|
this.config['collection'] = collection;
|
|
};
|
|
setAllDatabases() {
|
|
this.config['all-databases'] = 'true';
|
|
delete this.config['server.database'];
|
|
}
|
|
haveSetAllDatabases() {
|
|
return this.config.hasOwnProperty('all-databases');
|
|
}
|
|
toArgv() { return internal.toArgv(this.config); }
|
|
|
|
getExe() { return this.executable; }
|
|
|
|
print() {
|
|
print(this.executable);
|
|
print(this.config);
|
|
}
|
|
}
|
|
|
|
const createBaseConfigBuilder = function (type, options, instanceInfo, database = '_system') {
|
|
const cfg = new ConfigBuilder(type);
|
|
cfg.setAuth(options.username, options.password);
|
|
cfg.setDatabase(database);
|
|
cfg.setEndpoint(instanceInfo.endpoint);
|
|
cfg.setRootDir(instanceInfo.rootDir);
|
|
return cfg;
|
|
};
|
|
|
|
let executableExt = '';
|
|
if (platform.substr(0, 3) === 'win') {
|
|
executableExt = '.exe';
|
|
}
|
|
|
|
let serverCrashedLocal = false;
|
|
let serverFailMessagesLocal = "";
|
|
let cleanupDirectories = [];
|
|
|
|
let BIN_DIR;
|
|
let ARANGOBENCH_BIN;
|
|
let ARANGODUMP_BIN;
|
|
let ARANGOD_BIN;
|
|
let ARANGOIMPORT_BIN;
|
|
let ARANGORESTORE_BIN;
|
|
let ARANGOEXPORT_BIN;
|
|
let ARANGOSH_BIN;
|
|
let CONFIG_ARANGODB_DIR;
|
|
let CONFIG_RELATIVE_DIR;
|
|
let CONFIG_DIR;
|
|
let JS_DIR;
|
|
let JS_ENTERPRISE_DIR;
|
|
let LOGS_DIR;
|
|
let UNITTESTS_DIR;
|
|
|
|
const TOP_DIR = (function findTopDir () {
|
|
const topDir = fs.normalize(fs.makeAbsolute('.'));
|
|
|
|
if (!fs.exists('3rdParty') && !fs.exists('arangod') &&
|
|
!fs.exists('arangosh') && !fs.exists('tests')) {
|
|
throw new Error('Must be in ArangoDB topdir to execute tests.');
|
|
}
|
|
|
|
return topDir;
|
|
}());
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief calculates all the path locations
|
|
// / required to be called first.
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function setupBinaries (builddir, buildType, configDir) {
|
|
if (builddir === '') {
|
|
if (fs.exists('build') && fs.exists(fs.join('build', 'bin'))) {
|
|
builddir = 'build';
|
|
} else if (fs.exists('bin')) {
|
|
builddir = '.';
|
|
} else {
|
|
print('FATAL: cannot find binaries, use "--build"\n');
|
|
|
|
return {
|
|
status: false
|
|
};
|
|
}
|
|
}
|
|
|
|
BIN_DIR = fs.join(builddir, 'bin');
|
|
if (!fs.exists(BIN_DIR)) {
|
|
BIN_DIR = fs.join(TOP_DIR, BIN_DIR);
|
|
}
|
|
|
|
UNITTESTS_DIR = fs.join(fs.join(builddir, 'tests'));
|
|
if (!fs.exists(UNITTESTS_DIR)) {
|
|
UNITTESTS_DIR = fs.join(TOP_DIR, UNITTESTS_DIR);
|
|
}
|
|
|
|
if (buildType !== '') {
|
|
BIN_DIR = fs.join(BIN_DIR, buildType);
|
|
UNITTESTS_DIR = fs.join(UNITTESTS_DIR, buildType);
|
|
}
|
|
|
|
ARANGOBENCH_BIN = fs.join(BIN_DIR, 'arangobench' + executableExt);
|
|
ARANGODUMP_BIN = fs.join(BIN_DIR, 'arangodump' + executableExt);
|
|
ARANGOD_BIN = fs.join(BIN_DIR, 'arangod' + executableExt);
|
|
ARANGOIMPORT_BIN = fs.join(BIN_DIR, 'arangoimport' + executableExt);
|
|
ARANGORESTORE_BIN = fs.join(BIN_DIR, 'arangorestore' + executableExt);
|
|
ARANGOEXPORT_BIN = fs.join(BIN_DIR, 'arangoexport' + executableExt);
|
|
ARANGOSH_BIN = fs.join(BIN_DIR, 'arangosh' + executableExt);
|
|
|
|
CONFIG_ARANGODB_DIR = fs.join(builddir, 'etc', 'arangodb3');
|
|
if (!fs.exists(CONFIG_ARANGODB_DIR)) {
|
|
CONFIG_ARANGODB_DIR = fs.join(TOP_DIR, CONFIG_ARANGODB_DIR);
|
|
}
|
|
|
|
CONFIG_RELATIVE_DIR = fs.join(TOP_DIR, 'etc', 'relative');
|
|
CONFIG_DIR = fs.join(TOP_DIR, configDir);
|
|
|
|
JS_DIR = fs.join(TOP_DIR, 'js');
|
|
JS_ENTERPRISE_DIR = fs.join(TOP_DIR, 'enterprise/js');
|
|
|
|
LOGS_DIR = fs.join(TOP_DIR, 'logs');
|
|
|
|
let checkFiles = [
|
|
ARANGOBENCH_BIN,
|
|
ARANGODUMP_BIN,
|
|
ARANGOD_BIN,
|
|
ARANGOIMPORT_BIN,
|
|
ARANGORESTORE_BIN,
|
|
ARANGOEXPORT_BIN,
|
|
ARANGOSH_BIN];
|
|
for (let b = 0; b < checkFiles.length; ++b) {
|
|
if (!fs.isFile(checkFiles[b])) {
|
|
throw new Error('unable to locate ' + checkFiles[b]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief finds a free port
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function findFreePort (minPort, maxPort, usedPorts) {
|
|
if (typeof maxPort !== 'number') {
|
|
maxPort = 32768;
|
|
}
|
|
|
|
if (maxPort - minPort < 0) {
|
|
throw new Error('minPort ' + minPort + ' is smaller than maxPort ' + maxPort);
|
|
}
|
|
|
|
let tries = 0;
|
|
while (true) {
|
|
const port = Math.floor(Math.random() * (maxPort - minPort)) + minPort;
|
|
tries++;
|
|
if (tries > 20) {
|
|
throw new Error('Couldn\'t find a port after ' + tries + ' tries. portrange of ' + minPort + ', ' + maxPort + ' too narrow?');
|
|
}
|
|
if (Array.isArray(usedPorts) && usedPorts.indexOf(port) >= 0) {
|
|
continue;
|
|
}
|
|
const free = testPort('tcp://0.0.0.0:' + port);
|
|
|
|
if (free) {
|
|
return port;
|
|
}
|
|
|
|
internal.wait(0.1, false);
|
|
}
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief scans the log files for important infos
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function readImportantLogLines (logPath) {
|
|
const list = fs.list(logPath);
|
|
let importantLines = {};
|
|
|
|
for (let i = 0; i < list.length; i++) {
|
|
let fnLines = [];
|
|
|
|
if (list[i].slice(0, 3) === 'log') {
|
|
const buf = fs.readBuffer(fs.join(logPath, list[i]));
|
|
let lineStart = 0;
|
|
let maxBuffer = buf.length;
|
|
|
|
for (let j = 0; j < maxBuffer; j++) {
|
|
if (buf[j] === 10) { // \n
|
|
const line = buf.asciiSlice(lineStart, j);
|
|
lineStart = j + 1;
|
|
|
|
// filter out regular INFO lines, and test related messages
|
|
let warn = line.search('WARNING about to execute:') !== -1;
|
|
let info = line.search(' INFO ') !== -1;
|
|
|
|
if (warn || info) {
|
|
continue;
|
|
}
|
|
fnLines.push(line);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (fnLines.length > 0) {
|
|
importantLines[list[i]] = fnLines;
|
|
}
|
|
}
|
|
|
|
return importantLines;
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief cleans up the database direcory
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function cleanupLastDirectory (options) {
|
|
if (options.cleanup) {
|
|
while (cleanupDirectories.length) {
|
|
const cleanupDirectory = cleanupDirectories.shift();
|
|
if (options.extremeVerbosity === true) {
|
|
print(Date() + " Cleaning up: " + cleanupDirectory);
|
|
}
|
|
// Avoid attempting to remove the same directory multiple times
|
|
if ((cleanupDirectories.indexOf(cleanupDirectory) === -1) &&
|
|
(fs.exists(cleanupDirectory))) {
|
|
let i = 0;
|
|
while (i < 5) {
|
|
try {
|
|
fs.removeDirectoryRecursive(cleanupDirectory, true);
|
|
return;
|
|
} catch (x) {
|
|
print(Date() + ' failed to delete directory "' + cleanupDirectory + '" - "' +
|
|
x + '" - Will retry in 5 seconds"');
|
|
sleep(5);
|
|
}
|
|
i += 1;
|
|
}
|
|
print(Date() + ' failed to delete directory "' + cleanupDirectory + '" - "' +
|
|
'" - Deferring cleanup for test run end."');
|
|
cleanupDirectories.unshift(cleanupDirectory);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
function cleanupDBDirectories (options) {
|
|
if (options.cleanup) {
|
|
while (cleanupDirectories.length) {
|
|
const cleanupDirectory = cleanupDirectories.shift();
|
|
|
|
// Avoid attempting to remove the same directory multiple times
|
|
if ((cleanupDirectories.indexOf(cleanupDirectory) === -1) &&
|
|
(fs.exists(cleanupDirectory))) {
|
|
fs.removeDirectoryRecursive(cleanupDirectory, true);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function cleanupDBDirectoriesAppend (appendThis) {
|
|
cleanupDirectories.unshift(appendThis);
|
|
}
|
|
|
|
function getCleanupDBDirectories () {
|
|
return JSON.stringify(cleanupDirectories);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief adds authorization headers
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function makeAuthorizationHeaders (options) {
|
|
if (options['server.jwt-secret']) {
|
|
var jwt = crypto.jwtEncode(options['server.jwt-secret'],
|
|
{'server_id': 'none',
|
|
'iss': 'arangodb'}, 'HS256');
|
|
if (options.extremeVerbosity) {
|
|
print(Date() + ' Using jwt token: ' + jwt);
|
|
}
|
|
return {
|
|
'headers': {
|
|
'Authorization': 'bearer ' + jwt
|
|
}
|
|
};
|
|
} else {
|
|
return {
|
|
'headers': {
|
|
'Authorization': 'Basic ' + base64Encode(options.username + ':' +
|
|
options.password)
|
|
}
|
|
};
|
|
}
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief converts endpoints to URL
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function endpointToURL (endpoint) {
|
|
if (endpoint.substr(0, 6) === 'ssl://') {
|
|
return 'https://' + endpoint.substr(6);
|
|
}
|
|
|
|
const pos = endpoint.indexOf('://');
|
|
|
|
if (pos === -1) {
|
|
return 'http://' + endpoint;
|
|
}
|
|
|
|
return 'http' + endpoint.substr(pos);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief arguments for testing (server)
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function makeArgsArangod (options, appDir, role, tmpDir) {
|
|
console.assert(tmpDir !== undefined);
|
|
if (appDir === undefined) {
|
|
appDir = fs.getTempPath();
|
|
}
|
|
|
|
fs.makeDirectoryRecursive(appDir, true);
|
|
|
|
fs.makeDirectoryRecursive(tmpDir, true);
|
|
|
|
let config = 'arangod.conf';
|
|
|
|
if (role !== undefined && role !== null && role !== '') {
|
|
config = 'arangod-' + role + '.conf';
|
|
}
|
|
|
|
let args = {
|
|
'configuration': fs.join(CONFIG_DIR, config),
|
|
'define': 'TOP_DIR=' + TOP_DIR,
|
|
'wal.flush-timeout': options.walFlushTimeout,
|
|
'javascript.app-path': appDir,
|
|
'javascript.copy-installation': false,
|
|
'http.trusted-origin': options.httpTrustedOrigin || 'all',
|
|
'cluster.create-waits-for-sync-replication': false,
|
|
'temp.path': tmpDir
|
|
};
|
|
if (options.storageEngine !== undefined) {
|
|
args['server.storage-engine'] = options.storageEngine;
|
|
}
|
|
return args;
|
|
}
|
|
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief check whether process does bad on the wintendo
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function runProcdump (options, instanceInfo, rootDir, pid) {
|
|
let procdumpArgs = [ ];
|
|
let dumpFile = fs.join(rootDir, 'core_' + pid + '.dmp');
|
|
if (options.exceptionFilter != null) {
|
|
procdumpArgs = [
|
|
'-accepteula',
|
|
'-64',
|
|
'-e',
|
|
options.exceptionCount
|
|
];
|
|
let filters = options.exceptionFilter.split(',');
|
|
for (let which in filters) {
|
|
procdumpArgs.push('-f');
|
|
procdumpArgs.push(filters[which]);
|
|
}
|
|
procdumpArgs.push('-ma');
|
|
procdumpArgs.push(pid);
|
|
procdumpArgs.push(dumpFile);
|
|
} else {
|
|
procdumpArgs = [
|
|
'-accepteula',
|
|
'-e',
|
|
'-ma',
|
|
pid,
|
|
dumpFile
|
|
];
|
|
}
|
|
try {
|
|
if (options.extremeVerbosity) {
|
|
print(Date() + " Starting procdump: " + JSON.stringify(procdumpArgs));
|
|
}
|
|
instanceInfo.monitor = executeExternal('procdump', procdumpArgs);
|
|
instanceInfo.coreFilePattern = dumpFile;
|
|
} catch (x) {
|
|
print(Date() + ' failed to start procdump - is it installed?');
|
|
// throw x;
|
|
}
|
|
}
|
|
|
|
function stopProcdump (options, instanceInfo) {
|
|
if (instanceInfo.hasOwnProperty('monitor') &&
|
|
instanceInfo.monitor.pid !== null) {
|
|
print(Date() + " wating for procdump to exit");
|
|
statusExternal(instanceInfo.monitor.pid, true);
|
|
instanceInfo.monitor.pid = null;
|
|
}
|
|
}
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief executes a command and waits for result
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function executeAndWait (cmd, args, options, valgrindTest, rootDir, circumventCores, coreCheck = false) {
|
|
if (valgrindTest && options.valgrind) {
|
|
let valgrindOpts = {};
|
|
|
|
if (options.valgrindArgs) {
|
|
valgrindOpts = options.valgrindArgs;
|
|
}
|
|
|
|
let testfn = options.valgrindFileBase;
|
|
|
|
if (testfn.length > 0) {
|
|
testfn += '_';
|
|
}
|
|
|
|
testfn += valgrindTest;
|
|
|
|
if (valgrindOpts.xml === 'yes') {
|
|
valgrindOpts['xml-file'] = testfn + '.%p.xml';
|
|
}
|
|
|
|
valgrindOpts['log-file'] = testfn + '.%p.valgrind.log';
|
|
|
|
args = toArgv(valgrindOpts, true).concat([cmd]).concat(args);
|
|
cmd = options.valgrind;
|
|
}
|
|
|
|
if (circumventCores) {
|
|
if (platform.substr(0, 3) !== 'win') {
|
|
// this shellscript will prevent cores from being writen on macos and linux.
|
|
args.unshift(cmd);
|
|
cmd = TOP_DIR + '/scripts/disable-cores.sh';
|
|
}
|
|
}
|
|
|
|
if (options.extremeVerbosity) {
|
|
print(Date() + ' executeAndWait: cmd =', cmd, 'args =', args);
|
|
}
|
|
|
|
const startTime = time();
|
|
if ((typeof (cmd) !== 'string') || (cmd === 'true') || (cmd === 'false')) {
|
|
return {
|
|
status: false,
|
|
message: 'true or false as binary name for test cmd =' + cmd + 'args =' + args
|
|
};
|
|
}
|
|
|
|
let instanceInfo = {
|
|
rootDir: rootDir,
|
|
pid: 0,
|
|
exitStatus: {}
|
|
};
|
|
|
|
let res = {};
|
|
if (platform.substr(0, 3) === 'win' && !options.disableMonitor) {
|
|
res = executeExternal(cmd, args);
|
|
instanceInfo.pid = res.pid;
|
|
instanceInfo.exitStatus = res;
|
|
runProcdump(options, instanceInfo, rootDir, res.pid);
|
|
Object.assign(instanceInfo.exitStatus,
|
|
statusExternal(res.pid, true));
|
|
stopProcdump(options, instanceInfo);
|
|
} else {
|
|
res = executeExternalAndWait(cmd, args);
|
|
instanceInfo.pid = res.pid;
|
|
instanceInfo.exitStatus = res;
|
|
}
|
|
const deltaTime = time() - startTime;
|
|
|
|
let errorMessage = ' - ';
|
|
|
|
if (coreCheck &&
|
|
instanceInfo.exitStatus.hasOwnProperty('signal') &&
|
|
((instanceInfo.exitStatus.signal === 11) ||
|
|
(instanceInfo.exitStatus.signal === 6) ||
|
|
// Windows sometimes has random numbers in signal...
|
|
(platform.substr(0, 3) === 'win')
|
|
)
|
|
) {
|
|
print(Date() + " executeAndWait: Marking crashy - " + JSON.stringify(instanceInfo));
|
|
crashUtils.analyzeCrash(cmd,
|
|
instanceInfo,
|
|
options,
|
|
'execution of ' + cmd + ' - ' + instanceInfo.exitStatus.signal);
|
|
if (options.coreCheck) {
|
|
print(instanceInfo.exitStatus.gdbHint);
|
|
}
|
|
serverCrashedLocal = true;
|
|
}
|
|
|
|
if (instanceInfo.exitStatus.status === 'TERMINATED') {
|
|
const color = (instanceInfo.exitStatus.exit === 0 ? GREEN : RED);
|
|
|
|
print(color + Date() + ' Finished: ' + instanceInfo.exitStatus.status +
|
|
' exit code: ' + instanceInfo.exitStatus.exit +
|
|
' Time elapsed: ' + deltaTime + RESET);
|
|
|
|
if (instanceInfo.exitStatus.exit === 0) {
|
|
return {
|
|
status: true,
|
|
message: '',
|
|
duration: deltaTime
|
|
};
|
|
} else {
|
|
return {
|
|
status: false,
|
|
message: 'exit code was ' + instanceInfo.exitStatus.exit,
|
|
duration: deltaTime
|
|
};
|
|
}
|
|
} else if (instanceInfo.exitStatus.status === 'ABORTED') {
|
|
if (typeof (instanceInfo.exitStatus.errorMessage) !== 'undefined') {
|
|
errorMessage += instanceInfo.exitStatus.errorMessage;
|
|
}
|
|
|
|
print(Date() + ' Finished: ' + instanceInfo.exitStatus.status +
|
|
' Signal: ' + instanceInfo.exitStatus.signal +
|
|
' Time elapsed: ' + deltaTime + errorMessage);
|
|
|
|
return {
|
|
status: false,
|
|
message: 'irregular termination: ' + instanceInfo.exitStatus.status +
|
|
' exit signal: ' + instanceInfo.exitStatus.signal + errorMessage,
|
|
duration: deltaTime
|
|
};
|
|
} else {
|
|
if (typeof (instanceInfo.exitStatus.errorMessage) !== 'undefined') {
|
|
errorMessage += instanceInfo.exitStatus.errorMessage;
|
|
}
|
|
|
|
print(Date() + ' Finished: ' + instanceInfo.exitStatus.status +
|
|
' exit code: ' + instanceInfo.exitStatus.signal +
|
|
' Time elapsed: ' + deltaTime + errorMessage);
|
|
|
|
return {
|
|
status: false,
|
|
message: 'irregular termination: ' + instanceInfo.exitStatus.status +
|
|
' exit code: ' + instanceInfo.exitStatus.exit + errorMessage,
|
|
duration: deltaTime
|
|
};
|
|
}
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / operate the arango commandline utilities
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief arguments for testing (client)
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function makeArgsArangosh (options) {
|
|
return {
|
|
'configuration': fs.join(CONFIG_DIR, 'arangosh.conf'),
|
|
'javascript.startup-directory': JS_DIR,
|
|
'javascript.module-directory': JS_ENTERPRISE_DIR,
|
|
'server.username': options.username,
|
|
'server.password': options.password,
|
|
'flatCommands': ['--console.colors', 'false', '--quiet']
|
|
};
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief runs arangosh
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function runArangoshCmd (options, instanceInfo, addArgs, cmds, coreCheck = false) {
|
|
let args = makeArgsArangosh(options);
|
|
args['server.endpoint'] = instanceInfo.endpoint;
|
|
|
|
if (addArgs !== undefined) {
|
|
args = Object.assign(args, addArgs);
|
|
}
|
|
|
|
internal.env.INSTANCEINFO = JSON.stringify(instanceInfo);
|
|
const argv = toArgv(args).concat(cmds);
|
|
return executeAndWait(ARANGOSH_BIN, argv, options, 'arangoshcmd', instanceInfo.rootDir, false, coreCheck);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief runs arangoimport
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function runArangoImport (options, instanceInfo, what, coreCheck = false) {
|
|
let args = {
|
|
'log.foreground-tty': 'true',
|
|
'server.username': options.username,
|
|
'server.password': options.password,
|
|
'server.endpoint': instanceInfo.endpoint,
|
|
'file': fs.join(TOP_DIR, what.data),
|
|
'collection': what.coll,
|
|
'type': what.type,
|
|
'on-duplicate': what.onDuplicate || 'error',
|
|
'ignore-missing': what.ignoreMissing || false
|
|
};
|
|
|
|
if (what.skipLines !== undefined) {
|
|
args['skip-lines'] = what.skipLines;
|
|
}
|
|
|
|
if (what.create !== undefined) {
|
|
args['create-collection'] = what.create;
|
|
}
|
|
|
|
if (what.createDatabase !== undefined) {
|
|
args['create-database'] = what.createDatabase;
|
|
}
|
|
|
|
if (what.database !== undefined) {
|
|
args['server.database'] = what.database;
|
|
}
|
|
|
|
if (what.backslash !== undefined) {
|
|
args['backslash-escape'] = what.backslash;
|
|
}
|
|
|
|
if (what.separator !== undefined) {
|
|
args['separator'] = what.separator;
|
|
}
|
|
|
|
if (what.convert !== undefined) {
|
|
args['convert'] = what.convert ? 'true' : 'false';
|
|
}
|
|
if (what.removeAttribute !== undefined) {
|
|
args['remove-attribute'] = what.removeAttribute;
|
|
}
|
|
|
|
return executeAndWait(ARANGOIMPORT_BIN, toArgv(args), options, 'arangoimport', instanceInfo.rootDir, false, coreCheck);
|
|
}
|
|
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief runs arangodump or arangorestore based on config object
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function runArangoDumpRestoreCfg (config, options, rootDir, coreCheck) {
|
|
if (options.extremeVerbosity === true) {
|
|
config.print();
|
|
}
|
|
return executeAndWait(config.getExe(), config.toArgv(), options, 'arangorestore', rootDir, false, coreCheck);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief runs arangodump or arangorestore
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function runArangoDumpRestore (options, instanceInfo, which, database, rootDir, dumpDir = 'dump', includeSystem = true, coreCheck = false) {
|
|
const cfg = createBaseConfigBuilder(which, options, instanceInfo, database);
|
|
cfg.setIncludeSystem(includeSystem);
|
|
if (rootDir) { cfg.setRootDir(rootDir); }
|
|
|
|
if (which === 'dump') {
|
|
cfg.setOutputDirectory(dumpDir);
|
|
} else {
|
|
cfg.setInputDirectory(dumpDir, true);
|
|
}
|
|
|
|
if (options.encrypted) {
|
|
cfg.activateEncryption();
|
|
}
|
|
if (options.allDatabases) {
|
|
cfg.setAllDatabases();
|
|
}
|
|
return runArangoDumpRestoreCfg(cfg, options, rootDir, coreCheck);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief runs arangobench
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function runArangoBenchmark (options, instanceInfo, cmds, rootDir, coreCheck = false) {
|
|
let args = {
|
|
'configuration': fs.join(CONFIG_DIR, 'arangobench.conf'),
|
|
'log.foreground-tty': 'true',
|
|
'server.username': options.username,
|
|
'server.password': options.password,
|
|
'server.endpoint': instanceInfo.endpoint,
|
|
// 'server.request-timeout': 1200 // default now.
|
|
'server.connection-timeout': 10 // 5s default
|
|
};
|
|
|
|
args = Object.assign(args, cmds);
|
|
|
|
if (!args.hasOwnProperty('verbose')) {
|
|
args['log.level'] = 'warning';
|
|
args['flatCommands'] = ['--quiet'];
|
|
}
|
|
|
|
return executeAndWait(ARANGOBENCH_BIN, toArgv(args), options, 'arangobench', instanceInfo.rootDir, false, coreCheck);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / Server up/down utilities
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief dump the state of the agency to disk. if we still can get one.
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
function dumpAgency(instanceInfo, options) {
|
|
function dumpAgent(arangod, path, method, fn) {
|
|
let opts = {
|
|
method: method
|
|
};
|
|
if (instanceInfo.hasOwnProperty('authOpts')) {
|
|
opts['jwt'] = crypto.jwtEncode(instanceInfo.authOpts['server.jwt-secret'], {'server_id': 'none', 'iss': 'arangodb'}, 'HS256');
|
|
}
|
|
print('--------------------------------- '+ fn + ' -----------------------------------------------');
|
|
let agencyReply = download(arangod.url + path, method === 'POST' ? '[["/"]]' : '', opts);
|
|
if (agencyReply.code === 200) {
|
|
let agencyValue = JSON.parse(agencyReply.body);
|
|
fs.write(fs.join(options.testOutputDirectory, fn + '_' + arangod.pid + ".json"), JSON.stringify(agencyValue, null, 2));
|
|
} else {
|
|
print(agencyReply);
|
|
}
|
|
}
|
|
instanceInfo.arangods.forEach((arangod) => {
|
|
if (arangod.role === "agent") {
|
|
if (arangod.hasOwnProperty('exitStatus')) {
|
|
print(Date() + " this agent is already dead: " + JSON.stringify(arangod));
|
|
} else {
|
|
print(Date() + " Attempting to dump Agent: " + JSON.stringify(arangod));
|
|
dumpAgent(arangod, '/_api/agency/config', 'GET', 'agencyConfig');
|
|
|
|
dumpAgent(arangod, '/_api/agency/state', 'GET', 'agencyState');
|
|
|
|
dumpAgent(arangod, '/_api/agency/read', 'POST', 'agencyPlan');
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief the bad has happened, tell it the user and try to gather more
|
|
// / information about the incident. (arangod wrapper for the crash-utils)
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
function analyzeServerCrash (arangod, options, checkStr) {
|
|
return crashUtils.analyzeCrash(ARANGOD_BIN, arangod, options, checkStr);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief periodic checks whether spawned arangod processes are still alive
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
function checkArangoAlive (arangod, options) {
|
|
const res = statusExternal(arangod.pid, false);
|
|
const ret = res.status === 'RUNNING' && crashUtils.checkMonitorAlive(ARANGOD_BIN, arangod, options, res);
|
|
|
|
if (!ret) {
|
|
print(Date() + ' ArangoD with PID ' + arangod.pid + ' gone:');
|
|
if (!arangod.hasOwnProperty('exitStatus')) {
|
|
arangod.exitStatus = res;
|
|
}
|
|
print(arangod);
|
|
|
|
if (res.hasOwnProperty('signal') &&
|
|
((res.signal === 11) ||
|
|
(res.signal === 6) ||
|
|
// Windows sometimes has random numbers in signal...
|
|
(platform.substr(0, 3) === 'win')
|
|
)
|
|
) {
|
|
arangod.exitStatus = res;
|
|
analyzeServerCrash(arangod, options, 'health Check - ' + res.signal);
|
|
serverCrashedLocal = true;
|
|
print(Date() + " checkArangoAlive: Marking crashy - " + JSON.stringify(arangod));
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
function abortSurvivors(arangod, options) {
|
|
print(Date() + " Killing in the name of: ");
|
|
print(arangod);
|
|
if (!arangod.hasOwnProperty('exitStatus')) {
|
|
arangod.exitStatus = killExternal(arangod.pid, abortSignal);
|
|
}
|
|
}
|
|
|
|
function checkInstanceAlive (instanceInfo, options) {
|
|
if (options.activefailover && instanceInfo.hasOwnProperty('authOpts')) {
|
|
let d = detectCurrentLeader(instanceInfo);
|
|
if (instanceInfo.endpoint !== d.endpoint) {
|
|
print(Date() + ' failover has happened, leader is no more! Marking Crashy!');
|
|
serverCrashedLocal = true;
|
|
dumpAgency(instanceInfo, options);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
let rc = instanceInfo.arangods.reduce((previous, arangod) => {
|
|
return previous && checkArangoAlive(arangod, options);
|
|
}, true);
|
|
if (!rc) {
|
|
dumpAgency(instanceInfo, options);
|
|
print(Date() + ' If cluster - will now start killing the rest.');
|
|
instanceInfo.arangods.forEach((arangod) => {
|
|
abortSurvivors(arangod, options);
|
|
});
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief waits for garbage collection using /_admin/execute
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function waitOnServerForGC (instanceInfo, options, waitTime) {
|
|
try {
|
|
print(Date() + ' waiting ' + waitTime + ' for server GC');
|
|
const remoteCommand = 'require("internal").wait(' + waitTime + ', true);';
|
|
|
|
const requestOptions = makeAuthorizationHeaders(options);
|
|
requestOptions.method = 'POST';
|
|
requestOptions.timeout = waitTime * 10;
|
|
requestOptions.returnBodyOnError = true;
|
|
|
|
const reply = download(
|
|
instanceInfo.url + '/_admin/execute?returnAsJSON=true',
|
|
remoteCommand,
|
|
requestOptions);
|
|
|
|
print(Date() + ' waiting ' + waitTime + ' for server GC - done.');
|
|
|
|
if (!reply.error && reply.code === 200) {
|
|
return JSON.parse(reply.body);
|
|
} else {
|
|
return {
|
|
status: false,
|
|
message: yaml.safedump(reply.body)
|
|
};
|
|
}
|
|
} catch (ex) {
|
|
return {
|
|
status: false,
|
|
message: ex.message || String(ex),
|
|
stack: ex.stack
|
|
};
|
|
}
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief executes a command, possible with valgrind
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function executeArangod (cmd, args, options) {
|
|
if (options.valgrind) {
|
|
let valgrindOpts = {};
|
|
|
|
if (options.valgrindArgs) {
|
|
valgrindOpts = options.valgrindArgs;
|
|
}
|
|
|
|
let testfn = options.valgrindFileBase;
|
|
|
|
if (testfn.length > 0) {
|
|
testfn += '_';
|
|
}
|
|
|
|
if (valgrindOpts.xml === 'yes') {
|
|
valgrindOpts['xml-file'] = testfn + '.%p.xml';
|
|
}
|
|
|
|
valgrindOpts['log-file'] = testfn + '.%p.valgrind.log';
|
|
|
|
args = toArgv(valgrindOpts, true).concat([cmd]).concat(args);
|
|
cmd = options.valgrind;
|
|
} else if (options.rr) {
|
|
args = [cmd].concat(args);
|
|
cmd = 'rr';
|
|
}
|
|
|
|
if (options.extremeVerbosity) {
|
|
print(Date() + ' starting process ' + cmd + ' with arguments: ' + JSON.stringify(args));
|
|
}
|
|
return executeExternal(cmd, args);
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief on linux get a statistic about the sockets we used
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function getSockStat(arangod, options, preamble) {
|
|
if (options.getSockStat && (platform === 'linux')) {
|
|
let sockStat = preamble + arangod.pid + "\n";
|
|
try {
|
|
sockStat += fs.read("/proc/" + arangod.pid + "/net/sockstat");
|
|
return sockStat;
|
|
}
|
|
catch (e) {/* oops, process already gone? don't care. */ }
|
|
}
|
|
return "";
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief commands a server to shut down via webcall
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function shutdownArangod (arangod, options, forceTerminate) {
|
|
if (forceTerminate === undefined) {
|
|
forceTerminate = false;
|
|
}
|
|
if (options.hasOwnProperty('server')) {
|
|
print(Date() + ' running with external server');
|
|
return;
|
|
}
|
|
|
|
if (options.valgrind) {
|
|
waitOnServerForGC(arangod, options, 60);
|
|
}
|
|
if (options.rr && forceTerminate) {
|
|
forceTerminate = false;
|
|
options.useKillExternal = true;
|
|
}
|
|
if ((!arangod.hasOwnProperty('exitStatus')) ||
|
|
(arangod.exitStatus.status === 'RUNNING')) {
|
|
if (forceTerminate) {
|
|
let sockStat = getSockStat(arangod, options, "Force killing - sockstat before: ");
|
|
arangod.exitStatus = killExternal(arangod.pid, abortSignal);
|
|
analyzeServerCrash(arangod, options, 'shutdown timeout; instance forcefully KILLED because of fatal timeout in testrun ' + sockStat);
|
|
} else if (options.useKillExternal) {
|
|
let sockStat = getSockStat(arangod, options, "Shutdown by kill - sockstat before: ");
|
|
arangod.exitStatus = killExternal(arangod.pid);
|
|
print(sockStat);
|
|
} else {
|
|
const requestOptions = makeAuthorizationHeaders(options);
|
|
requestOptions.method = 'DELETE';
|
|
requestOptions.timeout = 60; // 60 seconds hopefully are enough for getting a response
|
|
print(Date() + ' ' + arangod.url + '/_admin/shutdown');
|
|
let sockStat = getSockStat(arangod, options, "Sock stat for: ");
|
|
const reply = download(arangod.url + '/_admin/shutdown', '', requestOptions);
|
|
if ((reply.code !== 200) && // if the server should reply, we expect 200 - if not:
|
|
!((reply.code === 500) &&
|
|
(
|
|
(reply.message === "Connection closed by remote") || // http connection
|
|
reply.message.includes('failed with #111') // https connection
|
|
))) {
|
|
serverCrashedLocal = true;
|
|
print(Date() + ' Wrong shutdown response: ' + JSON.stringify(reply) + "' " + sockStat + " continuing with hard kill!");
|
|
shutdownArangod(arangod, options, true);
|
|
}
|
|
else {
|
|
print(sockStat);
|
|
}
|
|
if (options.extremeVerbosity) {
|
|
print(Date() + ' Shutdown response: ' + JSON.stringify(reply));
|
|
}
|
|
}
|
|
} else {
|
|
print(Date() + ' Server already dead, doing nothing.');
|
|
}
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief shuts down an instance
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function shutdownInstance (instanceInfo, options, forceTerminate) {
|
|
if (forceTerminate === undefined) {
|
|
forceTerminate = false;
|
|
}
|
|
let shutdownSuccess = !forceTerminate;
|
|
|
|
// we need to find the leading server
|
|
if (options.activefailover) {
|
|
let d = detectCurrentLeader(instanceInfo);
|
|
if (instanceInfo.endpoint !== d.endpoint) {
|
|
print(Date() + ' failover has happened, leader is no more! Marking Crashy!');
|
|
serverCrashedLocal = true;
|
|
forceTerminate = true;
|
|
shutdownSuccess = false;
|
|
}
|
|
}
|
|
|
|
if (!checkInstanceAlive(instanceInfo, options)) {
|
|
print(Date() + ' Server already dead, doing nothing. This shouldn\'t happen?');
|
|
}
|
|
|
|
if (!forceTerminate) {
|
|
try {
|
|
// send a maintenance request to any of the coordinators, so that
|
|
// no failed server/failed follower jobs will be started on shutdown
|
|
let coords = instanceInfo.arangods.filter(arangod =>
|
|
arangod.role === 'coordinator' &&
|
|
!arangod.hasOwnProperty('exitStatus'));
|
|
if (coords.length > 0) {
|
|
let requestOptions = makeAuthorizationHeaders(options);
|
|
requestOptions.method = 'PUT';
|
|
|
|
print(coords[0].url + "/_admin/cluster/maintenance");
|
|
download(coords[0].url + "/_admin/cluster/maintenance", JSON.stringify("on"), requestOptions);
|
|
}
|
|
} catch (err) {
|
|
print(Date() + " error while setting cluster maintenance mode:", err);
|
|
shutdownSuccess = false;
|
|
}
|
|
}
|
|
|
|
// Shut down all non-agency servers:
|
|
const n = instanceInfo.arangods.length;
|
|
|
|
let toShutdown = instanceInfo.arangods.slice();
|
|
toShutdown.sort((a, b) => {
|
|
if (a.role === b.role) return 0;
|
|
if (a.role === 'coordinator' &&
|
|
b.role === 'dbserver') return -1;
|
|
if (b.role === 'coordinator' &&
|
|
a.role === 'dbserver') return 1;
|
|
if (a.role === 'agent') return 1;
|
|
if (b.role === 'agent') return -1;
|
|
return 0;
|
|
});
|
|
print(Date() + ' Shutdown order ' + JSON.stringify(toShutdown));
|
|
|
|
let nonAgenciesCount = instanceInfo.arangods
|
|
.filter(arangod => {
|
|
if (arangod.hasOwnProperty('exitStatus') &&
|
|
(arangod.exitStatus.status !== 'RUNNING')) {
|
|
return false;
|
|
}
|
|
return (arangod.role !== 'agent');
|
|
}).length;
|
|
|
|
let timeout = 666;
|
|
if (options.valgrind) {
|
|
timeout *= 10;
|
|
}
|
|
if (options.sanitizer) {
|
|
timeout *= 2;
|
|
}
|
|
|
|
if ((toShutdown.length > 0) && (options.agency === true) && (options.dumpAgencyOnError === true)) {
|
|
dumpAgency(instanceInfo, options);
|
|
}
|
|
var shutdownTime = internal.time();
|
|
while (toShutdown.length > 0) {
|
|
toShutdown = toShutdown.filter(arangod => {
|
|
if (!arangod.hasOwnProperty('exitStatus')) {
|
|
if ((nonAgenciesCount > 0) && (arangod.role === 'agent')) {
|
|
return true;
|
|
}
|
|
shutdownArangod(arangod, options, forceTerminate);
|
|
if (forceTerminate) {
|
|
print(Date() + " FORCED shut down: " + JSON.stringify(arangod));
|
|
} else {
|
|
arangod.exitStatus = {
|
|
status: 'RUNNING'
|
|
};
|
|
print(Date() + " Commanded shut down: " + JSON.stringify(arangod));
|
|
}
|
|
return true;
|
|
}
|
|
if (arangod.exitStatus.status === 'RUNNING') {
|
|
arangod.exitStatus = statusExternal(arangod.pid, false);
|
|
crashUtils.checkMonitorAlive(ARANGOD_BIN, arangod, options, arangod.exitStatus);
|
|
}
|
|
if (arangod.exitStatus.status === 'RUNNING') {
|
|
let localTimeout = timeout;
|
|
if (arangod.role === 'agent') {
|
|
localTimeout = localTimeout + 60;
|
|
}
|
|
if ((internal.time() - shutdownTime) > localTimeout) {
|
|
dumpAgency(instanceInfo, options);
|
|
print(Date() + ' forcefully terminating ' + yaml.safeDump(arangod) +
|
|
' after ' + timeout + 's grace period; marking crashy.');
|
|
serverCrashedLocal = true;
|
|
shutdownSuccess = false;
|
|
arangod.exitStatus = killExternal(arangod.pid, abortSignal);
|
|
analyzeServerCrash(arangod,
|
|
options,
|
|
'shutdown timeout; instance "' +
|
|
arangod.role +
|
|
'" forcefully KILLED after 60s - ' +
|
|
arangod.exitStatus.signal);
|
|
if (arangod.role !== 'agent') {
|
|
nonAgenciesCount --;
|
|
}
|
|
return false;
|
|
} else {
|
|
return true;
|
|
}
|
|
} else if (arangod.exitStatus.status !== 'TERMINATED') {
|
|
if (arangod.role !== 'agent') {
|
|
nonAgenciesCount --;
|
|
}
|
|
if (arangod.exitStatus.hasOwnProperty('signal') || arangod.exitStatus.hasOwnProperty('monitor')) {
|
|
analyzeServerCrash(arangod, options, 'instance "' + arangod.role + '" Shutdown - ' + arangod.exitStatus.signal);
|
|
print(Date() + " shutdownInstance: Marking crashy - " + JSON.stringify(arangod));
|
|
serverCrashedLocal = true;
|
|
shutdownSuccess = false;
|
|
}
|
|
stopProcdump(options, arangod);
|
|
} else {
|
|
if (arangod.role !== 'agent') {
|
|
nonAgenciesCount --;
|
|
}
|
|
print(Date() + ' Server "' + arangod.role + '" shutdown: Success: pid', arangod.pid);
|
|
stopProcdump(options, arangod);
|
|
return false;
|
|
}
|
|
});
|
|
if (toShutdown.length > 0) {
|
|
let roles = {};
|
|
toShutdown.forEach(arangod => {
|
|
if (!roles.hasOwnProperty(arangod.role)) {
|
|
roles[arangod.role] = 0;
|
|
}
|
|
++roles[arangod.role];
|
|
});
|
|
let roleNames = [];
|
|
for (let r in roles) {
|
|
// e.g. 2 + coordinator + (s)
|
|
roleNames.push(roles[r] + ' ' + r + '(s)');
|
|
}
|
|
print(roleNames.join(', ') + ' are still running...');
|
|
require('internal').wait(1, false);
|
|
}
|
|
}
|
|
|
|
if (!options.skipLogAnalysis) {
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
let errorEntries = readImportantLogLines(arangod.rootDir);
|
|
if (Object.keys(errorEntries).length > 0) {
|
|
print(Date() + ' Found messages in the server logs: \n' +
|
|
yaml.safeDump(errorEntries));
|
|
}
|
|
});
|
|
}
|
|
|
|
cleanupDirectories.unshift(instanceInfo.rootDir);
|
|
return shutdownSuccess;
|
|
}
|
|
|
|
function detectCurrentLeader(instanceInfo) {
|
|
let opts = {
|
|
method: 'POST',
|
|
jwt: crypto.jwtEncode(instanceInfo.authOpts['server.jwt-secret'], {'server_id': 'none', 'iss': 'arangodb'}, 'HS256'),
|
|
headers: {'content-type': 'application/json' }
|
|
};
|
|
let reply = download(instanceInfo.agencyUrl + '/_api/agency/read', '[["/arango/Plan/AsyncReplication/Leader"]]', opts);
|
|
|
|
if (!reply.error && reply.code === 200) {
|
|
let res = JSON.parse(reply.body);
|
|
//internal.print("Response ====> " + reply.body);
|
|
let leader = res[0].arango.Plan.AsyncReplication.Leader;
|
|
if (!leader) {
|
|
throw "Leader is not selected";
|
|
}
|
|
}
|
|
|
|
opts['method'] = 'GET';
|
|
reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', opts);
|
|
let res;
|
|
try {
|
|
res = JSON.parse(reply.body);
|
|
}
|
|
catch (x) {
|
|
throw "Failed to parse endpoints reply: " + JSON.stringify(reply);
|
|
}
|
|
let leader = res.endpoints[0].endpoint;
|
|
let leaderInstance;
|
|
instanceInfo.arangods.forEach(d => {
|
|
if (d.endpoint === leader) {
|
|
leaderInstance = d;
|
|
}
|
|
});
|
|
return leaderInstance;
|
|
}
|
|
|
|
function checkClusterAlive(options, instanceInfo, addArgs) {
|
|
// disabled because not in use (jslint)
|
|
// let coordinatorUrl = instanceInfo.url
|
|
// let response
|
|
let httpOptions = makeAuthorizationHeaders(options);
|
|
httpOptions.method = 'POST';
|
|
httpOptions.returnBodyOnError = true;
|
|
|
|
// scrape the jwt token
|
|
instanceInfo.authOpts = _.clone(options);
|
|
if (addArgs['server.jwt-secret'] && !instanceInfo.authOpts['server.jwt-secret']) {
|
|
instanceInfo.authOpts['server.jwt-secret'] = addArgs['server.jwt-secret'];
|
|
}
|
|
|
|
let count = 0;
|
|
while (true) {
|
|
++count;
|
|
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
const reply = download(arangod.url + '/_api/version', '', makeAuthorizationHeaders(instanceInfo.authOpts));
|
|
if (!reply.error && reply.code === 200) {
|
|
arangod.upAndRunning = true;
|
|
return true;
|
|
}
|
|
|
|
if (!checkArangoAlive(arangod, options)) {
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
if (!arangod.hasOwnProperty('exitStatus') ||
|
|
(arangod.exitStatus.status === 'RUNNING')) {
|
|
arangod.exitStatus = killExternal(arangod.pid, abortSignal);
|
|
}
|
|
analyzeServerCrash(arangod, options, 'startup timeout; forcefully terminating ' + arangod.role + ' with pid: ' + arangod.pid);
|
|
});
|
|
|
|
throw new Error(`cluster startup: pid ${arangod.pid} no longer alive! bailing out!`);
|
|
}
|
|
wait(0.5, false);
|
|
return true;
|
|
});
|
|
|
|
let upAndRunning = 0;
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
if (arangod.upAndRunning) {
|
|
upAndRunning += 1;
|
|
}
|
|
});
|
|
if (upAndRunning === instanceInfo.arangods.length) {
|
|
break;
|
|
}
|
|
|
|
// Didn't startup in 10 minutes? kill it, give up.
|
|
if (count > 1200) {
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
arangod.exitStatus = killExternal(arangod.pid, abortSignal);
|
|
analyzeServerCrash(arangod, options, 'startup timeout; forcefully terminating ' + arangod.role + ' with pid: ' + arangod.pid);
|
|
});
|
|
throw new Error('cluster startup timed out after 10 minutes!');
|
|
}
|
|
}
|
|
}
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief starts an instance
|
|
// /
|
|
// / protocol must be one of ["tcp", "ssl", "unix"]
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function startInstanceCluster (instanceInfo, protocol, options,
|
|
addArgs, rootDir) {
|
|
if (options.cluster && options.activefailover ||
|
|
!options.cluster && !options.activefailover) {
|
|
throw "invalid call to startInstanceCluster";
|
|
}
|
|
|
|
let makeArgs = function (name, role, args) {
|
|
args = args || {};
|
|
|
|
let subDir = fs.join(rootDir, name);
|
|
fs.makeDirectoryRecursive(subDir);
|
|
|
|
let subArgs = makeArgsArangod(options, fs.join(subDir, 'apps'), role, fs.join(subDir, 'tmp'));
|
|
// FIXME: someone should decide on the order of preferences
|
|
subArgs = Object.assign(subArgs, addArgs);
|
|
subArgs = Object.assign(subArgs, args);
|
|
|
|
return [subArgs, subDir];
|
|
};
|
|
|
|
options.agency = true;
|
|
options.agencyWaitForSync = false;
|
|
let usedPorts = [];
|
|
options.usedPorts = usedPorts;
|
|
startInstanceAgency(instanceInfo, protocol, options, ...makeArgs('agency', 'agency', {}));
|
|
|
|
let agencyEndpoint = instanceInfo.endpoint;
|
|
instanceInfo.agencyUrl = instanceInfo.url;
|
|
if (!checkInstanceAlive(instanceInfo, options)) {
|
|
throw new Error('startup of agency failed! bailing out!');
|
|
}
|
|
|
|
let i;
|
|
if (options.cluster) {
|
|
for (i = 0; i < options.dbServers; i++) {
|
|
let port = findFreePort(options.minPort, options.maxPort, usedPorts);
|
|
usedPorts.push(port);
|
|
let endpoint = protocol + '://127.0.0.1:' + port;
|
|
let primaryArgs = _.clone(options.extraArgs);
|
|
primaryArgs['server.endpoint'] = endpoint;
|
|
primaryArgs['cluster.my-address'] = endpoint;
|
|
primaryArgs['cluster.my-role'] = 'PRIMARY';
|
|
primaryArgs['cluster.agency-endpoint'] = agencyEndpoint;
|
|
primaryArgs['javascript.enabled'] = 'false';
|
|
|
|
startInstanceSingleServer(instanceInfo, protocol, options, ...makeArgs('dbserver' + i, 'dbserver', primaryArgs), 'dbserver');
|
|
}
|
|
|
|
for (i = 0; i < options.coordinators; i++) {
|
|
let port = findFreePort(options.minPort, options.maxPort, usedPorts);
|
|
usedPorts.push(port);
|
|
let endpoint = protocol + '://127.0.0.1:' + port;
|
|
let coordinatorArgs = _.clone(options.extraArgs);
|
|
coordinatorArgs['server.endpoint'] = endpoint;
|
|
coordinatorArgs['cluster.my-address'] = endpoint;
|
|
coordinatorArgs['cluster.my-role'] = 'COORDINATOR';
|
|
coordinatorArgs['cluster.agency-endpoint'] = agencyEndpoint;
|
|
|
|
startInstanceSingleServer(instanceInfo, protocol, options, ...makeArgs('coordinator' + i, 'coordinator', coordinatorArgs), 'coordinator');
|
|
}
|
|
} else if (options.activefailover) {
|
|
for (i = 0; i < options.singles; i++) {
|
|
let port = findFreePort(options.minPort, options.maxPort, usedPorts);
|
|
usedPorts.push(port);
|
|
let endpoint = protocol + '://127.0.0.1:' + port;
|
|
let singleArgs = _.clone(options.extraArgs);
|
|
singleArgs['server.endpoint'] = endpoint;
|
|
singleArgs['cluster.my-address'] = endpoint;
|
|
singleArgs['cluster.my-role'] = 'SINGLE';
|
|
singleArgs['cluster.agency-endpoint'] = agencyEndpoint;
|
|
singleArgs['replication.active-failover'] = true;
|
|
startInstanceSingleServer(instanceInfo, protocol, options, ...makeArgs('single' + i, 'single', singleArgs), 'single');
|
|
sleep(1.0);
|
|
}
|
|
}
|
|
checkClusterAlive(options, instanceInfo, addArgs);
|
|
|
|
// we need to find the leading server
|
|
if (options.activefailover) {
|
|
internal.wait(5.0, false);
|
|
let d = detectCurrentLeader(instanceInfo);
|
|
if (d === undefined) {
|
|
throw "failed to detect a leader";
|
|
}
|
|
instanceInfo.endpoint = d.endpoint;
|
|
instanceInfo.url = d.url;
|
|
}
|
|
|
|
arango.reconnect(instanceInfo.endpoint, '_system', 'root', '');
|
|
return true;
|
|
}
|
|
|
|
function launchFinalize(options, instanceInfo, startTime) {
|
|
if (!options.cluster) {
|
|
let count = 0;
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
while (true) {
|
|
wait(0.5, false);
|
|
if (options.useReconnect) {
|
|
try {
|
|
arango.reconnect(instanceInfo.endpoint,
|
|
'_system',
|
|
options.username,
|
|
options.password,
|
|
count > 50
|
|
);
|
|
break;
|
|
} catch (e) {
|
|
}
|
|
} else {
|
|
const reply = download(arangod.url + '/_api/version', '', makeAuthorizationHeaders(options));
|
|
|
|
if (!reply.error && reply.code === 200) {
|
|
break;
|
|
}
|
|
}
|
|
++count;
|
|
|
|
if (count % 60 === 0) {
|
|
if (!checkArangoAlive(arangod, options)) {
|
|
throw new Error('startup failed! bailing out!');
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
print(CYAN + Date() + ' up and running in ' + (time() - startTime) + ' seconds' + RESET);
|
|
var matchPort = /.*:.*:([0-9]*)/;
|
|
var ports = [];
|
|
var processInfo = [];
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
let res = matchPort.exec(arangod.endpoint);
|
|
if (!res) {
|
|
return;
|
|
}
|
|
var port = res[1];
|
|
ports.push('port ' + port);
|
|
processInfo.push(' [' + arangod.role + '] up with pid ' + arangod.pid + ' on port ' + port);
|
|
});
|
|
|
|
print(Date() + ' sniffing template:\n tcpdump -ni lo -s0 -w /tmp/out.pcap ' + ports.join(' or ') + '\n');
|
|
print(processInfo.join('\n') + '\n');
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief starts an instance
|
|
// /
|
|
// / protocol must be one of ["tcp", "ssl", "unix"]
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function startArango (protocol, options, addArgs, rootDir, role) {
|
|
const dataDir = fs.join(rootDir, 'data');
|
|
const appDir = fs.join(rootDir, 'apps');
|
|
const tmpDir = fs.join(rootDir, 'tmp');
|
|
|
|
fs.makeDirectoryRecursive(dataDir);
|
|
fs.makeDirectoryRecursive(appDir);
|
|
fs.makeDirectoryRecursive(tmpDir);
|
|
|
|
let args = makeArgsArangod(options, appDir, role, tmpDir);
|
|
let endpoint;
|
|
let port;
|
|
|
|
if (!addArgs['server.endpoint']) {
|
|
port = findFreePort(options.minPort, options.maxPort);
|
|
endpoint = protocol + '://127.0.0.1:' + port;
|
|
} else {
|
|
endpoint = addArgs['server.endpoint'];
|
|
port = endpoint.split(':').pop();
|
|
}
|
|
|
|
let instanceInfo = {
|
|
role,
|
|
port,
|
|
endpoint,
|
|
rootDir
|
|
};
|
|
|
|
args['server.endpoint'] = endpoint;
|
|
args['database.directory'] = dataDir;
|
|
args['log.file'] = fs.join(rootDir, 'log');
|
|
if (options.auditLoggingEnabled) {
|
|
args['audit.output'] = 'file://' + fs.join(rootDir, 'audit.log');
|
|
args['server.statistics'] = false;
|
|
args['foxx.queues'] = false;
|
|
}
|
|
|
|
if (protocol === 'ssl') {
|
|
args['ssl.keyfile'] = fs.join('UnitTests', 'server.pem');
|
|
}
|
|
|
|
args = Object.assign(args, options.extraArgs);
|
|
|
|
if (addArgs !== undefined) {
|
|
args = Object.assign(args, addArgs);
|
|
}
|
|
|
|
if (options.verbose) {
|
|
args['log.level'] = 'debug';
|
|
}
|
|
|
|
instanceInfo.url = endpointToURL(instanceInfo.endpoint);
|
|
instanceInfo.args = args;
|
|
try {
|
|
instanceInfo.pid = executeArangod(ARANGOD_BIN, toArgv(args), options).pid;
|
|
} catch (x) {
|
|
print(Date() + ' failed to run arangod - ' + JSON.stringify(x));
|
|
|
|
throw x;
|
|
}
|
|
instanceInfo.role = role;
|
|
|
|
if (platform.substr(0, 3) === 'win' && !options.disableMonitor) {
|
|
runProcdump(options, instanceInfo, rootDir, instanceInfo.pid);
|
|
}
|
|
return instanceInfo;
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief starts an agency instance
|
|
// /
|
|
// / protocol must be one of ["tcp", "ssl", "unix"]
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir) {
|
|
const dataDir = fs.join(rootDir, 'data');
|
|
|
|
const N = options.agencySize;
|
|
const S = options.agencySupervision;
|
|
if (options.agencyWaitForSync === undefined) {
|
|
options.agencyWaitForSync = false;
|
|
}
|
|
const wfs = options.agencyWaitForSync;
|
|
|
|
let usedPorts = options.usedPorts || [];
|
|
for (let i = 0; i < N; i++) {
|
|
let instanceArgs = _.clone(addArgs);
|
|
instanceArgs['log.file'] = fs.join(rootDir, 'log' + String(i));
|
|
instanceArgs['javascript.enabled'] = 'false';
|
|
instanceArgs['agency.activate'] = 'true';
|
|
instanceArgs['agency.size'] = String(N);
|
|
instanceArgs['agency.pool-size'] = String(N);
|
|
instanceArgs['agency.wait-for-sync'] = String(wfs);
|
|
instanceArgs['agency.supervision'] = String(S);
|
|
instanceArgs['database.directory'] = dataDir + String(i);
|
|
const port = findFreePort(options.minPort, options.maxPort, usedPorts);
|
|
usedPorts.push(port);
|
|
instanceArgs['server.endpoint'] = protocol + '://127.0.0.1:' + port;
|
|
instanceArgs['agency.my-address'] = protocol + '://127.0.0.1:' + port;
|
|
instanceArgs['agency.supervision-grace-period'] = '10.0';
|
|
instanceArgs['agency.supervision-frequency'] = '1.0';
|
|
|
|
if (i === N - 1) {
|
|
let l = [];
|
|
instanceInfo.arangods.forEach(arangod => {
|
|
l.push('--agency.endpoint');
|
|
l.push(arangod.endpoint);
|
|
});
|
|
l.push('--agency.endpoint');
|
|
l.push(protocol + '://127.0.0.1:' + port);
|
|
|
|
instanceArgs['flatCommands'] = l;
|
|
}
|
|
let dir = fs.join(rootDir, 'agency-' + i);
|
|
fs.makeDirectoryRecursive(dir);
|
|
fs.makeDirectoryRecursive(instanceArgs['database.directory']);
|
|
instanceInfo.arangods.push(startArango(protocol, options, instanceArgs, rootDir, 'agent'));
|
|
instanceInfo.endpoint = instanceInfo.arangods[instanceInfo.arangods.length - 1].endpoint;
|
|
instanceInfo.url = instanceInfo.arangods[instanceInfo.arangods.length - 1].url;
|
|
instanceInfo.role = 'agent';
|
|
print(Date() + ' Agency Endpoint: ' + instanceInfo.endpoint);
|
|
}
|
|
return instanceInfo;
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief starts a single server instance
|
|
// /
|
|
// / protocol must be one of ["tcp", "ssl", "unix"]
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function startInstanceSingleServer (instanceInfo, protocol, options,
|
|
addArgs, rootDir, role) {
|
|
instanceInfo.arangods.push(startArango(protocol, options, addArgs, rootDir, role));
|
|
|
|
instanceInfo.endpoint = instanceInfo.arangods[instanceInfo.arangods.length - 1].endpoint;
|
|
instanceInfo.url = instanceInfo.arangods[instanceInfo.arangods.length - 1].url;
|
|
|
|
return instanceInfo;
|
|
}
|
|
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
// / @brief starts any sort server instance
|
|
// /
|
|
// / protocol must be one of ["tcp", "ssl", "unix"]
|
|
// //////////////////////////////////////////////////////////////////////////////
|
|
|
|
function startInstance (protocol, options, addArgs, testname, tmpDir) {
|
|
let rootDir = fs.join(tmpDir || fs.getTempPath(), testname);
|
|
|
|
let instanceInfo = {
|
|
rootDir,
|
|
arangods: [],
|
|
protocol: protocol
|
|
};
|
|
|
|
const startTime = time();
|
|
try {
|
|
if (options.hasOwnProperty('server')) {
|
|
let rc = {
|
|
endpoint: options.server,
|
|
rootDir: options.serverRoot,
|
|
url: options.server.replace('tcp', 'http'),
|
|
arangods: []
|
|
};
|
|
arango.reconnect(rc.endpoint, '_system', 'root', '');
|
|
return rc;
|
|
} else if (options.cluster || options.activefailover) {
|
|
startInstanceCluster(instanceInfo, protocol, options,
|
|
addArgs, rootDir);
|
|
} else if (options.agency) {
|
|
startInstanceAgency(instanceInfo, protocol, options,
|
|
addArgs, rootDir);
|
|
} else {
|
|
startInstanceSingleServer(instanceInfo, protocol, options,
|
|
addArgs, rootDir, 'single');
|
|
}
|
|
|
|
launchFinalize(options, instanceInfo, startTime);
|
|
} catch (e) {
|
|
print(e, e.stack);
|
|
return false;
|
|
}
|
|
return instanceInfo;
|
|
}
|
|
|
|
function reStartInstance(options, instanceInfo, moreArgs) {
|
|
let launchInstance = function(options, oneInstanceInfo) {
|
|
try {
|
|
Object.assign(oneInstanceInfo.args, moreArgs);
|
|
oneInstanceInfo.pid = executeArangod(ARANGOD_BIN, toArgv(oneInstanceInfo.args), options).pid;
|
|
} catch (x) {
|
|
print(Date() + ' failed to run arangod - ' + JSON.stringify(x));
|
|
|
|
throw x;
|
|
}
|
|
if (platform.substr(0, 3) === 'win' && !options.disableMonitor) {
|
|
runProcdump(options, oneInstanceInfo, oneInstanceInfo.rootDir, oneInstanceInfo.pid);
|
|
}
|
|
};
|
|
|
|
const startTime = time();
|
|
|
|
instanceInfo.arangods.forEach(function (oneInstance, i) {
|
|
delete(oneInstance.exitStatus);
|
|
delete(oneInstance.pid);
|
|
oneInstance.upAndRunning = false;
|
|
});
|
|
|
|
if (options.cluster) {
|
|
let agencyInstance = {arangods: []};
|
|
instanceInfo.arangods.forEach(function (oneInstance, i) {
|
|
if (oneInstance.role === 'agent') {
|
|
print("relaunching: " + JSON.stringify(oneInstance));
|
|
launchInstance(options, oneInstance);
|
|
agencyInstance.arangods.push(_.clone(oneInstance));
|
|
}
|
|
});
|
|
let agencyEndpoint = instanceInfo.endpoint;
|
|
if (!checkInstanceAlive(agencyInstance, options)) {
|
|
throw new Error('startup of agency failed! bailing out!');
|
|
}
|
|
}
|
|
|
|
instanceInfo.arangods.forEach(function (oneInstance, i) {
|
|
if ((oneInstance.role === 'PRIMARY') ||
|
|
(oneInstance.role === 'primary') ||
|
|
(oneInstance.role === 'dbserver')) {
|
|
print("relaunching: " + JSON.stringify(oneInstance));
|
|
launchInstance(options, oneInstance);
|
|
}
|
|
});
|
|
instanceInfo.arangods.forEach(function (oneInstance, i) {
|
|
if ((oneInstance.role === 'COORDINATOR') || (oneInstance.role === 'coordinator')) {
|
|
print("relaunching: " + JSON.stringify(oneInstance));
|
|
launchInstance(options, oneInstance);
|
|
}
|
|
});
|
|
instanceInfo.arangods.forEach(function (oneInstance, i) {
|
|
if (oneInstance.role === 'single') {
|
|
launchInstance(options, oneInstance);
|
|
}
|
|
});
|
|
|
|
if (options.cluster) {
|
|
checkClusterAlive(options, instanceInfo, {}); // todo addArgs
|
|
arango.reconnect(instanceInfo.endpoint,
|
|
'_system',
|
|
options.username,
|
|
options.password,
|
|
false
|
|
);
|
|
}
|
|
|
|
launchFinalize(options, instanceInfo, startTime);
|
|
}
|
|
|
|
// exports.analyzeServerCrash = analyzeServerCrash;
|
|
exports.makeArgs = {
|
|
arangod: makeArgsArangod,
|
|
arangosh: makeArgsArangosh
|
|
};
|
|
|
|
exports.arangod = {
|
|
check: {
|
|
alive: checkArangoAlive,
|
|
instanceAlive: checkInstanceAlive
|
|
},
|
|
shutdown: shutdownArangod
|
|
};
|
|
|
|
exports.findFreePort = findFreePort;
|
|
|
|
exports.executeArangod = executeArangod;
|
|
exports.executeAndWait = executeAndWait;
|
|
exports.stopProcdump = stopProcdump;
|
|
|
|
exports.createBaseConfig = createBaseConfigBuilder;
|
|
exports.run = {
|
|
arangoshCmd: runArangoshCmd,
|
|
arangoImport: runArangoImport,
|
|
arangoDumpRestore: runArangoDumpRestore,
|
|
arangoDumpRestoreWithConfig: runArangoDumpRestoreCfg,
|
|
arangoBenchmark: runArangoBenchmark
|
|
};
|
|
|
|
exports.shutdownInstance = shutdownInstance;
|
|
exports.startArango = startArango;
|
|
exports.startInstance = startInstance;
|
|
exports.reStartInstance = reStartInstance;
|
|
exports.setupBinaries = setupBinaries;
|
|
exports.executableExt = executableExt;
|
|
exports.serverCrashed = serverCrashedLocal;
|
|
exports.serverFailMessages = serverFailMessagesLocal;
|
|
|
|
exports.cleanupDBDirectoriesAppend = cleanupDBDirectoriesAppend;
|
|
exports.cleanupDBDirectories = cleanupDBDirectories;
|
|
exports.cleanupLastDirectory = cleanupLastDirectory;
|
|
exports.getCleanupDBDirectories = getCleanupDBDirectories;
|
|
|
|
exports.makeAuthorizationHeaders = makeAuthorizationHeaders;
|
|
|
|
Object.defineProperty(exports, 'ARANGOEXPORT_BIN', {get: () => ARANGOEXPORT_BIN});
|
|
Object.defineProperty(exports, 'ARANGOD_BIN', {get: () => ARANGOD_BIN});
|
|
Object.defineProperty(exports, 'ARANGOSH_BIN', {get: () => ARANGOSH_BIN});
|
|
Object.defineProperty(exports, 'CONFIG_DIR', {get: () => CONFIG_DIR});
|
|
Object.defineProperty(exports, 'TOP_DIR', {get: () => TOP_DIR});
|
|
Object.defineProperty(exports, 'LOGS_DIR', {get: () => LOGS_DIR});
|
|
Object.defineProperty(exports, 'UNITTESTS_DIR', {get: () => UNITTESTS_DIR});
|
|
Object.defineProperty(exports, 'BIN_DIR', {get: () => BIN_DIR});
|
|
Object.defineProperty(exports, 'CONFIG_ARANGODB_DIR', {get: () => CONFIG_ARANGODB_DIR});
|
|
Object.defineProperty(exports, 'CONFIG_RELATIVE_DIR', {get: () => CONFIG_RELATIVE_DIR});
|
|
Object.defineProperty(exports, 'serverCrashed', {get: () => serverCrashedLocal, set: (value) => { serverCrashedLocal = value; } });
|
|
Object.defineProperty(exports, 'serverFailMessages', {get: () => serverFailMessagesLocal, set: (value) => { serverFailMessagesLocal = value; }});
|