mirror of https://gitee.com/bigwinds/arangodb
1500 lines
47 KiB
JavaScript
1500 lines
47 KiB
JavaScript
/*jshint strict: false, unused: false */
|
|
/*global ArangoServerState */
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief Cluster kickstarting functionality using dispatchers
|
|
///
|
|
/// @file
|
|
///
|
|
/// DISCLAIMER
|
|
///
|
|
/// Copyright 2014 triagens GmbH, Cologne, Germany
|
|
///
|
|
/// Licensed under the Apache License, Version 2.0 (the "License");
|
|
/// you may not use this file except in compliance with the License.
|
|
/// You may obtain a copy of the License at
|
|
///
|
|
/// http://www.apache.org/licenses/LICENSE-2.0
|
|
///
|
|
/// Unless required by applicable law or agreed to in writing, software
|
|
/// distributed under the License is distributed on an "AS IS" BASIS,
|
|
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
/// See the License for the specific language governing permissions and
|
|
/// limitations under the License.
|
|
///
|
|
/// Copyright holder is triAGENS GmbH, Cologne, Germany
|
|
///
|
|
/// @author Max Neunhoeffer
|
|
/// @author Copyright 2014, triAGENS GmbH, Cologne, Germany
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
|
|
var download = require("internal").download;
|
|
var executeExternal = require("internal").executeExternal;
|
|
var killExternal = require("internal").killExternal;
|
|
var statusExternal = require("internal").statusExternal;
|
|
var base64Encode = require("internal").base64Encode;
|
|
|
|
var fs = require("fs");
|
|
var wait = require("internal").wait;
|
|
var toArgv = require("internal").toArgv;
|
|
|
|
var exchangePort = require("@arangodb/cluster/planner").exchangePort;
|
|
var exchangeProtocol = require("@arangodb/cluster/planner").exchangeProtocol;
|
|
|
|
var console = require("console");
|
|
|
|
var launchActions = {};
|
|
var shutdownActions = {};
|
|
var cleanupActions = {};
|
|
var isHealthyActions = {};
|
|
var upgradeActions = {};
|
|
|
|
var getAddrPort = require("@arangodb/cluster/planner").getAddrPort;
|
|
var getPort = require("@arangodb/cluster/planner").getPort;
|
|
var endpointToURL = require("@arangodb/cluster/planner").endpointToURL;
|
|
|
|
function extractErrorMessage (result) {
|
|
if (result === undefined) {
|
|
return "unknown error";
|
|
}
|
|
|
|
if (result.hasOwnProperty("body") && result.body !== undefined) {
|
|
try {
|
|
var e = JSON.parse(result.body);
|
|
|
|
if (e.hasOwnProperty("errorMessage")) {
|
|
return e.errorMessage;
|
|
}
|
|
|
|
if (e.hasOwnProperty("errorNum")) {
|
|
return e.errorNum;
|
|
}
|
|
|
|
if (result.body !== "") {
|
|
return result.body;
|
|
}
|
|
}
|
|
catch (err) {
|
|
if (result.body !== "") {
|
|
return result.body;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (result.hasOwnProperty("message")) {
|
|
return result.message;
|
|
}
|
|
|
|
if (result.hasOwnProperty("code")) {
|
|
return String(result.code);
|
|
}
|
|
|
|
return "unknown error";
|
|
}
|
|
|
|
function makePath (path) {
|
|
return fs.join.apply(null,path.split("/"));
|
|
}
|
|
|
|
function encode (st) {
|
|
var st2 = "";
|
|
var i;
|
|
for (i = 0; i < st.length; i++) {
|
|
if (st[i] === "_") {
|
|
st2 += "@U";
|
|
}
|
|
else if (st[i] === "@") {
|
|
st2 += "@@";
|
|
}
|
|
else {
|
|
st2 += st[i];
|
|
}
|
|
}
|
|
return encodeURIComponent(st2);
|
|
}
|
|
|
|
function getAuthorizationHeader (username, passwd) {
|
|
return "Basic " + base64Encode(username + ":" + passwd);
|
|
}
|
|
|
|
function getAuthorization (dispatcher) {
|
|
return getAuthorizationHeader(dispatcher.username, dispatcher.passwd);
|
|
}
|
|
|
|
function waitForServerUp (cmd, endpoint, timeout) {
|
|
var url = endpointToURL(endpoint) + "/_api/version";
|
|
var time = 0;
|
|
while (true) {
|
|
var r = download(url, "", {});
|
|
if (r.code === 200 || r.code === 401) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Could talk to "+endpoint+" .");
|
|
}
|
|
return true;
|
|
}
|
|
if (time >= timeout) {
|
|
console.info("Could not talk to endpoint "+endpoint+", giving up.");
|
|
return false;
|
|
}
|
|
wait(0.5);
|
|
time += 0.5;
|
|
}
|
|
}
|
|
|
|
function waitForServerDown (cmd, endpoint, timeout) {
|
|
var url = endpointToURL(endpoint) + "/_api/version";
|
|
var time = 0;
|
|
while (true) {
|
|
var r = download(url, "", {});
|
|
if (r.code === 500 || r.code === 401) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Server at " + endpoint + " does not answer any more.");
|
|
}
|
|
return true;
|
|
}
|
|
if (time >= timeout) {
|
|
console.info("After timeout, server at " + endpoint +
|
|
"is still there, giving up.");
|
|
return false;
|
|
}
|
|
wait(0.5);
|
|
time += 0.5;
|
|
}
|
|
}
|
|
|
|
function sendToAgency (cmd, agencyURL, path, obj) {
|
|
var res;
|
|
var body;
|
|
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Sending %s to agency...", path);
|
|
}
|
|
if (typeof obj === "string") {
|
|
var count = 0;
|
|
while (count++ <= 2) {
|
|
body = "value=" + encodeURIComponent(obj);
|
|
res = download(agencyURL + path,body,
|
|
{"method":"PUT", "followRedirects": true,
|
|
"headers": { "Content-Type": "application/x-www-form-urlencoded"}});
|
|
if (res.code === 201 || res.code === 200) {
|
|
return true;
|
|
}
|
|
wait(3); // wait 3 seconds before trying again
|
|
}
|
|
return res;
|
|
}
|
|
if (typeof obj !== "object") {
|
|
return "Strange object found: not a string or object";
|
|
}
|
|
var keys = Object.keys(obj);
|
|
var i;
|
|
if (keys.length !== 0) {
|
|
for (i = 0; i < keys.length; i++) {
|
|
res = sendToAgency(cmd, agencyURL, path + "/" + encode(keys[i]), obj[keys[i]]);
|
|
if (res !== true) {
|
|
return res;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
// Create a directory
|
|
var count2 = 0;
|
|
while (count2++ <= 2) {
|
|
body = "dir=true";
|
|
res = download(agencyURL+path,body,
|
|
{"method": "PUT", "followRedirects": true,
|
|
"headers": { "Content-Type": "application/x-www-form-urlencoded"}});
|
|
if (res.code === 201 || res.code === 200) {
|
|
return true;
|
|
}
|
|
wait(3); // wait 3 seconds before trying again
|
|
}
|
|
return res;
|
|
}
|
|
|
|
launchActions.startAgent = function (dispatchers, cmd, isRelaunch) {
|
|
var yaml = require("js-yaml");
|
|
console.info("Starting agent...");
|
|
|
|
var dataPath = fs.makeAbsolute(cmd.dataPath);
|
|
if (dataPath !== cmd.dataPath) { // path was relative
|
|
dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
|
|
}
|
|
|
|
var agentDataDir = fs.join(dataPath, "agent"+cmd.agencyPrefix+cmd.extPort);
|
|
if (!isRelaunch) {
|
|
if (fs.exists(agentDataDir)) {
|
|
fs.removeDirectoryRecursive(agentDataDir,true);
|
|
}
|
|
}
|
|
|
|
var instanceName = "agent"+cmd.agencyPrefix+cmd.extPort;
|
|
var extEndpoint = getAddrPort(exchangePort(dispatchers[cmd.dispatcher].endpoint, cmd.extPort));
|
|
var extBind = cmd.onlyLocalhost ? ("127.0.0.1:"+cmd.extPort) : extEndpoint;
|
|
var clusterEndPoint = getAddrPort(exchangePort(dispatchers[cmd.dispatcher].endpoint, cmd.intPort));
|
|
var clusterBind = cmd.onlyLocalhost ? ("127.0.0.1:"+cmd.intPort) : clusterEndPoint;
|
|
|
|
var clusterUrl = "http://" + clusterEndPoint;
|
|
var agencyUrl = "http://" + extBind;
|
|
if (require("internal").platform.substr(0,3) === 'win') {
|
|
agentDataDir = agentDataDir.split("\\").join("/");
|
|
}
|
|
|
|
var args = {
|
|
"data-dir": agentDataDir,
|
|
"name": instanceName,
|
|
"bind-addr": extBind,
|
|
"addr": extEndpoint,
|
|
"peer-bind-addr": clusterBind,
|
|
"peer-addr": clusterEndPoint,
|
|
"initial-cluster-state": "new",
|
|
"initial-cluster": instanceName + "=" + clusterUrl
|
|
// the following might speed up etcd, but it might also
|
|
// make it more unstable:
|
|
// ,"peer-heartbeat-timeout": "10",
|
|
// "peer-election-timeout": "20"
|
|
};
|
|
var i;
|
|
if (cmd.peers.length > 0) {
|
|
var st = getAddrPort(cmd.peers[0]);
|
|
for (i = 1; i < cmd.peers.length; i++) {
|
|
st = st + "," + getAddrPort(cmd.peers[i]);
|
|
}
|
|
args.peers = st;
|
|
}
|
|
var agentPath = cmd.agentPath;
|
|
if (agentPath === "") {
|
|
agentPath = ArangoServerState.agentPath();
|
|
}
|
|
if (! fs.exists(agentPath)) {
|
|
return {"error":true, "isStartAgent": true,
|
|
"errorMessage": "agency binary not found at '" + agentPath + "'"};
|
|
}
|
|
|
|
var pid = executeExternal(agentPath, toArgv(args));
|
|
var res;
|
|
var count = 0;
|
|
while (++count < 20) {
|
|
wait(0.5); // Wait a bit to give it time to startup
|
|
res = download(agencyUrl + "/v2/stats/self");
|
|
if (res.code === 200) {
|
|
return {"error":false, "isStartAgent": true, "pid": pid,
|
|
"endpoint": "tcp://"+extEndpoint};
|
|
}
|
|
res = statusExternal(pid, false);
|
|
if (res.status !== "RUNNING") {
|
|
return {"error":true, "isStartAgent": true,
|
|
"errorMessage": "agency failed to start:\n" + yaml.safeDump({"exit status": res})};
|
|
}
|
|
}
|
|
killExternal(pid);
|
|
return {"error":true, "isStartAgent": true,
|
|
"errorMessage": "agency did not come alive on time, giving up."};
|
|
};
|
|
|
|
launchActions.sendConfiguration = function (dispatchers, cmd, isRelaunch) {
|
|
if (isRelaunch) {
|
|
// nothing to do here
|
|
console.info("Waiting 1 second for agency to come alive...");
|
|
wait(1);
|
|
return {"error":false, "isSendConfiguration": true};
|
|
}
|
|
var url = endpointToURL(cmd.agency.endpoints[0]) + "/v2/keys";
|
|
var res = sendToAgency(cmd, url, "", cmd.data);
|
|
if (res === true) {
|
|
return {"error":false, "isSendConfiguration": true};
|
|
}
|
|
var yaml = require("js-yaml");
|
|
return {"error":true, "isSendConfiguration": true, "suberror": res, errorMessage : yaml.safeDump(res)};
|
|
};
|
|
|
|
launchActions.startServers = function (dispatchers, cmd, isRelaunch) {
|
|
var dataPath = fs.makeAbsolute(cmd.dataPath);
|
|
if (dataPath !== cmd.dataPath) { // path was relative
|
|
dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
|
|
}
|
|
var logPath = fs.makeAbsolute(cmd.logPath);
|
|
if (logPath !== cmd.logPath) { // path was relative
|
|
logPath = fs.normalize(fs.join(ArangoServerState.logPath(), cmd.logPath));
|
|
}
|
|
|
|
var url = endpointToURL(cmd.agency.endpoints[0]) + "/v2/keys/"+
|
|
cmd.agency.agencyPrefix + "/";
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Downloading %sLaunchers/%s", url, encode(cmd.name));
|
|
}
|
|
var res = download(url + "Dispatcher/Launchers/" + encode(cmd.name), "", { method: "GET",
|
|
followRedirects: true });
|
|
if (res.code !== 200) {
|
|
return {"error": true, "isStartServers": true, "suberror": res};
|
|
}
|
|
var body = JSON.parse( res.body );
|
|
var info = JSON.parse(body.node.value);
|
|
var id, ep, args, pids, port, endpoints, endpointNames, roles;
|
|
|
|
console.info("Starting servers...");
|
|
var i;
|
|
var servers = info.DBservers.concat(info.Coordinators);
|
|
roles = [];
|
|
for (i = 0; i < info.DBservers.length; i++) {
|
|
roles.push("PRIMARY");
|
|
}
|
|
for (i = 0; i < info.Coordinators.length; i++) {
|
|
roles.push("COORDINATOR");
|
|
}
|
|
pids = [];
|
|
endpoints = [];
|
|
endpointNames = [];
|
|
for (i = 0; i < servers.length; i++) {
|
|
id = servers[i];
|
|
|
|
var serverUrl = url + "Dispatcher/Endpoints/" + encodeURIComponent(id);
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Downloading ", serverUrl);
|
|
}
|
|
res = download(serverUrl);
|
|
if (res.code !== 200) {
|
|
return {"error": true, "pids": pids,
|
|
"isStartServers": true, "suberror": res};
|
|
}
|
|
console.info("Starting server %s",id);
|
|
body = JSON.parse(res.body);
|
|
ep = body.node.value;
|
|
port = getPort(ep);
|
|
var useSSL = false;
|
|
if (roles[i] === "PRIMARY") {
|
|
args = ["--configuration", ArangoServerState.dbserverConfig()];
|
|
useSSL = cmd.useSSLonDBservers;
|
|
}
|
|
else {
|
|
args = ["--configuration", ArangoServerState.coordinatorConfig()];
|
|
useSSL = cmd.useSSLonCoordinators;
|
|
}
|
|
args = args.concat([
|
|
"--cluster.disable-dispatcher-kickstarter", "true",
|
|
"--cluster.disable-dispatcher-frontend", "true",
|
|
"--cluster.my-local-info", id,
|
|
"--cluster.my-role", roles[i],
|
|
"--cluster.my-address", ep,
|
|
"--cluster.agency-prefix", cmd.agency.agencyPrefix,
|
|
"--cluster.agency-endpoint", cmd.agency.endpoints[0],
|
|
"--server.endpoint"]);
|
|
if (cmd.onlyLocalhost) {
|
|
args.push(exchangeProtocol("tcp://127.0.0.1:" + port, useSSL));
|
|
}
|
|
else {
|
|
args.push(exchangeProtocol("tcp://0.0.0.0:" + port, useSSL));
|
|
}
|
|
args.push("--log.file");
|
|
var logfile = fs.join(logPath, "log-" + cmd.agency.agencyPrefix + "-" + id);
|
|
args.push(logfile);
|
|
if (!isRelaunch) {
|
|
if (!fs.exists(logPath)) {
|
|
fs.makeDirectoryRecursive(logPath);
|
|
}
|
|
if (fs.exists(logfile)) {
|
|
fs.remove(logfile);
|
|
}
|
|
}
|
|
var datadir = fs.join(dataPath, "data-" + cmd.agency.agencyPrefix + "-" + id);
|
|
if (!isRelaunch) {
|
|
if (!fs.exists(dataPath)) {
|
|
fs.makeDirectoryRecursive(dataPath);
|
|
}
|
|
if (fs.exists(datadir)) {
|
|
fs.removeDirectoryRecursive(datadir,true);
|
|
}
|
|
fs.makeDirectory(datadir);
|
|
}
|
|
args.push("--database.directory");
|
|
args.push(datadir);
|
|
args = args.concat(dispatchers[cmd.dispatcher].arangodExtraArgs);
|
|
var arangodPath = fs.makeAbsolute(cmd.arangodPath);
|
|
if (arangodPath !== cmd.arangodPath) {
|
|
arangodPath = ArangoServerState.arangodPath();
|
|
}
|
|
if ((cmd.valgrind !== '') &&
|
|
(cmd.valgrindHosts !== undefined) &&
|
|
(cmd.valgrindHosts.indexOf(roles[i]) > -1)) {
|
|
var valgrindOpts = cmd.valgrindOpts.concat(
|
|
["--xml-file=" + cmd.valgrindXmlFileBase + '_' + cmd.valgrindTestname + '_' + id + '.%p.xml',
|
|
"--log-file=" + cmd.valgrindXmlFileBase + '_' + cmd.valgrindTestname + '_' + id + '.%p.valgrind.log']);
|
|
var newargs = valgrindOpts.concat([arangodPath]).concat(args);
|
|
var cmdline = cmd.valgrind;
|
|
pids.push(executeExternal(cmdline, newargs));
|
|
}
|
|
else {
|
|
pids.push(executeExternal(arangodPath, args));
|
|
}
|
|
ep = exchangePort(dispatchers[cmd.dispatcher].endpoint,port);
|
|
ep = exchangeProtocol(ep,useSSL);
|
|
endpoints.push(ep);
|
|
endpointNames.push(id);
|
|
}
|
|
|
|
var error = false;
|
|
for (i = 0;i < endpoints.length;i++) {
|
|
var timeout = 50;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
if (! waitForServerUp(cmd, endpoints[i], timeout)) {
|
|
error = true;
|
|
}
|
|
}
|
|
|
|
return {"error": error,
|
|
"isStartServers": true,
|
|
"pids": pids,
|
|
"endpoints": endpoints,
|
|
"endpointNames": endpointNames,
|
|
"roles": roles};
|
|
};
|
|
|
|
launchActions.bootstrapServers = function (dispatchers, cmd, isRelaunch,
|
|
username, password) {
|
|
var coordinators = cmd.coordinators;
|
|
|
|
if (username === undefined && password === undefined) {
|
|
username = "root";
|
|
password = "";
|
|
}
|
|
|
|
// we need at least one coordinator
|
|
if (0 === coordinators.length) {
|
|
return {"error": true, "bootstrapServers": true, "errorMessage": "No coordinators to start"};
|
|
}
|
|
|
|
// authorization header for coordinator
|
|
var hdrs = {
|
|
Authorization: getAuthorizationHeader(username, password)
|
|
};
|
|
|
|
// default options
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var options = {
|
|
method: "POST",
|
|
timeout: timeout,
|
|
headers: hdrs,
|
|
returnBodyOnError: true
|
|
};
|
|
|
|
// execute bootstrap command on first server
|
|
var retryCount = 0;
|
|
var result;
|
|
var url = coordinators[0] + "/_admin/cluster/bootstrapDbServers";
|
|
var body = '{"isRelaunch": ' + (isRelaunch ? "true" : "false") + '}';
|
|
while (retryCount < timeout) {
|
|
|
|
result = download(url, body, options);
|
|
|
|
if ((result.code === 503) && (retryCount < 3)) {
|
|
wait(1);
|
|
retryCount+=1;
|
|
continue;
|
|
}
|
|
if (result.code !== 200) {
|
|
var err1 = "bootstrapping DB servers failed: " + extractErrorMessage(result);
|
|
console.error("%s", err1);
|
|
return {"error": true, "bootstrapServers": true, "errorMessage": err1};
|
|
}
|
|
break;
|
|
}
|
|
// execute cluster database upgrade
|
|
url = coordinators[0] + "/_admin/cluster/upgradeClusterDatabase";
|
|
|
|
result = download(url, body, options);
|
|
|
|
if (result.code !== 200) {
|
|
var err2 = "upgrading cluster database failed: " + extractErrorMessage(result);
|
|
console.error("%s", err2);
|
|
return {"error": true, "bootstrapServers": true, "errorMessage": err2};
|
|
}
|
|
|
|
// bootstrap coordinators
|
|
var i;
|
|
|
|
for (i = 0; i < coordinators.length; ++i) {
|
|
url = coordinators[i] + "/_admin/cluster/bootstrapCoordinator";
|
|
|
|
result = download(url, body, options);
|
|
|
|
if (result.code !== 200) {
|
|
var err = "bootstrapping coordinator " +
|
|
coordinators[i] + " failed: " +
|
|
extractErrorMessage(result);
|
|
console.error("%s", err);
|
|
return {"error": true, "bootstrapServers": true, "errorMessage": err};
|
|
}
|
|
}
|
|
|
|
return {"error": false, "bootstrapServers": true, "errorMessage": ''};
|
|
};
|
|
|
|
shutdownActions.startAgent = function (dispatchers, cmd, run) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Shutting down agent %s", JSON.stringify(run.pid));
|
|
}
|
|
killExternal(run.pid);
|
|
return {"error": false, "isStartAgent": true};
|
|
};
|
|
|
|
shutdownActions.sendConfiguration = function (dispatchers, cmd, run) {
|
|
return {"error": false, "isSendConfiguration": true};
|
|
};
|
|
|
|
shutdownActions.startServers = function (dispatchers, cmd, run) {
|
|
var i;
|
|
var url;
|
|
var r;
|
|
var serverStates = {};
|
|
var error = false;
|
|
|
|
for (i = 0;i < run.endpoints.length;i++) {
|
|
console.info("Using API to shutdown %s %s %s %s",
|
|
run.roles[i],
|
|
run.endpointNames[i],
|
|
run.endpoints[i],
|
|
JSON.stringify(run.pids[i]));
|
|
url = endpointToURL(run.endpoints[i])+"/_admin/shutdown";
|
|
// We use the cluster-internal authentication:
|
|
var hdrs = { Authorization : ArangoServerState.getClusterAuthentication() };
|
|
r = download(url,"",{method:"GET", headers: hdrs});
|
|
if (r.code !== 200) {
|
|
console.info("Shutdown API result: %s %s %s %s %s",
|
|
run.roles[i],
|
|
run.endpointNames[i],
|
|
run.endpoints[i],
|
|
JSON.stringify(run.pids[i]),
|
|
JSON.stringify(r));
|
|
}
|
|
}
|
|
for (i = 0;i < run.endpoints.length;i++) {
|
|
waitForServerDown(cmd, run.endpoints[i], 30);
|
|
// we cannot do much with the result...
|
|
}
|
|
|
|
var shutdownWait = 20;
|
|
if (cmd.valgrind !== '') {
|
|
shutdownWait *= 600; // should be enough, even with Valgrind
|
|
}
|
|
console.info("Waiting up to " + shutdownWait + " seconds for servers to shutdown gracefully...");
|
|
var j = 0;
|
|
var runpids = run.pids.length;
|
|
while ((j < shutdownWait) && (runpids > 0)) {
|
|
wait(1);
|
|
j++;
|
|
for (i = 0; i < run.pids.length; i++) {
|
|
if (serverStates[JSON.stringify(run.pids[i])] === undefined) {
|
|
var s = statusExternal(run.pids[i]);
|
|
|
|
if ((s.status === "NOT-FOUND") ||
|
|
(s.status === "TERMINATED") ||
|
|
s.hasOwnProperty('signal')) {
|
|
runpids -=1;
|
|
serverStates[JSON.stringify(run.pids[i])] = s;
|
|
error = true;
|
|
}
|
|
else if (j > shutdownWait) {
|
|
if (s.status !== "TERMINATED") {
|
|
if (s.hasOwnProperty('signal')) {
|
|
error = true;
|
|
console.error("shuting down %s %s done - with problems: " + s,
|
|
run.roles[i],
|
|
run.endpointNames[i],
|
|
JSON.stringify(run.pids[i]));
|
|
}
|
|
else {
|
|
console.info("Shutting down %s the hard way...",
|
|
JSON.stringify(run.pids[i]));
|
|
s.killedState = killExternal(run.pids[i]);
|
|
console.info("done.");
|
|
runpids -=1;
|
|
}
|
|
serverStates[JSON.stringify(run.pids[i])] = s;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return {"error": error, "isStartServers": true, "serverStates" : serverStates};
|
|
};
|
|
|
|
cleanupActions.startAgent = function (dispatchers, cmd) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Cleaning up agent...");
|
|
}
|
|
var dataPath = fs.makeAbsolute(cmd.dataPath);
|
|
if (dataPath !== cmd.dataPath) { // path was relative
|
|
dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
|
|
}
|
|
|
|
var agentDataDir = fs.join(dataPath, "agent"+cmd.agencyPrefix+cmd.extPort);
|
|
if (fs.exists(agentDataDir)) {
|
|
fs.removeDirectoryRecursive(agentDataDir,true);
|
|
}
|
|
return {"error":false, "isStartAgent": true};
|
|
};
|
|
|
|
cleanupActions.startServers = function (dispatchers, cmd, isRelaunch) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Cleaning up DBservers...");
|
|
}
|
|
var dataPath = fs.makeAbsolute(cmd.dataPath);
|
|
if (dataPath !== cmd.dataPath) { // path was relative
|
|
dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
|
|
}
|
|
var logPath = fs.makeAbsolute(cmd.logPath);
|
|
if (logPath !== cmd.logPath) { // path was relative
|
|
logPath = fs.normalize(fs.join(ArangoServerState.logPath(), cmd.logPath));
|
|
}
|
|
|
|
var servers = cmd.DBservers.concat(cmd.Coordinators);
|
|
var i;
|
|
var logfile, datadir;
|
|
for (i = 0; i < servers.length; i++) {
|
|
var id = servers[i];
|
|
logfile = fs.join(logPath,"log-"+cmd.agency.agencyPrefix+"-"+id);
|
|
if (fs.exists(logfile)) {
|
|
fs.remove(logfile);
|
|
}
|
|
datadir = fs.join(dataPath,"data-"+cmd.agency.agencyPrefix+"-"+id);
|
|
if (fs.exists(datadir)) {
|
|
fs.removeDirectoryRecursive(datadir,true);
|
|
}
|
|
}
|
|
return {"error": false, "isStartServers": true};
|
|
};
|
|
|
|
isHealthyActions.startAgent = function (dispatchers, cmd, run) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Checking health of agent %s", JSON.stringify(run.pid));
|
|
}
|
|
var r = statusExternal(run.pid);
|
|
r.isStartAgent = true;
|
|
r.error = false;
|
|
return r;
|
|
};
|
|
|
|
isHealthyActions.startServers = function (dispatchers, cmd, run) {
|
|
var i;
|
|
var r = [];
|
|
for (i = 0;i < run.pids.length;i++) {
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Checking health of %s %s %s %s",
|
|
run.roles[i],
|
|
run.endpointNames[i],
|
|
run.endpoints[i],
|
|
JSON.stringify(run.pids[i]));
|
|
}
|
|
r.push(statusExternal(run.pids[i]));
|
|
}
|
|
var s = [];
|
|
var x;
|
|
var error = false;
|
|
for (i = 0;i < run.endpoints.length;i++) {
|
|
x = waitForServerUp(cmd, run.endpoints[i], 0);
|
|
s.push(x);
|
|
if (x === false) {
|
|
error = true;
|
|
}
|
|
}
|
|
return {"error": error, "isStartServers": true, "status": r,
|
|
"answering": s};
|
|
};
|
|
|
|
// Upgrade for the agent is exactly as launch/relaunch:
|
|
|
|
upgradeActions.startAgent = launchActions.startAgent;
|
|
|
|
|
|
// Upgrade for the configuration
|
|
|
|
upgradeActions.sendConfiguration = launchActions.sendConfiguration;
|
|
|
|
|
|
// Upgrade for servers, copied from launchactions, but modified:
|
|
|
|
upgradeActions.startServers = function (dispatchers, cmd, isRelaunch) {
|
|
var dataPath = fs.makeAbsolute(cmd.dataPath);
|
|
if (dataPath !== cmd.dataPath) { // path was relative
|
|
dataPath = fs.normalize(fs.join(ArangoServerState.dataPath(),cmd.dataPath));
|
|
}
|
|
var logPath = fs.makeAbsolute(cmd.logPath);
|
|
if (logPath !== cmd.logPath) { // path was relative
|
|
logPath = fs.normalize(fs.join(ArangoServerState.logPath(), cmd.logPath));
|
|
}
|
|
|
|
var url = endpointToURL(cmd.agency.endpoints[0])+"/v2/keys/"+
|
|
cmd.agency.agencyPrefix+"/";
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Downloading %sLaunchers/%s", url, encode(cmd.name));
|
|
}
|
|
var res = download(url+"Launchers/" + encode(cmd.name), "", { method: "GET",
|
|
followRedirects: true });
|
|
if (res.code !== 200) {
|
|
return {"error": true, "isStartServers": true, "suberror": res};
|
|
}
|
|
var body = JSON.parse( res.body );
|
|
var info = JSON.parse(body.node.value);
|
|
var id,ep,args,pids,port,endpoints,endpointNames,roles;
|
|
|
|
console.info("Upgrading servers...");
|
|
var i;
|
|
var servers = info.DBservers.concat(info.Coordinators);
|
|
roles = [];
|
|
for (i = 0; i < info.DBservers.length; i++) {
|
|
roles.push("DBserver");
|
|
}
|
|
for (i = 0; i < info.Coordinators.length; i++) {
|
|
roles.push("Coordinator");
|
|
}
|
|
pids = [];
|
|
endpoints = [];
|
|
var useSSL;
|
|
var arangodPath;
|
|
var logfile, datadir;
|
|
for (i = 0; i < servers.length; i++) {
|
|
id = servers[i];
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Downloading %sTarget/MapIDToEndpoint/%s", url, id);
|
|
}
|
|
res = download(url + "Target/MapIDToEndpoint/" + id);
|
|
if (res.code !== 200) {
|
|
return {"error": true, "pids": pids,
|
|
"isStartServers": true, "suberror": res};
|
|
}
|
|
console.info("Upgrading server %s",id);
|
|
body = JSON.parse(res.body);
|
|
ep = JSON.parse(body.node.value);
|
|
port = getPort(ep);
|
|
useSSL = false;
|
|
if (roles[i] === "DBserver") {
|
|
args = ["--configuration", ArangoServerState.dbserverConfig()];
|
|
useSSL = cmd.useSSLonDBservers;
|
|
}
|
|
else {
|
|
args = ["--configuration", ArangoServerState.coordinatorConfig()];
|
|
useSSL = cmd.useSSLonCoordinators;
|
|
}
|
|
args = args.concat([
|
|
"--cluster.disable-dispatcher-kickstarter", "true",
|
|
"--cluster.disable-dispatcher-frontend", "true",
|
|
"--cluster.my-id", id,
|
|
"--cluster.agency-prefix", cmd.agency.agencyPrefix,
|
|
"--cluster.agency-endpoint", cmd.agency.endpoints[0],
|
|
"--server.endpoint"]);
|
|
if (cmd.onlyLocalhost) {
|
|
args.push(exchangeProtocol("tcp://127.0.0.1:"+port,useSSL));
|
|
}
|
|
else {
|
|
args.push(exchangeProtocol("tcp://0.0.0.0:"+port,useSSL));
|
|
}
|
|
args.push("--log.file");
|
|
logfile = fs.join(logPath,"log-"+cmd.agency.agencyPrefix+"-"+id);
|
|
args.push(logfile);
|
|
datadir = fs.join(dataPath,"data-"+cmd.agency.agencyPrefix+"-"+id);
|
|
args.push("--database.directory");
|
|
args.push(datadir);
|
|
args = args.concat(dispatchers[cmd.dispatcher].arangodExtraArgs);
|
|
args.push("--upgrade");
|
|
arangodPath = fs.makeAbsolute(cmd.arangodPath);
|
|
if (arangodPath !== cmd.arangodPath) {
|
|
arangodPath = ArangoServerState.arangodPath();
|
|
}
|
|
pids.push(executeExternal(arangodPath, args));
|
|
ep = exchangePort(dispatchers[cmd.dispatcher].endpoint,port);
|
|
ep = exchangeProtocol(ep,useSSL);
|
|
endpoints.push(ep);
|
|
}
|
|
|
|
var error = false;
|
|
for (i = 0;i < pids.length;i++) {
|
|
var s = statusExternal(pids[i], true);
|
|
if (s.status !== "TERMINATED" || s.exit !== 0) {
|
|
error = true;
|
|
console.info("Upgrade of server "+servers[i]+" went wrong.");
|
|
}
|
|
}
|
|
|
|
if (error === true) {
|
|
return {"error": error, "isStartServers": true,
|
|
"errorMessage": "error on upgrade"};
|
|
}
|
|
|
|
console.info("Starting servers...");
|
|
servers = info.DBservers.concat(info.Coordinators);
|
|
roles = [];
|
|
for (i = 0; i < info.DBservers.length; i++) {
|
|
roles.push("DBserver");
|
|
}
|
|
for (i = 0; i < info.Coordinators.length; i++) {
|
|
roles.push("Coordinator");
|
|
}
|
|
pids = [];
|
|
endpoints = [];
|
|
endpointNames = [];
|
|
for (i = 0; i < servers.length; i++) {
|
|
id = servers[i];
|
|
if (cmd.extremeVerbosity) {
|
|
console.info("Downloading %sTarget/MapIDToEndpoint/%s", url, id);
|
|
}
|
|
res = download(url + "Target/MapIDToEndpoint/" + id);
|
|
if (res.code !== 200) {
|
|
return {"error": true, "pids": pids,
|
|
"isStartServers": true, "suberror": res};
|
|
}
|
|
console.info("Starting server %s",id);
|
|
body = JSON.parse(res.body);
|
|
ep = JSON.parse(body.node.value);
|
|
port = getPort(ep);
|
|
useSSL = false;
|
|
if (roles[i] === "DBserver") {
|
|
args = ["--configuration", ArangoServerState.dbserverConfig()];
|
|
useSSL = cmd.useSSLonDBservers;
|
|
}
|
|
else {
|
|
args = ["--configuration", ArangoServerState.coordinatorConfig()];
|
|
useSSL = cmd.useSSLonCoordinators;
|
|
}
|
|
args = args.concat([
|
|
"--cluster.disable-dispatcher-kickstarter", "true",
|
|
"--cluster.disable-dispatcher-frontend", "true",
|
|
"--cluster.my-id", id,
|
|
"--cluster.agency-prefix", cmd.agency.agencyPrefix,
|
|
"--cluster.agency-endpoint", cmd.agency.endpoints[0],
|
|
"--server.endpoint"]);
|
|
if (cmd.onlyLocalhost) {
|
|
args.push(exchangeProtocol("tcp://127.0.0.1:"+port,useSSL));
|
|
}
|
|
else {
|
|
args.push(exchangeProtocol("tcp://0.0.0.0:"+port,useSSL));
|
|
}
|
|
args.push("--log.file");
|
|
logfile = fs.join(logPath,"log-"+cmd.agency.agencyPrefix+"-"+id);
|
|
args.push(logfile);
|
|
if (!isRelaunch) {
|
|
if (!fs.exists(logPath)) {
|
|
fs.makeDirectoryRecursive(logPath);
|
|
}
|
|
if (fs.exists(logfile)) {
|
|
fs.remove(logfile);
|
|
}
|
|
}
|
|
datadir = fs.join(dataPath,"data-"+cmd.agency.agencyPrefix+"-"+id);
|
|
args.push("--database.directory");
|
|
args.push(datadir);
|
|
if (!isRelaunch) {
|
|
if (!fs.exists(dataPath)) {
|
|
fs.makeDirectoryRecursive(dataPath);
|
|
}
|
|
if (fs.exists(datadir)) {
|
|
fs.removeDirectoryRecursive(datadir,true);
|
|
}
|
|
fs.makeDirectory(datadir);
|
|
}
|
|
args = args.concat(dispatchers[cmd.dispatcher].arangodExtraArgs);
|
|
arangodPath = fs.makeAbsolute(cmd.arangodPath);
|
|
if (arangodPath !== cmd.arangodPath) {
|
|
arangodPath = ArangoServerState.arangodPath();
|
|
}
|
|
pids.push(executeExternal(arangodPath, args));
|
|
ep = exchangePort(dispatchers[cmd.dispatcher].endpoint,port);
|
|
ep = exchangeProtocol(ep,useSSL);
|
|
endpoints.push(ep);
|
|
endpointNames.push(id);
|
|
}
|
|
|
|
error = false;
|
|
for (i = 0;i < endpoints.length;i++) {
|
|
if (! waitForServerUp(cmd, endpoints[i], 30)) {
|
|
error = true;
|
|
}
|
|
}
|
|
|
|
return {"error": error,
|
|
"isStartServers": true,
|
|
"pids": pids,
|
|
"endpoints": endpoints,
|
|
"endpointNames": endpointNames,
|
|
"roles": roles};
|
|
};
|
|
|
|
|
|
// Upgrade for the upgrade-database as in launch:
|
|
|
|
upgradeActions.bootstrapServers = function (dispatchers, cmd, isRelaunch,
|
|
username, password) {
|
|
var coordinators = cmd.coordinators;
|
|
|
|
// we need at least one coordinator
|
|
if (0 === coordinators.length) {
|
|
return {"error": true, "bootstrapServers": true};
|
|
}
|
|
|
|
// We wait another half second (we have already waited for all servers
|
|
// until they responded!), just to be on the safe side:
|
|
wait(0.5);
|
|
|
|
// authorization header for coordinator
|
|
var hdrs = {
|
|
Authorization: getAuthorizationHeader(username, password)
|
|
};
|
|
|
|
// default options
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var options = {
|
|
method: "POST",
|
|
timeout: timeout,
|
|
headers: hdrs,
|
|
returnBodyOnError: true
|
|
};
|
|
|
|
// execute bootstrap command on first server
|
|
var url = coordinators[0] + "/_admin/cluster/bootstrapDbServers";
|
|
var body = '{"isRelaunch": false, "upgrade": true}';
|
|
|
|
var result = download(url, body, options);
|
|
|
|
if (result.code !== 200) {
|
|
console.error("bootstrapping DB servers failed: %s", extractErrorMessage(result));
|
|
return {"error": true, "bootstrapServers": true};
|
|
}
|
|
|
|
// execute cluster database upgrade
|
|
console.info("Running upgrade script on whole cluster...");
|
|
|
|
url = coordinators[0] + "/_admin/cluster/upgradeClusterDatabase";
|
|
|
|
result = download(url, body, options);
|
|
|
|
if (result.code !== 200) {
|
|
console.error("upgrading cluster database failed: %s", extractErrorMessage(result));
|
|
return {"error": true, "bootstrapServers": true};
|
|
}
|
|
|
|
// bootstrap coordinators
|
|
var i;
|
|
|
|
for (i = 0; i < coordinators.length; ++i) {
|
|
url = coordinators[i] + "/_admin/cluster/bootstrapCoordinator";
|
|
|
|
result = download(url, body, options);
|
|
|
|
if (result.code !== 200) {
|
|
console.error("bootstrapping coordinator %s failed: %s",
|
|
coordinators[i],
|
|
extractErrorMessage(result));
|
|
return {"error": true, "bootstrapServers": true};
|
|
}
|
|
}
|
|
|
|
return {"error": false, "bootstrapServers": true};
|
|
};
|
|
|
|
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Cluster_Kickstarter_Constructor
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
function Kickstarter (clusterPlan, myname) {
|
|
this.clusterPlan = clusterPlan;
|
|
if (myname === undefined) {
|
|
this.myname = "me";
|
|
}
|
|
else {
|
|
this.myname = myname;
|
|
}
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Kickstarter_prototype_launch
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
Kickstarter.prototype.launch = function () {
|
|
var clusterPlan = this.clusterPlan;
|
|
var myname = this.myname;
|
|
var dispatchers = clusterPlan.dispatchers;
|
|
var cmds = clusterPlan.commands;
|
|
var results = [];
|
|
var cmd;
|
|
var errors = "";
|
|
var error = false;
|
|
var i;
|
|
var res;
|
|
for (i = 0; i < cmds.length; i++) {
|
|
cmd = cmds[i];
|
|
if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
|
|
if (launchActions.hasOwnProperty(cmd.action)) {
|
|
res = launchActions[cmd.action](dispatchers, cmd, false);
|
|
results.push(res);
|
|
if (res.error === true) {
|
|
errors += "Launchjob " + cmd.action + " failed: " + res.errorMessage;
|
|
error = true;
|
|
break;
|
|
}
|
|
}
|
|
else {
|
|
results.push({ error: false, action: cmd.action });
|
|
}
|
|
}
|
|
else {
|
|
var ep = dispatchers[cmd.dispatcher].endpoint;
|
|
var body = JSON.stringify({ "action": "launch",
|
|
"clusterPlan": {
|
|
"dispatchers": dispatchers,
|
|
"commands": [cmd] },
|
|
"myname": cmd.dispatcher });
|
|
var url = endpointToURL(ep) + "/_admin/clusterDispatch";
|
|
var hdrs = {};
|
|
if (dispatchers[cmd.dispatcher].username !== undefined &&
|
|
dispatchers[cmd.dispatcher].passwd !== undefined) {
|
|
hdrs.Authorization = getAuthorization(dispatchers[cmd.dispatcher]);
|
|
}
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var response = download(url, body, {method: "POST", headers: hdrs, timeout: timeout});
|
|
if (response.code !== 200) {
|
|
error = true;
|
|
results.push({"error":true, "errorMessage": "bad HTTP response code",
|
|
"response": response});
|
|
}
|
|
else {
|
|
try {
|
|
res = JSON.parse(response.body);
|
|
results.push(res.runInfo[0]);
|
|
}
|
|
catch (err) {
|
|
results.push({"error":true, "errorMessage": "exception in JSON.parse"});
|
|
error = true;
|
|
}
|
|
}
|
|
if (error) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
this.runInfo = results;
|
|
if (error) {
|
|
return {"error": true, "errorMessage": errors,
|
|
"runInfo": results};
|
|
}
|
|
return {"error": false, "errorMessage": "none",
|
|
"runInfo": results};
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Kickstarter_prototype_relaunch
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
Kickstarter.prototype.relaunch = function (username, password) {
|
|
var clusterPlan = this.clusterPlan;
|
|
var myname = this.myname;
|
|
var dispatchers = clusterPlan.dispatchers;
|
|
var cmds = clusterPlan.commands;
|
|
var results = [];
|
|
var cmd;
|
|
|
|
if (username === undefined && password === undefined) {
|
|
username = "root";
|
|
password = "";
|
|
}
|
|
|
|
var error = false;
|
|
var i;
|
|
var res;
|
|
for (i = 0; i < cmds.length; i++) {
|
|
cmd = cmds[i];
|
|
if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
|
|
if (launchActions.hasOwnProperty(cmd.action)) {
|
|
res = launchActions[cmd.action](dispatchers, cmd, true,
|
|
username, password);
|
|
results.push(res);
|
|
if (res.error === true) {
|
|
error = true;
|
|
break;
|
|
}
|
|
}
|
|
else {
|
|
results.push({ error: false, action: cmd.action });
|
|
}
|
|
}
|
|
else {
|
|
var ep = dispatchers[cmd.dispatcher].endpoint;
|
|
var body = JSON.stringify({ "action": "relaunch",
|
|
"clusterPlan": {
|
|
"dispatchers": dispatchers,
|
|
"commands": [cmd] },
|
|
"myname": cmd.dispatcher,
|
|
"username": username,
|
|
"password": password});
|
|
var url = endpointToURL(ep) + "/_admin/clusterDispatch";
|
|
var hdrs = {};
|
|
if (dispatchers[cmd.dispatcher].username !== undefined &&
|
|
dispatchers[cmd.dispatcher].passwd !== undefined) {
|
|
hdrs.Authorization = getAuthorization(dispatchers[cmd.dispatcher]);
|
|
}
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var response = download(url, body, {method: "POST", headers: hdrs, timeout: timeout});
|
|
if (response.code !== 200) {
|
|
error = true;
|
|
results.push({"error":true, "errorMessage": "bad HTTP response code",
|
|
"response": response});
|
|
}
|
|
else {
|
|
try {
|
|
res = JSON.parse(response.body);
|
|
results.push(res.runInfo[0]);
|
|
}
|
|
catch (err) {
|
|
results.push({"error":true, "errorMessage": "exception in JSON.parse"});
|
|
error = true;
|
|
}
|
|
}
|
|
if (error) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
this.runInfo = results;
|
|
if (error) {
|
|
return {"error": true, "errorMessage": "some error during relaunch",
|
|
"runInfo": results};
|
|
}
|
|
return {"error": false, "errorMessage": "none",
|
|
"runInfo": results};
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Kickstarter_prototype_shutdown
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
Kickstarter.prototype.shutdown = function() {
|
|
var clusterPlan = this.clusterPlan;
|
|
var myname = this.myname;
|
|
var dispatchers = clusterPlan.dispatchers;
|
|
var cmds = clusterPlan.commands;
|
|
var runInfo = this.runInfo;
|
|
var results = [];
|
|
var cmd;
|
|
|
|
var error = false;
|
|
var i;
|
|
var res;
|
|
for (i = cmds.length-1; i >= 0; i--) {
|
|
cmd = cmds[i];
|
|
var run = runInfo[i];
|
|
if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
|
|
if (shutdownActions.hasOwnProperty(cmd.action)) {
|
|
res = shutdownActions[cmd.action](dispatchers, cmd, run);
|
|
if (res.error === true) {
|
|
error = true;
|
|
}
|
|
results.push(res);
|
|
}
|
|
else {
|
|
results.push({ error: false, action: cmd.action });
|
|
}
|
|
}
|
|
else {
|
|
var ep = dispatchers[cmd.dispatcher].endpoint;
|
|
var body = JSON.stringify({ "action": "shutdown",
|
|
"clusterPlan": {
|
|
"dispatchers": dispatchers,
|
|
"commands": [cmd] },
|
|
"runInfo": [run],
|
|
"myname": cmd.dispatcher });
|
|
var url = endpointToURL(ep) + "/_admin/clusterDispatch";
|
|
var hdrs = {};
|
|
if (dispatchers[cmd.dispatcher].username !== undefined &&
|
|
dispatchers[cmd.dispatcher].passwd !== undefined) {
|
|
hdrs.Authorization = getAuthorization(dispatchers[cmd.dispatcher]);
|
|
}
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var response = download(url, body, {method: "POST", headers: hdrs, timeout: timeout});
|
|
if (response.code !== 200) {
|
|
error = true;
|
|
results.push({"error":true, "errorMessage": "bad HTTP response code",
|
|
"response": response});
|
|
}
|
|
else {
|
|
try {
|
|
res = JSON.parse(response.body);
|
|
results.push(res.results[0]);
|
|
}
|
|
catch (err) {
|
|
results.push({"error":true,
|
|
"errorMessage": "exception in JSON.parse"});
|
|
error = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
results = results.reverse();
|
|
if (error) {
|
|
return {"error": true, "errorMessage": "some error during shutdown",
|
|
"results": results};
|
|
}
|
|
return {"error": false, "errorMessage": "none", "results": results};
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Kickstarter_prototype_cleanup
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
Kickstarter.prototype.cleanup = function() {
|
|
var clusterPlan = this.clusterPlan;
|
|
var myname = this.myname;
|
|
var dispatchers = clusterPlan.dispatchers;
|
|
var cmds = clusterPlan.commands;
|
|
var results = [];
|
|
var cmd;
|
|
|
|
var error = false;
|
|
var i;
|
|
var res;
|
|
for (i = 0; i < cmds.length; i++) {
|
|
cmd = cmds[i];
|
|
if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
|
|
if (cleanupActions.hasOwnProperty(cmd.action)) {
|
|
res = cleanupActions[cmd.action](dispatchers, cmd);
|
|
results.push(res);
|
|
if (res.error === true) {
|
|
error = true;
|
|
}
|
|
}
|
|
else {
|
|
results.push({ error: false, action: cmd.action });
|
|
}
|
|
}
|
|
else {
|
|
var ep = dispatchers[cmd.dispatcher].endpoint;
|
|
var body = JSON.stringify({ "action": "cleanup",
|
|
"clusterPlan": {
|
|
"dispatchers": dispatchers,
|
|
"commands": [cmd] },
|
|
"myname": cmd.dispatcher });
|
|
var url = endpointToURL(ep) + "/_admin/clusterDispatch";
|
|
var hdrs = {};
|
|
if (dispatchers[cmd.dispatcher].username !== undefined &&
|
|
dispatchers[cmd.dispatcher].passwd !== undefined) {
|
|
hdrs.Authorization = getAuthorization(dispatchers[cmd.dispatcher]);
|
|
}
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var response = download(url, body, {method: "POST", headers: hdrs, timeout: timeout});
|
|
if (response.code !== 200) {
|
|
error = true;
|
|
results.push({"error":true, "errorMessage": "bad HTTP response code",
|
|
"response": response});
|
|
}
|
|
else {
|
|
try {
|
|
res = JSON.parse(response.body);
|
|
results.push(res.results[0]);
|
|
}
|
|
catch (err) {
|
|
results.push({"error":true,
|
|
"errorMessage": "exception in JSON.parse"});
|
|
error = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (error) {
|
|
return {"error": true, "errorMessage": "some error during cleanup",
|
|
"results": results};
|
|
}
|
|
return {"error": false, "errorMessage": "none", "results": results};
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Kickstarter_prototype_isHealthy
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
Kickstarter.prototype.isHealthy = function() {
|
|
var clusterPlan = this.clusterPlan;
|
|
var myname = this.myname;
|
|
var dispatchers = clusterPlan.dispatchers;
|
|
var cmds = clusterPlan.commands;
|
|
var runInfo = this.runInfo;
|
|
var results = [];
|
|
var cmd;
|
|
|
|
var error = false;
|
|
var i;
|
|
var res;
|
|
for (i = cmds.length-1; i >= 0; i--) {
|
|
cmd = cmds[i];
|
|
if (runInfo === undefined || ! Array.isArray(runInfo) ||
|
|
i >= runInfo.length) {
|
|
return {"error": true,
|
|
"errorMessage": "runInfo object not found or broken",
|
|
"results": results};
|
|
}
|
|
var run = runInfo[i];
|
|
if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
|
|
if (isHealthyActions.hasOwnProperty(cmd.action)) {
|
|
res = isHealthyActions[cmd.action](dispatchers, cmd, run);
|
|
if (res.error === true) {
|
|
error = true;
|
|
}
|
|
results.push(res);
|
|
}
|
|
else {
|
|
results.push({ error: false, action: cmd.action });
|
|
}
|
|
}
|
|
else {
|
|
var ep = dispatchers[cmd.dispatcher].endpoint;
|
|
var body = JSON.stringify({ "action": "isHealthy",
|
|
"clusterPlan": {
|
|
"dispatchers": dispatchers,
|
|
"commands": [cmd] },
|
|
"runInfo": [run],
|
|
"myname": cmd.dispatcher });
|
|
var url = "http" + ep.substr(3) + "/_admin/clusterDispatch";
|
|
var hdrs = {};
|
|
if (dispatchers[cmd.dispatcher].username !== undefined &&
|
|
dispatchers[cmd.dispatcher].passwd !== undefined) {
|
|
hdrs.Authorization = getAuthorization(dispatchers[cmd.dispatcher]);
|
|
}
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var response = download(url, body, {method: "POST", headers: hdrs, timeout: timeout});
|
|
if (response.code !== 200) {
|
|
error = true;
|
|
results.push({"error":true, "errorMessage": "bad HTTP response code",
|
|
"response": response});
|
|
}
|
|
else {
|
|
try {
|
|
res = JSON.parse(response.body);
|
|
results.push(res.results[0]);
|
|
if (res.results[0].error === true) {
|
|
error = true;
|
|
}
|
|
}
|
|
catch (err) {
|
|
results.push({"error":true,
|
|
"errorMessage": "exception in JSON.parse"});
|
|
error = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
results = results.reverse();
|
|
if (error) {
|
|
return {"error": true, "errorMessage": "some error during isHealthy check",
|
|
"results": results};
|
|
}
|
|
return {"error": false, "errorMessage": "none", "results": results};
|
|
};
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
/// @brief was docuBlock JSF_Kickstarter_prototype_upgrade
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
Kickstarter.prototype.upgrade = function (username, password) {
|
|
var clusterPlan = this.clusterPlan;
|
|
var myname = this.myname;
|
|
var dispatchers = clusterPlan.dispatchers;
|
|
var cmds = clusterPlan.commands;
|
|
var results = [];
|
|
var cmd;
|
|
|
|
if (username === undefined || password === undefined) {
|
|
username = "root";
|
|
password = "";
|
|
}
|
|
|
|
var error = false;
|
|
var i;
|
|
var res;
|
|
for (i = 0; i < cmds.length; i++) {
|
|
cmd = cmds[i];
|
|
if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
|
|
if (upgradeActions.hasOwnProperty(cmd.action)) {
|
|
res = upgradeActions[cmd.action](dispatchers, cmd, true,
|
|
username, password);
|
|
results.push(res);
|
|
if (res.error === true) {
|
|
error = true;
|
|
break;
|
|
}
|
|
}
|
|
else {
|
|
results.push({ error: false, action: cmd.action });
|
|
}
|
|
}
|
|
else {
|
|
var ep = dispatchers[cmd.dispatcher].endpoint;
|
|
var body = JSON.stringify({ "action": "upgrade",
|
|
"clusterPlan": {
|
|
"dispatchers": dispatchers,
|
|
"commands": [cmd] },
|
|
"myname": cmd.dispatcher,
|
|
"username": username,
|
|
"password": password});
|
|
var url = endpointToURL(ep) + "/_admin/clusterDispatch";
|
|
var hdrs = {};
|
|
if (dispatchers[cmd.dispatcher].username !== undefined &&
|
|
dispatchers[cmd.dispatcher].passwd !== undefined) {
|
|
hdrs.Authorization = getAuthorization(dispatchers[cmd.dispatcher]);
|
|
}
|
|
var timeout = 90;
|
|
if (cmd.valgrind !== '') {
|
|
timeout *= 10000;
|
|
}
|
|
var response = download(url, body, {method: "POST", headers: hdrs, timeout: timeout});
|
|
if (response.code !== 200) {
|
|
error = true;
|
|
results.push({"error":true, "errorMessage": "bad HTTP response code",
|
|
"response": response});
|
|
}
|
|
else {
|
|
try {
|
|
res = JSON.parse(response.body);
|
|
results.push(res.runInfo[0]);
|
|
}
|
|
catch (err) {
|
|
results.push({"error":true, "errorMessage": "exception in JSON.parse"});
|
|
error = true;
|
|
}
|
|
}
|
|
if (error) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
this.runInfo = results;
|
|
if (error) {
|
|
return {"error": true, "errorMessage": "some error during upgrade",
|
|
"runInfo": results};
|
|
}
|
|
return {"error": false, "errorMessage": "none",
|
|
"runInfo": results};
|
|
};
|
|
|
|
exports.Kickstarter = Kickstarter;
|