Add cleanup functionality for cluster kickstarter.

Documentation/RefManual/JSModuleCluster.md

@@ -43,4 +43,7 @@ Here are the details of the functionality:
 @anchor JSModuleClusterKickstarterRelaunch
 @copydetails JSF_Kickstarter_prototype_relaunch
 
+@anchor JSModuleClusterKickstarterCleanup
+@copydetails JSF_Kickstarter_prototype_cleanup
+
 @BNAVIGATE_JSModuleCluster

Documentation/RefManual/JSModuleClusterTOC.md

@@ -8,4 +8,5 @@ TOC {#JSModuleClusterTOC}
   - @ref JSModuleClusterKickstarterLaunch
   - @ref JSModuleClusterKickstarterShutdown
   - @ref JSModuleClusterKickstarterRelaunch
+  - @ref JSModuleClusterKickstarterCleanup
 

js/actions/api-cluster.js

@@ -307,8 +307,9 @@ actions.defineHttp({
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @fn JSF_cluster_dispatcher_POST
-/// @brief exposes the dispatcher functionality to start up a cluster 
-/// according to a startup plan as for example provided by the kickstarter.
+/// @brief exposes the dispatcher functionality to start up, shutdown,
+/// relaunch or cleanup a cluster according to a cluster plan as for
+/// example provided by the kickstarter.
 ///
 /// @RESTHEADER{POST /_admin/clusterDispatch,execute startup commands}
 ///
@@ -316,8 +317,28 @@ actions.defineHttp({
 ///
 /// @RESTBODYPARAM{body,json,required}
 ///
-/// @RESTDESCRIPTION Given a cluster plan (see JSF_cluster_planner_POST), this
-/// call executes the plan by either starting up processes personally
+/// @RESTDESCRIPTION The body must be an object with the following properties:
+/// 
+///   - `clusterPlan`: is a cluster plan (see JSF_cluster_planner_POST), 
+///   - `myname`: is the ID of this dispatcher, this is used to decide
+///     which commands are executed locally and which are forwarded
+///     to other dispatchers
+///   - `action`: can be one of the following:
+///
+///       - "launch": the cluster is launched for the first time, all
+///         data directories and log files are cleaned and created
+///       - "shutdown": the cluster is shut down, the additional property
+///         `runInfo` (see below) must be bound as well
+///       - "relaunch": the cluster is launched again, all data directories
+///         and log files are untouched and need to be there already
+///       - "cleanup": use this after a shutdown to remove all data in the
+///         data directories and all log files, use with caution
+///
+///   - `runInfo": this is needed for the "shutdown" action only and should
+///     be the structure that "launch" or "relaunch" returned. It contains
+///     runtime information like process IDs.
+///
+/// This call executes the plan by either doing the work personally
 /// or by delegating to other dispatchers.
 /// 
 /// @RESTRETURNCODES
@@ -403,6 +424,19 @@ actions.defineHttp({
         actions.resultException(req, res, error4, undefined, false);
       }
     }
+    else if (action === "cleanup") {
+      Kickstarter = require("org/arangodb/cluster/kickstarter").Kickstarter;
+      try {
+        k = new Kickstarter(input.clusterPlan, input.myname);
+        r = k.cleanup();
+        res.responseCode = actions.HTTP_OK;
+        res.contentType = "application/json; charset=utf-8";
+        res.body = JSON.stringify(r);
+      }
+      catch (error5) {
+        actions.resultException(req, res, error5, undefined, false);
+      }
+    }
     else {
       actions.resultError(req, res, actions.HTTP_BAD,
                           'Action '+action+' not yet implemented.');

js/server/modules/org/arangodb/cluster/kickstarter.js

@@ -45,6 +45,7 @@ var print = require("internal").print;
 
 var launchActions = {};
 var shutdownActions = {};
+var cleanupActions = {};
 
 function getAddrPort (endpoint) {
   var pos = endpoint.indexOf("://");
@@ -185,14 +186,17 @@ launchActions.startAgent = function (dispatchers, cmd, isRelaunch) {
   }
   var pid = executeExternal(agentPath, args);
   var res;
-  while (true) {
+  var count = 0;
+  while (++count < 20) {
     wait(0.5);   // Wait a bit to give it time to startup
     res = download("http://localhost:"+cmd.extPort+"/v2/keys/");
     if (res.code === 200) {
-      return {"error":false, "isAgent": true, "pid": pid, 
+      return {"error":false, "isStartAgent": true, "pid": pid, 
               "endpoint": extEndpoint};
     }
   }
+  return {"error":true, "isStartAgent": true, 
+          "errorMessage": "agency did not come alive"};
 };
 
 launchActions.sendConfiguration = function (dispatchers, cmd, isRelaunch) {
@@ -299,28 +303,81 @@ launchActions.startServers = function (dispatchers, cmd, isRelaunch) {
     pids.push(executeExternal(arangodPath, args));
     endpoints.push(exchangePort(dispatchers[cmd.dispatcher].endpoint,port));
   }
-  return {"error": false, "pids": pids, "endpoints": endpoints, "roles": roles};
+  return {"error": false, "isStartServers": true, 
+          "pids": pids, "endpoints": endpoints, "roles": roles};
 };
 
 shutdownActions.startAgent = function (dispatchers, cmd, run) {
-  print("Shutting down agent ", run.pid);
+  print("Shutting down agent", run.pid);
   killExternal(run.pid);
-  return {"error": false};
+  return {"error": false, "isStartAgent": true};
 };
 
 shutdownActions.sendConfiguration = function (dispatchers, cmd, run) {
   print("Waiting for 10 seconds for servers before shutting down agency.");
   wait(10);
-  return {"error": false};
+  return {"error": false, "isSendConfiguration": true};
 };
 
 shutdownActions.startServers = function (dispatchers, cmd, run) {
   var i;
   for (i = 0;i < run.pids.length;i++) {
-    print("Shutting down ", run.pids[i]);
+    print("Shutting down", run.pids[i]);
     killExternal(run.pids[i]);
   }
-  return {"error": false};
+  return {"error": false, "isStartServers": true};
+};
+
+cleanupActions.startAgent = function (dispatchers, cmd) {
+
+  print("Cleaning up agent...");
+
+  // First find out our own data directory:
+  var myDataDir = fs.normalize(fs.join(ArangoServerState.basePath(),".."));
+  var dataPath = fs.makeAbsolute(cmd.dataPath);
+  if (dataPath !== cmd.dataPath) {   // path was relative
+    dataPath = fs.normalize(fs.join(myDataDir,cmd.dataPath));
+  }
+
+  var agentDataDir = fs.join(dataPath, "agent"+cmd.agencyPrefix+cmd.extPort);
+  if (fs.exists(agentDataDir)) {
+    fs.removeDirectoryRecursive(agentDataDir,true);
+  }
+  return {"error":false, "isStartAgent": true};
+};
+
+cleanupActions.sendConfiguration = function (dispatchers, cmd) {
+  // nothing to do here
+  return {"error":false, "isSendConfiguration": true};
+};
+
+cleanupActions.startServers = function (dispatchers, cmd, isRelaunch) {
+
+  // First find out our own data directory to setup base for relative paths:
+  var myDataDir = fs.normalize(fs.join(ArangoServerState.basePath(),".."));
+  var dataPath = fs.makeAbsolute(cmd.dataPath);
+  if (dataPath !== cmd.dataPath) {   // path was relative
+    dataPath = fs.normalize(fs.join(myDataDir, cmd.dataPath));
+  }
+  var logPath = fs.makeAbsolute(cmd.logPath);
+  if (logPath !== cmd.logPath) {    // path was relative
+    logPath = fs.normalize(fs.join(myDataDir, cmd.logPath));
+  }
+
+  var servers = cmd.DBservers.concat(cmd.Coordinators);
+  var i;
+  for (i = 0; i < servers.length; i++) {
+    var id = servers[i];
+    var logfile = fs.join(logPath,"log-"+cmd.agency.agencyPrefix+"-"+id);
+    if (fs.exists(logfile)) {
+      fs.remove(logfile);
+    }
+    var datadir = fs.join(dataPath,"data-"+cmd.agency.agencyPrefix+"-"+id);
+    if (fs.exists(datadir)) {
+      fs.removeDirectoryRecursive(datadir,true);
+    }
+  }
+  return {"error": false, "isStartServers": true};
 };
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -588,6 +645,64 @@ Kickstarter.prototype.shutdown = function() {
   return {"error": false, "errorMessage": "none", "results": results};
 };
 
+////////////////////////////////////////////////////////////////////////////////
+/// @fn JSF_Kickstarter_prototype_cleanup
+/// @brief cleans up all the data of a cluster that has been shutdown
+///
+/// @FUN{@FA{Kickstarter}.cleanup()}
+///
+/// This cleans up all the data and logs of a previously shut down cluster.
+/// Use shutdown (see @ref JSF_Kickstarter_prototype_shutdown) first and
+/// use with caution, since potentially a lot of data is being erased with
+/// this call!
+////////////////////////////////////////////////////////////////////////////////
+
+Kickstarter.prototype.cleanup = function() {
+  var clusterPlan = this.clusterPlan;
+  var myname = this.myname;
+  var dispatchers = clusterPlan.dispatchers;
+  var cmds = clusterPlan.commands;
+  var results = [];
+  var cmd;
+
+  var error = false;
+  var i;
+  var res;
+  for (i = 0; i < cmds.length; i++) {
+    cmd = cmds[i];
+    if (cmd.dispatcher === undefined || cmd.dispatcher === myname) {
+      res = cleanupActions[cmd.action](dispatchers, cmd);
+      results.push(res);
+      if (res.error === true) {
+        error = true;
+      }
+    }
+    else {
+      var ep = dispatchers[cmd.dispatcher].endpoint;
+      var body = JSON.stringify({ "action": "cleanup",
+                                  "clusterPlan": {
+                                        "dispatchers": dispatchers,
+                                        "commands": [cmd] },
+                                  "myname": cmd.dispatcher });
+      var url = "http" + ep.substr(3) + "/_admin/clusterDispatch";
+      var response = download(url, body, {"method": "POST"});
+      if (response.code !== 200) {
+        error = true;
+        results.push({"error":true, "errorMessage": "bad HTTP response code",
+                      "response": response});
+      }
+      else {
+        results.push({"error":false});
+      }
+    }
+  }
+  if (error) {
+    return {"error": true, "errorMessage": "some error during cleanup",
+            "results": results};
+  }
+  return {"error": false, "errorMessage": "none", "results": results};
+};
+
 Kickstarter.prototype.isHealthy = function() {
   throw "not yet implemented";
 };

js/server/modules/org/arangodb/cluster/planner.js

@@ -163,9 +163,7 @@ PortFinder.prototype.next = function () {
       else {
         var url = "http" + this.dispatcher.endpoint.substr(3) + 
                   "/_admin/clusterCheckPort?port="+this.port;
-        print("Doing: ",url);
         var r = download(url, "", {"method": "GET"});
-        print("ResultCOde:", r.code);
         if (r.code === 200) {
           available = JSON.parse(r.body);
         }