Feature/arangodoc (#5476)

* introducing arangoinspect

arangod/GeneralServer/RestHandlerFactory.cpp

@@ -95,9 +95,11 @@ RestHandler* RestHandlerFactory::createHandler(
     case ServerState::Mode::TRYAGAIN: {
       if (path.find("/_admin/shutdown") == std::string::npos &&
           path.find("/_admin/cluster/health") == std::string::npos &&
+          path.find("/_admin/log") == std::string::npos &&
           path.find("/_admin/server/role") == std::string::npos &&
           path.find("/_admin/server/availability") == std::string::npos &&
           path.find("/_admin/status") == std::string::npos &&
+          path.find("/_admin/statistics") == std::string::npos &&
           path.find("/_api/agency/agency-callbacks") == std::string::npos &&
           path.find("/_api/cluster/") == std::string::npos &&
           path.find("/_api/replication") == std::string::npos &&

arangod/RestHandler/RestStatusHandler.cpp

@@ -33,6 +33,10 @@
 
 #include <iostream>
 
+#if defined(TRI_HAVE_POSIX_THREADS)
+#include <unistd.h>
+#endif
+
 #include <velocypack/Builder.h>
 #include <velocypack/velocypack-aliases.h>
 
@@ -56,6 +60,10 @@ RestStatus RestStatusHandler::execute() {
   result.add("server", VPackValue("arango"));
   result.add("version", VPackValue(ARANGODB_VERSION));
 
+#if defined(TRI_HAVE_POSIX_THREADS)
+  result.add("pid", VPackValue(getpid()));
+#endif
+  
 #ifdef USE_ENTERPRISE
   result.add("license", VPackValue("enterprise"));
 #else

arangosh/CMakeLists.txt

@@ -372,3 +372,13 @@ install_command_alias(${BIN_ARANGOSH}
   foxx-manager)
 
 install_config(foxx-manager)
+
+################################################################################
+## arangoinspect
+################################################################################
+
+install_command_alias(${BIN_ARANGOSH}
+  ${CMAKE_INSTALL_BINDIR}
+  arangoinspect)
+
+install_config(arangoinspect)

arangosh/Shell/ShellFeature.cpp

@@ -32,8 +32,8 @@ using namespace arangodb;
 using namespace arangodb::basics;
 using namespace arangodb::options;
 
-ShellFeature::ShellFeature(
+ShellFeature::ShellFeature(application_features::ApplicationServer* server,
-    application_features::ApplicationServer* server, int* result)
+                           int* result)
     : ApplicationFeature(server, "Shell"),
       _jslint(),
       _result(result),
@@ -123,15 +123,18 @@ void ShellFeature::validateOptions(
   }
 
   if (1 < n) {
-    LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "you cannot specify more than one type ("
+    LOG_TOPIC(ERR, arangodb::Logger::FIXME)
-             << "jslint, execute, execute-string, check-syntax, unit-tests)";
+        << "you cannot specify more than one type ("
+        << "jslint, execute, execute-string, check-syntax, unit-tests)";
   }
 }
 
 void ShellFeature::start() {
   *_result = EXIT_FAILURE;
 
-  V8ShellFeature* shell = application_features::ApplicationServer::getFeature<V8ShellFeature>("V8Shell");
+  V8ShellFeature* shell =
+      application_features::ApplicationServer::getFeature<V8ShellFeature>(
+          "V8Shell");
 
   bool ok = false;
 

arangosh/Shell/V8ClientConnection.cpp

@@ -49,7 +49,7 @@ using namespace arangodb::basics;
 using namespace arangodb::httpclient;
 using namespace arangodb::import;
 
-std::string V8ClientConnection::JWT_SECRET = "";
+std::shared_ptr<std::string> V8ClientConnection::JWT_SECRET = nullptr;
 
 std::string V8ClientConnection::jwtToken(std::string const& secret) {
   VPackBuilder headerBuilder;
@@ -106,8 +106,8 @@ void V8ClientConnection::init(
   params.setLocationRewriter(this, &rewriteLocation);
   params.setUserNamePassword("/", _username, _password);
 
-  if (!JWT_SECRET.empty()) {
+  if (JWT_SECRET != nullptr) {
-    params.setJwt(jwtToken(JWT_SECRET));
+    params.setJwt(jwtToken(*JWT_SECRET));
   }
 
   _client.reset(new SimpleHttpClient(connection, params));
@@ -434,20 +434,22 @@ static void ClientConnection_reconnect(
   std::string password;
 
   if (args.Length() < 4) {
-    ConsoleFeature* console =
+    if (V8ClientConnection::jwtSecret() == nullptr) {
+      ConsoleFeature* console =
         ApplicationServer::getFeature<ConsoleFeature>("Console");
-
+      
-    if (console->isEnabled()) {
+      if (console->isEnabled()) {
-      password = console->readPassword("Please specify a password: ");
+        password = console->readPassword("Please specify a password: ");
-    } else {
+      } else {
-      std::cout << "Please specify a password: " << std::flush;
+        std::cout << "Please specify a password: " << std::flush;
-      password = ConsoleFeature::readPassword();
+        password = ConsoleFeature::readPassword();
-      std::cout << std::endl << std::flush;
+        std::cout << std::endl << std::flush;
+      }
     }
   } else {
     password = TRI_ObjectToString(isolate, args[3]);
   }
-
+  
   bool warnConnect = true;
   if (args.Length() > 4) {
     warnConnect = TRI_ObjectToBoolean(args[4]);

arangosh/Shell/V8ClientConnection.h

@@ -49,11 +49,12 @@ class V8ClientConnection {
   V8ClientConnection& operator=(V8ClientConnection const&) = delete;
 
  public:
-  static void setJwtSecret(std::string const& jwtSecret) { JWT_SECRET = jwtSecret; }
+  static void setJwtSecret(std::string const& jwtSecret) { JWT_SECRET = std::make_shared<std::string>(jwtSecret); }
+  static std::shared_ptr<std::string> jwtSecret() { return JWT_SECRET; }
   static std::string jwtToken(std::string const& secret);
 
  private:
-  static std::string JWT_SECRET;
+  static std::shared_ptr<std::string> JWT_SECRET;
 
  public:
   V8ClientConnection(

arangosh/Shell/V8ShellFeature.cpp

@@ -54,10 +54,13 @@ using namespace arangodb::basics;
 using namespace arangodb::options;
 using namespace arangodb::rest;
 
+static std::string const DEFAULT_CLIENT_MODULE = "client.js";
+
 V8ShellFeature::V8ShellFeature(application_features::ApplicationServer* server,
                                std::string const& name)
     : ApplicationFeature(server, "V8Shell"),
       _startupDirectory("js"),
+      _clientModule(DEFAULT_CLIENT_MODULE),
       _currentModuleDirectory(true),
       _gcInterval(50),
       _name(name),
@@ -78,6 +81,10 @@ void V8ShellFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
                            "startup paths containing the Javascript files",
                            new StringParameter(&_startupDirectory));
 
+  options->addHiddenOption("--javascript.client-module",
+                           "client module to use at startup",
+                           new StringParameter(&_clientModule));
+
   options->addHiddenOption(
       "--javascript.module-directory",
       "additional paths containing JavaScript modules",
@@ -196,17 +203,18 @@ bool V8ShellFeature::printHello(V8ClientConnection* v8connection) {
   // http://www.network-science.de/ascii/   Font: ogre
 
   if (!_console->quiet()) {
-    std::string g = ShellColorsFeature::SHELL_COLOR_GREEN;
+    if (_clientModule == DEFAULT_CLIENT_MODULE) {
-    std::string r = ShellColorsFeature::SHELL_COLOR_RED;
+      std::string g = ShellColorsFeature::SHELL_COLOR_GREEN;
-    std::string z = ShellColorsFeature::SHELL_COLOR_RESET;
+      std::string r = ShellColorsFeature::SHELL_COLOR_RED;
+      std::string z = ShellColorsFeature::SHELL_COLOR_RESET;
 
-    if (!_console->colors()) {
+      if (!_console->colors()) {
-      g = "";
+        g = "";
-      r = "";
+        r = "";
-      z = "";
+        z = "";
-    }
+      }
 
-    // clang-format off
+      // clang-format off
 
     _console->printLine("");
     _console->printLine(g + "                                  "     + r + "     _     " + z);
@@ -217,17 +225,18 @@ bool V8ShellFeature::printHello(V8ClientConnection* v8connection) {
     _console->printLine(g + "                       |___/      "     + r + "           " + z);
     _console->printLine("");
 
-    // clang-format on
+      // clang-format on
 
-    std::ostringstream s;
+      std::ostringstream s;
 
-    s << "arangosh (" << rest::Version::getVerboseVersionString() << ")\n"
+      s << "arangosh (" << rest::Version::getVerboseVersionString() << ")\n"
-      << "Copyright (c) ArangoDB GmbH";
+        << "Copyright (c) ArangoDB GmbH";
 
-    _console->printLine(s.str());
+      _console->printLine(s.str());
-    _console->printLine("");
+      _console->printLine("");
 
-    _console->printWelcomeInfo();
+      _console->printWelcomeInfo();
+    }
 
     if (v8connection != nullptr) {
       if (v8connection->isConnected() &&
@@ -847,6 +856,25 @@ static void JS_VersionClient(v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_END
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief exit now
+////////////////////////////////////////////////////////////////////////////////
+
+static void JS_Exit(v8::FunctionCallbackInfo<v8::Value> const& args) {
+  TRI_V8_TRY_CATCH_BEGIN(isolate);
+  v8::HandleScope scope(isolate);
+
+  int64_t code = 0;
+
+  if (args.Length() > 0) {
+    code = TRI_ObjectToInt64(args[0]);
+  }
+
+  exit((int) code);
+
+  TRI_V8_TRY_CATCH_END
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief initializes global Javascript variables
 ////////////////////////////////////////////////////////////////////////////////
@@ -976,6 +1004,9 @@ void V8ShellFeature::initMode(ShellFeature::RunMode runMode,
   TRI_AddGlobalVariableVocbase(
       _isolate, TRI_V8_ASCII_STRING(_isolate, "IS_JS_LINT"),
       v8::Boolean::New(_isolate, runMode == ShellFeature::RunMode::JSLINT));
+
+  TRI_AddGlobalFunctionVocbase(
+      _isolate, TRI_V8_ASCII_STRING(_isolate, "SYS_EXIT"), JS_Exit);
 }
 
 void V8ShellFeature::loadModules(ShellFeature::RunMode runMode) {
@@ -1005,7 +1036,7 @@ void V8ShellFeature::loadModules(ShellFeature::RunMode runMode) {
   files.push_back(
       "common/bootstrap/modules.js");  // must come last before patches
 
-  files.push_back("client/client.js");  // needs internal
+  files.push_back("client/" + _clientModule);  // needs internal
 
   for (size_t i = 0; i < files.size(); ++i) {
     switch (loader.loadScript(_isolate, context, files[i], nullptr)) {

arangosh/Shell/V8ShellFeature.h

@@ -53,6 +53,7 @@ class V8ShellFeature final : public application_features::ApplicationFeature {
   
  private:
   std::string _startupDirectory;
+  std::string _clientModule;
   std::vector<std::string> _moduleDirectory;
   bool _currentModuleDirectory;
   uint64_t _gcInterval;

etc/arangodb3/arangoinspect.conf.in

@@ -0,0 +1,14 @@
+[console]
+pretty-print = true
+
+[log]
+@COMMENT_LOGFILE@file = -
+
+[server]
+endpoint = tcp://127.0.0.1:8529
+authentication = false
+ask-jwt-secret = true
+
+[javascript]
+startup-directory = @PKGDATADIR@/js
+client-module = inspector.js

etc/relative/arangoinspect.conf

@@ -0,0 +1,13 @@
+[console]
+pretty-print = true
+
+[server]
+authentication = false
+ask-jwt-secret = true
+
+[log]
+file = -
+
+[javascript]
+startup-directory = ./js
+client-module = inspector.js

js/client/inspector.js

@@ -0,0 +1,92 @@
+/* jshint -W051, -W020 */
+/* global global:true, window, require */
+'use strict';
+
+// /////////////////////////////////////////////////////////////////////////////
+// @brief ArangoDB Inspector
+// 
+// @file
+// 
+// DISCLAIMER
+// 
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+// 
+// Licensed under the Apache License, Version 2.0 (the "License")
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// 
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+// 
+// @author Dr. Frank Celler
+// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany
+// /////////////////////////////////////////////////////////////////////////////
+
+var arango;
+
+// global 'arango'
+global.arango = require('@arangodb').arango;
+
+// global 'db'
+global.db = require('@arangodb').db;
+
+// cleanup
+delete global.IS_EXECUTE_SCRIPT;
+delete global.IS_EXECUTE_STRING;
+delete global.IS_CHECK_SCRIPT;
+delete global.IS_UNIT_TESTS;
+delete global.IS_JS_LINT;
+
+// print banner
+(function() {
+  const internal = require('internal');
+
+  internal.print("    _                                  ___                           _");
+  internal.print("   / \\   _ __ __ _ _ __   __ _  ___   |_ _|_ __  ___ _ __   ___  ___| |_ ___  _ __");
+  internal.print("  / _ \\ | '__/ _` | '_ \\ / _` |/ _ \\   | || '_ \\/ __| '_ \\ / _ \\/ __| __/ _ \\| '__|");
+  internal.print(" / ___ \\| | | (_| | | | | (_| | (_) |  | || | | \\__ \\ |_) |  __/ (__| || (_) | |");
+  internal.print("/_/   \\_\\_|  \\__,_|_| |_|\\__, |\\___/  |___|_| |_|___/ .__/ \\___|\\___|\\__\\___/|_|");
+  internal.print("                         |___/                      |_|                         ");
+  internal.print("");
+})();
+
+// load rc file
+try {
+  // this will not work from within a browser
+  const __fs__ = require('fs');
+  const __rcf__ = __fs__.join(__fs__.home(), '.arangoinspect.rc');
+
+  if (__fs__.exists(__rcf__)) {
+    /* jshint evil: true */
+    const __content__ = __fs__.read(__rcf__);
+    eval(__content__);
+  }
+} catch (e) {
+  require('console').warn('arangoinspect.rc: %s', String(e));
+}
+
+// check connection success
+if (!arango.isConnected()) {
+  const internal = require('internal');
+  internal.print("FATAL cannot connect to server '" + arango.getEndpoint() + "'");
+  require("process").exit(1);
+}
+
+if (arango.lastErrorMessage()) {
+  const internal = require('internal');
+  internal.print("FATAL cannot connect to server '" + arango.getEndpoint() + "': "
+                 + arango.lastErrorMessage());
+  require("process").exit(1);
+}
+
+require("@arangodb/inspector");
+require("internal").exit();
+
+

js/client/modules/@arangodb/arangosh.js

@@ -1,4 +1,5 @@
 /* jshint strict: false */
+global.console = global.console || require('console');
 
 // //////////////////////////////////////////////////////////////////////////////
 // / @brief ArangoShell client API

js/client/modules/@arangodb/inspector.js

@@ -0,0 +1,732 @@
+/* global require, arango, print, db */
+'use strict';
+
+// /////////////////////////////////////////////////////////////////////////////
+// @brief ArangoDB Doctor
+// 
+// @file
+// 
+// DISCLAIMER
+// 
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+// 
+// Licensed under the Apache License, Version 2.0 (the "License")
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// 
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+// 
+// @author Dr. Frank Celler
+// @author Kaveh Vahedipour
+// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany
+// /////////////////////////////////////////////////////////////////////////////
+
+
+var internal = require('internal');
+var fs = require('fs');
+
+const executeExternalAndWait = internal.executeExternalAndWait;
+const _ = require("lodash");
+
+const servers = {};
+let possibleAgent = null;
+
+var healthRecord = {};
+
+function INFO() {
+  let args = Array.prototype.slice.call(arguments);
+  print.apply(print, ["INFO"].concat(args));
+}
+
+function WARN() {
+  let args = Array.prototype.slice.call(arguments);
+  print.apply(print, ["WARN"].concat(args));
+}
+
+function ERROR() {
+  let args = Array.prototype.slice.call(arguments);
+  print.apply(print, ["ERROR"].concat(args));
+}
+
+function loadAgencyConfig() {
+  var configuration = arango.GET("/_api/agency/config");
+  return configuration;
+}
+
+
+/**
+ * @brief Sort shard keys according to their numbers omitting startign 's' 
+ * 
+ * @param keys      Keys
+ */
+function sortShardKeys(keys) {
+  var ret = [];
+  // Get rid of 's' up front
+  keys.forEach(function(key, index, theArray) {
+    theArray[index] = key.substring(1);
+  });
+  // Sort keeping indices
+  var ind = range(0,keys.length-1);
+  ind.sort(function compare(i, j) {
+    return parseInt(keys[i]) > parseInt(keys[j]);
+  });
+  ind.forEach(function (i) {
+    ret.push('s'+keys[i]);
+  });
+  return ret;
+}
+
+
+function agencyInspector(obj) {
+
+  var nerrors = 0;
+  var nwarnings = 0;
+  var ndatabases = 0;
+  var ncollections = 0;
+  var nshards = 0;
+  var ncolinplannotincurrent = 0;
+
+  var report = {};
+
+  INFO('Analysing agency dump ...');
+  
+  // Must be array with length 1
+  if (obj.length !== 1) {
+    ERROR('agency dump must be an array with a single object inside');
+    process.exit(1);
+  }
+
+  // Check for /arango and assign
+  if (!obj[0].hasOwnProperty('arango')) {
+    ERROR('agency must have attribute "arango" as root');
+    process.exit(1);
+  }
+  const agency = obj[0]['arango'];
+
+  // Must have Plan, Current, Supervision, Target, Sync
+  if (!agency.hasOwnProperty('Plan')) {
+    ERROR('agency must have attribute "arango/Plan" object');
+    process.exit(1);  
+  }
+  const plan = agency.Plan;
+  if (!plan.hasOwnProperty('Version')) {
+    ERROR('plan has no version');
+    process.exit(1);    
+  }
+  if (!agency.hasOwnProperty('Current')) {
+    ERROR('agency must have attribute "arango/Current" object');
+    process.exit(1);  
+  }
+  const current = agency.Current;
+
+  // Start sanity of plan
+  INFO('Plan (version ' + plan.Version+ ')');
+
+  // Planned databases check if also in collections and current
+  INFO("  Databases");
+  report.Databases = {};
+  if (!plan.hasOwnProperty('Databases')) {
+    ERROR('no databases in plan');
+    return;
+  }
+  Object.keys(plan.Databases).forEach(function(database) {
+    INFO('    ' + database);
+    report.Databases[database] = {};
+    if (!plan.Collections.hasOwnProperty(database)) {
+      WARN('found planned database "' + database + '" without planned collectinos');
+    }
+    if (!current.Databases.hasOwnProperty(database)) {
+      WARN('found planned database "' + database + '" missing in "Current"');
+    }
+  });
+
+  INFO("  Collections");
+
+  // Planned collections
+  if (!plan.hasOwnProperty('Collections')) {
+    ERROR('no collections in plan');
+    process.exit(1);  
+  }
+
+  var warned = false;
+  Object.keys(plan.Collections).forEach(function(database) {
+    ++ndatabases;
+    INFO('    ' + database);
+
+    if (!plan.Databases.hasOwnProperty(database)) {
+      ERROR('found planned collections in unplanned database ' + database);
+    }
+
+    Object.keys(plan.Collections[database]).forEach(function(collection) {
+      report.Databases[database]
+      [plan.Collections[database][collection].name] = {};
+
+      ++ncollections;
+      const col = plan.Collections[database][collection];
+      INFO('      ' + col.name);
+      const distributeShardsLike = col.distributeShardsLike;
+      var myShardKeys = sortShardKeys(Object.keys(col.shards));
+
+      if (distributeShardsLike && distributeShardsLike !== "") {      
+        const prototype = plan.Collections[database][distributeShardsLike];
+        if (prototype.replicationFactor !== col.replicationFactor) {
+          ERROR('distributeShardsLike: replicationFactor mismatch');
+        }
+        if (prototype.numberOfShards !== col.numberOfShards) {
+          ERROR('distributeShardsLike: numberOfShards mismatch');
+        }      
+        var prototypeShardKeys = sortShardKeys(Object.keys(prototype.shards));
+        var ncshards = myShardKeys.length;
+        if (prototypeShardKeys.length !== ncshards) {
+          ERROR('distributeShardsLike: shard map mismatch');
+        }
+        for (var i = 0; i < ncshards; ++i) {
+          const shname = myShardKeys[i];
+          const ccul = current.Collections[database];     
+          if (JSON.stringify(col.shards[shname]) !==
+              JSON.stringify(prototype.shards[prototypeShardKeys[i]])) {
+            ERROR(
+              'distributeShardsLike: planned shard map mismatch between "/arango/Plan/Collections/' +
+                database + '/' + collection + '/shards/' + myShardKeys[i] +
+                '" and " /arango/Plan/Collections/'  + database + '/' + distributeShardsLike +
+                '/shards/' + prototypeShardKeys[i] + '"');
+            INFO('      ' + JSON.stringify(col.shards[shname]));
+            INFO('      ' + JSON.stringify(prototype.shards[prototypeShardKeys[i]]));
+          }
+        }
+      }
+
+      myShardKeys.forEach(function(shname) {
+        ++nshards;
+        if (current.Collections.hasOwnProperty(database)) {
+          if (!current.Collections[database].hasOwnProperty(collection)) {
+            WARN('missing collection "' + collection + '" in current');
+          } else {
+            if (!current.Collections[database][collection].hasOwnProperty(shname)) {
+              WARN('missing shard "' + shname + '" in current');           
+            }
+            const shard = current.Collections[database][collection][shname];
+            if (shard) {
+              if (JSON.stringify(shard.servers) !== JSON.stringify(col.shards[shname])) {
+                if (shard.servers[0] !== col.shards[shname][0]) {
+                  ERROR('/arango/Plan/Collections/' + database + '/' + collection + '/shards/'
+                        + shname + ' and /arango/Current/Collections/' + database + '/'
+                        + collection + '/' + shname + '/servers do not match');
+                } else {
+                  var sortedPlan = (shard.servers).sort();
+                  var sortedCurrent = (col.shards[shname]).sort();
+                  if (JSON.stringify(sortedPlan) === JSON.stringify(sortedCurrent)) {
+                    WARN('/arango/Plan/Collections/' + database + '/' + collection + '/shards/'
+                         + shname + ' and /arango/Current/Collections/' + database + '/'
+                         + collection + '/' + shname + '/servers follower do not match in order');
+                  }
+                }
+              }
+            }
+          }
+        } else {
+          if (!warned) {
+            WARN('planned database "' + database + '" missing entirely in current');
+            warned = true;
+          }
+        }
+      });
+
+    });
+  });
+
+  report.servers = {};
+  INFO('Server health');
+  INFO('  DB Servers');
+  report.servers.dbservers = {};
+  const supervision = agency.Supervision;
+  const target = agency.Target;
+  var servers = plan.DBServers;
+  Object.keys(servers).forEach(function(serverId) {
+    if (!target.MapUniqueToShortID.hasOwnProperty(serverId)) {
+      WARN('incomplete planned db server ' + serverId + ' is missing in "Target"');
+    } else { 
+      INFO('    ' + serverId + '(' + target.MapUniqueToShortID[serverId].ShortName + ')');
+      if (!supervision.Health.hasOwnProperty(serverId)) {
+        ERROR('planned db server ' + serverId + ' missing in supervision\'s health records.');
+      }
+      servers[serverId] = supervision.Health[serverId];
+      report.servers.dbservers[serverId] = {name: target.MapUniqueToShortID[serverId].ShortName, status : servers[serverId].Status};
+      if (servers[serverId].Status === "BAD") {
+        WARN('bad db server ' + serverId + '(' + servers[serverId].ShortName+ ')');
+      } else if (servers[serverId].Status === "FAILED") {
+        WARN('*** FAILED *** db server ' + serverId + '(' + servers[serverId].ShortName+ ')');
+      }
+    }
+  });
+
+  INFO('  Coordinators');
+  report.servers.coordinators = {};
+  servers = plan.Coordinators;
+  Object.keys(servers).forEach(function(serverId) {
+    if (!target.MapUniqueToShortID.hasOwnProperty(serverId)) {
+      WARN('incomplete planned db server ' + serverId + ' is missing in "Target"');
+    } else { 
+      INFO('    ' + serverId + '(' + target.MapUniqueToShortID[serverId].ShortName + ')');
+      report.servers.coordinators[serverId] = {name: target.MapUniqueToShortID[serverId].ShortName, status : servers[serverId].Status};
+
+      if (!supervision.Health.hasOwnProperty(serverId)) {
+        WARN('planned coordinator ' + serverId + ' missing in supervision\'s health records.');
+      }
+      servers[serverId] = supervision.Health[serverId];
+      if (servers[serverId].Status !== "GOOD") {
+        WARN('*** FAILED *** coordinator ' + serverId + '(' + servers[serverId].ShortName+ ')');
+      }
+    }
+  });
+
+  const jobs = {
+    ToDo: target.ToDo,
+    Pending: target.Pending,
+    Finished: target.Finished,
+    Failed: target.Failed
+  };
+  var njobs = [];
+  var nall;
+  Object.keys(jobs).forEach(function (state) {
+    njobs.push(Object.keys(jobs[state]).length);
+  });
+
+  var i = 0;
+  INFO('Supervision activity');
+  INFO('  Jobs: ' + nall + '(' +
+       'To do: ' + njobs[0] + ', ' +
+       'Pending: ' + njobs[1] + ', ' +
+       'Finished: ' + njobs[2] + ', ' +
+       'Failed: ' + njobs[3] + ')'
+      );
+
+  report.jobs = { todo: njobs[0], pending: njobs[1], finished: njobs[2], failed: njobs[3]};
+  
+  INFO('Summary');
+  if (nerrors > 0) {
+    ERROR('  ' + nerrors + ' errors');
+  }
+  if (nwarnings > 0) {
+    WARN('  ' + nwarnings + ' warnings');
+  }
+  INFO('  ' + ndatabases + ' databases');
+  INFO('  ' + ncollections + ' collections ');
+  INFO('  ' + nshards + ' shards ');
+  INFO('... agency analysis finished.');
+
+  return report;
+  
+}
+
+
+/**
+ * @brief Create an integer range as [start,end]
+ * 
+ * @param start     Start of range 
+ * @param end       End of range
+ */
+function range(start, end) {
+  return Array(end - start + 1).fill().map((_, idx) => start + idx);
+}
+
+function loadAgency(conn, seen) {
+
+  var agencyDump = conn.POST("/_api/agency/read", '[["/"]]');
+  seen[conn.getEndpoint()] = true;
+
+  if (agencyDump.code === 404) {
+    WARN("not talking to an agent, got: " + JSON.stringify(agencyDump));
+    return null;
+  }
+
+  if (agencyDump.code === 307) {
+    var red = conn.POST_RAW("/_api/agency/read", '[["/"]]');
+
+    if (red.code === 307) {
+      INFO("got redirect to " + red.headers.location);
+
+      let leader = red.headers.location;
+      let reg = /^(http|https):\/\/(.*)\/_api\/agency\/read$/;
+      let m = reg.exec(leader);
+
+      if (m === null) {
+        WARN("unknown redirect " + leader);
+        return null;
+      } else {
+        if (m[1] === "http") {
+          leader = "tcp://" + m[2];
+        }
+        else if (m[1] === "https") {
+          leader = "ssl://" + m[2];
+        }
+
+        if (leader in seen) {
+          WARN("cannot find leader, tried: " + Object.keys(seen).join(", "));
+          return null;
+        }
+
+        INFO("switching to " + leader);
+
+        console.log("http+" + leader);
+        console.log(conn.getEndpoint());
+        if ("http+" + leader != conn.getEndpoint()) {
+          conn.reconnect(leader, "_system");
+        }
+
+        return loadAgencyConfig(conn, seen);
+      }
+    }
+  }
+
+  if (agencyDump.code !== undefined) {
+    WARN("failed to load agency, got: " + JSON.stringify(agencyDump));
+    return null;
+  }
+
+  return agencyDump;
+  
+}
+
+function defineServer(type, id, source) {
+  if (id in servers) {
+    _.merge(servers[id].source, source);
+  } else {
+    servers[id] = {
+      type: type,
+      id: id,
+      source: source
+    };
+  }
+}
+
+function defineServerEndpoint(id, endpoint) {
+  const server = servers[id];
+
+  if ('endpoint' in server) {
+    if (server.endpoint !== endpoint) {
+      INFO("changing endpoint for " + id + " from "
+           + server.endpoint + " to " + endpoint);
+    }
+  }
+
+  server.endpoint = endpoint;
+}
+
+function defineServerStatus(id, status) {
+  const server = servers[id];
+
+  if ('status' in server) {
+    if (server.status !== status) {
+      INFO("changing status for " + id + " from "
+           + server.status + " to " + status);
+    }
+  }
+
+  server.status = status;
+}
+
+function defineAgentLeader(id, leading) {
+  const server = servers[id];
+
+  if ('leading' in server) {
+    if (server.leading !== leading) {
+      INFO("changing leading for " + id + " from "
+           + server.leading + " to " + leading);
+    }
+  }
+
+  server.leading = leading;
+}
+
+function defineAgentFromStatus(status, endpoint) {
+  let id = status.agent.id;
+  let leader = status.agent.leaderId;
+
+  defineServer('AGENT', id, { status: endpoint });
+  defineServerEndpoint(id, endpoint);
+
+  arango.reconnect(endpoint, "_system");
+
+  const cfg = db.configuration.toArray()[0].cfg;
+
+  _.forEach(cfg.active, function(id) {
+    defineServer('AGENT', id, { active: endpoint });
+  });
+
+  _.forEach(cfg.pool, function(loc, id) {
+    defineServer('AGENT', id, { pool: endpoint });
+    defineServerEndpoint(id, loc);
+    defineAgentLeader(id, id === leader);
+  });
+
+  defineAgentLeader(id,status.agent.leading);
+}
+
+function definePrimaryFromStatus(status, endpoint) {
+  let id = status.serverInfo.serverId;
+
+  defineServer('PRIMARY', id, { status: endpoint });
+  defineServerEndpoint(id, endpoint);
+
+  let agentEndpoints = status.agency.agencyComm.endpoints;
+
+  if (0 < agentEndpoints.length) {
+    possibleAgent = agentEndpoints[0];
+  } else {
+    console.error("Failed to find an agency endpoint");
+  }
+}
+
+function defineCoordinatorFromStatus(status, endpoint) {
+  let id = status.serverInfo.serverId;
+
+  defineServer('COORDINATOR', id, { status: endpoint });
+  defineServerEndpoint(id, endpoint);
+
+  let agentEndpoints = status.agency.agencyComm.endpoints;
+
+  if (0 < agentEndpoints.length) {
+    possibleAgent = agentEndpoints[0];
+  } else {
+    console.error("Failed to find an agency endpoint");
+  }
+}
+
+function defineSingleFromStatus(status, endpoint) {
+  defineServer('SINGLE', 'SINGLE', { status: endpoint });
+  defineServerEndpoint('SINGLE', endpoint);
+  if (status.hasOwnProperty('agency')) {
+    let agentEndpoints = status.agency.agencyComm.endpoints;
+    
+    if (0 < agentEndpoints.length) {
+      possibleAgent = agentEndpoints[0];
+    } else {
+      console.error("Failed to find an agency endpoint");
+    }
+  }
+}
+
+function serverBasics(conn) {
+  if (!conn) {
+    conn = arango;
+  }
+
+  const status = conn.GET("/_admin/status");
+  const role = status.serverInfo.role;
+
+  if (role === 'AGENT') {
+    defineAgentFromStatus(status, conn.getEndpoint());
+  } else if (role === 'PRIMARY') {
+    definePrimaryFromStatus(status, conn.getEndpoint());
+  } else if (role === 'COORDINATOR') {
+    defineCoordinatorFromStatus(status);
+  } else if (role === 'SINGLE') {
+    defineSingleFromStatus(status);
+  } else {
+    return "unknown";
+  }
+
+  return role;
+}
+
+function locateServers(plan) {
+  let health = plan[0].arango.Supervision.Health;
+  let cluster = plan[0].arango.Cluster;
+
+  _.forEach(health, function(info, id) {
+    const type = id.substr(0, 4);
+    
+    if (type === "PRMR") {
+      defineServer('PRIMARY', id, { supervision: cluster });
+      defineServerEndpoint(id, info.Endpoint);
+      defineServerStatus(id, info.Status);
+    } else if (type === "CRDN") {
+      defineServer('SINGLE', id, { supervision: cluster });
+      defineServerEndpoint(id, info.Endpoint);
+      defineServerStatus(id, info.Status);
+    } else if (type === "SNGL") {
+      defineServer('SINGLE', id, { supervision: cluster });
+      defineServerEndpoint(id, info.Endpoint);
+      defineServerStatus(id, info.Status);
+    } 
+  });
+}
+
+function listServers() {
+  return servers;
+}
+
+function getServerData(arango) {
+  var current = arango.getEndpoint();
+  var servers = listServers();
+  var report = {};
+  INFO('Collecting diagnostics from all servers ... ');
+  var nservers = Object.keys(servers).length;
+  Object.keys(servers).forEach(
+
+    function (server) {
+
+      if (nservers == 1 || servers.lengthserver !== "SINGLE") {
+        try {
+
+          if (servers[server].endpoint !== undefined) {
+            if (arango.getEndpoint() != "http+" + servers[server].endpoint) {
+              arango.reconnect(servers[server].endpoint, '_system');
+            }
+          }
+          
+          const version = arango.GET('_api/version'); // version api
+          const log = arango.GET('_admin/log').text;  // log api
+          const statistics = arango.GET('_admin/statistics').text;  // log api
+          var agencyConfig;
+          if (server.startsWith("AGNT")) {
+            agencyConfig = arango.GET('_api/agency/config');
+          }
+          const status = arango.GET('_admin/status');
+          var tmp = executeExternalAndWait(
+            '/bin/bash', ['-c', 'dmesg | tee /tmp/inspector-dmesg.out > /dev/null']);
+          const dmesg = fs.readFileSync('/tmp/inspector-dmesg.out', 'utf8');
+          tmp = executeExternalAndWait(
+            '/bin/bash', ['-c', 'df -h | tee /tmp/inspector-df.out > /dev/null']);
+          const df = fs.readFileSync('/tmp/inspector-df.out', 'utf8');
+          tmp = executeExternalAndWait(
+            '/bin/bash', ['-c', 'cat /proc/meminfo | tee /tmp/inspector-meminfo.out > /dev/null']);
+          const meminfo = fs.readFileSync('/tmp/inspector-meminfo.out', 'utf8');
+          tmp = executeExternalAndWait(
+            '/bin/bash', ['-c', 'uptime | tee /tmp/inspector-uptime.out > /dev/null']);
+          const uptime = fs.readFileSync('/tmp/inspector-uptime.out', 'utf8');
+          tmp = executeExternalAndWait(
+            '/bin/bash', ['-c', 'uname -a | tee /tmp/inspector-uname.out > /dev/null']);
+          const uname = fs.readFileSync('/tmp/inspector-uname.out', 'utf8');
+          var top;
+          if (status.pid !== undefined) {
+            tmp = executeExternalAndWait(
+              '/bin/bash', ['-c', 'top -b -H -p ' + status.pid + ' -n 1 | tee /tmp/inspector-top.out > /dev/null']);
+            top = fs.readFileSync('/tmp/inspector-top.out', 'utf8');
+          }
+
+          var local = {};
+          try {
+            var localDBs = db._databases();
+            localDBs.forEach( function(localDB) {
+              db._useDatabase(localDB);
+              local[localDB] = {};
+              var localCols = db._collections();
+              localCols.forEach( function(localCol) {
+                var colName = localCol.name();
+                local[localDB][colName] = {};
+                Object.keys(localCol.properties()).forEach( function(property) {
+                  local[localDB][colName][property] = localCol.properties()[property];
+                });
+                local[localDB][colName].index = localCol.getIndexes();
+                local[localDB][colName].count = localCol.count();
+              });});
+            db._useDatabase('_system');
+          } catch (e) {}
+          
+          // report this server
+          report[server] = {
+            version:version, log:log, dmesg:dmesg, statistics:statistics,
+            status:status, df:df, uptime:uptime, uname:uname, meminfo:meminfo,
+            local:local};
+
+          if (agencyConfig !==  undefined) {
+            report[server].config = agencyConfig;
+          }
+          if (top !==  undefined) {
+            report[server].top = top;
+          }
+
+        } catch (e) {
+          print(e);
+        }
+      }
+    });
+  if (Object.keys(servers).length > 1) {
+    if (current != arango.getEndpoint()) {
+      arango.reconnect(current, '_system');
+    }
+  }
+  INFO('... dignostics collected.');
+  return report;
+}
+
+(function() {
+  try {
+    var type = serverBasics();
+
+    if (type !== 'AGENT') {
+      if (possibleAgent !== null) {
+        arango.reconnect(possibleAgent, "_system");
+        serverBasics();
+      }
+    }
+
+    if (possibleAgent !== null) {
+      
+      // Agency dump and analysis
+      var agencyConfig = loadAgencyConfig();
+      var agencyDump = {};
+      var tries = 0;
+      while (true) {
+        if (agencyDump.leaderId !== "") {
+          if (agencyConfig.configuration.pool.hasOwnProperty(agencyConfig.leaderId)) {
+            if ("http+" + agencyConfig.configuration.pool[agencyConfig.leaderId] !=
+                arango.getEndpoint()) {
+              arango.reconnect(
+                agencyConfig.configuration.pool[agencyConfig.leaderId], "_system");
+            }
+            agencyDump = loadAgency(arango, {});
+            break;
+          } else { // Leader is not in pool? Fatal error;
+            console.error("Fatal error: " + agencyDump.leaderId +
+                          " is not a member of the agent pool " +
+                          JSON.stringify(agencyConfig.configuration.pool));
+            console.error("This deployment needs immediate administrative attention.");
+            possibleAgent = null;
+            return;
+          }
+        } else {
+          if (tries < 100) {
+            tries++;
+            internal.wait(1);
+          } else {
+            console.error("Error: Agency cannot elect a leader configured as " +
+                          JSON.stringify(agencyConfig));
+            console.error("This deployment needs immediate administrative attention.");
+            possibleAgent = null;
+            return;
+          }
+        }
+      }
+      
+      if (agencyDump !== null) {
+        locateServers(agencyDump);
+      }
+
+      healthRecord['analysis'] = agencyInspector(agencyDump);
+      healthRecord['agency'] = agencyDump[0].arango;
+
+    }
+    // Get all sorts of meta data from all servers
+    healthRecord['servers'] = getServerData(arango);
+
+    const ofname = 'arango-inspector.json';
+    require('fs').writeFileSync(ofname, JSON.stringify(healthRecord));
+
+    INFO("Report written to " + ofname + ".");
+    
+  } catch (e) {
+    print(e);
+  }
+}());
+

js/common/bootstrap/modules/internal.js

@@ -7,31 +7,33 @@ global.DEFINE_MODULE('internal', (function () {
 
   const exports = {};
 
-  // //////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////////////////////////////////////////////////////
-  // / @brief module "internal"
+  // @brief module "internal"
-  // /
+  //
-  // / @file
+  // @file
-  // /
+  //
-  // / DISCLAIMER
+  // DISCLAIMER
-  // /
+  //
-  // / Copyright 2004-2013 triAGENS GmbH, Cologne, Germany
+  // Copyright 2018 ArangoDB GmbH, Cologne, Germany
-  // /
+  // Copyright 2004-2013 triAGENS GmbH, Cologne, Germany
-  // / Licensed under the Apache License, Version 2.0 (the "License")
+  //
-  // / you may not use this file except in compliance with the License.
+  // Licensed under the Apache License, Version 2.0 (the "License")
-  // / You may obtain a copy of the License at
+  // you may not use this file except in compliance with the License.
-  // /
+  // You may obtain a copy of the License at
-  // /     http://www.apache.org/licenses/LICENSE-2.0
+  //
-  // /
+  //     http://www.apache.org/licenses/LICENSE-2.0
-  // / Unless required by applicable law or agreed to in writing, software
+  //
-  // / distributed under the License is distributed on an "AS IS" BASIS,
+  // Unless required by applicable law or agreed to in writing, software
-  // / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  // distributed under the License is distributed on an "AS IS" BASIS,
-  // / See the License for the specific language governing permissions and
+  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  // / limitations under the License.
+  // See the License for the specific language governing permissions and
-  // /
+  // limitations under the License.
-  // / Copyright holder is triAGENS GmbH, Cologne, Germany
+  //
-  // /
+  // Copyright holder is ArangoDB GmbH, Cologne, Germany
-  // / @author Dr. Frank Celler
+  //
-  // / @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany
+  // @author Dr. Frank Celler
+  // @author Copyright 2018, ArangoDB GmbH, Cologne, Germany
+  // @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany
   // //////////////////////////////////////////////////////////////////////////////
 
   // //////////////////////////////////////////////////////////////////////////////
@@ -748,6 +750,14 @@ global.DEFINE_MODULE('internal', (function () {
     global.SYS_UNIT_TESTS_RESULT = value;
   };
 
+  // end process
+  if (global.SYS_EXIT) {
+    exports.exit = global.SYS_EXIT;
+    delete global.SYS_EXIT;
+  } else {
+    exports.exit = function() {};
+  }
+
   // //////////////////////////////////////////////////////////////////////////////
   // / @brief structured to flat commandline arguments
   // / @param longOptsEqual whether long-options are in the type --opt=value

js/common/bootstrap/modules/process.js

@@ -12,19 +12,25 @@ global.DEFINE_MODULE('process', (function () {
   var exports = new EventEmitter();
 
   exports.env = internal.env;
+
   exports.argv = [];
+
   exports.stdout = {
     isTTY: internal.COLOR_OUTPUT,
     write(text) {
       console.infoLines(text);
     }
   };
+
   exports.cwd = function () {
     return fs.makeAbsolute('');
   };
+
   exports.nextTick = function (fn) {
     fn();
   };
 
+  exports.exit = internal.exit;
+
   return exports;
 }()));

scripts/startLeaderFollower.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 params=("$@")
 
-rm -rf cluster
+rm -rf active
 if [ -d cluster-init ];then
   echo "== creating cluster directory from existing cluster-init directory"
-  cp -a cluster-init cluster
+  cp -a active-init active
 else
   echo "== creating fresh directory"
-  mkdir -p cluster || { echo "failed to create cluster directory"; exit 1; }
+  mkdir -p active || { echo "failed to create cluster directory"; exit 1; }
   #if we want to restart we should probably store the parameters line wise
 fi
 
@@ -27,8 +27,8 @@ else
   exit 1
 fi
 
-if [[ -f cluster/startup_parameters ]];then
+if [[ -f active/startup_parameters ]];then
-    string="$(< cluster/startup_parameters)"
+    string="$(< active/startup_parameters)"
     if [[ -z "${params[@]}" ]]; then
         params=( $string )
     else
@@ -41,7 +41,7 @@ if [[ -f cluster/startup_parameters ]];then
 else
     #store parmeters
     if [[ -n "${params[@]}" ]]; then
-      echo "${params[@]}" > cluster/startup_parameters
+      echo "${params[@]}" > active/startup_parameters
     fi
 fi
 
@@ -88,11 +88,11 @@ NATH=$(( $NRSINGLESERVERS + $NRAGENTS ))
 ENDPOINT=[::]
 ADDRESS=[::1]
 
-rm -rf cluster
+rm -rf active
-if [ -d cluster-init ];then
+if [ -d active-init ];then
-  cp -a cluster-init cluster
+  cp -a active-init active
 fi
-mkdir -p cluster
+mkdir -p active
 
 if [ -z "$JWT_SECRET" ];then
   AUTHENTICATION="--server.authentication false"
@@ -139,17 +139,17 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do
         --agency.supervision-frequency $SFRE \
         --agency.supervision-grace-period 5.0 \
         --agency.wait-for-sync false \
-        --database.directory cluster/data$port \
+        --database.directory active/data$port \
         --javascript.enabled false \
         --server.endpoint $TRANSPORT://$ENDPOINT:$port \
         --server.statistics false \
         --server.threads 16 \
-        --log.file cluster/$port.log \
+        --log.file active/$port.log \
         --log.level $LOG_LEVEL_AGENCY \
         $STORAGE_ENGINE \
         $AUTHENTICATION \
         $SSLKEYFILE \
-        | tee cluster/$PORT.stdout 2>&1 &
+        | tee active/$PORT.stdout 2>&1 &
 done
 
 start() {
@@ -158,28 +158,28 @@ start() {
 
     TYPE=$1
     PORT=$2
-    mkdir cluster/data$PORT cluster/apps$PORT 
+    mkdir active/data$PORT active/apps$PORT 
     echo Starting $TYPE on port $PORT
     $CMD \
         -c none \
-        --database.directory cluster/data$PORT \
+        --database.directory active/data$PORT \
         --cluster.agency-endpoint $TRANSPORT://$ENDPOINT:$AG_BASE \
         --cluster.my-address $TRANSPORT://$ADDRESS:$PORT \
         --server.endpoint $TRANSPORT://$ENDPOINT:$PORT \
         --cluster.my-role $ROLE \
         --replication.active-failover true \
-        --log.file cluster/$PORT.log \
+        --log.file active/$PORT.log \
         --log.level $LOG_LEVEL \
         --server.statistics true \
         --server.threads 5 \
         --javascript.startup-directory $SRC_DIR/js \
         --javascript.module-directory $SRC_DIR/enterprise/js \
-        --javascript.app-path cluster/apps$PORT \
+        --javascript.app-path active/apps$PORT \
         --log.level $LOG_LEVEL_CLUSTER \
         $STORAGE_ENGINE \
         $AUTHENTICATION \
         $SSLKEYFILE \
-        | tee cluster/$PORT.stdout 2>&1 &
+        | tee active/$PORT.stdout 2>&1 &
 }
 
 PORTTOPDB=`expr $SS_BASE + $NRSINGLESERVERS - 1`
@@ -209,7 +209,7 @@ for p in `seq $SS_BASE $PORTTOPDB` ; do
     testServer $p
 done
 
-echo Done, your cluster is ready at
+echo Done, your active failover pair is ready at
 for p in `seq $SS_BASE $PORTTOPDB` ; do
     echo "   ${BUILD}/bin/arangosh --server.endpoint $TRANSPORT://[::1]:$p"
 done