diff --git a/arangod/GeneralServer/RestHandlerFactory.cpp b/arangod/GeneralServer/RestHandlerFactory.cpp index 0ed12b3504..fb7db4d73e 100644 --- a/arangod/GeneralServer/RestHandlerFactory.cpp +++ b/arangod/GeneralServer/RestHandlerFactory.cpp @@ -95,9 +95,11 @@ RestHandler* RestHandlerFactory::createHandler( case ServerState::Mode::TRYAGAIN: { if (path.find("/_admin/shutdown") == std::string::npos && path.find("/_admin/cluster/health") == std::string::npos && + path.find("/_admin/log") == std::string::npos && path.find("/_admin/server/role") == std::string::npos && path.find("/_admin/server/availability") == std::string::npos && path.find("/_admin/status") == std::string::npos && + path.find("/_admin/statistics") == std::string::npos && path.find("/_api/agency/agency-callbacks") == std::string::npos && path.find("/_api/cluster/") == std::string::npos && path.find("/_api/replication") == std::string::npos && diff --git a/arangod/RestHandler/RestStatusHandler.cpp b/arangod/RestHandler/RestStatusHandler.cpp index ba87e3cabd..2f4dbfaa2b 100644 --- a/arangod/RestHandler/RestStatusHandler.cpp +++ b/arangod/RestHandler/RestStatusHandler.cpp @@ -33,6 +33,10 @@ #include +#if defined(TRI_HAVE_POSIX_THREADS) +#include +#endif + #include #include @@ -56,6 +60,10 @@ RestStatus RestStatusHandler::execute() { result.add("server", VPackValue("arango")); result.add("version", VPackValue(ARANGODB_VERSION)); +#if defined(TRI_HAVE_POSIX_THREADS) + result.add("pid", VPackValue(getpid())); +#endif + #ifdef USE_ENTERPRISE result.add("license", VPackValue("enterprise")); #else diff --git a/arangosh/CMakeLists.txt b/arangosh/CMakeLists.txt index cd049c5681..62a1153437 100644 --- a/arangosh/CMakeLists.txt +++ b/arangosh/CMakeLists.txt @@ -372,3 +372,13 @@ install_command_alias(${BIN_ARANGOSH} foxx-manager) install_config(foxx-manager) + +################################################################################ +## arangoinspect +################################################################################ + +install_command_alias(${BIN_ARANGOSH} + ${CMAKE_INSTALL_BINDIR} + arangoinspect) + +install_config(arangoinspect) diff --git a/arangosh/Shell/ShellFeature.cpp b/arangosh/Shell/ShellFeature.cpp index 31c751544d..4547f72581 100644 --- a/arangosh/Shell/ShellFeature.cpp +++ b/arangosh/Shell/ShellFeature.cpp @@ -32,8 +32,8 @@ using namespace arangodb; using namespace arangodb::basics; using namespace arangodb::options; -ShellFeature::ShellFeature( - application_features::ApplicationServer* server, int* result) +ShellFeature::ShellFeature(application_features::ApplicationServer* server, + int* result) : ApplicationFeature(server, "Shell"), _jslint(), _result(result), @@ -123,15 +123,18 @@ void ShellFeature::validateOptions( } if (1 < n) { - LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "you cannot specify more than one type (" - << "jslint, execute, execute-string, check-syntax, unit-tests)"; + LOG_TOPIC(ERR, arangodb::Logger::FIXME) + << "you cannot specify more than one type (" + << "jslint, execute, execute-string, check-syntax, unit-tests)"; } } void ShellFeature::start() { *_result = EXIT_FAILURE; - V8ShellFeature* shell = application_features::ApplicationServer::getFeature("V8Shell"); + V8ShellFeature* shell = + application_features::ApplicationServer::getFeature( + "V8Shell"); bool ok = false; diff --git a/arangosh/Shell/V8ClientConnection.cpp b/arangosh/Shell/V8ClientConnection.cpp index 02b2e54751..382e4383a0 100644 --- a/arangosh/Shell/V8ClientConnection.cpp +++ b/arangosh/Shell/V8ClientConnection.cpp @@ -49,7 +49,7 @@ using namespace arangodb::basics; using namespace arangodb::httpclient; using namespace arangodb::import; -std::string V8ClientConnection::JWT_SECRET = ""; +std::shared_ptr V8ClientConnection::JWT_SECRET = nullptr; std::string V8ClientConnection::jwtToken(std::string const& secret) { VPackBuilder headerBuilder; @@ -106,8 +106,8 @@ void V8ClientConnection::init( params.setLocationRewriter(this, &rewriteLocation); params.setUserNamePassword("/", _username, _password); - if (!JWT_SECRET.empty()) { - params.setJwt(jwtToken(JWT_SECRET)); + if (JWT_SECRET != nullptr) { + params.setJwt(jwtToken(*JWT_SECRET)); } _client.reset(new SimpleHttpClient(connection, params)); @@ -434,20 +434,22 @@ static void ClientConnection_reconnect( std::string password; if (args.Length() < 4) { - ConsoleFeature* console = + if (V8ClientConnection::jwtSecret() == nullptr) { + ConsoleFeature* console = ApplicationServer::getFeature("Console"); - - if (console->isEnabled()) { - password = console->readPassword("Please specify a password: "); - } else { - std::cout << "Please specify a password: " << std::flush; - password = ConsoleFeature::readPassword(); - std::cout << std::endl << std::flush; + + if (console->isEnabled()) { + password = console->readPassword("Please specify a password: "); + } else { + std::cout << "Please specify a password: " << std::flush; + password = ConsoleFeature::readPassword(); + std::cout << std::endl << std::flush; + } } } else { password = TRI_ObjectToString(isolate, args[3]); } - + bool warnConnect = true; if (args.Length() > 4) { warnConnect = TRI_ObjectToBoolean(args[4]); diff --git a/arangosh/Shell/V8ClientConnection.h b/arangosh/Shell/V8ClientConnection.h index e4d368d8ed..9cbdd02c5b 100644 --- a/arangosh/Shell/V8ClientConnection.h +++ b/arangosh/Shell/V8ClientConnection.h @@ -49,11 +49,12 @@ class V8ClientConnection { V8ClientConnection& operator=(V8ClientConnection const&) = delete; public: - static void setJwtSecret(std::string const& jwtSecret) { JWT_SECRET = jwtSecret; } + static void setJwtSecret(std::string const& jwtSecret) { JWT_SECRET = std::make_shared(jwtSecret); } + static std::shared_ptr jwtSecret() { return JWT_SECRET; } static std::string jwtToken(std::string const& secret); private: - static std::string JWT_SECRET; + static std::shared_ptr JWT_SECRET; public: V8ClientConnection( diff --git a/arangosh/Shell/V8ShellFeature.cpp b/arangosh/Shell/V8ShellFeature.cpp index 06fc717344..02b1cf0afd 100644 --- a/arangosh/Shell/V8ShellFeature.cpp +++ b/arangosh/Shell/V8ShellFeature.cpp @@ -54,10 +54,13 @@ using namespace arangodb::basics; using namespace arangodb::options; using namespace arangodb::rest; +static std::string const DEFAULT_CLIENT_MODULE = "client.js"; + V8ShellFeature::V8ShellFeature(application_features::ApplicationServer* server, std::string const& name) : ApplicationFeature(server, "V8Shell"), _startupDirectory("js"), + _clientModule(DEFAULT_CLIENT_MODULE), _currentModuleDirectory(true), _gcInterval(50), _name(name), @@ -78,6 +81,10 @@ void V8ShellFeature::collectOptions(std::shared_ptr options) { "startup paths containing the Javascript files", new StringParameter(&_startupDirectory)); + options->addHiddenOption("--javascript.client-module", + "client module to use at startup", + new StringParameter(&_clientModule)); + options->addHiddenOption( "--javascript.module-directory", "additional paths containing JavaScript modules", @@ -196,17 +203,18 @@ bool V8ShellFeature::printHello(V8ClientConnection* v8connection) { // http://www.network-science.de/ascii/ Font: ogre if (!_console->quiet()) { - std::string g = ShellColorsFeature::SHELL_COLOR_GREEN; - std::string r = ShellColorsFeature::SHELL_COLOR_RED; - std::string z = ShellColorsFeature::SHELL_COLOR_RESET; + if (_clientModule == DEFAULT_CLIENT_MODULE) { + std::string g = ShellColorsFeature::SHELL_COLOR_GREEN; + std::string r = ShellColorsFeature::SHELL_COLOR_RED; + std::string z = ShellColorsFeature::SHELL_COLOR_RESET; - if (!_console->colors()) { - g = ""; - r = ""; - z = ""; - } + if (!_console->colors()) { + g = ""; + r = ""; + z = ""; + } - // clang-format off + // clang-format off _console->printLine(""); _console->printLine(g + " " + r + " _ " + z); @@ -217,17 +225,18 @@ bool V8ShellFeature::printHello(V8ClientConnection* v8connection) { _console->printLine(g + " |___/ " + r + " " + z); _console->printLine(""); - // clang-format on + // clang-format on - std::ostringstream s; + std::ostringstream s; - s << "arangosh (" << rest::Version::getVerboseVersionString() << ")\n" - << "Copyright (c) ArangoDB GmbH"; + s << "arangosh (" << rest::Version::getVerboseVersionString() << ")\n" + << "Copyright (c) ArangoDB GmbH"; - _console->printLine(s.str()); - _console->printLine(""); + _console->printLine(s.str()); + _console->printLine(""); - _console->printWelcomeInfo(); + _console->printWelcomeInfo(); + } if (v8connection != nullptr) { if (v8connection->isConnected() && @@ -847,6 +856,25 @@ static void JS_VersionClient(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_END } +//////////////////////////////////////////////////////////////////////////////// +/// @brief exit now +//////////////////////////////////////////////////////////////////////////////// + +static void JS_Exit(v8::FunctionCallbackInfo const& args) { + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + int64_t code = 0; + + if (args.Length() > 0) { + code = TRI_ObjectToInt64(args[0]); + } + + exit((int) code); + + TRI_V8_TRY_CATCH_END +} + //////////////////////////////////////////////////////////////////////////////// /// @brief initializes global Javascript variables //////////////////////////////////////////////////////////////////////////////// @@ -976,6 +1004,9 @@ void V8ShellFeature::initMode(ShellFeature::RunMode runMode, TRI_AddGlobalVariableVocbase( _isolate, TRI_V8_ASCII_STRING(_isolate, "IS_JS_LINT"), v8::Boolean::New(_isolate, runMode == ShellFeature::RunMode::JSLINT)); + + TRI_AddGlobalFunctionVocbase( + _isolate, TRI_V8_ASCII_STRING(_isolate, "SYS_EXIT"), JS_Exit); } void V8ShellFeature::loadModules(ShellFeature::RunMode runMode) { @@ -1005,7 +1036,7 @@ void V8ShellFeature::loadModules(ShellFeature::RunMode runMode) { files.push_back( "common/bootstrap/modules.js"); // must come last before patches - files.push_back("client/client.js"); // needs internal + files.push_back("client/" + _clientModule); // needs internal for (size_t i = 0; i < files.size(); ++i) { switch (loader.loadScript(_isolate, context, files[i], nullptr)) { diff --git a/arangosh/Shell/V8ShellFeature.h b/arangosh/Shell/V8ShellFeature.h index 1fe61a36ea..7fd371ca72 100644 --- a/arangosh/Shell/V8ShellFeature.h +++ b/arangosh/Shell/V8ShellFeature.h @@ -53,6 +53,7 @@ class V8ShellFeature final : public application_features::ApplicationFeature { private: std::string _startupDirectory; + std::string _clientModule; std::vector _moduleDirectory; bool _currentModuleDirectory; uint64_t _gcInterval; diff --git a/etc/arangodb3/arangoinspect.conf.in b/etc/arangodb3/arangoinspect.conf.in new file mode 100644 index 0000000000..a58637cf11 --- /dev/null +++ b/etc/arangodb3/arangoinspect.conf.in @@ -0,0 +1,14 @@ +[console] +pretty-print = true + +[log] +@COMMENT_LOGFILE@file = - + +[server] +endpoint = tcp://127.0.0.1:8529 +authentication = false +ask-jwt-secret = true + +[javascript] +startup-directory = @PKGDATADIR@/js +client-module = inspector.js diff --git a/etc/relative/arangoinspect.conf b/etc/relative/arangoinspect.conf new file mode 100644 index 0000000000..d711119bd6 --- /dev/null +++ b/etc/relative/arangoinspect.conf @@ -0,0 +1,13 @@ +[console] +pretty-print = true + +[server] +authentication = false +ask-jwt-secret = true + +[log] +file = - + +[javascript] +startup-directory = ./js +client-module = inspector.js diff --git a/js/client/inspector.js b/js/client/inspector.js new file mode 100644 index 0000000000..94ead1db24 --- /dev/null +++ b/js/client/inspector.js @@ -0,0 +1,92 @@ +/* jshint -W051, -W020 */ +/* global global:true, window, require */ +'use strict'; + +// ///////////////////////////////////////////////////////////////////////////// +// @brief ArangoDB Inspector +// +// @file +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// @author Dr. Frank Celler +// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany +// ///////////////////////////////////////////////////////////////////////////// + +var arango; + +// global 'arango' +global.arango = require('@arangodb').arango; + +// global 'db' +global.db = require('@arangodb').db; + +// cleanup +delete global.IS_EXECUTE_SCRIPT; +delete global.IS_EXECUTE_STRING; +delete global.IS_CHECK_SCRIPT; +delete global.IS_UNIT_TESTS; +delete global.IS_JS_LINT; + +// print banner +(function() { + const internal = require('internal'); + + internal.print(" _ ___ _"); + internal.print(" / \\ _ __ __ _ _ __ __ _ ___ |_ _|_ __ ___ _ __ ___ ___| |_ ___ _ __"); + internal.print(" / _ \\ | '__/ _` | '_ \\ / _` |/ _ \\ | || '_ \\/ __| '_ \\ / _ \\/ __| __/ _ \\| '__|"); + internal.print(" / ___ \\| | | (_| | | | | (_| | (_) | | || | | \\__ \\ |_) | __/ (__| || (_) | |"); + internal.print("/_/ \\_\\_| \\__,_|_| |_|\\__, |\\___/ |___|_| |_|___/ .__/ \\___|\\___|\\__\\___/|_|"); + internal.print(" |___/ |_| "); + internal.print(""); +})(); + +// load rc file +try { + // this will not work from within a browser + const __fs__ = require('fs'); + const __rcf__ = __fs__.join(__fs__.home(), '.arangoinspect.rc'); + + if (__fs__.exists(__rcf__)) { + /* jshint evil: true */ + const __content__ = __fs__.read(__rcf__); + eval(__content__); + } +} catch (e) { + require('console').warn('arangoinspect.rc: %s', String(e)); +} + +// check connection success +if (!arango.isConnected()) { + const internal = require('internal'); + internal.print("FATAL cannot connect to server '" + arango.getEndpoint() + "'"); + require("process").exit(1); +} + +if (arango.lastErrorMessage()) { + const internal = require('internal'); + internal.print("FATAL cannot connect to server '" + arango.getEndpoint() + "': " + + arango.lastErrorMessage()); + require("process").exit(1); +} + +require("@arangodb/inspector"); +require("internal").exit(); + + diff --git a/js/client/modules/@arangodb/arangosh.js b/js/client/modules/@arangodb/arangosh.js index f92154a0b7..08bf48f6c0 100644 --- a/js/client/modules/@arangodb/arangosh.js +++ b/js/client/modules/@arangodb/arangosh.js @@ -1,4 +1,5 @@ /* jshint strict: false */ +global.console = global.console || require('console'); // ////////////////////////////////////////////////////////////////////////////// // / @brief ArangoShell client API diff --git a/js/client/modules/@arangodb/inspector.js b/js/client/modules/@arangodb/inspector.js new file mode 100644 index 0000000000..c4e4e5e906 --- /dev/null +++ b/js/client/modules/@arangodb/inspector.js @@ -0,0 +1,732 @@ +/* global require, arango, print, db */ +'use strict'; + +// ///////////////////////////////////////////////////////////////////////////// +// @brief ArangoDB Doctor +// +// @file +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// @author Dr. Frank Celler +// @author Kaveh Vahedipour +// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany +// ///////////////////////////////////////////////////////////////////////////// + + +var internal = require('internal'); +var fs = require('fs'); + +const executeExternalAndWait = internal.executeExternalAndWait; +const _ = require("lodash"); + +const servers = {}; +let possibleAgent = null; + +var healthRecord = {}; + +function INFO() { + let args = Array.prototype.slice.call(arguments); + print.apply(print, ["INFO"].concat(args)); +} + +function WARN() { + let args = Array.prototype.slice.call(arguments); + print.apply(print, ["WARN"].concat(args)); +} + +function ERROR() { + let args = Array.prototype.slice.call(arguments); + print.apply(print, ["ERROR"].concat(args)); +} + +function loadAgencyConfig() { + var configuration = arango.GET("/_api/agency/config"); + return configuration; +} + + +/** + * @brief Sort shard keys according to their numbers omitting startign 's' + * + * @param keys Keys + */ +function sortShardKeys(keys) { + var ret = []; + // Get rid of 's' up front + keys.forEach(function(key, index, theArray) { + theArray[index] = key.substring(1); + }); + // Sort keeping indices + var ind = range(0,keys.length-1); + ind.sort(function compare(i, j) { + return parseInt(keys[i]) > parseInt(keys[j]); + }); + ind.forEach(function (i) { + ret.push('s'+keys[i]); + }); + return ret; +} + + +function agencyInspector(obj) { + + var nerrors = 0; + var nwarnings = 0; + var ndatabases = 0; + var ncollections = 0; + var nshards = 0; + var ncolinplannotincurrent = 0; + + var report = {}; + + INFO('Analysing agency dump ...'); + + // Must be array with length 1 + if (obj.length !== 1) { + ERROR('agency dump must be an array with a single object inside'); + process.exit(1); + } + + // Check for /arango and assign + if (!obj[0].hasOwnProperty('arango')) { + ERROR('agency must have attribute "arango" as root'); + process.exit(1); + } + const agency = obj[0]['arango']; + + // Must have Plan, Current, Supervision, Target, Sync + if (!agency.hasOwnProperty('Plan')) { + ERROR('agency must have attribute "arango/Plan" object'); + process.exit(1); + } + const plan = agency.Plan; + if (!plan.hasOwnProperty('Version')) { + ERROR('plan has no version'); + process.exit(1); + } + if (!agency.hasOwnProperty('Current')) { + ERROR('agency must have attribute "arango/Current" object'); + process.exit(1); + } + const current = agency.Current; + + // Start sanity of plan + INFO('Plan (version ' + plan.Version+ ')'); + + // Planned databases check if also in collections and current + INFO(" Databases"); + report.Databases = {}; + if (!plan.hasOwnProperty('Databases')) { + ERROR('no databases in plan'); + return; + } + Object.keys(plan.Databases).forEach(function(database) { + INFO(' ' + database); + report.Databases[database] = {}; + if (!plan.Collections.hasOwnProperty(database)) { + WARN('found planned database "' + database + '" without planned collectinos'); + } + if (!current.Databases.hasOwnProperty(database)) { + WARN('found planned database "' + database + '" missing in "Current"'); + } + }); + + INFO(" Collections"); + + // Planned collections + if (!plan.hasOwnProperty('Collections')) { + ERROR('no collections in plan'); + process.exit(1); + } + + var warned = false; + Object.keys(plan.Collections).forEach(function(database) { + ++ndatabases; + INFO(' ' + database); + + if (!plan.Databases.hasOwnProperty(database)) { + ERROR('found planned collections in unplanned database ' + database); + } + + Object.keys(plan.Collections[database]).forEach(function(collection) { + report.Databases[database] + [plan.Collections[database][collection].name] = {}; + + ++ncollections; + const col = plan.Collections[database][collection]; + INFO(' ' + col.name); + const distributeShardsLike = col.distributeShardsLike; + var myShardKeys = sortShardKeys(Object.keys(col.shards)); + + if (distributeShardsLike && distributeShardsLike !== "") { + const prototype = plan.Collections[database][distributeShardsLike]; + if (prototype.replicationFactor !== col.replicationFactor) { + ERROR('distributeShardsLike: replicationFactor mismatch'); + } + if (prototype.numberOfShards !== col.numberOfShards) { + ERROR('distributeShardsLike: numberOfShards mismatch'); + } + var prototypeShardKeys = sortShardKeys(Object.keys(prototype.shards)); + var ncshards = myShardKeys.length; + if (prototypeShardKeys.length !== ncshards) { + ERROR('distributeShardsLike: shard map mismatch'); + } + for (var i = 0; i < ncshards; ++i) { + const shname = myShardKeys[i]; + const ccul = current.Collections[database]; + if (JSON.stringify(col.shards[shname]) !== + JSON.stringify(prototype.shards[prototypeShardKeys[i]])) { + ERROR( + 'distributeShardsLike: planned shard map mismatch between "/arango/Plan/Collections/' + + database + '/' + collection + '/shards/' + myShardKeys[i] + + '" and " /arango/Plan/Collections/' + database + '/' + distributeShardsLike + + '/shards/' + prototypeShardKeys[i] + '"'); + INFO(' ' + JSON.stringify(col.shards[shname])); + INFO(' ' + JSON.stringify(prototype.shards[prototypeShardKeys[i]])); + } + } + } + + myShardKeys.forEach(function(shname) { + ++nshards; + if (current.Collections.hasOwnProperty(database)) { + if (!current.Collections[database].hasOwnProperty(collection)) { + WARN('missing collection "' + collection + '" in current'); + } else { + if (!current.Collections[database][collection].hasOwnProperty(shname)) { + WARN('missing shard "' + shname + '" in current'); + } + const shard = current.Collections[database][collection][shname]; + if (shard) { + if (JSON.stringify(shard.servers) !== JSON.stringify(col.shards[shname])) { + if (shard.servers[0] !== col.shards[shname][0]) { + ERROR('/arango/Plan/Collections/' + database + '/' + collection + '/shards/' + + shname + ' and /arango/Current/Collections/' + database + '/' + + collection + '/' + shname + '/servers do not match'); + } else { + var sortedPlan = (shard.servers).sort(); + var sortedCurrent = (col.shards[shname]).sort(); + if (JSON.stringify(sortedPlan) === JSON.stringify(sortedCurrent)) { + WARN('/arango/Plan/Collections/' + database + '/' + collection + '/shards/' + + shname + ' and /arango/Current/Collections/' + database + '/' + + collection + '/' + shname + '/servers follower do not match in order'); + } + } + } + } + } + } else { + if (!warned) { + WARN('planned database "' + database + '" missing entirely in current'); + warned = true; + } + } + }); + + }); + }); + + report.servers = {}; + INFO('Server health'); + INFO(' DB Servers'); + report.servers.dbservers = {}; + const supervision = agency.Supervision; + const target = agency.Target; + var servers = plan.DBServers; + Object.keys(servers).forEach(function(serverId) { + if (!target.MapUniqueToShortID.hasOwnProperty(serverId)) { + WARN('incomplete planned db server ' + serverId + ' is missing in "Target"'); + } else { + INFO(' ' + serverId + '(' + target.MapUniqueToShortID[serverId].ShortName + ')'); + if (!supervision.Health.hasOwnProperty(serverId)) { + ERROR('planned db server ' + serverId + ' missing in supervision\'s health records.'); + } + servers[serverId] = supervision.Health[serverId]; + report.servers.dbservers[serverId] = {name: target.MapUniqueToShortID[serverId].ShortName, status : servers[serverId].Status}; + if (servers[serverId].Status === "BAD") { + WARN('bad db server ' + serverId + '(' + servers[serverId].ShortName+ ')'); + } else if (servers[serverId].Status === "FAILED") { + WARN('*** FAILED *** db server ' + serverId + '(' + servers[serverId].ShortName+ ')'); + } + } + }); + + INFO(' Coordinators'); + report.servers.coordinators = {}; + servers = plan.Coordinators; + Object.keys(servers).forEach(function(serverId) { + if (!target.MapUniqueToShortID.hasOwnProperty(serverId)) { + WARN('incomplete planned db server ' + serverId + ' is missing in "Target"'); + } else { + INFO(' ' + serverId + '(' + target.MapUniqueToShortID[serverId].ShortName + ')'); + report.servers.coordinators[serverId] = {name: target.MapUniqueToShortID[serverId].ShortName, status : servers[serverId].Status}; + + if (!supervision.Health.hasOwnProperty(serverId)) { + WARN('planned coordinator ' + serverId + ' missing in supervision\'s health records.'); + } + servers[serverId] = supervision.Health[serverId]; + if (servers[serverId].Status !== "GOOD") { + WARN('*** FAILED *** coordinator ' + serverId + '(' + servers[serverId].ShortName+ ')'); + } + } + }); + + const jobs = { + ToDo: target.ToDo, + Pending: target.Pending, + Finished: target.Finished, + Failed: target.Failed + }; + var njobs = []; + var nall; + Object.keys(jobs).forEach(function (state) { + njobs.push(Object.keys(jobs[state]).length); + }); + + var i = 0; + INFO('Supervision activity'); + INFO(' Jobs: ' + nall + '(' + + 'To do: ' + njobs[0] + ', ' + + 'Pending: ' + njobs[1] + ', ' + + 'Finished: ' + njobs[2] + ', ' + + 'Failed: ' + njobs[3] + ')' + ); + + report.jobs = { todo: njobs[0], pending: njobs[1], finished: njobs[2], failed: njobs[3]}; + + INFO('Summary'); + if (nerrors > 0) { + ERROR(' ' + nerrors + ' errors'); + } + if (nwarnings > 0) { + WARN(' ' + nwarnings + ' warnings'); + } + INFO(' ' + ndatabases + ' databases'); + INFO(' ' + ncollections + ' collections '); + INFO(' ' + nshards + ' shards '); + INFO('... agency analysis finished.'); + + return report; + +} + + +/** + * @brief Create an integer range as [start,end] + * + * @param start Start of range + * @param end End of range + */ +function range(start, end) { + return Array(end - start + 1).fill().map((_, idx) => start + idx); +} + +function loadAgency(conn, seen) { + + var agencyDump = conn.POST("/_api/agency/read", '[["/"]]'); + seen[conn.getEndpoint()] = true; + + if (agencyDump.code === 404) { + WARN("not talking to an agent, got: " + JSON.stringify(agencyDump)); + return null; + } + + if (agencyDump.code === 307) { + var red = conn.POST_RAW("/_api/agency/read", '[["/"]]'); + + if (red.code === 307) { + INFO("got redirect to " + red.headers.location); + + let leader = red.headers.location; + let reg = /^(http|https):\/\/(.*)\/_api\/agency\/read$/; + let m = reg.exec(leader); + + if (m === null) { + WARN("unknown redirect " + leader); + return null; + } else { + if (m[1] === "http") { + leader = "tcp://" + m[2]; + } + else if (m[1] === "https") { + leader = "ssl://" + m[2]; + } + + if (leader in seen) { + WARN("cannot find leader, tried: " + Object.keys(seen).join(", ")); + return null; + } + + INFO("switching to " + leader); + + console.log("http+" + leader); + console.log(conn.getEndpoint()); + if ("http+" + leader != conn.getEndpoint()) { + conn.reconnect(leader, "_system"); + } + + return loadAgencyConfig(conn, seen); + } + } + } + + if (agencyDump.code !== undefined) { + WARN("failed to load agency, got: " + JSON.stringify(agencyDump)); + return null; + } + + return agencyDump; + +} + +function defineServer(type, id, source) { + if (id in servers) { + _.merge(servers[id].source, source); + } else { + servers[id] = { + type: type, + id: id, + source: source + }; + } +} + +function defineServerEndpoint(id, endpoint) { + const server = servers[id]; + + if ('endpoint' in server) { + if (server.endpoint !== endpoint) { + INFO("changing endpoint for " + id + " from " + + server.endpoint + " to " + endpoint); + } + } + + server.endpoint = endpoint; +} + +function defineServerStatus(id, status) { + const server = servers[id]; + + if ('status' in server) { + if (server.status !== status) { + INFO("changing status for " + id + " from " + + server.status + " to " + status); + } + } + + server.status = status; +} + +function defineAgentLeader(id, leading) { + const server = servers[id]; + + if ('leading' in server) { + if (server.leading !== leading) { + INFO("changing leading for " + id + " from " + + server.leading + " to " + leading); + } + } + + server.leading = leading; +} + +function defineAgentFromStatus(status, endpoint) { + let id = status.agent.id; + let leader = status.agent.leaderId; + + defineServer('AGENT', id, { status: endpoint }); + defineServerEndpoint(id, endpoint); + + arango.reconnect(endpoint, "_system"); + + const cfg = db.configuration.toArray()[0].cfg; + + _.forEach(cfg.active, function(id) { + defineServer('AGENT', id, { active: endpoint }); + }); + + _.forEach(cfg.pool, function(loc, id) { + defineServer('AGENT', id, { pool: endpoint }); + defineServerEndpoint(id, loc); + defineAgentLeader(id, id === leader); + }); + + defineAgentLeader(id,status.agent.leading); +} + +function definePrimaryFromStatus(status, endpoint) { + let id = status.serverInfo.serverId; + + defineServer('PRIMARY', id, { status: endpoint }); + defineServerEndpoint(id, endpoint); + + let agentEndpoints = status.agency.agencyComm.endpoints; + + if (0 < agentEndpoints.length) { + possibleAgent = agentEndpoints[0]; + } else { + console.error("Failed to find an agency endpoint"); + } +} + +function defineCoordinatorFromStatus(status, endpoint) { + let id = status.serverInfo.serverId; + + defineServer('COORDINATOR', id, { status: endpoint }); + defineServerEndpoint(id, endpoint); + + let agentEndpoints = status.agency.agencyComm.endpoints; + + if (0 < agentEndpoints.length) { + possibleAgent = agentEndpoints[0]; + } else { + console.error("Failed to find an agency endpoint"); + } +} + +function defineSingleFromStatus(status, endpoint) { + defineServer('SINGLE', 'SINGLE', { status: endpoint }); + defineServerEndpoint('SINGLE', endpoint); + if (status.hasOwnProperty('agency')) { + let agentEndpoints = status.agency.agencyComm.endpoints; + + if (0 < agentEndpoints.length) { + possibleAgent = agentEndpoints[0]; + } else { + console.error("Failed to find an agency endpoint"); + } + } +} + +function serverBasics(conn) { + if (!conn) { + conn = arango; + } + + const status = conn.GET("/_admin/status"); + const role = status.serverInfo.role; + + if (role === 'AGENT') { + defineAgentFromStatus(status, conn.getEndpoint()); + } else if (role === 'PRIMARY') { + definePrimaryFromStatus(status, conn.getEndpoint()); + } else if (role === 'COORDINATOR') { + defineCoordinatorFromStatus(status); + } else if (role === 'SINGLE') { + defineSingleFromStatus(status); + } else { + return "unknown"; + } + + return role; +} + +function locateServers(plan) { + let health = plan[0].arango.Supervision.Health; + let cluster = plan[0].arango.Cluster; + + _.forEach(health, function(info, id) { + const type = id.substr(0, 4); + + if (type === "PRMR") { + defineServer('PRIMARY', id, { supervision: cluster }); + defineServerEndpoint(id, info.Endpoint); + defineServerStatus(id, info.Status); + } else if (type === "CRDN") { + defineServer('SINGLE', id, { supervision: cluster }); + defineServerEndpoint(id, info.Endpoint); + defineServerStatus(id, info.Status); + } else if (type === "SNGL") { + defineServer('SINGLE', id, { supervision: cluster }); + defineServerEndpoint(id, info.Endpoint); + defineServerStatus(id, info.Status); + } + }); +} + +function listServers() { + return servers; +} + +function getServerData(arango) { + var current = arango.getEndpoint(); + var servers = listServers(); + var report = {}; + INFO('Collecting diagnostics from all servers ... '); + var nservers = Object.keys(servers).length; + Object.keys(servers).forEach( + + function (server) { + + if (nservers == 1 || servers.lengthserver !== "SINGLE") { + try { + + if (servers[server].endpoint !== undefined) { + if (arango.getEndpoint() != "http+" + servers[server].endpoint) { + arango.reconnect(servers[server].endpoint, '_system'); + } + } + + const version = arango.GET('_api/version'); // version api + const log = arango.GET('_admin/log').text; // log api + const statistics = arango.GET('_admin/statistics').text; // log api + var agencyConfig; + if (server.startsWith("AGNT")) { + agencyConfig = arango.GET('_api/agency/config'); + } + const status = arango.GET('_admin/status'); + var tmp = executeExternalAndWait( + '/bin/bash', ['-c', 'dmesg | tee /tmp/inspector-dmesg.out > /dev/null']); + const dmesg = fs.readFileSync('/tmp/inspector-dmesg.out', 'utf8'); + tmp = executeExternalAndWait( + '/bin/bash', ['-c', 'df -h | tee /tmp/inspector-df.out > /dev/null']); + const df = fs.readFileSync('/tmp/inspector-df.out', 'utf8'); + tmp = executeExternalAndWait( + '/bin/bash', ['-c', 'cat /proc/meminfo | tee /tmp/inspector-meminfo.out > /dev/null']); + const meminfo = fs.readFileSync('/tmp/inspector-meminfo.out', 'utf8'); + tmp = executeExternalAndWait( + '/bin/bash', ['-c', 'uptime | tee /tmp/inspector-uptime.out > /dev/null']); + const uptime = fs.readFileSync('/tmp/inspector-uptime.out', 'utf8'); + tmp = executeExternalAndWait( + '/bin/bash', ['-c', 'uname -a | tee /tmp/inspector-uname.out > /dev/null']); + const uname = fs.readFileSync('/tmp/inspector-uname.out', 'utf8'); + var top; + if (status.pid !== undefined) { + tmp = executeExternalAndWait( + '/bin/bash', ['-c', 'top -b -H -p ' + status.pid + ' -n 1 | tee /tmp/inspector-top.out > /dev/null']); + top = fs.readFileSync('/tmp/inspector-top.out', 'utf8'); + } + + var local = {}; + try { + var localDBs = db._databases(); + localDBs.forEach( function(localDB) { + db._useDatabase(localDB); + local[localDB] = {}; + var localCols = db._collections(); + localCols.forEach( function(localCol) { + var colName = localCol.name(); + local[localDB][colName] = {}; + Object.keys(localCol.properties()).forEach( function(property) { + local[localDB][colName][property] = localCol.properties()[property]; + }); + local[localDB][colName].index = localCol.getIndexes(); + local[localDB][colName].count = localCol.count(); + });}); + db._useDatabase('_system'); + } catch (e) {} + + // report this server + report[server] = { + version:version, log:log, dmesg:dmesg, statistics:statistics, + status:status, df:df, uptime:uptime, uname:uname, meminfo:meminfo, + local:local}; + + if (agencyConfig !== undefined) { + report[server].config = agencyConfig; + } + if (top !== undefined) { + report[server].top = top; + } + + } catch (e) { + print(e); + } + } + }); + if (Object.keys(servers).length > 1) { + if (current != arango.getEndpoint()) { + arango.reconnect(current, '_system'); + } + } + INFO('... dignostics collected.'); + return report; +} + +(function() { + try { + var type = serverBasics(); + + if (type !== 'AGENT') { + if (possibleAgent !== null) { + arango.reconnect(possibleAgent, "_system"); + serverBasics(); + } + } + + if (possibleAgent !== null) { + + // Agency dump and analysis + var agencyConfig = loadAgencyConfig(); + var agencyDump = {}; + var tries = 0; + while (true) { + if (agencyDump.leaderId !== "") { + if (agencyConfig.configuration.pool.hasOwnProperty(agencyConfig.leaderId)) { + if ("http+" + agencyConfig.configuration.pool[agencyConfig.leaderId] != + arango.getEndpoint()) { + arango.reconnect( + agencyConfig.configuration.pool[agencyConfig.leaderId], "_system"); + } + agencyDump = loadAgency(arango, {}); + break; + } else { // Leader is not in pool? Fatal error; + console.error("Fatal error: " + agencyDump.leaderId + + " is not a member of the agent pool " + + JSON.stringify(agencyConfig.configuration.pool)); + console.error("This deployment needs immediate administrative attention."); + possibleAgent = null; + return; + } + } else { + if (tries < 100) { + tries++; + internal.wait(1); + } else { + console.error("Error: Agency cannot elect a leader configured as " + + JSON.stringify(agencyConfig)); + console.error("This deployment needs immediate administrative attention."); + possibleAgent = null; + return; + } + } + } + + if (agencyDump !== null) { + locateServers(agencyDump); + } + + healthRecord['analysis'] = agencyInspector(agencyDump); + healthRecord['agency'] = agencyDump[0].arango; + + } + // Get all sorts of meta data from all servers + healthRecord['servers'] = getServerData(arango); + + const ofname = 'arango-inspector.json'; + require('fs').writeFileSync(ofname, JSON.stringify(healthRecord)); + + INFO("Report written to " + ofname + "."); + + } catch (e) { + print(e); + } +}()); + diff --git a/js/common/bootstrap/modules/internal.js b/js/common/bootstrap/modules/internal.js index 10c03ea76a..5bc164f74d 100644 --- a/js/common/bootstrap/modules/internal.js +++ b/js/common/bootstrap/modules/internal.js @@ -7,31 +7,33 @@ global.DEFINE_MODULE('internal', (function () { const exports = {}; - // ////////////////////////////////////////////////////////////////////////////// - // / @brief module "internal" - // / - // / @file - // / - // / DISCLAIMER - // / - // / Copyright 2004-2013 triAGENS GmbH, Cologne, Germany - // / - // / Licensed under the Apache License, Version 2.0 (the "License") - // / you may not use this file except in compliance with the License. - // / You may obtain a copy of the License at - // / - // / http://www.apache.org/licenses/LICENSE-2.0 - // / - // / Unless required by applicable law or agreed to in writing, software - // / distributed under the License is distributed on an "AS IS" BASIS, - // / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - // / See the License for the specific language governing permissions and - // / limitations under the License. - // / - // / Copyright holder is triAGENS GmbH, Cologne, Germany - // / - // / @author Dr. Frank Celler - // / @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany + /////////////////////////////////////////////////////////////////////////////// + // @brief module "internal" + // + // @file + // + // DISCLAIMER + // + // Copyright 2018 ArangoDB GmbH, Cologne, Germany + // Copyright 2004-2013 triAGENS GmbH, Cologne, Germany + // + // Licensed under the Apache License, Version 2.0 (the "License") + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License. + // + // Copyright holder is ArangoDB GmbH, Cologne, Germany + // + // @author Dr. Frank Celler + // @author Copyright 2018, ArangoDB GmbH, Cologne, Germany + // @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany // ////////////////////////////////////////////////////////////////////////////// // ////////////////////////////////////////////////////////////////////////////// @@ -748,6 +750,14 @@ global.DEFINE_MODULE('internal', (function () { global.SYS_UNIT_TESTS_RESULT = value; }; + // end process + if (global.SYS_EXIT) { + exports.exit = global.SYS_EXIT; + delete global.SYS_EXIT; + } else { + exports.exit = function() {}; + } + // ////////////////////////////////////////////////////////////////////////////// // / @brief structured to flat commandline arguments // / @param longOptsEqual whether long-options are in the type --opt=value diff --git a/js/common/bootstrap/modules/process.js b/js/common/bootstrap/modules/process.js index 1abb948b3a..6e507c8808 100644 --- a/js/common/bootstrap/modules/process.js +++ b/js/common/bootstrap/modules/process.js @@ -12,19 +12,25 @@ global.DEFINE_MODULE('process', (function () { var exports = new EventEmitter(); exports.env = internal.env; + exports.argv = []; + exports.stdout = { isTTY: internal.COLOR_OUTPUT, write(text) { console.infoLines(text); } }; + exports.cwd = function () { return fs.makeAbsolute(''); }; + exports.nextTick = function (fn) { fn(); }; + exports.exit = internal.exit; + return exports; }())); diff --git a/scripts/startLeaderFollower.sh b/scripts/startLeaderFollower.sh index 9c553e2eb7..f34a94f223 100755 --- a/scripts/startLeaderFollower.sh +++ b/scripts/startLeaderFollower.sh @@ -1,13 +1,13 @@ #!/bin/bash params=("$@") -rm -rf cluster +rm -rf active if [ -d cluster-init ];then echo "== creating cluster directory from existing cluster-init directory" - cp -a cluster-init cluster + cp -a active-init active else echo "== creating fresh directory" - mkdir -p cluster || { echo "failed to create cluster directory"; exit 1; } + mkdir -p active || { echo "failed to create cluster directory"; exit 1; } #if we want to restart we should probably store the parameters line wise fi @@ -27,8 +27,8 @@ else exit 1 fi -if [[ -f cluster/startup_parameters ]];then - string="$(< cluster/startup_parameters)" +if [[ -f active/startup_parameters ]];then + string="$(< active/startup_parameters)" if [[ -z "${params[@]}" ]]; then params=( $string ) else @@ -41,7 +41,7 @@ if [[ -f cluster/startup_parameters ]];then else #store parmeters if [[ -n "${params[@]}" ]]; then - echo "${params[@]}" > cluster/startup_parameters + echo "${params[@]}" > active/startup_parameters fi fi @@ -88,11 +88,11 @@ NATH=$(( $NRSINGLESERVERS + $NRAGENTS )) ENDPOINT=[::] ADDRESS=[::1] -rm -rf cluster -if [ -d cluster-init ];then - cp -a cluster-init cluster +rm -rf active +if [ -d active-init ];then + cp -a active-init active fi -mkdir -p cluster +mkdir -p active if [ -z "$JWT_SECRET" ];then AUTHENTICATION="--server.authentication false" @@ -139,17 +139,17 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do --agency.supervision-frequency $SFRE \ --agency.supervision-grace-period 5.0 \ --agency.wait-for-sync false \ - --database.directory cluster/data$port \ + --database.directory active/data$port \ --javascript.enabled false \ --server.endpoint $TRANSPORT://$ENDPOINT:$port \ --server.statistics false \ --server.threads 16 \ - --log.file cluster/$port.log \ + --log.file active/$port.log \ --log.level $LOG_LEVEL_AGENCY \ $STORAGE_ENGINE \ $AUTHENTICATION \ $SSLKEYFILE \ - | tee cluster/$PORT.stdout 2>&1 & + | tee active/$PORT.stdout 2>&1 & done start() { @@ -158,28 +158,28 @@ start() { TYPE=$1 PORT=$2 - mkdir cluster/data$PORT cluster/apps$PORT + mkdir active/data$PORT active/apps$PORT echo Starting $TYPE on port $PORT $CMD \ -c none \ - --database.directory cluster/data$PORT \ + --database.directory active/data$PORT \ --cluster.agency-endpoint $TRANSPORT://$ENDPOINT:$AG_BASE \ --cluster.my-address $TRANSPORT://$ADDRESS:$PORT \ --server.endpoint $TRANSPORT://$ENDPOINT:$PORT \ --cluster.my-role $ROLE \ --replication.active-failover true \ - --log.file cluster/$PORT.log \ + --log.file active/$PORT.log \ --log.level $LOG_LEVEL \ --server.statistics true \ --server.threads 5 \ --javascript.startup-directory $SRC_DIR/js \ --javascript.module-directory $SRC_DIR/enterprise/js \ - --javascript.app-path cluster/apps$PORT \ + --javascript.app-path active/apps$PORT \ --log.level $LOG_LEVEL_CLUSTER \ $STORAGE_ENGINE \ $AUTHENTICATION \ $SSLKEYFILE \ - | tee cluster/$PORT.stdout 2>&1 & + | tee active/$PORT.stdout 2>&1 & } PORTTOPDB=`expr $SS_BASE + $NRSINGLESERVERS - 1` @@ -209,7 +209,7 @@ for p in `seq $SS_BASE $PORTTOPDB` ; do testServer $p done -echo Done, your cluster is ready at +echo Done, your active failover pair is ready at for p in `seq $SS_BASE $PORTTOPDB` ; do echo " ${BUILD}/bin/arangosh --server.endpoint $TRANSPORT://[::1]:$p" done