Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/arangodoc (#5476) 

* introducing arangoinspect
This commit is contained in:
Kaveh Vahedipour 2018-06-05 15:38:50 +02:00 committed by Max Neunhöffer
parent 3e983adb2d
commit 03edbf44d7
16 changed files with 1006 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arangod/GeneralServer/RestHandlerFactory.cpp
View File

@ -95,9 +95,11 @@ RestHandler* RestHandlerFactory::createHandler(
case ServerState::Mode::TRYAGAIN: {
if (path.find("/_admin/shutdown") == std::string::npos &&
path.find("/_admin/cluster/health") == std::string::npos &&
path.find("/_admin/log") == std::string::npos &&
path.find("/_admin/server/role") == std::string::npos &&
path.find("/_admin/server/availability") == std::string::npos &&
path.find("/_admin/status") == std::string::npos &&
path.find("/_admin/statistics") == std::string::npos &&
path.find("/_api/agency/agency-callbacks") == std::string::npos &&
path.find("/_api/cluster/") == std::string::npos &&
path.find("/_api/replication") == std::string::npos &&

8
arangod/RestHandler/RestStatusHandler.cpp
View File

@ -33,6 +33,10 @@
#include <iostream>
#if defined(TRI_HAVE_POSIX_THREADS)
#include <unistd.h>
#endif
#include <velocypack/Builder.h>
#include <velocypack/velocypack-aliases.h>
@ -56,6 +60,10 @@ RestStatus RestStatusHandler::execute() {
result.add("server", VPackValue("arango"));
result.add("version", VPackValue(ARANGODB_VERSION));
#if defined(TRI_HAVE_POSIX_THREADS)
result.add("pid", VPackValue(getpid()));
#endif
#ifdef USE_ENTERPRISE
result.add("license", VPackValue("enterprise"));
#else

10
arangosh/CMakeLists.txt
View File

@ -372,3 +372,13 @@ install_command_alias(${BIN_ARANGOSH}
foxx-manager)
install_config(foxx-manager)
################################################################################
## arangoinspect
################################################################################
install_command_alias(${BIN_ARANGOSH}
${CMAKE_INSTALL_BINDIR}
arangoinspect)
install_config(arangoinspect)

11
arangosh/Shell/ShellFeature.cpp
View File

@ -32,8 +32,8 @@ using namespace arangodb;
using namespace arangodb::basics;
using namespace arangodb::options;
ShellFeature::ShellFeature(
application_features::ApplicationServer* server, int* result)
ShellFeature::ShellFeature(application_features::ApplicationServer* server,
int* result)
: ApplicationFeature(server, "Shell"),
_jslint(),
_result(result),
@ -123,7 +123,8 @@ void ShellFeature::validateOptions(
}
if (1 < n) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "you cannot specify more than one type ("
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "you cannot specify more than one type ("
<< "jslint, execute, execute-string, check-syntax, unit-tests)";
}
}
@ -131,7 +132,9 @@ void ShellFeature::validateOptions(
void ShellFeature::start() {
*_result = EXIT_FAILURE;
V8ShellFeature* shell = application_features::ApplicationServer::getFeature<V8ShellFeature>("V8Shell");
V8ShellFeature* shell =
application_features::ApplicationServer::getFeature<V8ShellFeature>(
"V8Shell");
bool ok = false;

8
arangosh/Shell/V8ClientConnection.cpp
View File

@ -49,7 +49,7 @@ using namespace arangodb::basics;
using namespace arangodb::httpclient;
using namespace arangodb::import;
std::string V8ClientConnection::JWT_SECRET = "";
std::shared_ptr<std::string> V8ClientConnection::JWT_SECRET = nullptr;
std::string V8ClientConnection::jwtToken(std::string const& secret) {
VPackBuilder headerBuilder;
@ -106,8 +106,8 @@ void V8ClientConnection::init(
params.setLocationRewriter(this, &rewriteLocation);
params.setUserNamePassword("/", _username, _password);
if (!JWT_SECRET.empty()) {
params.setJwt(jwtToken(JWT_SECRET));
if (JWT_SECRET != nullptr) {
params.setJwt(jwtToken(*JWT_SECRET));
}
_client.reset(new SimpleHttpClient(connection, params));
@ -434,6 +434,7 @@ static void ClientConnection_reconnect(
std::string password;
if (args.Length() < 4) {
if (V8ClientConnection::jwtSecret() == nullptr) {
ConsoleFeature* console =
ApplicationServer::getFeature<ConsoleFeature>("Console");
@ -444,6 +445,7 @@ static void ClientConnection_reconnect(
password = ConsoleFeature::readPassword();
std::cout << std::endl << std::flush;
}
}
} else {
password = TRI_ObjectToString(isolate, args[3]);
}

5
arangosh/Shell/V8ClientConnection.h
View File

@ -49,11 +49,12 @@ class V8ClientConnection {
V8ClientConnection& operator=(V8ClientConnection const&) = delete;
public:
static void setJwtSecret(std::string const& jwtSecret) { JWT_SECRET = jwtSecret; }
static void setJwtSecret(std::string const& jwtSecret) { JWT_SECRET = std::make_shared<std::string>(jwtSecret); }
static std::shared_ptr<std::string> jwtSecret() { return JWT_SECRET; }
static std::string jwtToken(std::string const& secret);
private:
static std::string JWT_SECRET;
static std::shared_ptr<std::string> JWT_SECRET;
public:
V8ClientConnection(

33
arangosh/Shell/V8ShellFeature.cpp
View File

@ -54,10 +54,13 @@ using namespace arangodb::basics;
using namespace arangodb::options;
using namespace arangodb::rest;
static std::string const DEFAULT_CLIENT_MODULE = "client.js";
V8ShellFeature::V8ShellFeature(application_features::ApplicationServer* server,
std::string const& name)
: ApplicationFeature(server, "V8Shell"),
_startupDirectory("js"),
_clientModule(DEFAULT_CLIENT_MODULE),
_currentModuleDirectory(true),
_gcInterval(50),
_name(name),
@ -78,6 +81,10 @@ void V8ShellFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
"startup paths containing the Javascript files",
new StringParameter(&_startupDirectory));
options->addHiddenOption("--javascript.client-module",
"client module to use at startup",
new StringParameter(&_clientModule));
options->addHiddenOption(
"--javascript.module-directory",
"additional paths containing JavaScript modules",
@ -196,6 +203,7 @@ bool V8ShellFeature::printHello(V8ClientConnection* v8connection) {
// http://www.network-science.de/ascii/ Font: ogre
if (!_console->quiet()) {
if (_clientModule == DEFAULT_CLIENT_MODULE) {
std::string g = ShellColorsFeature::SHELL_COLOR_GREEN;
std::string r = ShellColorsFeature::SHELL_COLOR_RED;
std::string z = ShellColorsFeature::SHELL_COLOR_RESET;
@ -228,6 +236,7 @@ bool V8ShellFeature::printHello(V8ClientConnection* v8connection) {
_console->printLine("");
_console->printWelcomeInfo();
}
if (v8connection != nullptr) {
if (v8connection->isConnected() &&
@ -847,6 +856,25 @@ static void JS_VersionClient(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_TRY_CATCH_END
}
////////////////////////////////////////////////////////////////////////////////
/// @brief exit now
////////////////////////////////////////////////////////////////////////////////
static void JS_Exit(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_TRY_CATCH_BEGIN(isolate);
v8::HandleScope scope(isolate);
int64_t code = 0;
if (args.Length() > 0) {
code = TRI_ObjectToInt64(args[0]);
}
exit((int) code);
TRI_V8_TRY_CATCH_END
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initializes global Javascript variables
////////////////////////////////////////////////////////////////////////////////
@ -976,6 +1004,9 @@ void V8ShellFeature::initMode(ShellFeature::RunMode runMode,
TRI_AddGlobalVariableVocbase(
_isolate, TRI_V8_ASCII_STRING(_isolate, "IS_JS_LINT"),
v8::Boolean::New(_isolate, runMode == ShellFeature::RunMode::JSLINT));
TRI_AddGlobalFunctionVocbase(
_isolate, TRI_V8_ASCII_STRING(_isolate, "SYS_EXIT"), JS_Exit);
}
void V8ShellFeature::loadModules(ShellFeature::RunMode runMode) {
@ -1005,7 +1036,7 @@ void V8ShellFeature::loadModules(ShellFeature::RunMode runMode) {
files.push_back(
"common/bootstrap/modules.js"); // must come last before patches
files.push_back("client/client.js"); // needs internal
files.push_back("client/" + _clientModule); // needs internal
for (size_t i = 0; i < files.size(); ++i) {
switch (loader.loadScript(_isolate, context, files[i], nullptr)) {

1
arangosh/Shell/V8ShellFeature.h
View File

@ -53,6 +53,7 @@ class V8ShellFeature final : public application_features::ApplicationFeature {
private:
std::string _startupDirectory;
std::string _clientModule;
std::vector<std::string> _moduleDirectory;
bool _currentModuleDirectory;
uint64_t _gcInterval;

14
etc/arangodb3/arangoinspect.conf.in Normal file
View File

@ -0,0 +1,14 @@
[console]
pretty-print = true
[log]
@COMMENT_LOGFILE@file = -
[server]
endpoint = tcp://127.0.0.1:8529
authentication = false
ask-jwt-secret = true
[javascript]
startup-directory = @PKGDATADIR@/js
client-module = inspector.js

13
etc/relative/arangoinspect.conf Normal file
View File

@ -0,0 +1,13 @@
[console]
pretty-print = true
[server]
authentication = false
ask-jwt-secret = true
[log]
file = -
[javascript]
startup-directory = ./js
client-module = inspector.js

92
js/client/inspector.js Normal file
View File

@ -0,0 +1,92 @@
/* jshint -W051, -W020 */
/* global global:true, window, require */
'use strict';
// /////////////////////////////////////////////////////////////////////////////
// @brief ArangoDB Inspector
//
// @file
//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Dr. Frank Celler
// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany
// /////////////////////////////////////////////////////////////////////////////
var arango;
// global 'arango'
global.arango = require('@arangodb').arango;
// global 'db'
global.db = require('@arangodb').db;
// cleanup
delete global.IS_EXECUTE_SCRIPT;
delete global.IS_EXECUTE_STRING;
delete global.IS_CHECK_SCRIPT;
delete global.IS_UNIT_TESTS;
delete global.IS_JS_LINT;
// print banner
(function() {
const internal = require('internal');
internal.print(" _ ___ _");
internal.print(" / \\ _ __ __ _ _ __ __ _ ___ |_ _|_ __ ___ _ __ ___ ___| |_ ___ _ __");
internal.print(" / _ \\ | '__/ _` | '_ \\ / _` |/ _ \\ | || '_ \\/ __| '_ \\ / _ \\/ __| __/ _ \\| '__|");
internal.print(" / ___ \\| | | (_| | | | | (_| | (_) | | || | | \\__ \\ |_) | __/ (__| || (_) | |");
internal.print("/_/ \\_\\_| \\__,_|_| |_|\\__, |\\___/ |___|_| |_|___/ .__/ \\___|\\___|\\__\\___/|_|");
internal.print(" |___/ |_| ");
internal.print("");
})();
// load rc file
try {
// this will not work from within a browser
const __fs__ = require('fs');
const __rcf__ = __fs__.join(__fs__.home(), '.arangoinspect.rc');
if (__fs__.exists(__rcf__)) {
/* jshint evil: true */
const __content__ = __fs__.read(__rcf__);
eval(__content__);
}
} catch (e) {
require('console').warn('arangoinspect.rc: %s', String(e));
}
// check connection success
if (!arango.isConnected()) {
const internal = require('internal');
internal.print("FATAL cannot connect to server '" + arango.getEndpoint() + "'");
require("process").exit(1);
}
if (arango.lastErrorMessage()) {
const internal = require('internal');
internal.print("FATAL cannot connect to server '" + arango.getEndpoint() + "': "
+ arango.lastErrorMessage());
require("process").exit(1);
}
require("@arangodb/inspector");
require("internal").exit();

1
js/client/modules/@arangodb/arangosh.js
View File

@ -1,4 +1,5 @@
/* jshint strict: false */
global.console = global.console || require('console');
// //////////////////////////////////////////////////////////////////////////////
// / @brief ArangoShell client API

732
js/client/modules/@arangodb/inspector.js Normal file
View File

@ -0,0 +1,732 @@
/* global require, arango, print, db */
'use strict';
// /////////////////////////////////////////////////////////////////////////////
// @brief ArangoDB Doctor
//
// @file
//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Dr. Frank Celler
// @author Kaveh Vahedipour
// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany
// /////////////////////////////////////////////////////////////////////////////
var internal = require('internal');
var fs = require('fs');
const executeExternalAndWait = internal.executeExternalAndWait;
const _ = require("lodash");
const servers = {};
let possibleAgent = null;
var healthRecord = {};
function INFO() {
let args = Array.prototype.slice.call(arguments);
print.apply(print, ["INFO"].concat(args));
}
function WARN() {
let args = Array.prototype.slice.call(arguments);
print.apply(print, ["WARN"].concat(args));
}
function ERROR() {
let args = Array.prototype.slice.call(arguments);
print.apply(print, ["ERROR"].concat(args));
}
function loadAgencyConfig() {
var configuration = arango.GET("/_api/agency/config");
return configuration;
}
/**
* @brief Sort shard keys according to their numbers omitting startign 's'
*
* @param keys Keys
*/
function sortShardKeys(keys) {
var ret = [];
// Get rid of 's' up front
keys.forEach(function(key, index, theArray) {
theArray[index] = key.substring(1);
});
// Sort keeping indices
var ind = range(0,keys.length-1);
ind.sort(function compare(i, j) {
return parseInt(keys[i]) > parseInt(keys[j]);
});
ind.forEach(function (i) {
ret.push('s'+keys[i]);
});
return ret;
}
function agencyInspector(obj) {
var nerrors = 0;
var nwarnings = 0;
var ndatabases = 0;
var ncollections = 0;
var nshards = 0;
var ncolinplannotincurrent = 0;
var report = {};
INFO('Analysing agency dump ...');
// Must be array with length 1
if (obj.length !== 1) {
ERROR('agency dump must be an array with a single object inside');
process.exit(1);
}
// Check for /arango and assign
if (!obj[0].hasOwnProperty('arango')) {
ERROR('agency must have attribute "arango" as root');
process.exit(1);
}
const agency = obj[0]['arango'];
// Must have Plan, Current, Supervision, Target, Sync
if (!agency.hasOwnProperty('Plan')) {
ERROR('agency must have attribute "arango/Plan" object');
process.exit(1);
}
const plan = agency.Plan;
if (!plan.hasOwnProperty('Version')) {
ERROR('plan has no version');
process.exit(1);
}
if (!agency.hasOwnProperty('Current')) {
ERROR('agency must have attribute "arango/Current" object');
process.exit(1);
}
const current = agency.Current;
// Start sanity of plan
INFO('Plan (version ' + plan.Version+ ')');
// Planned databases check if also in collections and current
INFO(" Databases");
report.Databases = {};
if (!plan.hasOwnProperty('Databases')) {
ERROR('no databases in plan');
return;
}
Object.keys(plan.Databases).forEach(function(database) {
INFO(' ' + database);
report.Databases[database] = {};
if (!plan.Collections.hasOwnProperty(database)) {
WARN('found planned database "' + database + '" without planned collectinos');
}
if (!current.Databases.hasOwnProperty(database)) {
WARN('found planned database "' + database + '" missing in "Current"');
}
});
INFO(" Collections");
// Planned collections
if (!plan.hasOwnProperty('Collections')) {
ERROR('no collections in plan');
process.exit(1);
}
var warned = false;
Object.keys(plan.Collections).forEach(function(database) {
++ndatabases;
INFO(' ' + database);
if (!plan.Databases.hasOwnProperty(database)) {
ERROR('found planned collections in unplanned database ' + database);
}
Object.keys(plan.Collections[database]).forEach(function(collection) {
report.Databases[database]
[plan.Collections[database][collection].name] = {};
++ncollections;
const col = plan.Collections[database][collection];
INFO(' ' + col.name);
const distributeShardsLike = col.distributeShardsLike;
var myShardKeys = sortShardKeys(Object.keys(col.shards));
if (distributeShardsLike && distributeShardsLike !== "") {
const prototype = plan.Collections[database][distributeShardsLike];
if (prototype.replicationFactor !== col.replicationFactor) {
ERROR('distributeShardsLike: replicationFactor mismatch');
}
if (prototype.numberOfShards !== col.numberOfShards) {
ERROR('distributeShardsLike: numberOfShards mismatch');
}
var prototypeShardKeys = sortShardKeys(Object.keys(prototype.shards));
var ncshards = myShardKeys.length;
if (prototypeShardKeys.length !== ncshards) {
ERROR('distributeShardsLike: shard map mismatch');
}
for (var i = 0; i < ncshards; ++i) {
const shname = myShardKeys[i];
const ccul = current.Collections[database];
if (JSON.stringify(col.shards[shname]) !==
JSON.stringify(prototype.shards[prototypeShardKeys[i]])) {
ERROR(
'distributeShardsLike: planned shard map mismatch between "/arango/Plan/Collections/' +
database + '/' + collection + '/shards/' + myShardKeys[i] +
'" and " /arango/Plan/Collections/' + database + '/' + distributeShardsLike +
'/shards/' + prototypeShardKeys[i] + '"');
INFO(' ' + JSON.stringify(col.shards[shname]));
INFO(' ' + JSON.stringify(prototype.shards[prototypeShardKeys[i]]));
}
}
}
myShardKeys.forEach(function(shname) {
++nshards;
if (current.Collections.hasOwnProperty(database)) {
if (!current.Collections[database].hasOwnProperty(collection)) {
WARN('missing collection "' + collection + '" in current');
} else {
if (!current.Collections[database][collection].hasOwnProperty(shname)) {
WARN('missing shard "' + shname + '" in current');
}
const shard = current.Collections[database][collection][shname];
if (shard) {
if (JSON.stringify(shard.servers) !== JSON.stringify(col.shards[shname])) {
if (shard.servers[0] !== col.shards[shname][0]) {
ERROR('/arango/Plan/Collections/' + database + '/' + collection + '/shards/'
+ shname + ' and /arango/Current/Collections/' + database + '/'
+ collection + '/' + shname + '/servers do not match');
} else {
var sortedPlan = (shard.servers).sort();
var sortedCurrent = (col.shards[shname]).sort();
if (JSON.stringify(sortedPlan) === JSON.stringify(sortedCurrent)) {
WARN('/arango/Plan/Collections/' + database + '/' + collection + '/shards/'
+ shname + ' and /arango/Current/Collections/' + database + '/'
+ collection + '/' + shname + '/servers follower do not match in order');
}
}
}
}
}
} else {
if (!warned) {
WARN('planned database "' + database + '" missing entirely in current');
warned = true;
}
}
});
});
});
report.servers = {};
INFO('Server health');
INFO(' DB Servers');
report.servers.dbservers = {};
const supervision = agency.Supervision;
const target = agency.Target;
var servers = plan.DBServers;
Object.keys(servers).forEach(function(serverId) {
if (!target.MapUniqueToShortID.hasOwnProperty(serverId)) {
WARN('incomplete planned db server ' + serverId + ' is missing in "Target"');
} else {
INFO(' ' + serverId + '(' + target.MapUniqueToShortID[serverId].ShortName + ')');
if (!supervision.Health.hasOwnProperty(serverId)) {
ERROR('planned db server ' + serverId + ' missing in supervision\'s health records.');
}
servers[serverId] = supervision.Health[serverId];
report.servers.dbservers[serverId] = {name: target.MapUniqueToShortID[serverId].ShortName, status : servers[serverId].Status};
if (servers[serverId].Status === "BAD") {
WARN('bad db server ' + serverId + '(' + servers[serverId].ShortName+ ')');
} else if (servers[serverId].Status === "FAILED") {
WARN('*** FAILED *** db server ' + serverId + '(' + servers[serverId].ShortName+ ')');
}
}
});
INFO(' Coordinators');
report.servers.coordinators = {};
servers = plan.Coordinators;
Object.keys(servers).forEach(function(serverId) {
if (!target.MapUniqueToShortID.hasOwnProperty(serverId)) {
WARN('incomplete planned db server ' + serverId + ' is missing in "Target"');
} else {
INFO(' ' + serverId + '(' + target.MapUniqueToShortID[serverId].ShortName + ')');
report.servers.coordinators[serverId] = {name: target.MapUniqueToShortID[serverId].ShortName, status : servers[serverId].Status};
if (!supervision.Health.hasOwnProperty(serverId)) {
WARN('planned coordinator ' + serverId + ' missing in supervision\'s health records.');
}
servers[serverId] = supervision.Health[serverId];
if (servers[serverId].Status !== "GOOD") {
WARN('*** FAILED *** coordinator ' + serverId + '(' + servers[serverId].ShortName+ ')');
}
}
});
const jobs = {
ToDo: target.ToDo,
Pending: target.Pending,
Finished: target.Finished,
Failed: target.Failed
};
var njobs = [];
var nall;
Object.keys(jobs).forEach(function (state) {
njobs.push(Object.keys(jobs[state]).length);
});
var i = 0;
INFO('Supervision activity');
INFO(' Jobs: ' + nall + '(' +
'To do: ' + njobs[0] + ', ' +
'Pending: ' + njobs[1] + ', ' +
'Finished: ' + njobs[2] + ', ' +
'Failed: ' + njobs[3] + ')'
);
report.jobs = { todo: njobs[0], pending: njobs[1], finished: njobs[2], failed: njobs[3]};
INFO('Summary');
if (nerrors > 0) {
ERROR(' ' + nerrors + ' errors');
}
if (nwarnings > 0) {
WARN(' ' + nwarnings + ' warnings');
}
INFO(' ' + ndatabases + ' databases');
INFO(' ' + ncollections + ' collections ');
INFO(' ' + nshards + ' shards ');
INFO('... agency analysis finished.');
return report;
}
/**
* @brief Create an integer range as [start,end]
*
* @param start Start of range
* @param end End of range
*/
function range(start, end) {
return Array(end - start + 1).fill().map((_, idx) => start + idx);
}
function loadAgency(conn, seen) {
var agencyDump = conn.POST("/_api/agency/read", '[["/"]]');
seen[conn.getEndpoint()] = true;
if (agencyDump.code === 404) {
WARN("not talking to an agent, got: " + JSON.stringify(agencyDump));
return null;
}
if (agencyDump.code === 307) {
var red = conn.POST_RAW("/_api/agency/read", '[["/"]]');
if (red.code === 307) {
INFO("got redirect to " + red.headers.location);
let leader = red.headers.location;
let reg = /^(http|https):\/\/(.*)\/_api\/agency\/read$/;
let m = reg.exec(leader);
if (m === null) {
WARN("unknown redirect " + leader);
return null;
} else {
if (m[1] === "http") {
leader = "tcp://" + m[2];
}
else if (m[1] === "https") {
leader = "ssl://" + m[2];
}
if (leader in seen) {
WARN("cannot find leader, tried: " + Object.keys(seen).join(", "));
return null;
}
INFO("switching to " + leader);
console.log("http+" + leader);
console.log(conn.getEndpoint());
if ("http+" + leader != conn.getEndpoint()) {
conn.reconnect(leader, "_system");
}
return loadAgencyConfig(conn, seen);
}
}
}
if (agencyDump.code !== undefined) {
WARN("failed to load agency, got: " + JSON.stringify(agencyDump));
return null;
}
return agencyDump;
}
function defineServer(type, id, source) {
if (id in servers) {
_.merge(servers[id].source, source);
} else {
servers[id] = {
type: type,
id: id,
source: source
};
}
}
function defineServerEndpoint(id, endpoint) {
const server = servers[id];
if ('endpoint' in server) {
if (server.endpoint !== endpoint) {
INFO("changing endpoint for " + id + " from "
+ server.endpoint + " to " + endpoint);
}
}
server.endpoint = endpoint;
}
function defineServerStatus(id, status) {
const server = servers[id];
if ('status' in server) {
if (server.status !== status) {
INFO("changing status for " + id + " from "
+ server.status + " to " + status);
}
}
server.status = status;
}
function defineAgentLeader(id, leading) {
const server = servers[id];
if ('leading' in server) {
if (server.leading !== leading) {
INFO("changing leading for " + id + " from "
+ server.leading + " to " + leading);
}
}
server.leading = leading;
}
function defineAgentFromStatus(status, endpoint) {
let id = status.agent.id;
let leader = status.agent.leaderId;
defineServer('AGENT', id, { status: endpoint });
defineServerEndpoint(id, endpoint);
arango.reconnect(endpoint, "_system");
const cfg = db.configuration.toArray()[0].cfg;
_.forEach(cfg.active, function(id) {
defineServer('AGENT', id, { active: endpoint });
});
_.forEach(cfg.pool, function(loc, id) {
defineServer('AGENT', id, { pool: endpoint });
defineServerEndpoint(id, loc);
defineAgentLeader(id, id === leader);
});
defineAgentLeader(id,status.agent.leading);
}
function definePrimaryFromStatus(status, endpoint) {
let id = status.serverInfo.serverId;
defineServer('PRIMARY', id, { status: endpoint });
defineServerEndpoint(id, endpoint);
let agentEndpoints = status.agency.agencyComm.endpoints;
if (0 < agentEndpoints.length) {
possibleAgent = agentEndpoints[0];
} else {
console.error("Failed to find an agency endpoint");
}
}
function defineCoordinatorFromStatus(status, endpoint) {
let id = status.serverInfo.serverId;
defineServer('COORDINATOR', id, { status: endpoint });
defineServerEndpoint(id, endpoint);
let agentEndpoints = status.agency.agencyComm.endpoints;
if (0 < agentEndpoints.length) {
possibleAgent = agentEndpoints[0];
} else {
console.error("Failed to find an agency endpoint");
}
}
function defineSingleFromStatus(status, endpoint) {
defineServer('SINGLE', 'SINGLE', { status: endpoint });
defineServerEndpoint('SINGLE', endpoint);
if (status.hasOwnProperty('agency')) {
let agentEndpoints = status.agency.agencyComm.endpoints;
if (0 < agentEndpoints.length) {
possibleAgent = agentEndpoints[0];
} else {
console.error("Failed to find an agency endpoint");
}
}
}
function serverBasics(conn) {
if (!conn) {
conn = arango;
}
const status = conn.GET("/_admin/status");
const role = status.serverInfo.role;
if (role === 'AGENT') {
defineAgentFromStatus(status, conn.getEndpoint());
} else if (role === 'PRIMARY') {
definePrimaryFromStatus(status, conn.getEndpoint());
} else if (role === 'COORDINATOR') {
defineCoordinatorFromStatus(status);
} else if (role === 'SINGLE') {
defineSingleFromStatus(status);
} else {
return "unknown";
}
return role;
}
function locateServers(plan) {
let health = plan[0].arango.Supervision.Health;
let cluster = plan[0].arango.Cluster;
_.forEach(health, function(info, id) {
const type = id.substr(0, 4);
if (type === "PRMR") {
defineServer('PRIMARY', id, { supervision: cluster });
defineServerEndpoint(id, info.Endpoint);
defineServerStatus(id, info.Status);
} else if (type === "CRDN") {
defineServer('SINGLE', id, { supervision: cluster });
defineServerEndpoint(id, info.Endpoint);
defineServerStatus(id, info.Status);
} else if (type === "SNGL") {
defineServer('SINGLE', id, { supervision: cluster });
defineServerEndpoint(id, info.Endpoint);
defineServerStatus(id, info.Status);
}
});
}
function listServers() {
return servers;
}
function getServerData(arango) {
var current = arango.getEndpoint();
var servers = listServers();
var report = {};
INFO('Collecting diagnostics from all servers ... ');
var nservers = Object.keys(servers).length;
Object.keys(servers).forEach(
function (server) {
if (nservers == 1 || servers.lengthserver !== "SINGLE") {
try {
if (servers[server].endpoint !== undefined) {
if (arango.getEndpoint() != "http+" + servers[server].endpoint) {
arango.reconnect(servers[server].endpoint, '_system');
}
}
const version = arango.GET('_api/version'); // version api
const log = arango.GET('_admin/log').text; // log api
const statistics = arango.GET('_admin/statistics').text; // log api
var agencyConfig;
if (server.startsWith("AGNT")) {
agencyConfig = arango.GET('_api/agency/config');
}
const status = arango.GET('_admin/status');
var tmp = executeExternalAndWait(
'/bin/bash', ['-c', 'dmesg | tee /tmp/inspector-dmesg.out > /dev/null']);
const dmesg = fs.readFileSync('/tmp/inspector-dmesg.out', 'utf8');
tmp = executeExternalAndWait(
'/bin/bash', ['-c', 'df -h | tee /tmp/inspector-df.out > /dev/null']);
const df = fs.readFileSync('/tmp/inspector-df.out', 'utf8');
tmp = executeExternalAndWait(
'/bin/bash', ['-c', 'cat /proc/meminfo | tee /tmp/inspector-meminfo.out > /dev/null']);
const meminfo = fs.readFileSync('/tmp/inspector-meminfo.out', 'utf8');
tmp = executeExternalAndWait(
'/bin/bash', ['-c', 'uptime | tee /tmp/inspector-uptime.out > /dev/null']);
const uptime = fs.readFileSync('/tmp/inspector-uptime.out', 'utf8');
tmp = executeExternalAndWait(
'/bin/bash', ['-c', 'uname -a | tee /tmp/inspector-uname.out > /dev/null']);
const uname = fs.readFileSync('/tmp/inspector-uname.out', 'utf8');
var top;
if (status.pid !== undefined) {
tmp = executeExternalAndWait(
'/bin/bash', ['-c', 'top -b -H -p ' + status.pid + ' -n 1 | tee /tmp/inspector-top.out > /dev/null']);
top = fs.readFileSync('/tmp/inspector-top.out', 'utf8');
}
var local = {};
try {
var localDBs = db._databases();
localDBs.forEach( function(localDB) {
db._useDatabase(localDB);
local[localDB] = {};
var localCols = db._collections();
localCols.forEach( function(localCol) {
var colName = localCol.name();
local[localDB][colName] = {};
Object.keys(localCol.properties()).forEach( function(property) {
local[localDB][colName][property] = localCol.properties()[property];
});
local[localDB][colName].index = localCol.getIndexes();
local[localDB][colName].count = localCol.count();
});});
db._useDatabase('_system');
} catch (e) {}
// report this server
report[server] = {
version:version, log:log, dmesg:dmesg, statistics:statistics,
status:status, df:df, uptime:uptime, uname:uname, meminfo:meminfo,
local:local};
if (agencyConfig !== undefined) {
report[server].config = agencyConfig;
}
if (top !== undefined) {
report[server].top = top;
}
} catch (e) {
print(e);
}
}
});
if (Object.keys(servers).length > 1) {
if (current != arango.getEndpoint()) {
arango.reconnect(current, '_system');
}
}
INFO('... dignostics collected.');
return report;
}
(function() {
try {
var type = serverBasics();
if (type !== 'AGENT') {
if (possibleAgent !== null) {
arango.reconnect(possibleAgent, "_system");
serverBasics();
}
}
if (possibleAgent !== null) {
// Agency dump and analysis
var agencyConfig = loadAgencyConfig();
var agencyDump = {};
var tries = 0;
while (true) {
if (agencyDump.leaderId !== "") {
if (agencyConfig.configuration.pool.hasOwnProperty(agencyConfig.leaderId)) {
if ("http+" + agencyConfig.configuration.pool[agencyConfig.leaderId] !=
arango.getEndpoint()) {
arango.reconnect(
agencyConfig.configuration.pool[agencyConfig.leaderId], "_system");
}
agencyDump = loadAgency(arango, {});
break;
} else { // Leader is not in pool? Fatal error;
console.error("Fatal error: " + agencyDump.leaderId +
" is not a member of the agent pool " +
JSON.stringify(agencyConfig.configuration.pool));
console.error("This deployment needs immediate administrative attention.");
possibleAgent = null;
return;
}
} else {
if (tries < 100) {
tries++;
internal.wait(1);
} else {
console.error("Error: Agency cannot elect a leader configured as " +
JSON.stringify(agencyConfig));
console.error("This deployment needs immediate administrative attention.");
possibleAgent = null;
return;
}
}
}
if (agencyDump !== null) {
locateServers(agencyDump);
}
healthRecord['analysis'] = agencyInspector(agencyDump);
healthRecord['agency'] = agencyDump[0].arango;
}
// Get all sorts of meta data from all servers
healthRecord['servers'] = getServerData(arango);
const ofname = 'arango-inspector.json';
require('fs').writeFileSync(ofname, JSON.stringify(healthRecord));
INFO("Report written to " + ofname + ".");
} catch (e) {
print(e);
}
}());

60
js/common/bootstrap/modules/internal.js
View File

@ -7,31 +7,33 @@ global.DEFINE_MODULE('internal', (function () {
const exports = {};
// //////////////////////////////////////////////////////////////////////////////
// / @brief module "internal"
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2004-2013 triAGENS GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Dr. Frank Celler
// / @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany
///////////////////////////////////////////////////////////////////////////////
// @brief module "internal"
//
// @file
//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
// Copyright 2004-2013 triAGENS GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License")
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Dr. Frank Celler
// @author Copyright 2018, ArangoDB GmbH, Cologne, Germany
// @author Copyright 2010-2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////
@ -748,6 +750,14 @@ global.DEFINE_MODULE('internal', (function () {
global.SYS_UNIT_TESTS_RESULT = value;
};
// end process
if (global.SYS_EXIT) {
exports.exit = global.SYS_EXIT;
delete global.SYS_EXIT;
} else {
exports.exit = function() {};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief structured to flat commandline arguments
// / @param longOptsEqual whether long-options are in the type --opt=value

6
js/common/bootstrap/modules/process.js
View File

@ -12,19 +12,25 @@ global.DEFINE_MODULE('process', (function () {
var exports = new EventEmitter();
exports.env = internal.env;
exports.argv = [];
exports.stdout = {
isTTY: internal.COLOR_OUTPUT,
write(text) {
console.infoLines(text);
}
};
exports.cwd = function () {
return fs.makeAbsolute('');
};
exports.nextTick = function (fn) {
fn();
};
exports.exit = internal.exit;
return exports;
}()));

38
scripts/startLeaderFollower.sh
View File

@ -1,13 +1,13 @@
#!/bin/bash
params=("$@")
rm -rf cluster
rm -rf active
if [ -d cluster-init ];then
echo "== creating cluster directory from existing cluster-init directory"
cp -a cluster-init cluster
cp -a active-init active
else
echo "== creating fresh directory"
mkdir -p cluster || { echo "failed to create cluster directory"; exit 1; }
mkdir -p active || { echo "failed to create cluster directory"; exit 1; }
#if we want to restart we should probably store the parameters line wise
fi
@ -27,8 +27,8 @@ else
exit 1
fi
if [[ -f cluster/startup_parameters ]];then
string="$(< cluster/startup_parameters)"
if [[ -f active/startup_parameters ]];then
string="$(< active/startup_parameters)"
if [[ -z "${params[@]}" ]]; then
params=( $string )
else
@ -41,7 +41,7 @@ if [[ -f cluster/startup_parameters ]];then
else
#store parmeters
if [[ -n "${params[@]}" ]]; then
echo "${params[@]}" > cluster/startup_parameters
echo "${params[@]}" > active/startup_parameters
fi
fi
@ -88,11 +88,11 @@ NATH=$(( $NRSINGLESERVERS + $NRAGENTS ))
ENDPOINT=[::]
ADDRESS=[::1]
rm -rf cluster
if [ -d cluster-init ];then
cp -a cluster-init cluster
rm -rf active
if [ -d active-init ];then
cp -a active-init active
fi
mkdir -p cluster
mkdir -p active
if [ -z "$JWT_SECRET" ];then
AUTHENTICATION="--server.authentication false"
@ -139,17 +139,17 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do
--agency.supervision-frequency $SFRE \
--agency.supervision-grace-period 5.0 \
--agency.wait-for-sync false \
--database.directory cluster/data$port \
--database.directory active/data$port \
--javascript.enabled false \
--server.endpoint $TRANSPORT://$ENDPOINT:$port \
--server.statistics false \
--server.threads 16 \
--log.file cluster/$port.log \
--log.file active/$port.log \
--log.level $LOG_LEVEL_AGENCY \
$STORAGE_ENGINE \
$AUTHENTICATION \
$SSLKEYFILE \
| tee cluster/$PORT.stdout 2>&1 &
| tee active/$PORT.stdout 2>&1 &
done
start() {
@ -158,28 +158,28 @@ start() {
TYPE=$1
PORT=$2
mkdir cluster/data$PORT cluster/apps$PORT
mkdir active/data$PORT active/apps$PORT
echo Starting $TYPE on port $PORT
$CMD \
-c none \
--database.directory cluster/data$PORT \
--database.directory active/data$PORT \
--cluster.agency-endpoint $TRANSPORT://$ENDPOINT:$AG_BASE \
--cluster.my-address $TRANSPORT://$ADDRESS:$PORT \
--server.endpoint $TRANSPORT://$ENDPOINT:$PORT \
--cluster.my-role $ROLE \
--replication.active-failover true \
--log.file cluster/$PORT.log \
--log.file active/$PORT.log \
--log.level $LOG_LEVEL \
--server.statistics true \
--server.threads 5 \
--javascript.startup-directory $SRC_DIR/js \
--javascript.module-directory $SRC_DIR/enterprise/js \
--javascript.app-path cluster/apps$PORT \
--javascript.app-path active/apps$PORT \
--log.level $LOG_LEVEL_CLUSTER \
$STORAGE_ENGINE \
$AUTHENTICATION \
$SSLKEYFILE \
| tee cluster/$PORT.stdout 2>&1 &
| tee active/$PORT.stdout 2>&1 &
}
PORTTOPDB=`expr $SS_BASE + $NRSINGLESERVERS - 1`
@ -209,7 +209,7 @@ for p in `seq $SS_BASE $PORTTOPDB` ; do
testServer $p
done
echo Done, your cluster is ready at
echo Done, your active failover pair is ready at
for p in `seq $SS_BASE $PORTTOPDB` ; do
echo " ${BUILD}/bin/arangosh --server.endpoint $TRANSPORT://[::1]:$p"
done