1
0
Fork 0

Feature 3.5/sharding config (#10111)

* added options

`--cluster.max-number-of-shards`
`--cluster.default-replication-factor`
`--cluster.min-replication-factor`
`--cluster.max-replication-factor`

* fix creating satellite collections

* added CHANGELOG entry and tests

* guard against invalid replicationFactor changes when changing properties

* fix validation

* remove stray print

* Update arangod/Cluster/ClusterFeature.h

Co-Authored-By: Michael Hackstein <michael@arangodb.com>
This commit is contained in:
Jan 2019-10-01 15:43:57 +02:00 committed by KVS85
parent 932def9cf1
commit 13dae8cace
22 changed files with 390 additions and 106 deletions

View File

@ -1,6 +1,31 @@
v3.5.1 (XXXX-XX-XX)
-------------------
* Added startup option `--cluster.max-number-of-shards` for restricting the
maximum number of shards when creating new collections. The default
value for this setting is `1000`, which is also the previously hard-coded
built-in limit. A value of `0` for this option means "unrestricted".
When the setting is adjusted, it will not affect already existing
collections, but only collections that are created or restored
afterwards.
* Added startup options for managing the replication factor for newly
created collections:
- `--cluster.min-replication-factor`: this settings controls the minimum
replication factor value that is permitted when creating new collections.
No collections can be created which have a replication factor value
below this setting's value. The default value is 1.
- `--cluster.max-replication-factor`: this settings controls the maximum
replication factor value that is permitted when creating new collections.
No collections can be created which have a replication factor value
above this setting's value. The default value is 10.
- `--cluster.default-replication-factor`: this settings controls the default
replication factor value that is used when creating new collections and
no value of replication factor has been specified.
If no value is set for this option, the value of the option
`--cluster.min-replication-factor` will be used.
* Fixed unintended multiple unlock commands from coordinator to
transaction locked db servers.

View File

@ -124,10 +124,22 @@ void ClusterFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
"this server's advertised endpoint (e.g. external IP "
"address or load balancer, optional)",
new StringParameter(&_myAdvertisedEndpoint));
options->addOption("--cluster.default-replication-factor",
"default replication factor for new collections",
new UInt32Parameter(&_defaultReplicationFactor)).setIntroducedIn(30501);
options->addOption("--cluster.system-replication-factor",
"replication factor for system collections",
new UInt32Parameter(&_systemReplicationFactor));
options->addOption("--cluster.min-replication-factor",
"minimum replication factor for new collections",
new UInt32Parameter(&_minReplicationFactor)).setIntroducedIn(30501);
options->addOption("--cluster.max-replication-factor",
"maximum replication factor for new collections (0 = unrestricted)",
new UInt32Parameter(&_maxReplicationFactor)).setIntroducedIn(30501);
options->addOption(
"--cluster.create-waits-for-sync-replication",
@ -135,6 +147,10 @@ void ClusterFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
new BooleanParameter(&_createWaitsForSyncReplication),
arangodb::options::makeFlags(arangodb::options::Flags::Hidden));
options->addOption("--cluster.max-number-of-shards",
"maximum number of shards when creating new collections (0 = unrestricted)",
new UInt32Parameter(&_maxNumberOfShards)).setIntroducedIn(30501);
options->addOption(
"--cluster.index-create-timeout",
"amount of time (in seconds) the coordinator will wait for an index to "
@ -155,6 +171,49 @@ void ClusterFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
<< "details.";
FATAL_ERROR_EXIT();
}
if (_minReplicationFactor == 0) {
// min replication factor must not be 0
LOG_TOPIC("2fbdd", FATAL, arangodb::Logger::CLUSTER)
<< "Invalid value for `--cluster.min-replication-factor`. The value must be at least 1";
FATAL_ERROR_EXIT();
}
if (_maxReplicationFactor > 10) {
// 10 is a hard-coded limit for the replication factor
LOG_TOPIC("886c6", FATAL, arangodb::Logger::CLUSTER)
<< "Invalid value for `--cluster.max-replication-factor`. The value must not exceed 10";
FATAL_ERROR_EXIT();
}
TRI_ASSERT(_minReplicationFactor > 0);
if (!options->processingResult().touched("cluster.default-replication-factor")) {
// no default replication factor set. now use the minimum value, which is
// guaranteed to be at least 1
_defaultReplicationFactor = _minReplicationFactor;
}
if (_defaultReplicationFactor == 0) {
// default replication factor must not be 0
LOG_TOPIC("fc8a9", FATAL, arangodb::Logger::CLUSTER)
<< "Invalid value for `--cluster.default-replication-factor`. The value must be at least 1";
FATAL_ERROR_EXIT();
}
if (_defaultReplicationFactor > 0 &&
_maxReplicationFactor > 0 &&
_defaultReplicationFactor > _maxReplicationFactor) {
LOG_TOPIC("6cf0c", FATAL, arangodb::Logger::CLUSTER)
<< "Invalid value for `--cluster.default-replication-factor`. Must not be higher than `--cluster.max-replication-factor`";
FATAL_ERROR_EXIT();
}
if (_defaultReplicationFactor > 0 &&
_defaultReplicationFactor < _minReplicationFactor) {
LOG_TOPIC("b9aea", FATAL, arangodb::Logger::CLUSTER)
<< "Invalid value for `--cluster.default-replication-factor`. Must not be lower than `--cluster.min-replication-factor`";
FATAL_ERROR_EXIT();
}
// check if the cluster is enabled
_enableCluster = !_agencyEndpoints.empty();

View File

@ -66,7 +66,11 @@ class ClusterFeature : public application_features::ApplicationFeature {
std::string _myRole;
std::string _myEndpoint;
std::string _myAdvertisedEndpoint;
uint32_t _systemReplicationFactor = 2;
std::uint32_t _defaultReplicationFactor = 0; // a value of 0 means it will use the min replication factor
std::uint32_t _systemReplicationFactor = 2;
std::uint32_t _minReplicationFactor = 1;
std::uint32_t _maxReplicationFactor = 10; // maximum replication factor (0 = unrestricted)
std::uint32_t _maxNumberOfShards = 1000; // maximum number of shards (0 = unrestricted)
bool _createWaitsForSyncReplication = true;
double _indexCreationTimeout = 3600.0;
@ -88,7 +92,11 @@ class ClusterFeature : public application_features::ApplicationFeature {
return _createWaitsForSyncReplication;
};
double indexCreationTimeout() const { return _indexCreationTimeout; }
uint32_t systemReplicationFactor() { return _systemReplicationFactor; };
uint32_t defaultReplicationFactor() { return _defaultReplicationFactor; }
uint32_t systemReplicationFactor() { return _systemReplicationFactor; }
std::uint32_t maxNumberOfShards() const { return _maxNumberOfShards; }
std::uint32_t minReplicationFactor() const { return _minReplicationFactor; }
std::uint32_t maxReplicationFactor() const { return _maxReplicationFactor; }
void stop() override final;

View File

@ -49,6 +49,7 @@
#include "RestServer/DatabaseFeature.h"
#include "RestServer/QueryRegistryFeature.h"
#include "RestServer/ServerIdFeature.h"
#include "Sharding/ShardingInfo.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "StorageEngine/PhysicalCollection.h"
#include "StorageEngine/StorageEngine.h"
@ -1090,6 +1091,11 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
return Result();
}
Result res = ShardingInfo::validateShardsAndReplicationFactor(parameters);
if (res.fail()) {
return res;
}
auto& dbName = _vocbase.name();
ClusterInfo* ci = ClusterInfo::instance();
@ -1185,7 +1191,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
if (!isValidReplFactorSlice) {
if (replicationFactor == 0) {
replicationFactor = 1;
replicationFactor = application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster")->defaultReplicationFactor();
if (replicationFactor == 0) {
replicationFactor = 1;
}
}
TRI_ASSERT(replicationFactor > 0);
toMerge.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor));

View File

@ -26,6 +26,7 @@
#include "Basics/Exceptions.h"
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h"
#include "Cluster/ClusterFeature.h"
#include "Cluster/ServerState.h"
#include "Logger/Logger.h"
#include "Sharding/ShardingFeature.h"
@ -61,10 +62,18 @@ ShardingInfo::ShardingInfo(arangodb::velocypack::Slice info, LogicalCollection*
VPackSlice shardKeysSlice = info.get(StaticStrings::ShardKeys);
if (ServerState::instance()->isCoordinator()) {
if ((_numberOfShards == 0 && !isSmart) || _numberOfShards > 1000) {
if (_numberOfShards == 0 && !isSmart) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"invalid number of shards");
}
// intentionally no call to validateNumberOfShards here,
// because this constructor is called from the constructor of
// LogicalCollection, and we want LogicalCollection to be created
// with any configured number of shards in case the maximum allowed
// number of shards is set or decreased in a cluster with already
// existing collections that would violate the setting.
// so we validate the number of shards against the maximum only
// when a collection is created by a user, and on a restore
}
VPackSlice distributeShardsLike = info.get(StaticStrings::DistributeShardsLike);
@ -88,7 +97,7 @@ ShardingInfo::ShardingInfo(arangodb::velocypack::Slice info, LogicalCollection*
_avoidServers.push_back(i.copyString());
} else {
LOG_TOPIC("e5bc6", ERR, arangodb::Logger::FIXME)
<< "avoidServers must be a vector of strings we got "
<< "avoidServers must be a vector of strings, we got "
<< avoidServersSlice.toJson() << ". discarding!";
_avoidServers.clear();
break;
@ -97,13 +106,14 @@ ShardingInfo::ShardingInfo(arangodb::velocypack::Slice info, LogicalCollection*
}
}
bool isSatellite = false;
auto replicationFactorSlice = info.get(StaticStrings::ReplicationFactor);
if (!replicationFactorSlice.isNone()) {
bool isError = true;
if (replicationFactorSlice.isNumber()) {
_replicationFactor = replicationFactorSlice.getNumber<size_t>();
// mop: only allow satellite collections to be created explicitly
if (_replicationFactor > 0 && _replicationFactor <= 10) {
if (_replicationFactor > 0) {
isError = false;
#ifdef USE_ENTERPRISE
} else if (_replicationFactor == 0) {
@ -120,6 +130,7 @@ ShardingInfo::ShardingInfo(arangodb::velocypack::Slice info, LogicalCollection*
_distributeShardsLike = "";
_avoidServers.clear();
isError = false;
isSatellite = true;
}
#endif
if (isError) {
@ -128,25 +139,27 @@ ShardingInfo::ShardingInfo(arangodb::velocypack::Slice info, LogicalCollection*
}
}
auto minReplicationFactorSlice = info.get(StaticStrings::MinReplicationFactor);
if (!minReplicationFactorSlice.isNone()) {
if (minReplicationFactorSlice.isNumber()) {
_minReplicationFactor = minReplicationFactorSlice.getNumber<size_t>();
if (_minReplicationFactor > _replicationFactor) {
if (!isSatellite) {
auto minReplicationFactorSlice = info.get(StaticStrings::MinReplicationFactor);
if (!minReplicationFactorSlice.isNone()) {
if (minReplicationFactorSlice.isNumber()) {
_minReplicationFactor = minReplicationFactorSlice.getNumber<size_t>();
if (_minReplicationFactor > _replicationFactor) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor cannot be larger than replicationFactor (" +
basics::StringUtils::itoa(_minReplicationFactor) + " > " +
basics::StringUtils::itoa(_replicationFactor) + ")");
}
if (_minReplicationFactor == 0 && _replicationFactor != 0) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor cannot be 0");
}
} else {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor cannot be larger then replicationFactor (" +
basics::StringUtils::itoa(_minReplicationFactor) + " > " +
basics::StringUtils::itoa(_replicationFactor) + ")");
"minReplicationFactor needs to be an integer number");
}
if (_minReplicationFactor == 0 && _replicationFactor != 0) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor cannot be 0");
}
} else {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor needs to be an integer number");
}
}
@ -363,7 +376,7 @@ void ShardingInfo::replicationFactor(size_t replicationFactor) {
if (replicationFactor < _minReplicationFactor) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"replicationFactor cannot be smaller then minReplicationFactor (" +
"replicationFactor cannot be smaller than minReplicationFactor (" +
basics::StringUtils::itoa(_replicationFactor) + " < " +
basics::StringUtils::itoa(_minReplicationFactor) + ")");
}
@ -379,7 +392,7 @@ void ShardingInfo::minReplicationFactor(size_t minReplicationFactor) {
if (minReplicationFactor > _replicationFactor) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor cannot be larger then replicationFactor (" +
"minReplicationFactor cannot be larger than replicationFactor (" +
basics::StringUtils::itoa(_minReplicationFactor) + " > " +
basics::StringUtils::itoa(_replicationFactor) + ")");
}
@ -390,7 +403,7 @@ void ShardingInfo::setMinAndMaxReplicationFactor(size_t minimal, size_t maximal)
if (minimal > maximal) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"minReplicationFactor cannot be larger then replicationFactor (" +
"minReplicationFactor cannot be larger than replicationFactor (" +
basics::StringUtils::itoa(minimal) + " > " +
basics::StringUtils::itoa(maximal) + ")");
}
@ -459,3 +472,45 @@ int ShardingInfo::getResponsibleShard(arangodb::velocypack::Slice slice, bool do
return _shardingStrategy->getResponsibleShard(slice, docComplete, shardID,
usesDefaultShardKeys, key);
}
Result ShardingInfo::validateShardsAndReplicationFactor(arangodb::velocypack::Slice slice) {
Result res;
auto const* cl = application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster");
auto numberOfShardsSlice = slice.get(StaticStrings::NumberOfShards);
if (numberOfShardsSlice.isNumber()) {
uint32_t const maxNumberOfShards = cl->maxNumberOfShards();
uint32_t numberOfShards = numberOfShardsSlice.getNumber<uint32_t>();
if (maxNumberOfShards > 0 &&
numberOfShards > maxNumberOfShards) {
res.reset(TRI_ERROR_CLUSTER_TOO_MANY_SHARDS,
std::string("too many shards. maximum number of shards is ") + std::to_string(maxNumberOfShards));
}
}
auto replicationFactorSlice = slice.get(StaticStrings::ReplicationFactor);
if (replicationFactorSlice.isNumber()) {
uint32_t const minReplicationFactor = cl->minReplicationFactor();
uint32_t const maxReplicationFactor = cl->maxReplicationFactor();
int64_t replicationFactorProbe = replicationFactorSlice.getNumber<int64_t>();
if (replicationFactorProbe <= 0) {
res.reset(TRI_ERROR_BAD_PARAMETER, "invalid value for replicationFactor");
} else {
uint32_t replicationFactor = replicationFactorSlice.getNumber<uint32_t>();
if (replicationFactor > maxReplicationFactor &&
maxReplicationFactor > 0) {
res.reset(TRI_ERROR_BAD_PARAMETER,
std::string("replicationFactor must not be higher than maximum allowed replicationFactor (") + std::to_string(maxReplicationFactor) + ")");
} else if (replicationFactor < minReplicationFactor &&
minReplicationFactor > 0) {
res.reset(TRI_ERROR_BAD_PARAMETER,
std::string("replicationFactor must not be lower than minimum allowed replicationFactor (") + std::to_string(minReplicationFactor) + ")");
}
}
}
return res;
}

View File

@ -24,6 +24,8 @@
#ifndef ARANGOD_CLUSTER_SHARDING_INFO_H
#define ARANGOD_CLUSTER_SHARDING_INFO_H 1
#include "Basics/Result.h"
#include <velocypack/Builder.h>
#include <velocypack/Slice.h>
#include <unordered_set>
@ -75,6 +77,10 @@ class ShardingInfo {
/// in its constructor, after the shardingInfo has been set up already
void numberOfShards(size_t numberOfShards);
/// @brief validates the number of shards and the replication factor
/// in slice against the minimum and maximum configured values
static Result validateShardsAndReplicationFactor(arangodb::velocypack::Slice slice);
bool usesDefaultShardKeys() const;
std::vector<std::string> const& shardKeys() const;

View File

@ -50,6 +50,7 @@
#include "Basics/Utf8Helper.h"
#include "Basics/conversions.h"
#include "Basics/tri-strings.h"
#include "Cluster/ClusterFeature.h"
#include "Cluster/ClusterInfo.h"
#include "Cluster/ServerState.h"
#include "GeneralServer/AuthenticationFeature.h"
@ -2065,6 +2066,36 @@ void TRI_InitV8VocBridge(v8::Isolate* isolate, v8::Handle<v8::Context> context,
v8::Boolean::New(isolate,
StatisticsFeature::enabled()))
.FromMaybe(false); // ignore result //, v8::ReadOnly);
// replication factors
context->Global()
->DefineOwnProperty(TRI_IGETC,
TRI_V8_ASCII_STRING(isolate, "DEFAULT_REPLICATION_FACTOR"),
v8::Number::New(isolate,
application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster")->defaultReplicationFactor()), v8::ReadOnly)
.FromMaybe(false); // ignore result
context->Global()
->DefineOwnProperty(TRI_IGETC,
TRI_V8_ASCII_STRING(isolate, "MIN_REPLICATION_FACTOR"),
v8::Number::New(isolate,
application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster")->minReplicationFactor()), v8::ReadOnly)
.FromMaybe(false); // ignore result
context->Global()
->DefineOwnProperty(TRI_IGETC,
TRI_V8_ASCII_STRING(isolate, "MAX_REPLICATION_FACTOR"),
v8::Number::New(isolate,
application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster")->maxReplicationFactor()), v8::ReadOnly)
.FromMaybe(false); // ignore result
// max number of shards
context->Global()
->DefineOwnProperty(TRI_IGETC,
TRI_V8_ASCII_STRING(isolate, "MAX_NUMBER_OF_SHARDS"),
v8::Number::New(isolate,
application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster")->maxNumberOfShards()), v8::ReadOnly)
.FromMaybe(false); // ignore result
// a thread-global variable that will is supposed to contain the AQL module
// do not remove this, otherwise AQL queries will break

View File

@ -40,6 +40,7 @@
#include "Scheduler/Scheduler.h"
#include "Scheduler/SchedulerFeature.h"
#include "Sharding/ShardingFeature.h"
#include "Sharding/ShardingInfo.h"
#include "StorageEngine/PhysicalCollection.h"
#include "Transaction/V8Context.h"
#include "Utils/Events.h"
@ -259,6 +260,13 @@ Result Collections::create(TRI_vocbase_t& vocbase,
builder.openArray();
for (auto const& info : infos) {
TRI_ASSERT(builder.isOpenArray());
if (ServerState::instance()->isCoordinator()) {
Result res = ShardingInfo::validateShardsAndReplicationFactor(info.properties);
if (res.fail()) {
return res;
}
}
if (info.name.empty()) {
events::CreateCollection(vocbase.name(), info.name, TRI_ERROR_ARANGO_ILLEGAL_NAME);
@ -275,6 +283,16 @@ Result Collections::create(TRI_vocbase_t& vocbase,
arangodb::velocypack::Value(static_cast<int>(info.collectionType)));
helper.add(arangodb::StaticStrings::DataSourceName,
arangodb::velocypack::Value(info.name));
if (ServerState::instance()->isCoordinator()) {
// patch default replicationFactor into the data
VPackSlice replicationFactorSlice = info.properties.get(arangodb::StaticStrings::ReplicationFactor);
if (replicationFactorSlice.isNone()) {
helper.add(arangodb::StaticStrings::ReplicationFactor,
VPackValue(application_features::ApplicationServer::getFeature<ClusterFeature>("Cluster")->defaultReplicationFactor()));
}
}
helper.close();
VPackBuilder merged =
VPackCollection::merge(info.properties, helper.slice(), false, true);
@ -566,6 +584,11 @@ Result Collections::updateProperties(LogicalCollection& collection,
auto info = ci->getCollection(collection.vocbase().name(),
std::to_string(collection.id()));
Result res = ShardingInfo::validateShardsAndReplicationFactor(props);
if (res.fail()) {
return res;
}
return info->properties(props, partialUpdate);
} else {
auto ctx = transaction::V8Context::CreateWhenRequired(collection.vocbase(), false);

View File

@ -90,7 +90,11 @@ router.get('/config.js', function (req, res) {
engine: db._engine().name,
statisticsEnabled: internal.enabledStatistics(),
foxxStoreEnabled: !internal.isFoxxStoreDisabled(),
foxxApiEnabled: !internal.isFoxxApiDisabled()
foxxApiEnabled: !internal.isFoxxApiDisabled(),
minReplicationFactor: internal.minReplicationFactor,
maxReplicationFactor: internal.maxReplicationFactor,
defaultReplicationFactor: internal.defaultReplicationFactor,
maxNumberOfShards: internal.maxNumberOfShards
})}`
);
})

File diff suppressed because one or more lines are too long

View File

@ -1 +1 @@
<!DOCTYPE html><html lang="en" xmlns="http://www.w3.org/1999/html"><head><meta charset="utf-8"><title>ArangoDB Web Interface</title><meta name="description" content="ArangoDB Admin Web Interface"><meta name="author" content="Heiko Kernbach, Michael Hackstein"><meta name="viewport" content="width=device-width,initial-scale=1"><link href="css/style.css" rel="stylesheet"><link href="css/sass.css" rel="stylesheet"><link rel="shortcut icon" type="image/x-icon" href="favicon.ico"><script src="config.js"></script></head><body><nav class="navbar" style="display: none"><div class="primary"><div class="navlogo"><a class="logo big" href="#"><img id="ArangoDBLogo" class="arangodbLogo" src="img/arangodb-edition-optimized.svg"></a><a class="logo small" href="#"><img class="arangodbLogo" src="img/arangodb_logo_small.png"></a><a class="version"><span id="currentVersion"></span></a></div><div class="statmenu" id="statisticBar"></div><div class="navmenu" id="navigationBar"></div></div></nav><div id="modalPlaceholder"></div><div class="bodyWrapper" style="display: none"><div class="centralRow"><div id="navbar2" class="navbarWrapper secondary"><div class="subnavmenu" id="subNavigationBar"></div></div><div class="resizecontainer contentWrapper"><div id="loadingScreen" class="loadingScreen" style="display: none"><i class="fa fa-circle-o-notch fa-spin fa-3x fa-fw margin-bottom"></i> <span class="sr-only">Loading...</span></div><div id="content" class="centralContent"></div><footer class="footer"><div id="footerBar"></div></footer></div></div></div><div id="progressPlaceholder" style="display:none"></div><div id="spotlightPlaceholder" style="display:none"></div><div id="graphSettingsContent" style="display: none"></div><div id="filterSelectDiv" style="display:none"></div><div id="offlinePlaceholder" style="display:none"><div class="offline-div"><div class="pure-u"><div class="pure-u-1-4"></div><div class="pure-u-1-2 offline-window"><div class="offline-header"><h3>You have been disconnected from the server</h3></div><div class="offline-body"><p>The connection to the server has been lost. The server may be under heavy load.</p><p>Trying to reconnect in <span id="offlineSeconds">10</span> seconds.</p><p class="animation_state"><span><button class="button-success">Reconnect now</button></span></p></div></div><div class="pure-u-1-4"></div></div></div></div><div class="arangoFrame" style=""><div class="outerDiv"><div class="innerDiv"></div></div></div><script src="libs.js?version=1565792175175"></script><script src="app.js?version=1565792175175"></script><script src="templates.js?version=1565792175175"></script></body></html>
<!DOCTYPE html><html lang="en" xmlns="http://www.w3.org/1999/html"><head><meta charset="utf-8"><title>ArangoDB Web Interface</title><meta name="description" content="ArangoDB Admin Web Interface"><meta name="author" content="Heiko Kernbach, Michael Hackstein"><meta name="viewport" content="width=device-width,initial-scale=1"><link href="css/style.css" rel="stylesheet"><link href="css/sass.css" rel="stylesheet"><link rel="shortcut icon" type="image/x-icon" href="favicon.ico"><script src="config.js"></script></head><body><nav class="navbar" style="display: none"><div class="primary"><div class="navlogo"><a class="logo big" href="#"><img id="ArangoDBLogo" class="arangodbLogo" src="img/arangodb-edition-optimized.svg"></a><a class="logo small" href="#"><img class="arangodbLogo" src="img/arangodb_logo_small.png"></a><a class="version"><span id="currentVersion"></span></a></div><div class="statmenu" id="statisticBar"></div><div class="navmenu" id="navigationBar"></div></div></nav><div id="modalPlaceholder"></div><div class="bodyWrapper" style="display: none"><div class="centralRow"><div id="navbar2" class="navbarWrapper secondary"><div class="subnavmenu" id="subNavigationBar"></div></div><div class="resizecontainer contentWrapper"><div id="loadingScreen" class="loadingScreen" style="display: none"><i class="fa fa-circle-o-notch fa-spin fa-3x fa-fw margin-bottom"></i> <span class="sr-only">Loading...</span></div><div id="content" class="centralContent"></div><footer class="footer"><div id="footerBar"></div></footer></div></div></div><div id="progressPlaceholder" style="display:none"></div><div id="spotlightPlaceholder" style="display:none"></div><div id="graphSettingsContent" style="display: none"></div><div id="filterSelectDiv" style="display:none"></div><div id="offlinePlaceholder" style="display:none"><div class="offline-div"><div class="pure-u"><div class="pure-u-1-4"></div><div class="pure-u-1-2 offline-window"><div class="offline-header"><h3>You have been disconnected from the server</h3></div><div class="offline-body"><p>The connection to the server has been lost. The server may be under heavy load.</p><p>Trying to reconnect in <span id="offlineSeconds">10</span> seconds.</p><p class="animation_state"><span><button class="button-success">Reconnect now</button></span></p></div></div><div class="pure-u-1-4"></div></div></div></div><div class="arangoFrame" style=""><div class="outerDiv"><div class="innerDiv"></div></div></div><script src="libs.js?version=1569844092461"></script><script src="app.js?version=1569844092461"></script><script src="templates.js?version=1569844092461"></script></body></html>

File diff suppressed because one or more lines are too long

View File

@ -8,6 +8,10 @@
el: '#content',
el2: '#collectionsThumbnailsIn',
readOnly: false,
defaultReplicationFactor: 0,
minReplicationFactor: 0,
maxReplicationFactor: 0,
maxNumberOfShards: 0,
searchTimeout: null,
refreshRate: 10000,
@ -88,7 +92,11 @@
}
},
initialize: function () {
initialize: function (options) {
this.defaultReplicationFactor = frontendConfig.defaultReplicationFactor;
this.minReplicationFactor = frontendConfig.minReplicationFactor;
this.maxReplicationFactor = frontendConfig.maxReplicationFactor;
this.maxNumberOfShards = frontendConfig.maxNumberOfShards;
var self = this;
window.setInterval(function () {
@ -360,7 +368,11 @@
var distributeShardsLike = '';
if (replicationFactor === '') {
replicationFactor = 1;
if (self.defaultReplicationFactor) {
replicationFactor = self.defaultReplicationFactor;
} else {
replicationFactor = 1;
}
}
if (minReplicationFactor === '') {
minReplicationFactor = 1;
@ -519,9 +531,9 @@
tableContent.push(
window.modalView.createTextEntry(
'new-collection-shards',
'Shards',
'',
'The number of shards to create. You cannot change this afterwards. ',
'Number of shards',
this.maxNumberOfShards === 1 ? String(this.maxNumberOfShards) : 0,
'The number of shards to create. The maximum value is ' + this.maxNumberOfShards + '. You cannot change this afterwards. ',
'',
true
)
@ -537,6 +549,29 @@
false
)
);
tableContent.push(
window.modalView.createTextEntry(
'new-replication-factor',
'Replication factor',
this.defaultReplicationFactor,
'Numeric value. Must be between ' +
(this.minReplicationFactor ? this.minReplicationFactor : 1) +
' and ' +
(this.maxReplicationFactor ? this.maxReplicationFactor : 10) +
'. Total number of copies of the data in the cluster',
'',
false,
[
{
rule: Joi.string().allow('').optional().regex(/^[1-9][0-9]*$/),
msg: 'Must be a number between ' +
(this.minReplicationFactor ? this.minReplicationFactor : 1) +
' and ' +
(this.maxReplicationFactor ? this.maxReplicationFactor : 10) + '.'
}
]
)
);
}
buttons.push(
@ -581,22 +616,6 @@
)
);
}
advancedTableContent.push(
window.modalView.createTextEntry(
'new-replication-factor',
'Replication factor',
'',
'Numeric value. Must be at least 1. Total number of copies of the data in the cluster',
'',
false,
[
{
rule: Joi.string().allow('').optional().regex(/^([1-9]|10)$/),
msg: 'Must be a number between 1 and 10.'
}
]
)
);
advancedTableContent.push(
window.modalView.createTextEntry(
'new-min-replication-factor',

View File

@ -132,6 +132,7 @@
"ERROR_REPLICATION_WRONG_CHECKSUM" : { "code" : 1416, "message" : "wrong checksum" },
"ERROR_REPLICATION_SHARD_NONEMPTY" : { "code" : 1417, "message" : "shard not empty" },
"ERROR_CLUSTER_SERVER_UNKNOWN" : { "code" : 1449, "message" : "got a request from an unkown server" },
"ERROR_CLUSTER_TOO_MANY_SHARDS" : { "code" : 1450, "message" : "too many shards" },
"ERROR_CLUSTER_COLLECTION_ID_EXISTS" : { "code" : 1453, "message" : "collection ID already exists" },
"ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION_IN_PLAN" : { "code" : 1454, "message" : "could not create collection in plan" },
"ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION" : { "code" : 1456, "message" : "could not create collection" },

View File

@ -527,6 +527,27 @@
delete global.SYS_DEBUG_CAN_USE_FAILAT;
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief replicationFactor(s) and number of shards
// //////////////////////////////////////////////////////////////////////////////
if (global.DEFAULT_REPLICATION_FACTOR) {
exports.defaultReplicationFactor = global.DEFAULT_REPLICATION_FACTOR;
delete global.DEFAULT_REPLICATION_FACTOR;
}
if (global.MIN_REPLICATION_FACTOR) {
exports.minReplicationFactor = global.MIN_REPLICATION_FACTOR;
delete global.MIN_REPLICATION_FACTOR;
}
if (global.MAX_REPLICATION_FACTOR) {
exports.maxReplicationFactor = global.MAX_REPLICATION_FACTOR;
delete global.MAX_REPLICATION_FACTOR;
}
if (global.MAX_NUMBER_OF_SHARDS) {
exports.maxNumberOfShards = global.MAX_NUMBER_OF_SHARDS;
delete global.MAX_NUMBER_OF_SHARDS;
}
// /////////////////////////////////////////////////////////////////////////////
// / @brief whether or not clustering is enabled
// /////////////////////////////////////////////////////////////////////////////

View File

@ -170,6 +170,7 @@ ERROR_REPLICATION_SHARD_NONEMPTY,1417,"shard not empty","Will be raised when a s
################################################################################
ERROR_CLUSTER_SERVER_UNKNOWN,1449,"got a request from an unkown server","Will be raised on some occasions when one server gets a request from another, which has not (yet?) been made known via the agency."
ERROR_CLUSTER_TOO_MANY_SHARDS,1450,"too many shards","Will be raised when the number of shards for a collection is higher than allowed."
ERROR_CLUSTER_COLLECTION_ID_EXISTS,1453,"collection ID already exists","Will be raised when a coordinator in a cluster tries to create a collection and the collection ID already exists."
ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION_IN_PLAN,1454,"could not create collection in plan","Will be raised when a coordinator in a cluster cannot create an entry for a new collection in the Plan hierarchy in the agency."
ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION,1456,"could not create collection","Will be raised when a coordinator in a cluster notices that some DBServers report problems when creating shards for a new collection."

View File

@ -131,6 +131,7 @@ void TRI_InitializeErrorMessages() {
REG_ERROR(ERROR_REPLICATION_WRONG_CHECKSUM, "wrong checksum");
REG_ERROR(ERROR_REPLICATION_SHARD_NONEMPTY, "shard not empty");
REG_ERROR(ERROR_CLUSTER_SERVER_UNKNOWN, "got a request from an unkown server");
REG_ERROR(ERROR_CLUSTER_TOO_MANY_SHARDS, "too many shards");
REG_ERROR(ERROR_CLUSTER_COLLECTION_ID_EXISTS, "collection ID already exists");
REG_ERROR(ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION_IN_PLAN, "could not create collection in plan");
REG_ERROR(ERROR_CLUSTER_COULD_NOT_CREATE_COLLECTION, "could not create collection");

View File

@ -664,6 +664,12 @@ constexpr int TRI_ERROR_REPLICATION_SHARD_NONEMPTY
/// another, which has not (yet?) been made known via the agency.
constexpr int TRI_ERROR_CLUSTER_SERVER_UNKNOWN = 1449;
/// 1450: ERROR_CLUSTER_TOO_MANY_SHARDS
/// "too many shards"
/// Will be raised when the number of shards for a collection is higher than
/// allowed.
constexpr int TRI_ERROR_CLUSTER_TOO_MANY_SHARDS = 1450;
/// 1453: ERROR_CLUSTER_COLLECTION_ID_EXISTS
/// "collection ID already exists"
/// Will be raised when a coordinator in a cluster tries to create a collection

View File

@ -32,6 +32,7 @@ var jsunity = require("jsunity");
var arangodb = require("@arangodb");
var ERRORS = arangodb.errors;
var db = arangodb.db;
let internal = require("internal");
////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
@ -407,58 +408,6 @@ function ClusterCollectionSuite () {
assertNull(db._collection("UnitTestsClusterCrud"));
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test create
////////////////////////////////////////////////////////////////////////////////
testCreateInvalidShardKeys1 : function () {
try {
db._create("UnitTestsClusterCrud", { shardKeys: [ ] });
}
catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test create
////////////////////////////////////////////////////////////////////////////////
testCreateInvalidShardKeys2 : function () {
try {
db._create("UnitTestsClusterCrud", { shardKeys: [ "_rev" ] });
}
catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test create
////////////////////////////////////////////////////////////////////////////////
testCreateInvalidShardKeys3 : function () {
try {
db._create("UnitTestsClusterCrud", { shardKeys: [ "", "foo" ] });
}
catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test create
////////////////////////////////////////////////////////////////////////////////
testCreateInvalidShardKeys4 : function () {
try {
db._create("UnitTestsClusterCrud", { shardKeys: [ "a", "_from" ] });
}
catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test create
////////////////////////////////////////////////////////////////////////////////
@ -466,8 +415,8 @@ function ClusterCollectionSuite () {
testCreateInvalidNumberOfShards1 : function () {
try {
db._create("UnitTestsClusterCrud", { numberOfShards : 0 });
}
catch (err) {
fail();
} catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
},
@ -479,9 +428,75 @@ function ClusterCollectionSuite () {
testCreateInvalidNumberOfShards2 : function () {
try {
db._create("UnitTestsClusterCrud", { numberOfShards : 1024 * 1024 });
fail();
} catch (err) {
assertEqual(ERRORS.ERROR_CLUSTER_TOO_MANY_SHARDS.code, err.errorNum);
}
catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test numberOfShards
////////////////////////////////////////////////////////////////////////////////
testCreateAsManyShardsAsAllowed : function () {
let max = internal.maxNumberOfShards;
if (max > 0) {
db._create("UnitTestsClusterCrud", { numberOfShards : max });
let properties = db["UnitTestsClusterCrud"].properties();
assertEqual(max, properties.numberOfShards);
}
},
testCreateMoreShardsThanAllowed : function () {
let max = internal.maxNumberOfShards;
if (max > 0) {
try {
db._create("UnitTestsClusterCrud", { numberOfShards : max + 1 });
fail();
} catch (err) {
assertEqual(ERRORS.ERROR_CLUSTER_TOO_MANY_SHARDS.code, err.errorNum);
}
}
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test replicationFactor
////////////////////////////////////////////////////////////////////////////////
testMinReplicationFactor : function () {
let min = internal.minReplicationFactor;
if (min > 0) {
db._create("UnitTestsClusterCrud", { replicationFactor: min });
let properties = db["UnitTestsClusterCrud"].properties();
assertEqual(min, properties.replicationFactor);
try {
db["UnitTestsClusterCrud"].properties({ replicationFactor: min - 1 });
fail();
} catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
}
},
testMaxReplicationFactor : function () {
let max = internal.maxReplicationFactor;
if (max > 0) {
try {
db._create("UnitTestsClusterCrud", { replicationFactor: max });
let properties = db["UnitTestsClusterCrud"].properties();
assertEqual(max, properties.replicationFactor);
try {
db["UnitTestsClusterCrud"].properties({ replicationFactor: max + 1 });
fail();
} catch (err) {
assertEqual(ERRORS.ERROR_BAD_PARAMETER.code, err.errorNum);
}
} catch (err) {
// if creation fails, then it must have been exactly this error
assertEqual(ERRORS.ERROR_CLUSTER_INSUFFICIENT_DBSERVERS.code, err.errorNum);
}
}
},