1
0
Fork 0

Merge branch 'devel' of https://github.com/arangodb/arangodb into bug-fix/install-js-server-files

* 'devel' of https://github.com/arangodb/arangodb:
  Bug fix/fix internal issue 4451 (#10540)
  Bug fix/increase test timeout (#10596)
This commit is contained in:
Jan Christoph Uhde 2019-12-02 10:17:08 +01:00
commit 51679472dd
29 changed files with 421 additions and 177 deletions

View File

@ -1,6 +1,10 @@
devel
-----
* Fixed permissions for dump/restore.
* The _users collection is now properly restored when using arangorestore.
* Updated arangosync to 0.7.1.
* rename `minReplicationFactor` into `writeConcern` to make it consistent with

View File

@ -393,6 +393,12 @@ VPackBuilder auth::UserManager::allUsers() {
return result;
}
void auth::UserManager::triggerCacheRevalidation() {
triggerLocalReload();
triggerGlobalReload();
loadFromDB();
}
/// Trigger eventual reload, user facing API call
void auth::UserManager::triggerGlobalReload() {
if (!ServerState::instance()->isCoordinator()) {

View File

@ -95,6 +95,9 @@ class UserManager {
/// Trigger eventual reload on all other coordinators (and in TokenCache)
void triggerGlobalReload();
/// Trigger cache revalidation after user restore
void triggerCacheRevalidation();
/// Create the root user with a default password, will fail if the user
/// already exists. Only ever call if you can guarantee to be in charge
void createRootUser();

View File

@ -224,7 +224,6 @@ void ClusterInfo::cleanup() {
_plannedViews.clear();
_plannedCollections.clear();
_shards.clear();
_shardKeys.clear();
_shardIds.clear();
_currentCollections.clear();
}
@ -235,6 +234,7 @@ void ClusterInfo::triggerBackgroundGetIds() {
_uniqid._nextUpperValue = 0ULL;
try {
_idLock.assertLockedByCurrentThread();
if (_uniqid._backgroundJobIsRunning) {
return;
}
@ -598,7 +598,7 @@ void ClusterInfo::loadPlan() {
// >
decltype(_shards) newShards;
decltype(_shardServers) newShardServers;
decltype(_shardKeys) newShardKeys;
decltype(_shardToName) newShardToName;
bool swapDatabases = false;
bool swapCollections = false;
@ -957,17 +957,16 @@ void ClusterInfo::loadPlan() {
databaseCollections.try_emplace(collectionId, newCollection);
}
newShardKeys.try_emplace(collectionId, std::make_shared<std::vector<std::string>>(
newCollection->shardKeys()));
auto shardIDs = newCollection->shardIds();
auto shards = std::make_shared<std::vector<std::string>>();
shards->reserve(shardIDs->size());
newShardToName.reserve(shardIDs->size());
for (auto const& p : *shardIDs) {
TRI_ASSERT(p.first.size() >= 2);
shards->push_back(p.first);
newShardServers.try_emplace(p.first, p.second);
newShardToName.try_emplace(p.first, newCollection->name());
}
// Sort by the number in the shard ID ("s0000001" for example):
@ -1053,8 +1052,8 @@ void ClusterInfo::loadPlan() {
if (swapCollections) {
_plannedCollections.swap(newCollections);
_shards.swap(newShards);
_shardKeys.swap(newShardKeys);
_shardServers.swap(newShardServers);
_shardToName.swap(newShardToName);
}
if (swapViews) {
@ -4288,6 +4287,16 @@ arangodb::Result ClusterInfo::getShardServers(ShardID const& shardId,
return arangodb::Result(TRI_ERROR_FAILED);
}
CollectionID ClusterInfo::getCollectionNameForShard(ShardID const& shardId) {
READ_LOCKER(readLocker, _planProt.lock);
auto it = _shardToName.find(shardId);
if (it != _shardToName.end()) {
return it->second;
}
return StaticStrings::Empty;
}
arangodb::Result ClusterInfo::agencyDump(std::shared_ptr<VPackBuilder> body) {
AgencyCommResult dump = _agency.dump();

View File

@ -815,6 +815,9 @@ class ClusterInfo final {
* @return List of DB servers serving the shard
*/
arangodb::Result getShardServers(ShardID const& shardId, std::vector<ServerID>&);
/// @brief map shardId to collection name (not ID)
CollectionID getCollectionNameForShard(ShardID const& shardId);
/**
* @brief Lock agency's hot backup with TTL 60 seconds
@ -956,8 +959,7 @@ class ClusterInfo final {
ProtectionData _planProt;
uint64_t _planVersion; // This is the version in the Plan which underlies
// the data in _plannedCollections, _shards and
// _shardKeys
// the data in _plannedCollections and _shards
uint64_t _currentVersion; // This is the version in Current which underlies
// the data in _currentDatabases,
// _currentCollections and _shardsIds
@ -978,11 +980,10 @@ class ClusterInfo final {
std::shared_ptr<std::vector<std::string>>>
_shards; // from Plan/Collections/
// (may later come from Current/Collections/ )
std::unordered_map<CollectionID,
std::shared_ptr<std::vector<std::string>>>
_shardKeys; // from Plan/Collections/
// planned shard => servers map
std::unordered_map<ShardID, std::vector<ServerID>> _shardServers;
// planned shard ID => collection name
std::unordered_map<ShardID, CollectionID> _shardToName;
AllViews _plannedViews; // from Plan/Views/
AllViews _newPlannedViews; // views that have been created during `loadPlan`

View File

@ -113,7 +113,20 @@ futures::Future<Result> RestHandler::forwardRequest(bool& forwarded) {
return futures::makeFuture(Result());
}
std::string serverId = forwardingTarget();
ResultT forwardResult = forwardingTarget();
if (forwardResult.fail()) {
return futures::makeFuture(forwardResult.result());
}
auto forwardContent = forwardResult.get();
std::string serverId = std::get<0>(forwardContent);
bool removeHeader = std::get<1>(forwardContent);
if (removeHeader) {
_request->removeHeader(StaticStrings::Authorization);
_request->setUser("");
}
if (serverId.empty()) {
// no need to actually forward
return futures::makeFuture(Result());

View File

@ -30,6 +30,7 @@
#include "Network/Methods.h"
#include "Rest/GeneralResponse.h"
#include <Cluster/ResultT.h>
#include <atomic>
#include <thread>
@ -40,13 +41,13 @@ class ApplicationServer;
namespace basics {
class Exception;
}
namespace futures {
template<typename T>
template <typename T>
class Future;
template<typename T>
template <typename T>
class Try;
}
} // namespace futures
class GeneralRequest;
class RequestStatistics;
@ -129,9 +130,7 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
// you might need to implment this in you handler
// if it will be executed in an async job
virtual void cancel() {
_canceled.store(true);
}
virtual void cancel() { _canceled.store(true); }
virtual void handleError(basics::Exception const&) = 0;
@ -143,7 +142,11 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
/// handled by this server, the method should return an empty string.
/// Otherwise, this method should return a valid short name for the
/// target server.
virtual std::string forwardingTarget() { return ""; }
/// std::string -> empty string or valid short name
/// boolean -> should auth header and user be removed in that request
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() {
return {std::make_pair(StaticStrings::Empty, false)};
}
void resetResponse(rest::ResponseCode);
@ -154,11 +157,11 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
// generates an error
void generateError(arangodb::Result const&);
template<typename T>
template <typename T>
RestStatus waitForFuture(futures::Future<T>&& f) {
if (f.isReady()) { // fast-path out
f.result().throwIfFailed(); // just throw the error upwards
if (f.isReady()) { // fast-path out
f.result().throwIfFailed(); // just throw the error upwards
return RestStatus::DONE;
}
bool done = false;
@ -172,7 +175,7 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
});
return done ? RestStatus::DONE : RestStatus::WAITING;
}
enum class HandlerState : uint8_t {
PREPARE = 0,
EXECUTE,
@ -182,11 +185,9 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
DONE,
FAILED
};
/// handler state machine
HandlerState state() const {
return _state;
}
HandlerState state() const { return _state; }
private:
void runHandlerStateMachine();
@ -198,16 +199,14 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
/// otherwise execute() will be called
void executeEngine(bool isContinue);
void compressResponse();
protected:
protected:
std::unique_ptr<GeneralRequest> _request;
std::unique_ptr<GeneralResponse> _response;
application_features::ApplicationServer& _server;
RequestStatistics* _statistics;
private:
mutable Mutex _executionMutex;
std::function<void(rest::RestHandler*)> _callback;
@ -219,7 +218,6 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
HandlerState _state;
protected:
std::atomic<bool> _canceled;
bool _allowDirectExecution = false;

View File

@ -75,26 +75,26 @@ RestStatus RestControlPregelHandler::execute() {
}
/// @brief returns the short id of the server which should handle this request
std::string RestControlPregelHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestControlPregelHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::POST && type != rest::RequestType::GET &&
type != rest::RequestType::DELETE_REQ) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}
void RestControlPregelHandler::startExecution() {
@ -107,7 +107,7 @@ void RestControlPregelHandler::startExecution() {
// algorithm
std::string algorithm =
VelocyPackHelper::getStringValue(body, "algorithm", "");
VelocyPackHelper::getStringValue(body, "algorithm", StaticStrings::Empty);
if ("" == algorithm) {
generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND,
"invalid algorithm");

View File

@ -38,7 +38,7 @@ class RestControlPregelHandler : public arangodb::RestVocbaseBaseHandler {
RestStatus execute() override;
protected:
virtual std::string forwardingTarget() override;
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private:
void startExecution();

View File

@ -381,25 +381,25 @@ RestStatus RestCursorHandler::handleQueryResult() {
}
/// @brief returns the short id of the server which should handle this request
std::string RestCursorHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestCursorHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}
////////////////////////////////////////////////////////////////////////////////

View File

@ -88,7 +88,7 @@ class RestCursorHandler : public RestVocbaseBaseHandler {
RestStatus processQuery();
/// @brief returns the short id of the server which should handle this request
virtual std::string forwardingTarget() override;
ResultT<std::pair<std::string, bool>> forwardingTarget() override;
//////////////////////////////////////////////////////////////////////////////
/// @brief unregister the currently running query

View File

@ -120,9 +120,9 @@ void RestDocumentHandler::shutdownExecute(bool isFinalized) noexcept {
}
/// @brief returns the short id of the server which should handle this request
std::string RestDocumentHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestDocumentHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
bool found = false;
@ -131,17 +131,17 @@ std::string RestDocumentHandler::forwardingTarget() {
uint64_t tid = basics::StringUtils::uint64(value);
if (!transaction::isCoordinatorTransactionId(tid)) {
TRI_ASSERT(transaction::isLegacyTransactionId(tid));
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
////////////////////////////////////////////////////////////////////////////////

View File

@ -56,7 +56,7 @@ class RestDocumentHandler : public RestVocbaseBaseHandler {
void shutdownExecute(bool isFinalized) noexcept override final;
protected:
std::string forwardingTarget() override final;
ResultT<std::pair<std::string, bool>> forwardingTarget() override final;
private:
// inserts a document

View File

@ -245,24 +245,24 @@ void RestJobHandler::deleteJob() {
}
/// @brief returns the short id of the server which should handle this request
std::string RestJobHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestJobHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::GET && type != rest::RequestType::PUT &&
type != rest::RequestType::DELETE_REQ) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}

View File

@ -85,7 +85,7 @@ class RestJobHandler : public RestBaseHandler {
void deleteJob();
protected:
virtual std::string forwardingTarget() override;
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private:
//////////////////////////////////////////////////////////////////////////////

View File

@ -346,9 +346,9 @@ bool RestQueryHandler::parseQuery() {
}
/// @brief returns the short id of the server which should handle this request
std::string RestQueryHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestQueryHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
bool found = false;
@ -357,15 +357,15 @@ std::string RestQueryHandler::forwardingTarget() {
uint64_t tid = basics::StringUtils::uint64(value);
if (!transaction::isCoordinatorTransactionId(tid)) {
TRI_ASSERT(transaction::isLegacyTransactionId(tid));
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}

View File

@ -96,7 +96,7 @@ class RestQueryHandler : public RestVocbaseBaseHandler {
bool parseQuery();
virtual std::string forwardingTarget() override;
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
};
} // namespace arangodb

View File

@ -29,9 +29,9 @@
#include "Basics/ReadLocker.h"
#include "Basics/Result.h"
#include "Basics/RocksDBUtils.h"
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h"
#include "Basics/WriteLocker.h"
#include "Basics/StaticStrings.h"
#include "Cluster/ClusterFeature.h"
#include "Cluster/ClusterHelpers.h"
#include "Cluster/ClusterMethods.h"
@ -280,6 +280,11 @@ std::string const RestReplicationHandler::HoldReadLockCollection =
// main function that dispatches the different routes and commands
RestStatus RestReplicationHandler::execute() {
auto res = testPermissions();
if (!res.ok()) {
generateError(res);
return RestStatus::DONE;
}
// extract the request type
auto const type = _request->requestType();
auto const& suffixes = _request->suffixes();
@ -589,27 +594,113 @@ BAD_CALL:
return RestStatus::DONE;
}
/// @brief returns the short id of the server which should handle this request
std::string RestReplicationHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) {
return "";
Result RestReplicationHandler::testPermissions() {
if (!_request->authenticated()) {
return TRI_ERROR_NO_ERROR;
}
std::string user = _request->user();
auto const& suffixes = _request->suffixes();
size_t const len = suffixes.size();
if (len >= 1) {
auto const type = _request->requestType();
std::string const& command = suffixes[0];
if ((command == Batch) || (command == Inventory && type == rest::RequestType::GET) ||
(command == Dump && type == rest::RequestType::GET)) {
ServerID const& DBserver = _request->value("DBserver");
if (!DBserver.empty()) {
return DBserver;
(command == Dump && type == rest::RequestType::GET) ||
(command == RestoreCollection && type == rest::RequestType::PUT)) {
if (command == Dump) {
// check dump collection permissions (at least ro needed)
std::string collectionName = _request->value("collection");
if (ServerState::instance()->isCoordinator()) {
// We have a shard id, need to translate
ClusterInfo& ci = server().getFeature<ClusterFeature>().clusterInfo();
collectionName = ci.getCollectionNameForShard(collectionName);
}
if (!collectionName.empty()) {
auto& exec = ExecContext::current();
ExecContextSuperuserScope escope(exec.isAdminUser());
if (!exec.isAdminUser() &&
!exec.canUseCollection(collectionName, auth::Level::RO)) {
// not enough rights
return Result(TRI_ERROR_FORBIDDEN);
}
} else {
// not found, return 404
return Result(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND);
}
} else if (command == RestoreCollection) {
VPackSlice const slice = _request->payload();
VPackSlice const parameters = slice.get("parameters");
if (parameters.isObject()) {
if (parameters.get("name").isString()) {
std::string collectionName = parameters.get("name").copyString();
if (!collectionName.empty()) {
std::string dbName = _request->databaseName();
DatabaseFeature& databaseFeature =
_vocbase.server().getFeature<DatabaseFeature>();
TRI_vocbase_t* vocbase = databaseFeature.lookupDatabase(dbName);
if (vocbase == nullptr) {
return Result(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
}
std::string const& overwriteCollection =
_request->value("overwrite");
auto& exec = ExecContext::current();
ExecContextSuperuserScope escope(exec.isAdminUser());
if (overwriteCollection == "true" ||
vocbase->lookupCollection(collectionName) == nullptr) {
// 1.) re-create collection, means: overwrite=true (rw database)
// OR 2.) not existing, new collection (rw database)
if (!exec.isAdminUser() && !exec.canUseDatabase(dbName, auth::Level::RW)) {
return Result(TRI_ERROR_FORBIDDEN);
}
} else {
// 3.) Existing collection (ro database, rw collection)
// no overwrite. restoring into an existing collection
if (!exec.isAdminUser() &&
!exec.canUseCollection(collectionName, auth::Level::RW)) {
return Result(TRI_ERROR_FORBIDDEN);
}
}
} else {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER,
"empty collection name");
}
} else {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid collection name type");
}
} else {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid collection parameter type");
}
}
}
}
return Result(TRI_ERROR_NO_ERROR);
}
return "";
/// @brief returns the short id of the server which should handle this request
ResultT<std::pair<std::string, bool>> RestReplicationHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) {
return {std::make_pair("", false)};
}
auto res = testPermissions();
if (!res.ok()) {
return res;
}
ServerID const& DBserver = _request->value("DBserver");
if (!DBserver.empty()) {
// if DBserver property present, remove auth header
return std::make_pair(DBserver, true);
}
return {std::make_pair(StaticStrings::Empty, false)};
}
////////////////////////////////////////////////////////////////////////////////
@ -712,8 +803,16 @@ void RestReplicationHandler::handleCommandClusterInventory() {
vocbase->toVelocyPack(resultBuilder);
}
auto& exec = ExecContext::current();
ExecContextSuperuserScope escope(exec.isAdminUser());
resultBuilder.add("collections", VPackValue(VPackValueType::Array));
for (std::shared_ptr<LogicalCollection> const& c : cols) {
if (!exec.isAdminUser() &&
!exec.canUseCollection(vocbase->name(), c->name(), auth::Level::RO)) {
continue;
}
// We want to check if the collection is usable and all followers
// are in sync:
std::shared_ptr<ShardMap> shardMap = c->shardIds();
@ -784,7 +883,7 @@ void RestReplicationHandler::handleCommandRestoreCollection() {
bool force = _request->parsedValue<bool>("force", false);
bool ignoreDistributeShardsLikeErrors =
_request->parsedValue<bool>("ignoreDistributeShardsLikeErrors", false);
Result res;
if (ServerState::instance()->isCoordinator()) {
res = processRestoreCollectionCoordinator(slice, overwrite, force,
@ -1080,7 +1179,7 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
}
// now re-create the collection
// Build up new information that we need to merge with the given one
VPackBuilder toMerge;
toMerge.openObject();
@ -1094,9 +1193,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
// force one shard, and force distributeShardsLike to be "_graphs"
toMerge.add(StaticStrings::NumberOfShards, VPackValue(1));
if (!_vocbase.IsSystemName(name)) {
// system-collections will be sharded normally. only user collections will get
// the forced sharding
toMerge.add(StaticStrings::DistributeShardsLike, VPackValue(_vocbase.shardingPrototypeName()));
// system-collections will be sharded normally. only user collections will
// get the forced sharding
toMerge.add(StaticStrings::DistributeShardsLike,
VPackValue(_vocbase.shardingPrototypeName()));
}
} else {
size_t numberOfShards = 1;
@ -1116,10 +1216,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
if (_vocbase.sharding() == "single" &&
parameters.get(StaticStrings::DistributeShardsLike).isNone() &&
!_vocbase.IsSystemName(name) &&
numberOfShards <= 1) {
!_vocbase.IsSystemName(name) && numberOfShards <= 1) {
// shard like _graphs
toMerge.add(StaticStrings::DistributeShardsLike, VPackValue(_vocbase.shardingPrototypeName()));
toMerge.add(StaticStrings::DistributeShardsLike,
VPackValue(_vocbase.shardingPrototypeName()));
}
}
@ -1128,13 +1228,14 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
// not an error: for historical reasons the write concern is read from the
// variable "minReplicationFactor"
VPackSlice writeConcernSlice = parameters.get(StaticStrings::WriteConcern);
if (writeConcernSlice.isNone()) { // minReplicationFactor is deprecated in 3.6
if (writeConcernSlice.isNone()) { // minReplicationFactor is deprecated in 3.6
writeConcernSlice = parameters.get(StaticStrings::MinReplicationFactor);
}
bool isValidReplicationFactorSlice = replicationFactorSlice.isInteger() ||
(replicationFactorSlice.isString() &&
replicationFactorSlice.isEqualString(StaticStrings::Satellite));
bool isValidReplicationFactorSlice =
replicationFactorSlice.isInteger() ||
(replicationFactorSlice.isString() &&
replicationFactorSlice.isEqualString(StaticStrings::Satellite));
bool isValidWriteConcernSlice =
replicationFactorSlice.isInteger() && writeConcernSlice.isInteger() &&
@ -1142,7 +1243,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
writeConcernSlice.getInt() > 0;
if (!isValidReplicationFactorSlice) {
size_t replicationFactor = _vocbase.server().getFeature<ClusterFeature>().defaultReplicationFactor();
size_t replicationFactor =
_vocbase.server().getFeature<ClusterFeature>().defaultReplicationFactor();
if (replicationFactor == 0) {
replicationFactor = 1;
}
@ -1255,7 +1357,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
// not desired, so it is hardcoded to false
auto cols =
ClusterMethods::createCollectionOnCoordinator(_vocbase, merged, ignoreDistributeShardsLikeErrors,
createWaitsForSyncReplication, false, false, nullptr);
createWaitsForSyncReplication,
false, false, nullptr);
ExecContext const& exec = ExecContext::current();
TRI_ASSERT(cols.size() == 1);
if (name[0] != '_' && !exec.isSuperuser()) {
@ -1402,7 +1505,7 @@ Result RestReplicationHandler::processRestoreUsersBatch(std::string const& colle
VPackSlice allMarkersSlice = allMarkers.slice();
std::string aql(
"FOR u IN @restored UPSERT {name: u.name} INSERT u REPLACE u "
"FOR u IN @restored UPSERT {user: u.user} INSERT u REPLACE u "
"INTO @@collection OPTIONS {ignoreErrors: true, silent: true, "
"waitForSync: false, isRestore: true}");
@ -1455,8 +1558,7 @@ Result RestReplicationHandler::processRestoreUsersBatch(std::string const& colle
AuthenticationFeature* af = AuthenticationFeature::instance();
TRI_ASSERT(af->userManager() != nullptr);
if (af->userManager() != nullptr) {
af->userManager()->triggerLocalReload();
af->userManager()->triggerGlobalReload();
af->userManager()->triggerCacheRevalidation();
}
return queryResult.result;
@ -1917,8 +2019,8 @@ void RestReplicationHandler::handleCommandRestoreView() {
if (!overwrite) {
generateError(GeneralResponse::responseCode(TRI_ERROR_ARANGO_DUPLICATE_NAME),
TRI_ERROR_ARANGO_DUPLICATE_NAME,
std::string("unable to restore view '") + nameSlice.copyString() + ": " +
TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME));
std::string("unable to restore view '") + nameSlice.copyString() +
": " + TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME));
return;
}
@ -2524,7 +2626,7 @@ void RestReplicationHandler::handleCommandHoldReadLockCollection() {
"'ttl' and 'id'");
return;
}
VPackSlice collection = body.get("collection");
VPackSlice ttlSlice = body.get("ttl");
VPackSlice idSlice = body.get("id");
@ -2535,8 +2637,9 @@ void RestReplicationHandler::handleCommandHoldReadLockCollection() {
rebootId = RebootId(body.get(StaticStrings::RebootId).getNumber<uint64_t>());
serverId = body.get("serverId").copyString();
} else {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"'rebootId' must be accompanied by string attribute 'serverId'");
generateError(
rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"'rebootId' must be accompanied by string attribute 'serverId'");
return;
}
} else {
@ -2903,7 +3006,6 @@ uint64_t RestReplicationHandler::determineChunkSize() const {
return chunkSize;
}
ReplicationApplier* RestReplicationHandler::getApplier(bool& global) {
global = _request->parsedValue("global", false);
@ -2922,11 +3024,9 @@ ReplicationApplier* RestReplicationHandler::getApplier(bool& global) {
}
}
Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id,
LogicalCollection& col, double ttl,
AccessMode::Type access,
RebootId const& rebootId,
std::string const& serverId) {
Result RestReplicationHandler::createBlockingTransaction(
aql::QueryId id, LogicalCollection& col, double ttl, AccessMode::Type access,
RebootId const& rebootId, std::string const& serverId) {
// This is a constant JSON structure for Queries.
// we actually do not need a plan, as we only want the query registry to have
// a hold of our transaction
@ -2963,25 +3063,25 @@ Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id,
std::string vn = _vocbase.name();
try {
std::function<void(void)> f =
[=]() {
try {
// Code does not matter, read only access, so we can roll back.
QueryRegistryFeature::registry()->destroy(vn, id, TRI_ERROR_QUERY_KILLED, false);
} catch (...) {
// All errors that show up here can only be
// triggered if the query is destroyed in between.
}
};
std::function<void(void)> f = [=]() {
try {
// Code does not matter, read only access, so we can roll back.
QueryRegistryFeature::registry()->destroy(vn, id, TRI_ERROR_QUERY_KILLED, false);
} catch (...) {
// All errors that show up here can only be
// triggered if the query is destroyed in between.
}
};
std::string comment = std::string("SynchronizeShard from ") + serverId +
" for " + col.name() + " access mode " + AccessMode::typeString(access);
" for " + col.name() + " access mode " +
AccessMode::typeString(access);
auto rGuard = std::make_unique<CallbackGuard>(
ci.rebootTracker().callMeOnChange(
RebootTracker::PeerState(serverId, rebootId), f, comment));
ci.rebootTracker().callMeOnChange(RebootTracker::PeerState(serverId, rebootId),
f, comment));
queryRegistry->insert(id, query.get(), ttl, true, true, std::move(rGuard));
} catch (...) {
// For compatibility we only return this error
return {TRI_ERROR_TRANSACTION_INTERNAL, "cannot begin read transaction"};
@ -3150,8 +3250,8 @@ void RestReplicationHandler::registerTombstone(aql::QueryId id) const {
std::string key = IdToTombstoneKey(_vocbase, id);
{
WRITE_LOCKER(writeLocker, RestReplicationHandler::_tombLock);
RestReplicationHandler::_tombstones.try_emplace(key, std::chrono::steady_clock::now() +
RestReplicationHandler::_tombstoneTimeout);
RestReplicationHandler::_tombstones.try_emplace(
key, std::chrono::steady_clock::now() + RestReplicationHandler::_tombstoneTimeout);
}
timeoutTombstones();
}

View File

@ -28,8 +28,8 @@
#include "Aql/types.h"
#include "Basics/Common.h"
#include "Basics/Result.h"
#include "Cluster/ResultT.h"
#include "Cluster/ClusterTypes.h"
#include "Cluster/ResultT.h"
#include "Replication/Syncer.h"
#include "Replication/common-defines.h"
#include "RestHandler/RestVocbaseBaseHandler.h"
@ -107,7 +107,7 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
static std::string const HoldReadLockCollection;
protected:
std::string forwardingTarget() override final;
ResultT<std::pair<std::string, bool>> forwardingTarget() override final;
//////////////////////////////////////////////////////////////////////////////
/// @brief creates an error if called on a coordinator server
@ -309,8 +309,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// @brief restores the structure of a collection, coordinator case
//////////////////////////////////////////////////////////////////////////////
Result processRestoreCollectionCoordinator(VPackSlice const&, bool overwrite,
bool force,
Result processRestoreCollectionCoordinator(VPackSlice const& collection,
bool dropExisting, bool force,
bool ignoreDistributeShardsLikeErrors);
//////////////////////////////////////////////////////////////////////////////
@ -474,9 +474,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// It will be registered with the given id, and it will have
/// the given time to live.
//////////////////////////////////////////////////////////////////////////////
Result createBlockingTransaction(aql::QueryId id, LogicalCollection& col,
double ttl, AccessMode::Type access,
RebootId const& rebootId,
Result createBlockingTransaction(aql::QueryId id, LogicalCollection& col, double ttl,
AccessMode::Type access, RebootId const& rebootId,
std::string const& serverId);
//////////////////////////////////////////////////////////////////////////////
@ -504,6 +503,13 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
//////////////////////////////////////////////////////////////////////////////
ResultT<bool> cancelBlockingTransaction(aql::QueryId id) const;
//////////////////////////////////////////////////////////////////////////////
/// @brief Validate that the requesting user has access rights to this route
/// Will return TRI_ERROR_NO_ERROR if user has access
/// Will return error code otherwise.
//////////////////////////////////////////////////////////////////////////////
Result testPermissions();
};
} // namespace arangodb
#endif

View File

@ -74,26 +74,26 @@ RestStatus RestTasksHandler::execute() {
}
/// @brief returns the short id of the server which should handle this request
std::string RestTasksHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestTasksHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::POST && type != rest::RequestType::PUT &&
type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}
void RestTasksHandler::getTasks() {

View File

@ -38,7 +38,7 @@ class RestTasksHandler : public arangodb::RestVocbaseBaseHandler {
RestStatus execute() override;
protected:
virtual std::string forwardingTarget() override;
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private:
void getTasks();

View File

@ -341,24 +341,24 @@ void RestTransactionHandler::cancel() {
}
/// @brief returns the short id of the server which should handle this request
std::string RestTransactionHandler::forwardingTarget() {
ResultT<std::pair<std::string, bool>> RestTransactionHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::GET && type != rest::RequestType::PUT &&
type != rest::RequestType::DELETE_REQ) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) {
return "";
return {std::make_pair(StaticStrings::Empty, false)};
}
auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer);
return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
}

View File

@ -48,7 +48,7 @@ class RestTransactionHandler : public arangodb::RestVocbaseBaseHandler {
void cancel() override final;
protected:
virtual std::string forwardingTarget() override;
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private:
void executeGetState();

View File

@ -1141,24 +1141,42 @@ arangodb::Result processInputDirectory(
arangodb::Result processJob(arangodb::httpclient::SimpleHttpClient& httpClient,
arangodb::RestoreFeature::JobData& jobData) {
arangodb::Result result;
if (jobData.options.indexesFirst && jobData.options.importStructure) {
// restore indexes first if we are using rocksdb
VPackSlice const parameters = jobData.collection.get("parameters");
std::string const cname =
arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", "");
if (cname == "_users") {
// special case: never restore data in the _users collection first as it could
// potentially change user permissions. In that case index creation will fail.
result = ::restoreIndexes(httpClient, jobData);
if (result.fail()) {
return result;
}
}
if (jobData.options.importData) {
result = ::restoreData(httpClient, jobData);
if (result.fail()) {
return result;
}
}
if (!jobData.options.indexesFirst && jobData.options.importStructure) {
// restore indexes second if we are using mmfiles
result = ::restoreIndexes(httpClient, jobData);
if (result.fail()) {
return result;
} else {
if (jobData.options.indexesFirst && jobData.options.importStructure) {
// restore indexes first if we are using rocksdb
result = ::restoreIndexes(httpClient, jobData);
if (result.fail()) {
return result;
}
}
if (jobData.options.importData) {
result = ::restoreData(httpClient, jobData);
if (result.fail()) {
return result;
}
}
if (!jobData.options.indexesFirst && jobData.options.importStructure) {
// restore indexes second if we are using mmfiles
result = ::restoreIndexes(httpClient, jobData);
if (result.fail()) {
return result;
}
}
}

View File

@ -113,6 +113,13 @@ class DumpRestoreHelper {
print(CYAN + Date() + ': ' + this.which + ' and Restore - ' + s + RESET);
}
adjustRestoreToDump()
{
this.restoreOptions = this.dumpOptions;
this.restoreConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo);
this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.restoreOptions, this.instanceInfo.rootDir, this.options.coreCheck);
}
isAlive() {
return pu.arangod.check.instanceAlive(this.instanceInfo, this.options);
}
@ -157,8 +164,14 @@ class DumpRestoreHelper {
return this.validate(this.results.setup);
}
dumpFrom(database) {
dumpFrom(database, separateDir = false) {
this.print('dump');
if (separateDir) {
if (!fs.exists(fs.join(this.instanceInfo.rootDir, 'dump'))) {
fs.makeDirectory(fs.join(this.instanceInfo.rootDir, 'dump'));
}
this.dumpConfig.setOutputDirectory('dump' + fs.pathSeparator + database);
}
if (!this.dumpConfig.haveSetAllDatabases()) {
this.dumpConfig.setDatabase(database);
}
@ -166,8 +179,19 @@ class DumpRestoreHelper {
return this.validate(this.results.dump);
}
restoreTo(database) {
restoreTo(database, options = { separate: false, fromDir: '' }) {
this.print('restore');
if (options.hasOwnProperty('separate') && options.separate === true) {
if (!options.hasOwnProperty('fromDir') || typeof options.fromDir !== 'string') {
options.fromDir = database;
}
if (!fs.exists(fs.join(this.instanceInfo.rootDir, 'dump'))) {
fs.makeDirectory(fs.join(this.instanceInfo.rootDir, 'dump'));
}
this.restoreConfig.setInputDirectory('dump' + fs.pathSeparator + options.fromDir, true);
}
if (!this.restoreConfig.haveSetAllDatabases()) {
this.restoreConfig.setDatabase(database);
}
@ -213,6 +237,7 @@ class DumpRestoreHelper {
restoreFoxxComplete(database) {
this.print('Foxx Apps with full restore');
this.restoreConfig.setDatabase(database);
this.restoreConfig.setIncludeSystem(true);
this.results.restoreFoxxComplete = this.arangorestore();
return this.validate(this.results.restoreFoxxComplete);
}
@ -343,14 +368,28 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore
const cleanupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpCleanup));
const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain));
const tearDownFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpTearDown));
if (
!helper.runSetupSuite(setupFile) ||
!helper.dumpFrom('UnitTestsDumpSrc') ||
!helper.runCleanupSuite(cleanupFile) ||
!helper.restoreTo('UnitTestsDumpDst') ||
!helper.runTests(testFile,'UnitTestsDumpDst') ||
!helper.tearDown(tearDownFile)) {
return helper.extractResults();
if (options.hasOwnProperty("multipleDumps") && options.multipleDumps) {
if (!helper.runSetupSuite(setupFile) ||
!helper.dumpFrom('_system', true) ||
!helper.dumpFrom('UnitTestsDumpSrc', true) ||
!helper.runCleanupSuite(cleanupFile) ||
!helper.restoreTo('UnitTestsDumpDst', { separate: true, fromDir: 'UnitTestsDumpSrc'}) ||
!helper.restoreTo('_system', { separate: true }) ||
!helper.runTests(testFile,'UnitTestsDumpDst') ||
!helper.tearDown(tearDownFile)) {
return helper.extractResults();
}
}
else {
if (!helper.runSetupSuite(setupFile) ||
!helper.dumpFrom('UnitTestsDumpSrc') ||
!helper.runCleanupSuite(cleanupFile) ||
!helper.restoreTo('UnitTestsDumpDst') ||
!helper.runTests(testFile,'UnitTestsDumpDst') ||
!helper.tearDown(tearDownFile)) {
return helper.extractResults();
}
}
if (tstFiles.hasOwnProperty("dumpCheckGraph")) {
@ -365,6 +404,10 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore
if (tstFiles.hasOwnProperty("foxxTest")) {
const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest));
if (options.hasOwnProperty("multipleDumps") && options.multipleDumps) {
helper.adjustRestoreToDump();
helper.restoreConfig.setInputDirectory(fs.join('dump','UnitTestsDumpSrc'), true);
}
if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') ||
!helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') ||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') ||
@ -410,20 +453,6 @@ function dumpMultiple (options) {
}
function dumpAuthentication (options) {
if (options.cluster) {
if (options.extremeVerbosity) {
print(CYAN + 'Skipped because of cluster.' + RESET);
}
return {
'dump_authentication': {
'status': true,
'message': 'skipped because of cluster',
'skipped': true
}
};
}
const clientAuth = {
'server.authentication': 'true'
};
@ -438,16 +467,26 @@ function dumpAuthentication (options) {
password: 'foobarpasswd'
};
let restoreAuthOpts = {
username: 'foobaruser',
password: 'pinus'
};
_.defaults(dumpAuthOpts, options);
_.defaults(restoreAuthOpts, options);
let tstFiles = {
dumpSetup: 'dump-authentication-setup.js',
dumpCleanup: 'cleanup-nothing.js',
dumpCleanup: 'cleanup-alter-user.js',
dumpAgain: 'dump-authentication.js',
dumpTearDown: 'dump-teardown.js',
foxxTest: 'check-foxx.js'
};
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, dumpAuthOpts, 'dump_authentication', tstFiles, function(){});
options.multipleDumps = true;
options['server.jwt-secret'] = 'haxxmann';
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, restoreAuthOpts, 'dump_authentication', tstFiles, function(){});
}
function dumpEncrypted (options) {

View File

@ -167,6 +167,10 @@ class GeneralRequest {
return _headers;
}
void removeHeader(std::string key) {
_headers.erase(key);
}
#ifdef ARANGODB_USE_GOOGLE_TESTS
void addHeader(std::string key, std::string value) {
_headers.try_emplace(std::move(key), std::move(value));

View File

@ -0,0 +1,41 @@
/*jshint globalstrict:false, strict:false */
/* global db */
////////////////////////////////////////////////////////////////////////////////
/// @brief teardown for dump/reload tests
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB Inc, Cologne, Germany
///
/// @author Wilfried Goesgens
/// @author Copyright 2019, ArangoDB Inc, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
(function () {
'use strict';
var users = require("@arangodb/users");
users.update("foobaruser", "pinus", true);
})();
return {
status: true
};

View File

@ -163,6 +163,8 @@ function dumpTestSuite () {
assertEqual(users.permission(uName, "_system"), 'rw');
assertEqual(users.permission(uName, "UnitTestsDumpSrc"), 'rw');
assertEqual(users.permission(uName, "UnitTestsDumpEmpty"), 'rw');
assertTrue(users.isValid("foobaruser", "foobarpasswd"));
}
};

View File

@ -19,7 +19,7 @@ describe ArangoDB do
@longQuery = "FOR x IN 1..1 LET y = SLEEP(0.2) LET a = y LET b = a LET c = b LET d = c LET e = d LET f = e RETURN x"
@longQueryBody = JSON.dump({query: @longQuery})
@queryWithBind = "FOR x IN 1..5 LET y = SLEEP(@value) RETURN x"
@queryWithBindBody = JSON.dump({query: @queryWithBind, bindVars: {value: 2}})
@queryWithBindBody = JSON.dump({query: @queryWithBind, bindVars: {value: 4}})
@queryEndpoint ="/_api/cursor"
@queryPrefix = "api-cursor"
end
@ -135,7 +135,7 @@ describe ArangoDB do
found.should have_key("query")
found["query"].should eq(@queryWithBind)
found.should have_key("bindVars")
found["bindVars"].should eq({"value" => 2})
found["bindVars"].should eq({"value" => 4})
found.should have_key("runTime")
found["runTime"].should be_kind_of(Numeric)
found.should have_key("started")