1
0
Fork 0

Merge branch 'devel' of https://github.com/arangodb/arangodb into bug-fix/install-js-server-files

* 'devel' of https://github.com/arangodb/arangodb:
  Bug fix/fix internal issue 4451 (#10540)
  Bug fix/increase test timeout (#10596)
This commit is contained in:
Jan Christoph Uhde 2019-12-02 10:17:08 +01:00
commit 51679472dd
29 changed files with 421 additions and 177 deletions

View File

@ -1,6 +1,10 @@
devel devel
----- -----
* Fixed permissions for dump/restore.
* The _users collection is now properly restored when using arangorestore.
* Updated arangosync to 0.7.1. * Updated arangosync to 0.7.1.
* rename `minReplicationFactor` into `writeConcern` to make it consistent with * rename `minReplicationFactor` into `writeConcern` to make it consistent with

View File

@ -393,6 +393,12 @@ VPackBuilder auth::UserManager::allUsers() {
return result; return result;
} }
void auth::UserManager::triggerCacheRevalidation() {
triggerLocalReload();
triggerGlobalReload();
loadFromDB();
}
/// Trigger eventual reload, user facing API call /// Trigger eventual reload, user facing API call
void auth::UserManager::triggerGlobalReload() { void auth::UserManager::triggerGlobalReload() {
if (!ServerState::instance()->isCoordinator()) { if (!ServerState::instance()->isCoordinator()) {

View File

@ -95,6 +95,9 @@ class UserManager {
/// Trigger eventual reload on all other coordinators (and in TokenCache) /// Trigger eventual reload on all other coordinators (and in TokenCache)
void triggerGlobalReload(); void triggerGlobalReload();
/// Trigger cache revalidation after user restore
void triggerCacheRevalidation();
/// Create the root user with a default password, will fail if the user /// Create the root user with a default password, will fail if the user
/// already exists. Only ever call if you can guarantee to be in charge /// already exists. Only ever call if you can guarantee to be in charge
void createRootUser(); void createRootUser();

View File

@ -224,7 +224,6 @@ void ClusterInfo::cleanup() {
_plannedViews.clear(); _plannedViews.clear();
_plannedCollections.clear(); _plannedCollections.clear();
_shards.clear(); _shards.clear();
_shardKeys.clear();
_shardIds.clear(); _shardIds.clear();
_currentCollections.clear(); _currentCollections.clear();
} }
@ -235,6 +234,7 @@ void ClusterInfo::triggerBackgroundGetIds() {
_uniqid._nextUpperValue = 0ULL; _uniqid._nextUpperValue = 0ULL;
try { try {
_idLock.assertLockedByCurrentThread();
if (_uniqid._backgroundJobIsRunning) { if (_uniqid._backgroundJobIsRunning) {
return; return;
} }
@ -598,7 +598,7 @@ void ClusterInfo::loadPlan() {
// > // >
decltype(_shards) newShards; decltype(_shards) newShards;
decltype(_shardServers) newShardServers; decltype(_shardServers) newShardServers;
decltype(_shardKeys) newShardKeys; decltype(_shardToName) newShardToName;
bool swapDatabases = false; bool swapDatabases = false;
bool swapCollections = false; bool swapCollections = false;
@ -957,17 +957,16 @@ void ClusterInfo::loadPlan() {
databaseCollections.try_emplace(collectionId, newCollection); databaseCollections.try_emplace(collectionId, newCollection);
} }
newShardKeys.try_emplace(collectionId, std::make_shared<std::vector<std::string>>(
newCollection->shardKeys()));
auto shardIDs = newCollection->shardIds(); auto shardIDs = newCollection->shardIds();
auto shards = std::make_shared<std::vector<std::string>>(); auto shards = std::make_shared<std::vector<std::string>>();
shards->reserve(shardIDs->size()); shards->reserve(shardIDs->size());
newShardToName.reserve(shardIDs->size());
for (auto const& p : *shardIDs) { for (auto const& p : *shardIDs) {
TRI_ASSERT(p.first.size() >= 2); TRI_ASSERT(p.first.size() >= 2);
shards->push_back(p.first); shards->push_back(p.first);
newShardServers.try_emplace(p.first, p.second); newShardServers.try_emplace(p.first, p.second);
newShardToName.try_emplace(p.first, newCollection->name());
} }
// Sort by the number in the shard ID ("s0000001" for example): // Sort by the number in the shard ID ("s0000001" for example):
@ -1053,8 +1052,8 @@ void ClusterInfo::loadPlan() {
if (swapCollections) { if (swapCollections) {
_plannedCollections.swap(newCollections); _plannedCollections.swap(newCollections);
_shards.swap(newShards); _shards.swap(newShards);
_shardKeys.swap(newShardKeys);
_shardServers.swap(newShardServers); _shardServers.swap(newShardServers);
_shardToName.swap(newShardToName);
} }
if (swapViews) { if (swapViews) {
@ -4288,6 +4287,16 @@ arangodb::Result ClusterInfo::getShardServers(ShardID const& shardId,
return arangodb::Result(TRI_ERROR_FAILED); return arangodb::Result(TRI_ERROR_FAILED);
} }
CollectionID ClusterInfo::getCollectionNameForShard(ShardID const& shardId) {
READ_LOCKER(readLocker, _planProt.lock);
auto it = _shardToName.find(shardId);
if (it != _shardToName.end()) {
return it->second;
}
return StaticStrings::Empty;
}
arangodb::Result ClusterInfo::agencyDump(std::shared_ptr<VPackBuilder> body) { arangodb::Result ClusterInfo::agencyDump(std::shared_ptr<VPackBuilder> body) {
AgencyCommResult dump = _agency.dump(); AgencyCommResult dump = _agency.dump();

View File

@ -816,6 +816,9 @@ class ClusterInfo final {
*/ */
arangodb::Result getShardServers(ShardID const& shardId, std::vector<ServerID>&); arangodb::Result getShardServers(ShardID const& shardId, std::vector<ServerID>&);
/// @brief map shardId to collection name (not ID)
CollectionID getCollectionNameForShard(ShardID const& shardId);
/** /**
* @brief Lock agency's hot backup with TTL 60 seconds * @brief Lock agency's hot backup with TTL 60 seconds
* *
@ -956,8 +959,7 @@ class ClusterInfo final {
ProtectionData _planProt; ProtectionData _planProt;
uint64_t _planVersion; // This is the version in the Plan which underlies uint64_t _planVersion; // This is the version in the Plan which underlies
// the data in _plannedCollections, _shards and // the data in _plannedCollections and _shards
// _shardKeys
uint64_t _currentVersion; // This is the version in Current which underlies uint64_t _currentVersion; // This is the version in Current which underlies
// the data in _currentDatabases, // the data in _currentDatabases,
// _currentCollections and _shardsIds // _currentCollections and _shardsIds
@ -978,11 +980,10 @@ class ClusterInfo final {
std::shared_ptr<std::vector<std::string>>> std::shared_ptr<std::vector<std::string>>>
_shards; // from Plan/Collections/ _shards; // from Plan/Collections/
// (may later come from Current/Collections/ ) // (may later come from Current/Collections/ )
std::unordered_map<CollectionID,
std::shared_ptr<std::vector<std::string>>>
_shardKeys; // from Plan/Collections/
// planned shard => servers map // planned shard => servers map
std::unordered_map<ShardID, std::vector<ServerID>> _shardServers; std::unordered_map<ShardID, std::vector<ServerID>> _shardServers;
// planned shard ID => collection name
std::unordered_map<ShardID, CollectionID> _shardToName;
AllViews _plannedViews; // from Plan/Views/ AllViews _plannedViews; // from Plan/Views/
AllViews _newPlannedViews; // views that have been created during `loadPlan` AllViews _newPlannedViews; // views that have been created during `loadPlan`

View File

@ -113,7 +113,20 @@ futures::Future<Result> RestHandler::forwardRequest(bool& forwarded) {
return futures::makeFuture(Result()); return futures::makeFuture(Result());
} }
std::string serverId = forwardingTarget(); ResultT forwardResult = forwardingTarget();
if (forwardResult.fail()) {
return futures::makeFuture(forwardResult.result());
}
auto forwardContent = forwardResult.get();
std::string serverId = std::get<0>(forwardContent);
bool removeHeader = std::get<1>(forwardContent);
if (removeHeader) {
_request->removeHeader(StaticStrings::Authorization);
_request->setUser("");
}
if (serverId.empty()) { if (serverId.empty()) {
// no need to actually forward // no need to actually forward
return futures::makeFuture(Result()); return futures::makeFuture(Result());

View File

@ -30,6 +30,7 @@
#include "Network/Methods.h" #include "Network/Methods.h"
#include "Rest/GeneralResponse.h" #include "Rest/GeneralResponse.h"
#include <Cluster/ResultT.h>
#include <atomic> #include <atomic>
#include <thread> #include <thread>
@ -46,7 +47,7 @@ template<typename T>
class Future; class Future;
template <typename T> template <typename T>
class Try; class Try;
} } // namespace futures
class GeneralRequest; class GeneralRequest;
class RequestStatistics; class RequestStatistics;
@ -129,9 +130,7 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
// you might need to implment this in you handler // you might need to implment this in you handler
// if it will be executed in an async job // if it will be executed in an async job
virtual void cancel() { virtual void cancel() { _canceled.store(true); }
_canceled.store(true);
}
virtual void handleError(basics::Exception const&) = 0; virtual void handleError(basics::Exception const&) = 0;
@ -143,7 +142,11 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
/// handled by this server, the method should return an empty string. /// handled by this server, the method should return an empty string.
/// Otherwise, this method should return a valid short name for the /// Otherwise, this method should return a valid short name for the
/// target server. /// target server.
virtual std::string forwardingTarget() { return ""; } /// std::string -> empty string or valid short name
/// boolean -> should auth header and user be removed in that request
virtual ResultT<std::pair<std::string, bool>> forwardingTarget() {
return {std::make_pair(StaticStrings::Empty, false)};
}
void resetResponse(rest::ResponseCode); void resetResponse(rest::ResponseCode);
@ -184,9 +187,7 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
}; };
/// handler state machine /// handler state machine
HandlerState state() const { HandlerState state() const { return _state; }
return _state;
}
private: private:
void runHandlerStateMachine(); void runHandlerStateMachine();
@ -200,14 +201,12 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
void compressResponse(); void compressResponse();
protected: protected:
std::unique_ptr<GeneralRequest> _request; std::unique_ptr<GeneralRequest> _request;
std::unique_ptr<GeneralResponse> _response; std::unique_ptr<GeneralResponse> _response;
application_features::ApplicationServer& _server; application_features::ApplicationServer& _server;
RequestStatistics* _statistics; RequestStatistics* _statistics;
private: private:
mutable Mutex _executionMutex; mutable Mutex _executionMutex;
std::function<void(rest::RestHandler*)> _callback; std::function<void(rest::RestHandler*)> _callback;
@ -219,7 +218,6 @@ class RestHandler : public std::enable_shared_from_this<RestHandler> {
HandlerState _state; HandlerState _state;
protected: protected:
std::atomic<bool> _canceled; std::atomic<bool> _canceled;
bool _allowDirectExecution = false; bool _allowDirectExecution = false;

View File

@ -75,26 +75,26 @@ RestStatus RestControlPregelHandler::execute() {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestControlPregelHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestControlPregelHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType(); rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::POST && type != rest::RequestType::GET && if (type != rest::RequestType::POST && type != rest::RequestType::GET &&
type != rest::RequestType::DELETE_REQ) { type != rest::RequestType::DELETE_REQ) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
std::vector<std::string> const& suffixes = _request->suffixes(); std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) { if (suffixes.size() < 1) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }
void RestControlPregelHandler::startExecution() { void RestControlPregelHandler::startExecution() {
@ -107,7 +107,7 @@ void RestControlPregelHandler::startExecution() {
// algorithm // algorithm
std::string algorithm = std::string algorithm =
VelocyPackHelper::getStringValue(body, "algorithm", ""); VelocyPackHelper::getStringValue(body, "algorithm", StaticStrings::Empty);
if ("" == algorithm) { if ("" == algorithm) {
generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND, generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND,
"invalid algorithm"); "invalid algorithm");

View File

@ -38,7 +38,7 @@ class RestControlPregelHandler : public arangodb::RestVocbaseBaseHandler {
RestStatus execute() override; RestStatus execute() override;
protected: protected:
virtual std::string forwardingTarget() override; virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private: private:
void startExecution(); void startExecution();

View File

@ -381,25 +381,25 @@ RestStatus RestCursorHandler::handleQueryResult() {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestCursorHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestCursorHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType(); rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { if (type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
std::vector<std::string> const& suffixes = _request->suffixes(); std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) { if (suffixes.size() < 1) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////

View File

@ -88,7 +88,7 @@ class RestCursorHandler : public RestVocbaseBaseHandler {
RestStatus processQuery(); RestStatus processQuery();
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
virtual std::string forwardingTarget() override; ResultT<std::pair<std::string, bool>> forwardingTarget() override;
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
/// @brief unregister the currently running query /// @brief unregister the currently running query

View File

@ -120,9 +120,9 @@ void RestDocumentHandler::shutdownExecute(bool isFinalized) noexcept {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestDocumentHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestDocumentHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) { if (!ServerState::instance()->isCoordinator()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
bool found = false; bool found = false;
@ -131,17 +131,17 @@ std::string RestDocumentHandler::forwardingTarget() {
uint64_t tid = basics::StringUtils::uint64(value); uint64_t tid = basics::StringUtils::uint64(value);
if (!transaction::isCoordinatorTransactionId(tid)) { if (!transaction::isCoordinatorTransactionId(tid)) {
TRI_ASSERT(transaction::isLegacyTransactionId(tid)); TRI_ASSERT(transaction::isLegacyTransactionId(tid));
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////

View File

@ -56,7 +56,7 @@ class RestDocumentHandler : public RestVocbaseBaseHandler {
void shutdownExecute(bool isFinalized) noexcept override final; void shutdownExecute(bool isFinalized) noexcept override final;
protected: protected:
std::string forwardingTarget() override final; ResultT<std::pair<std::string, bool>> forwardingTarget() override final;
private: private:
// inserts a document // inserts a document

View File

@ -245,24 +245,24 @@ void RestJobHandler::deleteJob() {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestJobHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestJobHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType(); rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::GET && type != rest::RequestType::PUT && if (type != rest::RequestType::GET && type != rest::RequestType::PUT &&
type != rest::RequestType::DELETE_REQ) { type != rest::RequestType::DELETE_REQ) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
std::vector<std::string> const& suffixes = _request->suffixes(); std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) { if (suffixes.size() < 1) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }

View File

@ -85,7 +85,7 @@ class RestJobHandler : public RestBaseHandler {
void deleteJob(); void deleteJob();
protected: protected:
virtual std::string forwardingTarget() override; virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private: private:
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////

View File

@ -346,9 +346,9 @@ bool RestQueryHandler::parseQuery() {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestQueryHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestQueryHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) { if (!ServerState::instance()->isCoordinator()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
bool found = false; bool found = false;
@ -357,15 +357,15 @@ std::string RestQueryHandler::forwardingTarget() {
uint64_t tid = basics::StringUtils::uint64(value); uint64_t tid = basics::StringUtils::uint64(value);
if (!transaction::isCoordinatorTransactionId(tid)) { if (!transaction::isCoordinatorTransactionId(tid)) {
TRI_ASSERT(transaction::isLegacyTransactionId(tid)); TRI_ASSERT(transaction::isLegacyTransactionId(tid));
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }

View File

@ -96,7 +96,7 @@ class RestQueryHandler : public RestVocbaseBaseHandler {
bool parseQuery(); bool parseQuery();
virtual std::string forwardingTarget() override; virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
}; };
} // namespace arangodb } // namespace arangodb

View File

@ -29,9 +29,9 @@
#include "Basics/ReadLocker.h" #include "Basics/ReadLocker.h"
#include "Basics/Result.h" #include "Basics/Result.h"
#include "Basics/RocksDBUtils.h" #include "Basics/RocksDBUtils.h"
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h" #include "Basics/VelocyPackHelper.h"
#include "Basics/WriteLocker.h" #include "Basics/WriteLocker.h"
#include "Basics/StaticStrings.h"
#include "Cluster/ClusterFeature.h" #include "Cluster/ClusterFeature.h"
#include "Cluster/ClusterHelpers.h" #include "Cluster/ClusterHelpers.h"
#include "Cluster/ClusterMethods.h" #include "Cluster/ClusterMethods.h"
@ -280,6 +280,11 @@ std::string const RestReplicationHandler::HoldReadLockCollection =
// main function that dispatches the different routes and commands // main function that dispatches the different routes and commands
RestStatus RestReplicationHandler::execute() { RestStatus RestReplicationHandler::execute() {
auto res = testPermissions();
if (!res.ok()) {
generateError(res);
return RestStatus::DONE;
}
// extract the request type // extract the request type
auto const type = _request->requestType(); auto const type = _request->requestType();
auto const& suffixes = _request->suffixes(); auto const& suffixes = _request->suffixes();
@ -589,27 +594,113 @@ BAD_CALL:
return RestStatus::DONE; return RestStatus::DONE;
} }
/// @brief returns the short id of the server which should handle this request Result RestReplicationHandler::testPermissions() {
std::string RestReplicationHandler::forwardingTarget() { if (!_request->authenticated()) {
if (!ServerState::instance()->isCoordinator()) { return TRI_ERROR_NO_ERROR;
return "";
} }
std::string user = _request->user();
auto const& suffixes = _request->suffixes(); auto const& suffixes = _request->suffixes();
size_t const len = suffixes.size(); size_t const len = suffixes.size();
if (len >= 1) { if (len >= 1) {
auto const type = _request->requestType(); auto const type = _request->requestType();
std::string const& command = suffixes[0]; std::string const& command = suffixes[0];
if ((command == Batch) || (command == Inventory && type == rest::RequestType::GET) || if ((command == Batch) || (command == Inventory && type == rest::RequestType::GET) ||
(command == Dump && type == rest::RequestType::GET)) { (command == Dump && type == rest::RequestType::GET) ||
ServerID const& DBserver = _request->value("DBserver"); (command == RestoreCollection && type == rest::RequestType::PUT)) {
if (!DBserver.empty()) { if (command == Dump) {
return DBserver; // check dump collection permissions (at least ro needed)
} std::string collectionName = _request->value("collection");
}
if (ServerState::instance()->isCoordinator()) {
// We have a shard id, need to translate
ClusterInfo& ci = server().getFeature<ClusterFeature>().clusterInfo();
collectionName = ci.getCollectionNameForShard(collectionName);
} }
return ""; if (!collectionName.empty()) {
auto& exec = ExecContext::current();
ExecContextSuperuserScope escope(exec.isAdminUser());
if (!exec.isAdminUser() &&
!exec.canUseCollection(collectionName, auth::Level::RO)) {
// not enough rights
return Result(TRI_ERROR_FORBIDDEN);
}
} else {
// not found, return 404
return Result(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND);
}
} else if (command == RestoreCollection) {
VPackSlice const slice = _request->payload();
VPackSlice const parameters = slice.get("parameters");
if (parameters.isObject()) {
if (parameters.get("name").isString()) {
std::string collectionName = parameters.get("name").copyString();
if (!collectionName.empty()) {
std::string dbName = _request->databaseName();
DatabaseFeature& databaseFeature =
_vocbase.server().getFeature<DatabaseFeature>();
TRI_vocbase_t* vocbase = databaseFeature.lookupDatabase(dbName);
if (vocbase == nullptr) {
return Result(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND);
}
std::string const& overwriteCollection =
_request->value("overwrite");
auto& exec = ExecContext::current();
ExecContextSuperuserScope escope(exec.isAdminUser());
if (overwriteCollection == "true" ||
vocbase->lookupCollection(collectionName) == nullptr) {
// 1.) re-create collection, means: overwrite=true (rw database)
// OR 2.) not existing, new collection (rw database)
if (!exec.isAdminUser() && !exec.canUseDatabase(dbName, auth::Level::RW)) {
return Result(TRI_ERROR_FORBIDDEN);
}
} else {
// 3.) Existing collection (ro database, rw collection)
// no overwrite. restoring into an existing collection
if (!exec.isAdminUser() &&
!exec.canUseCollection(collectionName, auth::Level::RW)) {
return Result(TRI_ERROR_FORBIDDEN);
}
}
} else {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER,
"empty collection name");
}
} else {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid collection name type");
}
} else {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid collection parameter type");
}
}
}
}
return Result(TRI_ERROR_NO_ERROR);
}
/// @brief returns the short id of the server which should handle this request
ResultT<std::pair<std::string, bool>> RestReplicationHandler::forwardingTarget() {
if (!ServerState::instance()->isCoordinator()) {
return {std::make_pair("", false)};
}
auto res = testPermissions();
if (!res.ok()) {
return res;
}
ServerID const& DBserver = _request->value("DBserver");
if (!DBserver.empty()) {
// if DBserver property present, remove auth header
return std::make_pair(DBserver, true);
}
return {std::make_pair(StaticStrings::Empty, false)};
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
@ -712,8 +803,16 @@ void RestReplicationHandler::handleCommandClusterInventory() {
vocbase->toVelocyPack(resultBuilder); vocbase->toVelocyPack(resultBuilder);
} }
auto& exec = ExecContext::current();
ExecContextSuperuserScope escope(exec.isAdminUser());
resultBuilder.add("collections", VPackValue(VPackValueType::Array)); resultBuilder.add("collections", VPackValue(VPackValueType::Array));
for (std::shared_ptr<LogicalCollection> const& c : cols) { for (std::shared_ptr<LogicalCollection> const& c : cols) {
if (!exec.isAdminUser() &&
!exec.canUseCollection(vocbase->name(), c->name(), auth::Level::RO)) {
continue;
}
// We want to check if the collection is usable and all followers // We want to check if the collection is usable and all followers
// are in sync: // are in sync:
std::shared_ptr<ShardMap> shardMap = c->shardIds(); std::shared_ptr<ShardMap> shardMap = c->shardIds();
@ -1094,9 +1193,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
// force one shard, and force distributeShardsLike to be "_graphs" // force one shard, and force distributeShardsLike to be "_graphs"
toMerge.add(StaticStrings::NumberOfShards, VPackValue(1)); toMerge.add(StaticStrings::NumberOfShards, VPackValue(1));
if (!_vocbase.IsSystemName(name)) { if (!_vocbase.IsSystemName(name)) {
// system-collections will be sharded normally. only user collections will get // system-collections will be sharded normally. only user collections will
// the forced sharding // get the forced sharding
toMerge.add(StaticStrings::DistributeShardsLike, VPackValue(_vocbase.shardingPrototypeName())); toMerge.add(StaticStrings::DistributeShardsLike,
VPackValue(_vocbase.shardingPrototypeName()));
} }
} else { } else {
size_t numberOfShards = 1; size_t numberOfShards = 1;
@ -1116,10 +1216,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
if (_vocbase.sharding() == "single" && if (_vocbase.sharding() == "single" &&
parameters.get(StaticStrings::DistributeShardsLike).isNone() && parameters.get(StaticStrings::DistributeShardsLike).isNone() &&
!_vocbase.IsSystemName(name) && !_vocbase.IsSystemName(name) && numberOfShards <= 1) {
numberOfShards <= 1) {
// shard like _graphs // shard like _graphs
toMerge.add(StaticStrings::DistributeShardsLike, VPackValue(_vocbase.shardingPrototypeName())); toMerge.add(StaticStrings::DistributeShardsLike,
VPackValue(_vocbase.shardingPrototypeName()));
} }
} }
@ -1132,7 +1232,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
writeConcernSlice = parameters.get(StaticStrings::MinReplicationFactor); writeConcernSlice = parameters.get(StaticStrings::MinReplicationFactor);
} }
bool isValidReplicationFactorSlice = replicationFactorSlice.isInteger() || bool isValidReplicationFactorSlice =
replicationFactorSlice.isInteger() ||
(replicationFactorSlice.isString() && (replicationFactorSlice.isString() &&
replicationFactorSlice.isEqualString(StaticStrings::Satellite)); replicationFactorSlice.isEqualString(StaticStrings::Satellite));
@ -1142,7 +1243,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
writeConcernSlice.getInt() > 0; writeConcernSlice.getInt() > 0;
if (!isValidReplicationFactorSlice) { if (!isValidReplicationFactorSlice) {
size_t replicationFactor = _vocbase.server().getFeature<ClusterFeature>().defaultReplicationFactor(); size_t replicationFactor =
_vocbase.server().getFeature<ClusterFeature>().defaultReplicationFactor();
if (replicationFactor == 0) { if (replicationFactor == 0) {
replicationFactor = 1; replicationFactor = 1;
} }
@ -1255,7 +1357,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator(
// not desired, so it is hardcoded to false // not desired, so it is hardcoded to false
auto cols = auto cols =
ClusterMethods::createCollectionOnCoordinator(_vocbase, merged, ignoreDistributeShardsLikeErrors, ClusterMethods::createCollectionOnCoordinator(_vocbase, merged, ignoreDistributeShardsLikeErrors,
createWaitsForSyncReplication, false, false, nullptr); createWaitsForSyncReplication,
false, false, nullptr);
ExecContext const& exec = ExecContext::current(); ExecContext const& exec = ExecContext::current();
TRI_ASSERT(cols.size() == 1); TRI_ASSERT(cols.size() == 1);
if (name[0] != '_' && !exec.isSuperuser()) { if (name[0] != '_' && !exec.isSuperuser()) {
@ -1402,7 +1505,7 @@ Result RestReplicationHandler::processRestoreUsersBatch(std::string const& colle
VPackSlice allMarkersSlice = allMarkers.slice(); VPackSlice allMarkersSlice = allMarkers.slice();
std::string aql( std::string aql(
"FOR u IN @restored UPSERT {name: u.name} INSERT u REPLACE u " "FOR u IN @restored UPSERT {user: u.user} INSERT u REPLACE u "
"INTO @@collection OPTIONS {ignoreErrors: true, silent: true, " "INTO @@collection OPTIONS {ignoreErrors: true, silent: true, "
"waitForSync: false, isRestore: true}"); "waitForSync: false, isRestore: true}");
@ -1455,8 +1558,7 @@ Result RestReplicationHandler::processRestoreUsersBatch(std::string const& colle
AuthenticationFeature* af = AuthenticationFeature::instance(); AuthenticationFeature* af = AuthenticationFeature::instance();
TRI_ASSERT(af->userManager() != nullptr); TRI_ASSERT(af->userManager() != nullptr);
if (af->userManager() != nullptr) { if (af->userManager() != nullptr) {
af->userManager()->triggerLocalReload(); af->userManager()->triggerCacheRevalidation();
af->userManager()->triggerGlobalReload();
} }
return queryResult.result; return queryResult.result;
@ -1917,8 +2019,8 @@ void RestReplicationHandler::handleCommandRestoreView() {
if (!overwrite) { if (!overwrite) {
generateError(GeneralResponse::responseCode(TRI_ERROR_ARANGO_DUPLICATE_NAME), generateError(GeneralResponse::responseCode(TRI_ERROR_ARANGO_DUPLICATE_NAME),
TRI_ERROR_ARANGO_DUPLICATE_NAME, TRI_ERROR_ARANGO_DUPLICATE_NAME,
std::string("unable to restore view '") + nameSlice.copyString() + ": " + std::string("unable to restore view '") + nameSlice.copyString() +
TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME)); ": " + TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME));
return; return;
} }
@ -2535,7 +2637,8 @@ void RestReplicationHandler::handleCommandHoldReadLockCollection() {
rebootId = RebootId(body.get(StaticStrings::RebootId).getNumber<uint64_t>()); rebootId = RebootId(body.get(StaticStrings::RebootId).getNumber<uint64_t>());
serverId = body.get("serverId").copyString(); serverId = body.get("serverId").copyString();
} else { } else {
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, generateError(
rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
"'rebootId' must be accompanied by string attribute 'serverId'"); "'rebootId' must be accompanied by string attribute 'serverId'");
return; return;
} }
@ -2903,7 +3006,6 @@ uint64_t RestReplicationHandler::determineChunkSize() const {
return chunkSize; return chunkSize;
} }
ReplicationApplier* RestReplicationHandler::getApplier(bool& global) { ReplicationApplier* RestReplicationHandler::getApplier(bool& global) {
global = _request->parsedValue("global", false); global = _request->parsedValue("global", false);
@ -2922,11 +3024,9 @@ ReplicationApplier* RestReplicationHandler::getApplier(bool& global) {
} }
} }
Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id, Result RestReplicationHandler::createBlockingTransaction(
LogicalCollection& col, double ttl, aql::QueryId id, LogicalCollection& col, double ttl, AccessMode::Type access,
AccessMode::Type access, RebootId const& rebootId, std::string const& serverId) {
RebootId const& rebootId,
std::string const& serverId) {
// This is a constant JSON structure for Queries. // This is a constant JSON structure for Queries.
// we actually do not need a plan, as we only want the query registry to have // we actually do not need a plan, as we only want the query registry to have
// a hold of our transaction // a hold of our transaction
@ -2963,8 +3063,7 @@ Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id,
std::string vn = _vocbase.name(); std::string vn = _vocbase.name();
try { try {
std::function<void(void)> f = std::function<void(void)> f = [=]() {
[=]() {
try { try {
// Code does not matter, read only access, so we can roll back. // Code does not matter, read only access, so we can roll back.
QueryRegistryFeature::registry()->destroy(vn, id, TRI_ERROR_QUERY_KILLED, false); QueryRegistryFeature::registry()->destroy(vn, id, TRI_ERROR_QUERY_KILLED, false);
@ -2975,10 +3074,11 @@ Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id,
}; };
std::string comment = std::string("SynchronizeShard from ") + serverId + std::string comment = std::string("SynchronizeShard from ") + serverId +
" for " + col.name() + " access mode " + AccessMode::typeString(access); " for " + col.name() + " access mode " +
AccessMode::typeString(access);
auto rGuard = std::make_unique<CallbackGuard>( auto rGuard = std::make_unique<CallbackGuard>(
ci.rebootTracker().callMeOnChange( ci.rebootTracker().callMeOnChange(RebootTracker::PeerState(serverId, rebootId),
RebootTracker::PeerState(serverId, rebootId), f, comment)); f, comment));
queryRegistry->insert(id, query.get(), ttl, true, true, std::move(rGuard)); queryRegistry->insert(id, query.get(), ttl, true, true, std::move(rGuard));
@ -3150,8 +3250,8 @@ void RestReplicationHandler::registerTombstone(aql::QueryId id) const {
std::string key = IdToTombstoneKey(_vocbase, id); std::string key = IdToTombstoneKey(_vocbase, id);
{ {
WRITE_LOCKER(writeLocker, RestReplicationHandler::_tombLock); WRITE_LOCKER(writeLocker, RestReplicationHandler::_tombLock);
RestReplicationHandler::_tombstones.try_emplace(key, std::chrono::steady_clock::now() + RestReplicationHandler::_tombstones.try_emplace(
RestReplicationHandler::_tombstoneTimeout); key, std::chrono::steady_clock::now() + RestReplicationHandler::_tombstoneTimeout);
} }
timeoutTombstones(); timeoutTombstones();
} }

View File

@ -28,8 +28,8 @@
#include "Aql/types.h" #include "Aql/types.h"
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Basics/Result.h" #include "Basics/Result.h"
#include "Cluster/ResultT.h"
#include "Cluster/ClusterTypes.h" #include "Cluster/ClusterTypes.h"
#include "Cluster/ResultT.h"
#include "Replication/Syncer.h" #include "Replication/Syncer.h"
#include "Replication/common-defines.h" #include "Replication/common-defines.h"
#include "RestHandler/RestVocbaseBaseHandler.h" #include "RestHandler/RestVocbaseBaseHandler.h"
@ -107,7 +107,7 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
static std::string const HoldReadLockCollection; static std::string const HoldReadLockCollection;
protected: protected:
std::string forwardingTarget() override final; ResultT<std::pair<std::string, bool>> forwardingTarget() override final;
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
/// @brief creates an error if called on a coordinator server /// @brief creates an error if called on a coordinator server
@ -309,8 +309,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// @brief restores the structure of a collection, coordinator case /// @brief restores the structure of a collection, coordinator case
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
Result processRestoreCollectionCoordinator(VPackSlice const&, bool overwrite, Result processRestoreCollectionCoordinator(VPackSlice const& collection,
bool force, bool dropExisting, bool force,
bool ignoreDistributeShardsLikeErrors); bool ignoreDistributeShardsLikeErrors);
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
@ -474,9 +474,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// It will be registered with the given id, and it will have /// It will be registered with the given id, and it will have
/// the given time to live. /// the given time to live.
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
Result createBlockingTransaction(aql::QueryId id, LogicalCollection& col, Result createBlockingTransaction(aql::QueryId id, LogicalCollection& col, double ttl,
double ttl, AccessMode::Type access, AccessMode::Type access, RebootId const& rebootId,
RebootId const& rebootId,
std::string const& serverId); std::string const& serverId);
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
@ -504,6 +503,13 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
ResultT<bool> cancelBlockingTransaction(aql::QueryId id) const; ResultT<bool> cancelBlockingTransaction(aql::QueryId id) const;
//////////////////////////////////////////////////////////////////////////////
/// @brief Validate that the requesting user has access rights to this route
/// Will return TRI_ERROR_NO_ERROR if user has access
/// Will return error code otherwise.
//////////////////////////////////////////////////////////////////////////////
Result testPermissions();
}; };
} // namespace arangodb } // namespace arangodb
#endif #endif

View File

@ -74,26 +74,26 @@ RestStatus RestTasksHandler::execute() {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestTasksHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestTasksHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType(); rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::POST && type != rest::RequestType::PUT && if (type != rest::RequestType::POST && type != rest::RequestType::PUT &&
type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) { type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
std::vector<std::string> const& suffixes = _request->suffixes(); std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) { if (suffixes.size() < 1) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }
void RestTasksHandler::getTasks() { void RestTasksHandler::getTasks() {

View File

@ -38,7 +38,7 @@ class RestTasksHandler : public arangodb::RestVocbaseBaseHandler {
RestStatus execute() override; RestStatus execute() override;
protected: protected:
virtual std::string forwardingTarget() override; virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private: private:
void getTasks(); void getTasks();

View File

@ -341,24 +341,24 @@ void RestTransactionHandler::cancel() {
} }
/// @brief returns the short id of the server which should handle this request /// @brief returns the short id of the server which should handle this request
std::string RestTransactionHandler::forwardingTarget() { ResultT<std::pair<std::string, bool>> RestTransactionHandler::forwardingTarget() {
rest::RequestType const type = _request->requestType(); rest::RequestType const type = _request->requestType();
if (type != rest::RequestType::GET && type != rest::RequestType::PUT && if (type != rest::RequestType::GET && type != rest::RequestType::PUT &&
type != rest::RequestType::DELETE_REQ) { type != rest::RequestType::DELETE_REQ) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
std::vector<std::string> const& suffixes = _request->suffixes(); std::vector<std::string> const& suffixes = _request->suffixes();
if (suffixes.size() < 1) { if (suffixes.size() < 1) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]);
uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick);
if (sourceServer == ServerState::instance()->getShortId()) { if (sourceServer == ServerState::instance()->getShortId()) {
return ""; return {std::make_pair(StaticStrings::Empty, false)};
} }
auto& ci = server().getFeature<ClusterFeature>().clusterInfo(); auto& ci = server().getFeature<ClusterFeature>().clusterInfo();
return ci.getCoordinatorByShortID(sourceServer); return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)};
} }

View File

@ -48,7 +48,7 @@ class RestTransactionHandler : public arangodb::RestVocbaseBaseHandler {
void cancel() override final; void cancel() override final;
protected: protected:
virtual std::string forwardingTarget() override; virtual ResultT<std::pair<std::string, bool>> forwardingTarget() override;
private: private:
void executeGetState(); void executeGetState();

View File

@ -1141,6 +1141,23 @@ arangodb::Result processInputDirectory(
arangodb::Result processJob(arangodb::httpclient::SimpleHttpClient& httpClient, arangodb::Result processJob(arangodb::httpclient::SimpleHttpClient& httpClient,
arangodb::RestoreFeature::JobData& jobData) { arangodb::RestoreFeature::JobData& jobData) {
arangodb::Result result; arangodb::Result result;
VPackSlice const parameters = jobData.collection.get("parameters");
std::string const cname =
arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", "");
if (cname == "_users") {
// special case: never restore data in the _users collection first as it could
// potentially change user permissions. In that case index creation will fail.
result = ::restoreIndexes(httpClient, jobData);
if (result.fail()) {
return result;
}
result = ::restoreData(httpClient, jobData);
if (result.fail()) {
return result;
}
} else {
if (jobData.options.indexesFirst && jobData.options.importStructure) { if (jobData.options.indexesFirst && jobData.options.importStructure) {
// restore indexes first if we are using rocksdb // restore indexes first if we are using rocksdb
result = ::restoreIndexes(httpClient, jobData); result = ::restoreIndexes(httpClient, jobData);
@ -1161,6 +1178,7 @@ arangodb::Result processJob(arangodb::httpclient::SimpleHttpClient& httpClient,
return result; return result;
} }
} }
}
++jobData.stats.restoredCollections; ++jobData.stats.restoredCollections;

View File

@ -113,6 +113,13 @@ class DumpRestoreHelper {
print(CYAN + Date() + ': ' + this.which + ' and Restore - ' + s + RESET); print(CYAN + Date() + ': ' + this.which + ' and Restore - ' + s + RESET);
} }
adjustRestoreToDump()
{
this.restoreOptions = this.dumpOptions;
this.restoreConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo);
this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.restoreOptions, this.instanceInfo.rootDir, this.options.coreCheck);
}
isAlive() { isAlive() {
return pu.arangod.check.instanceAlive(this.instanceInfo, this.options); return pu.arangod.check.instanceAlive(this.instanceInfo, this.options);
} }
@ -157,8 +164,14 @@ class DumpRestoreHelper {
return this.validate(this.results.setup); return this.validate(this.results.setup);
} }
dumpFrom(database) { dumpFrom(database, separateDir = false) {
this.print('dump'); this.print('dump');
if (separateDir) {
if (!fs.exists(fs.join(this.instanceInfo.rootDir, 'dump'))) {
fs.makeDirectory(fs.join(this.instanceInfo.rootDir, 'dump'));
}
this.dumpConfig.setOutputDirectory('dump' + fs.pathSeparator + database);
}
if (!this.dumpConfig.haveSetAllDatabases()) { if (!this.dumpConfig.haveSetAllDatabases()) {
this.dumpConfig.setDatabase(database); this.dumpConfig.setDatabase(database);
} }
@ -166,8 +179,19 @@ class DumpRestoreHelper {
return this.validate(this.results.dump); return this.validate(this.results.dump);
} }
restoreTo(database) { restoreTo(database, options = { separate: false, fromDir: '' }) {
this.print('restore'); this.print('restore');
if (options.hasOwnProperty('separate') && options.separate === true) {
if (!options.hasOwnProperty('fromDir') || typeof options.fromDir !== 'string') {
options.fromDir = database;
}
if (!fs.exists(fs.join(this.instanceInfo.rootDir, 'dump'))) {
fs.makeDirectory(fs.join(this.instanceInfo.rootDir, 'dump'));
}
this.restoreConfig.setInputDirectory('dump' + fs.pathSeparator + options.fromDir, true);
}
if (!this.restoreConfig.haveSetAllDatabases()) { if (!this.restoreConfig.haveSetAllDatabases()) {
this.restoreConfig.setDatabase(database); this.restoreConfig.setDatabase(database);
} }
@ -213,6 +237,7 @@ class DumpRestoreHelper {
restoreFoxxComplete(database) { restoreFoxxComplete(database) {
this.print('Foxx Apps with full restore'); this.print('Foxx Apps with full restore');
this.restoreConfig.setDatabase(database); this.restoreConfig.setDatabase(database);
this.restoreConfig.setIncludeSystem(true);
this.results.restoreFoxxComplete = this.arangorestore(); this.results.restoreFoxxComplete = this.arangorestore();
return this.validate(this.results.restoreFoxxComplete); return this.validate(this.results.restoreFoxxComplete);
} }
@ -343,8 +368,21 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore
const cleanupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpCleanup)); const cleanupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpCleanup));
const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain)); const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain));
const tearDownFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpTearDown)); const tearDownFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpTearDown));
if (
!helper.runSetupSuite(setupFile) || if (options.hasOwnProperty("multipleDumps") && options.multipleDumps) {
if (!helper.runSetupSuite(setupFile) ||
!helper.dumpFrom('_system', true) ||
!helper.dumpFrom('UnitTestsDumpSrc', true) ||
!helper.runCleanupSuite(cleanupFile) ||
!helper.restoreTo('UnitTestsDumpDst', { separate: true, fromDir: 'UnitTestsDumpSrc'}) ||
!helper.restoreTo('_system', { separate: true }) ||
!helper.runTests(testFile,'UnitTestsDumpDst') ||
!helper.tearDown(tearDownFile)) {
return helper.extractResults();
}
}
else {
if (!helper.runSetupSuite(setupFile) ||
!helper.dumpFrom('UnitTestsDumpSrc') || !helper.dumpFrom('UnitTestsDumpSrc') ||
!helper.runCleanupSuite(cleanupFile) || !helper.runCleanupSuite(cleanupFile) ||
!helper.restoreTo('UnitTestsDumpDst') || !helper.restoreTo('UnitTestsDumpDst') ||
@ -352,6 +390,7 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore
!helper.tearDown(tearDownFile)) { !helper.tearDown(tearDownFile)) {
return helper.extractResults(); return helper.extractResults();
} }
}
if (tstFiles.hasOwnProperty("dumpCheckGraph")) { if (tstFiles.hasOwnProperty("dumpCheckGraph")) {
const notCluster = getClusterStrings(options).notCluster; const notCluster = getClusterStrings(options).notCluster;
@ -365,6 +404,10 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore
if (tstFiles.hasOwnProperty("foxxTest")) { if (tstFiles.hasOwnProperty("foxxTest")) {
const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest)); const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest));
if (options.hasOwnProperty("multipleDumps") && options.multipleDumps) {
helper.adjustRestoreToDump();
helper.restoreConfig.setInputDirectory(fs.join('dump','UnitTestsDumpSrc'), true);
}
if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') || if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') ||
!helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') || !helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') ||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') || !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') ||
@ -410,20 +453,6 @@ function dumpMultiple (options) {
} }
function dumpAuthentication (options) { function dumpAuthentication (options) {
if (options.cluster) {
if (options.extremeVerbosity) {
print(CYAN + 'Skipped because of cluster.' + RESET);
}
return {
'dump_authentication': {
'status': true,
'message': 'skipped because of cluster',
'skipped': true
}
};
}
const clientAuth = { const clientAuth = {
'server.authentication': 'true' 'server.authentication': 'true'
}; };
@ -438,16 +467,26 @@ function dumpAuthentication (options) {
password: 'foobarpasswd' password: 'foobarpasswd'
}; };
let restoreAuthOpts = {
username: 'foobaruser',
password: 'pinus'
};
_.defaults(dumpAuthOpts, options); _.defaults(dumpAuthOpts, options);
_.defaults(restoreAuthOpts, options);
let tstFiles = { let tstFiles = {
dumpSetup: 'dump-authentication-setup.js', dumpSetup: 'dump-authentication-setup.js',
dumpCleanup: 'cleanup-nothing.js', dumpCleanup: 'cleanup-alter-user.js',
dumpAgain: 'dump-authentication.js', dumpAgain: 'dump-authentication.js',
dumpTearDown: 'dump-teardown.js', dumpTearDown: 'dump-teardown.js',
foxxTest: 'check-foxx.js' foxxTest: 'check-foxx.js'
}; };
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, dumpAuthOpts, 'dump_authentication', tstFiles, function(){}); options.multipleDumps = true;
options['server.jwt-secret'] = 'haxxmann';
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, restoreAuthOpts, 'dump_authentication', tstFiles, function(){});
} }
function dumpEncrypted (options) { function dumpEncrypted (options) {

View File

@ -167,6 +167,10 @@ class GeneralRequest {
return _headers; return _headers;
} }
void removeHeader(std::string key) {
_headers.erase(key);
}
#ifdef ARANGODB_USE_GOOGLE_TESTS #ifdef ARANGODB_USE_GOOGLE_TESTS
void addHeader(std::string key, std::string value) { void addHeader(std::string key, std::string value) {
_headers.try_emplace(std::move(key), std::move(value)); _headers.try_emplace(std::move(key), std::move(value));

View File

@ -0,0 +1,41 @@
/*jshint globalstrict:false, strict:false */
/* global db */
////////////////////////////////////////////////////////////////////////////////
/// @brief teardown for dump/reload tests
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB Inc, Cologne, Germany
///
/// @author Wilfried Goesgens
/// @author Copyright 2019, ArangoDB Inc, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
(function () {
'use strict';
var users = require("@arangodb/users");
users.update("foobaruser", "pinus", true);
})();
return {
status: true
};

View File

@ -163,6 +163,8 @@ function dumpTestSuite () {
assertEqual(users.permission(uName, "_system"), 'rw'); assertEqual(users.permission(uName, "_system"), 'rw');
assertEqual(users.permission(uName, "UnitTestsDumpSrc"), 'rw'); assertEqual(users.permission(uName, "UnitTestsDumpSrc"), 'rw');
assertEqual(users.permission(uName, "UnitTestsDumpEmpty"), 'rw'); assertEqual(users.permission(uName, "UnitTestsDumpEmpty"), 'rw');
assertTrue(users.isValid("foobaruser", "foobarpasswd"));
} }
}; };

View File

@ -19,7 +19,7 @@ describe ArangoDB do
@longQuery = "FOR x IN 1..1 LET y = SLEEP(0.2) LET a = y LET b = a LET c = b LET d = c LET e = d LET f = e RETURN x" @longQuery = "FOR x IN 1..1 LET y = SLEEP(0.2) LET a = y LET b = a LET c = b LET d = c LET e = d LET f = e RETURN x"
@longQueryBody = JSON.dump({query: @longQuery}) @longQueryBody = JSON.dump({query: @longQuery})
@queryWithBind = "FOR x IN 1..5 LET y = SLEEP(@value) RETURN x" @queryWithBind = "FOR x IN 1..5 LET y = SLEEP(@value) RETURN x"
@queryWithBindBody = JSON.dump({query: @queryWithBind, bindVars: {value: 2}}) @queryWithBindBody = JSON.dump({query: @queryWithBind, bindVars: {value: 4}})
@queryEndpoint ="/_api/cursor" @queryEndpoint ="/_api/cursor"
@queryPrefix = "api-cursor" @queryPrefix = "api-cursor"
end end
@ -135,7 +135,7 @@ describe ArangoDB do
found.should have_key("query") found.should have_key("query")
found["query"].should eq(@queryWithBind) found["query"].should eq(@queryWithBind)
found.should have_key("bindVars") found.should have_key("bindVars")
found["bindVars"].should eq({"value" => 2}) found["bindVars"].should eq({"value" => 4})
found.should have_key("runTime") found.should have_key("runTime")
found["runTime"].should be_kind_of(Numeric) found["runTime"].should be_kind_of(Numeric)
found.should have_key("started") found.should have_key("started")