From 2bf269a842c813f22466b397e9368fafc1a7930a Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 29 Nov 2019 18:46:33 +0100 Subject: [PATCH 1/2] Bug fix/increase test timeout (#10596) * add quoting for better readability of the errormessage * increase sleep for stability --- tests/rb/HttpInterface/api-query-analysis-spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rb/HttpInterface/api-query-analysis-spec.rb b/tests/rb/HttpInterface/api-query-analysis-spec.rb index b5d02aae20..15c4b96ffa 100644 --- a/tests/rb/HttpInterface/api-query-analysis-spec.rb +++ b/tests/rb/HttpInterface/api-query-analysis-spec.rb @@ -19,7 +19,7 @@ describe ArangoDB do @longQuery = "FOR x IN 1..1 LET y = SLEEP(0.2) LET a = y LET b = a LET c = b LET d = c LET e = d LET f = e RETURN x" @longQueryBody = JSON.dump({query: @longQuery}) @queryWithBind = "FOR x IN 1..5 LET y = SLEEP(@value) RETURN x" - @queryWithBindBody = JSON.dump({query: @queryWithBind, bindVars: {value: 2}}) + @queryWithBindBody = JSON.dump({query: @queryWithBind, bindVars: {value: 4}}) @queryEndpoint ="/_api/cursor" @queryPrefix = "api-cursor" end @@ -135,7 +135,7 @@ describe ArangoDB do found.should have_key("query") found["query"].should eq(@queryWithBind) found.should have_key("bindVars") - found["bindVars"].should eq({"value" => 2}) + found["bindVars"].should eq({"value" => 4}) found.should have_key("runTime") found["runTime"].should be_kind_of(Numeric) found.should have_key("started") From a472758986b8181b3fc18c59412d773949987de1 Mon Sep 17 00:00:00 2001 From: Heiko Date: Sat, 30 Nov 2019 02:40:28 +0100 Subject: [PATCH 2/2] Bug fix/fix internal issue 4451 (#10540) * Fix dump_authentication suite * user the correct attribute name * properly reload user permissions after _users collection restore * fixed foxx restore test * changelog * changed the order of index creation during restore for _users collection * changed interface of forwarding target in general server rest handler * added remove header function to general request class * implemented forwardingRequest interface changes * added method to find collection in vocbase through shard id * added co perm verification * Revert "added method to find collection in vocbase through shard id" This reverts commit af28442c01432224cfe4d777b6134d0f685e38ba. * added shard to name map to cluster info * return ResultT in forwarding * fixed test * fixed compile issue * changelog * Improved changelog formatting. * Revert "fixed test" This reverts commit 3f63d94ff099a94e56addcb432bd0fe733d1bcc6. * Added authentication dump to singleserver and handle admin users. * restore perms --- CHANGELOG | 4 + arangod/Auth/UserManager.cpp | 6 + arangod/Auth/UserManager.h | 3 + arangod/Cluster/ClusterInfo.cpp | 21 +- arangod/Cluster/ClusterInfo.h | 11 +- arangod/GeneralServer/RestHandler.cpp | 15 +- arangod/GeneralServer/RestHandler.h | 40 ++-- .../RestHandler/RestControlPregelHandler.cpp | 12 +- .../RestHandler/RestControlPregelHandler.h | 2 +- arangod/RestHandler/RestCursorHandler.cpp | 10 +- arangod/RestHandler/RestCursorHandler.h | 2 +- arangod/RestHandler/RestDocumentHandler.cpp | 12 +- arangod/RestHandler/RestDocumentHandler.h | 2 +- arangod/RestHandler/RestJobHandler.cpp | 10 +- arangod/RestHandler/RestJobHandler.h | 2 +- arangod/RestHandler/RestQueryHandler.cpp | 12 +- arangod/RestHandler/RestQueryHandler.h | 2 +- .../RestHandler/RestReplicationHandler.cpp | 208 +++++++++++++----- arangod/RestHandler/RestReplicationHandler.h | 20 +- arangod/RestHandler/RestTasksHandler.cpp | 10 +- arangod/RestHandler/RestTasksHandler.h | 2 +- .../RestHandler/RestTransactionHandler.cpp | 10 +- arangod/RestHandler/RestTransactionHandler.h | 2 +- arangosh/Restore/RestoreFeature.cpp | 38 +++- .../modules/@arangodb/testsuites/dump.js | 91 +++++--- lib/Rest/GeneralRequest.h | 4 + tests/js/server/dump/cleanup-alter-user.js | 41 ++++ tests/js/server/dump/dump-authentication.js | 2 + 28 files changed, 419 insertions(+), 175 deletions(-) create mode 100644 tests/js/server/dump/cleanup-alter-user.js diff --git a/CHANGELOG b/CHANGELOG index 5a585f2e04..fd43f67e16 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,10 @@ devel ----- +* Fixed permissions for dump/restore. + +* The _users collection is now properly restored when using arangorestore. + * Updated arangosync to 0.7.1. * rename `minReplicationFactor` into `writeConcern` to make it consistent with diff --git a/arangod/Auth/UserManager.cpp b/arangod/Auth/UserManager.cpp index 2e9fe1ef2a..8d39ffbbb8 100644 --- a/arangod/Auth/UserManager.cpp +++ b/arangod/Auth/UserManager.cpp @@ -393,6 +393,12 @@ VPackBuilder auth::UserManager::allUsers() { return result; } +void auth::UserManager::triggerCacheRevalidation() { + triggerLocalReload(); + triggerGlobalReload(); + loadFromDB(); +} + /// Trigger eventual reload, user facing API call void auth::UserManager::triggerGlobalReload() { if (!ServerState::instance()->isCoordinator()) { diff --git a/arangod/Auth/UserManager.h b/arangod/Auth/UserManager.h index a2d15e9cb7..20458004c0 100644 --- a/arangod/Auth/UserManager.h +++ b/arangod/Auth/UserManager.h @@ -95,6 +95,9 @@ class UserManager { /// Trigger eventual reload on all other coordinators (and in TokenCache) void triggerGlobalReload(); + /// Trigger cache revalidation after user restore + void triggerCacheRevalidation(); + /// Create the root user with a default password, will fail if the user /// already exists. Only ever call if you can guarantee to be in charge void createRootUser(); diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index 0610b45921..c785091a7b 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -224,7 +224,6 @@ void ClusterInfo::cleanup() { _plannedViews.clear(); _plannedCollections.clear(); _shards.clear(); - _shardKeys.clear(); _shardIds.clear(); _currentCollections.clear(); } @@ -235,6 +234,7 @@ void ClusterInfo::triggerBackgroundGetIds() { _uniqid._nextUpperValue = 0ULL; try { + _idLock.assertLockedByCurrentThread(); if (_uniqid._backgroundJobIsRunning) { return; } @@ -598,7 +598,7 @@ void ClusterInfo::loadPlan() { // > decltype(_shards) newShards; decltype(_shardServers) newShardServers; - decltype(_shardKeys) newShardKeys; + decltype(_shardToName) newShardToName; bool swapDatabases = false; bool swapCollections = false; @@ -957,17 +957,16 @@ void ClusterInfo::loadPlan() { databaseCollections.try_emplace(collectionId, newCollection); } - newShardKeys.try_emplace(collectionId, std::make_shared>( - newCollection->shardKeys())); - auto shardIDs = newCollection->shardIds(); auto shards = std::make_shared>(); shards->reserve(shardIDs->size()); + newShardToName.reserve(shardIDs->size()); for (auto const& p : *shardIDs) { TRI_ASSERT(p.first.size() >= 2); shards->push_back(p.first); newShardServers.try_emplace(p.first, p.second); + newShardToName.try_emplace(p.first, newCollection->name()); } // Sort by the number in the shard ID ("s0000001" for example): @@ -1053,8 +1052,8 @@ void ClusterInfo::loadPlan() { if (swapCollections) { _plannedCollections.swap(newCollections); _shards.swap(newShards); - _shardKeys.swap(newShardKeys); _shardServers.swap(newShardServers); + _shardToName.swap(newShardToName); } if (swapViews) { @@ -4288,6 +4287,16 @@ arangodb::Result ClusterInfo::getShardServers(ShardID const& shardId, return arangodb::Result(TRI_ERROR_FAILED); } +CollectionID ClusterInfo::getCollectionNameForShard(ShardID const& shardId) { + READ_LOCKER(readLocker, _planProt.lock); + + auto it = _shardToName.find(shardId); + if (it != _shardToName.end()) { + return it->second; + } + return StaticStrings::Empty; +} + arangodb::Result ClusterInfo::agencyDump(std::shared_ptr body) { AgencyCommResult dump = _agency.dump(); diff --git a/arangod/Cluster/ClusterInfo.h b/arangod/Cluster/ClusterInfo.h index c607ff5180..8fa862a139 100644 --- a/arangod/Cluster/ClusterInfo.h +++ b/arangod/Cluster/ClusterInfo.h @@ -815,6 +815,9 @@ class ClusterInfo final { * @return List of DB servers serving the shard */ arangodb::Result getShardServers(ShardID const& shardId, std::vector&); + + /// @brief map shardId to collection name (not ID) + CollectionID getCollectionNameForShard(ShardID const& shardId); /** * @brief Lock agency's hot backup with TTL 60 seconds @@ -956,8 +959,7 @@ class ClusterInfo final { ProtectionData _planProt; uint64_t _planVersion; // This is the version in the Plan which underlies - // the data in _plannedCollections, _shards and - // _shardKeys + // the data in _plannedCollections and _shards uint64_t _currentVersion; // This is the version in Current which underlies // the data in _currentDatabases, // _currentCollections and _shardsIds @@ -978,11 +980,10 @@ class ClusterInfo final { std::shared_ptr>> _shards; // from Plan/Collections/ // (may later come from Current/Collections/ ) - std::unordered_map>> - _shardKeys; // from Plan/Collections/ // planned shard => servers map std::unordered_map> _shardServers; + // planned shard ID => collection name + std::unordered_map _shardToName; AllViews _plannedViews; // from Plan/Views/ AllViews _newPlannedViews; // views that have been created during `loadPlan` diff --git a/arangod/GeneralServer/RestHandler.cpp b/arangod/GeneralServer/RestHandler.cpp index bbd2778d93..5db82656b2 100644 --- a/arangod/GeneralServer/RestHandler.cpp +++ b/arangod/GeneralServer/RestHandler.cpp @@ -113,7 +113,20 @@ futures::Future RestHandler::forwardRequest(bool& forwarded) { return futures::makeFuture(Result()); } - std::string serverId = forwardingTarget(); + ResultT forwardResult = forwardingTarget(); + if (forwardResult.fail()) { + return futures::makeFuture(forwardResult.result()); + } + + auto forwardContent = forwardResult.get(); + std::string serverId = std::get<0>(forwardContent); + bool removeHeader = std::get<1>(forwardContent); + + if (removeHeader) { + _request->removeHeader(StaticStrings::Authorization); + _request->setUser(""); + } + if (serverId.empty()) { // no need to actually forward return futures::makeFuture(Result()); diff --git a/arangod/GeneralServer/RestHandler.h b/arangod/GeneralServer/RestHandler.h index 4484f5193a..1c6d156979 100644 --- a/arangod/GeneralServer/RestHandler.h +++ b/arangod/GeneralServer/RestHandler.h @@ -30,6 +30,7 @@ #include "Network/Methods.h" #include "Rest/GeneralResponse.h" +#include #include #include @@ -40,13 +41,13 @@ class ApplicationServer; namespace basics { class Exception; } - + namespace futures { -template +template class Future; -template +template class Try; -} +} // namespace futures class GeneralRequest; class RequestStatistics; @@ -129,9 +130,7 @@ class RestHandler : public std::enable_shared_from_this { // you might need to implment this in you handler // if it will be executed in an async job - virtual void cancel() { - _canceled.store(true); - } + virtual void cancel() { _canceled.store(true); } virtual void handleError(basics::Exception const&) = 0; @@ -143,7 +142,11 @@ class RestHandler : public std::enable_shared_from_this { /// handled by this server, the method should return an empty string. /// Otherwise, this method should return a valid short name for the /// target server. - virtual std::string forwardingTarget() { return ""; } + /// std::string -> empty string or valid short name + /// boolean -> should auth header and user be removed in that request + virtual ResultT> forwardingTarget() { + return {std::make_pair(StaticStrings::Empty, false)}; + } void resetResponse(rest::ResponseCode); @@ -154,11 +157,11 @@ class RestHandler : public std::enable_shared_from_this { // generates an error void generateError(arangodb::Result const&); - - template + + template RestStatus waitForFuture(futures::Future&& f) { - if (f.isReady()) { // fast-path out - f.result().throwIfFailed(); // just throw the error upwards + if (f.isReady()) { // fast-path out + f.result().throwIfFailed(); // just throw the error upwards return RestStatus::DONE; } bool done = false; @@ -172,7 +175,7 @@ class RestHandler : public std::enable_shared_from_this { }); return done ? RestStatus::DONE : RestStatus::WAITING; } - + enum class HandlerState : uint8_t { PREPARE = 0, EXECUTE, @@ -182,11 +185,9 @@ class RestHandler : public std::enable_shared_from_this { DONE, FAILED }; - + /// handler state machine - HandlerState state() const { - return _state; - } + HandlerState state() const { return _state; } private: void runHandlerStateMachine(); @@ -198,16 +199,14 @@ class RestHandler : public std::enable_shared_from_this { /// otherwise execute() will be called void executeEngine(bool isContinue); void compressResponse(); - - protected: + protected: std::unique_ptr _request; std::unique_ptr _response; application_features::ApplicationServer& _server; RequestStatistics* _statistics; private: - mutable Mutex _executionMutex; std::function _callback; @@ -219,7 +218,6 @@ class RestHandler : public std::enable_shared_from_this { HandlerState _state; protected: - std::atomic _canceled; bool _allowDirectExecution = false; diff --git a/arangod/RestHandler/RestControlPregelHandler.cpp b/arangod/RestHandler/RestControlPregelHandler.cpp index 8ce1a21b23..8587669960 100644 --- a/arangod/RestHandler/RestControlPregelHandler.cpp +++ b/arangod/RestHandler/RestControlPregelHandler.cpp @@ -75,26 +75,26 @@ RestStatus RestControlPregelHandler::execute() { } /// @brief returns the short id of the server which should handle this request -std::string RestControlPregelHandler::forwardingTarget() { +ResultT> RestControlPregelHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::POST && type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } void RestControlPregelHandler::startExecution() { @@ -107,7 +107,7 @@ void RestControlPregelHandler::startExecution() { // algorithm std::string algorithm = - VelocyPackHelper::getStringValue(body, "algorithm", ""); + VelocyPackHelper::getStringValue(body, "algorithm", StaticStrings::Empty); if ("" == algorithm) { generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND, "invalid algorithm"); diff --git a/arangod/RestHandler/RestControlPregelHandler.h b/arangod/RestHandler/RestControlPregelHandler.h index a0ee5cf45e..cce767e718 100644 --- a/arangod/RestHandler/RestControlPregelHandler.h +++ b/arangod/RestHandler/RestControlPregelHandler.h @@ -38,7 +38,7 @@ class RestControlPregelHandler : public arangodb::RestVocbaseBaseHandler { RestStatus execute() override; protected: - virtual std::string forwardingTarget() override; + virtual ResultT> forwardingTarget() override; private: void startExecution(); diff --git a/arangod/RestHandler/RestCursorHandler.cpp b/arangod/RestHandler/RestCursorHandler.cpp index 99df841478..2f72b09f74 100644 --- a/arangod/RestHandler/RestCursorHandler.cpp +++ b/arangod/RestHandler/RestCursorHandler.cpp @@ -381,25 +381,25 @@ RestStatus RestCursorHandler::handleQueryResult() { } /// @brief returns the short id of the server which should handle this request -std::string RestCursorHandler::forwardingTarget() { +ResultT> RestCursorHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestCursorHandler.h b/arangod/RestHandler/RestCursorHandler.h index ede68666f0..4af2ac1d0b 100644 --- a/arangod/RestHandler/RestCursorHandler.h +++ b/arangod/RestHandler/RestCursorHandler.h @@ -88,7 +88,7 @@ class RestCursorHandler : public RestVocbaseBaseHandler { RestStatus processQuery(); /// @brief returns the short id of the server which should handle this request - virtual std::string forwardingTarget() override; + ResultT> forwardingTarget() override; ////////////////////////////////////////////////////////////////////////////// /// @brief unregister the currently running query diff --git a/arangod/RestHandler/RestDocumentHandler.cpp b/arangod/RestHandler/RestDocumentHandler.cpp index 78f2440192..2923b56fee 100644 --- a/arangod/RestHandler/RestDocumentHandler.cpp +++ b/arangod/RestHandler/RestDocumentHandler.cpp @@ -120,9 +120,9 @@ void RestDocumentHandler::shutdownExecute(bool isFinalized) noexcept { } /// @brief returns the short id of the server which should handle this request -std::string RestDocumentHandler::forwardingTarget() { +ResultT> RestDocumentHandler::forwardingTarget() { if (!ServerState::instance()->isCoordinator()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } bool found = false; @@ -131,17 +131,17 @@ std::string RestDocumentHandler::forwardingTarget() { uint64_t tid = basics::StringUtils::uint64(value); if (!transaction::isCoordinatorTransactionId(tid)) { TRI_ASSERT(transaction::isLegacyTransactionId(tid)); - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestDocumentHandler.h b/arangod/RestHandler/RestDocumentHandler.h index a6d2d4d554..af6ffba9a3 100644 --- a/arangod/RestHandler/RestDocumentHandler.h +++ b/arangod/RestHandler/RestDocumentHandler.h @@ -56,7 +56,7 @@ class RestDocumentHandler : public RestVocbaseBaseHandler { void shutdownExecute(bool isFinalized) noexcept override final; protected: - std::string forwardingTarget() override final; + ResultT> forwardingTarget() override final; private: // inserts a document diff --git a/arangod/RestHandler/RestJobHandler.cpp b/arangod/RestHandler/RestJobHandler.cpp index 973e2acaaf..d5a22d39f4 100644 --- a/arangod/RestHandler/RestJobHandler.cpp +++ b/arangod/RestHandler/RestJobHandler.cpp @@ -245,24 +245,24 @@ void RestJobHandler::deleteJob() { } /// @brief returns the short id of the server which should handle this request -std::string RestJobHandler::forwardingTarget() { +ResultT> RestJobHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::GET && type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } diff --git a/arangod/RestHandler/RestJobHandler.h b/arangod/RestHandler/RestJobHandler.h index b9b52465f4..4122c68079 100644 --- a/arangod/RestHandler/RestJobHandler.h +++ b/arangod/RestHandler/RestJobHandler.h @@ -85,7 +85,7 @@ class RestJobHandler : public RestBaseHandler { void deleteJob(); protected: - virtual std::string forwardingTarget() override; + virtual ResultT> forwardingTarget() override; private: ////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestQueryHandler.cpp b/arangod/RestHandler/RestQueryHandler.cpp index f00515a239..23cf585bcb 100644 --- a/arangod/RestHandler/RestQueryHandler.cpp +++ b/arangod/RestHandler/RestQueryHandler.cpp @@ -346,9 +346,9 @@ bool RestQueryHandler::parseQuery() { } /// @brief returns the short id of the server which should handle this request -std::string RestQueryHandler::forwardingTarget() { +ResultT> RestQueryHandler::forwardingTarget() { if (!ServerState::instance()->isCoordinator()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } bool found = false; @@ -357,15 +357,15 @@ std::string RestQueryHandler::forwardingTarget() { uint64_t tid = basics::StringUtils::uint64(value); if (!transaction::isCoordinatorTransactionId(tid)) { TRI_ASSERT(transaction::isLegacyTransactionId(tid)); - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint32_t sourceServer = TRI_ExtractServerIdFromTick(tid); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } diff --git a/arangod/RestHandler/RestQueryHandler.h b/arangod/RestHandler/RestQueryHandler.h index 30690cf029..c73cba3cb4 100644 --- a/arangod/RestHandler/RestQueryHandler.h +++ b/arangod/RestHandler/RestQueryHandler.h @@ -96,7 +96,7 @@ class RestQueryHandler : public RestVocbaseBaseHandler { bool parseQuery(); - virtual std::string forwardingTarget() override; + virtual ResultT> forwardingTarget() override; }; } // namespace arangodb diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 6cbd0c6955..f92de48169 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -29,9 +29,9 @@ #include "Basics/ReadLocker.h" #include "Basics/Result.h" #include "Basics/RocksDBUtils.h" +#include "Basics/StaticStrings.h" #include "Basics/VelocyPackHelper.h" #include "Basics/WriteLocker.h" -#include "Basics/StaticStrings.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterHelpers.h" #include "Cluster/ClusterMethods.h" @@ -280,6 +280,11 @@ std::string const RestReplicationHandler::HoldReadLockCollection = // main function that dispatches the different routes and commands RestStatus RestReplicationHandler::execute() { + auto res = testPermissions(); + if (!res.ok()) { + generateError(res); + return RestStatus::DONE; + } // extract the request type auto const type = _request->requestType(); auto const& suffixes = _request->suffixes(); @@ -589,27 +594,113 @@ BAD_CALL: return RestStatus::DONE; } -/// @brief returns the short id of the server which should handle this request -std::string RestReplicationHandler::forwardingTarget() { - if (!ServerState::instance()->isCoordinator()) { - return ""; +Result RestReplicationHandler::testPermissions() { + if (!_request->authenticated()) { + return TRI_ERROR_NO_ERROR; } + std::string user = _request->user(); auto const& suffixes = _request->suffixes(); size_t const len = suffixes.size(); if (len >= 1) { auto const type = _request->requestType(); std::string const& command = suffixes[0]; if ((command == Batch) || (command == Inventory && type == rest::RequestType::GET) || - (command == Dump && type == rest::RequestType::GET)) { - ServerID const& DBserver = _request->value("DBserver"); - if (!DBserver.empty()) { - return DBserver; + (command == Dump && type == rest::RequestType::GET) || + (command == RestoreCollection && type == rest::RequestType::PUT)) { + if (command == Dump) { + // check dump collection permissions (at least ro needed) + std::string collectionName = _request->value("collection"); + + if (ServerState::instance()->isCoordinator()) { + // We have a shard id, need to translate + ClusterInfo& ci = server().getFeature().clusterInfo(); + collectionName = ci.getCollectionNameForShard(collectionName); + } + + if (!collectionName.empty()) { + auto& exec = ExecContext::current(); + ExecContextSuperuserScope escope(exec.isAdminUser()); + if (!exec.isAdminUser() && + !exec.canUseCollection(collectionName, auth::Level::RO)) { + // not enough rights + return Result(TRI_ERROR_FORBIDDEN); + } + } else { + // not found, return 404 + return Result(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); + } + } else if (command == RestoreCollection) { + VPackSlice const slice = _request->payload(); + VPackSlice const parameters = slice.get("parameters"); + if (parameters.isObject()) { + if (parameters.get("name").isString()) { + std::string collectionName = parameters.get("name").copyString(); + if (!collectionName.empty()) { + std::string dbName = _request->databaseName(); + DatabaseFeature& databaseFeature = + _vocbase.server().getFeature(); + TRI_vocbase_t* vocbase = databaseFeature.lookupDatabase(dbName); + if (vocbase == nullptr) { + return Result(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND); + } + + std::string const& overwriteCollection = + _request->value("overwrite"); + + auto& exec = ExecContext::current(); + ExecContextSuperuserScope escope(exec.isAdminUser()); + + if (overwriteCollection == "true" || + vocbase->lookupCollection(collectionName) == nullptr) { + // 1.) re-create collection, means: overwrite=true (rw database) + // OR 2.) not existing, new collection (rw database) + if (!exec.isAdminUser() && !exec.canUseDatabase(dbName, auth::Level::RW)) { + return Result(TRI_ERROR_FORBIDDEN); + } + } else { + // 3.) Existing collection (ro database, rw collection) + // no overwrite. restoring into an existing collection + if (!exec.isAdminUser() && + !exec.canUseCollection(collectionName, auth::Level::RW)) { + return Result(TRI_ERROR_FORBIDDEN); + } + } + } else { + return Result(TRI_ERROR_HTTP_BAD_PARAMETER, + "empty collection name"); + } + } else { + return Result(TRI_ERROR_HTTP_BAD_PARAMETER, + "invalid collection name type"); + } + } else { + return Result(TRI_ERROR_HTTP_BAD_PARAMETER, + "invalid collection parameter type"); + } } } } + return Result(TRI_ERROR_NO_ERROR); +} - return ""; +/// @brief returns the short id of the server which should handle this request +ResultT> RestReplicationHandler::forwardingTarget() { + if (!ServerState::instance()->isCoordinator()) { + return {std::make_pair("", false)}; + } + auto res = testPermissions(); + if (!res.ok()) { + return res; + } + + ServerID const& DBserver = _request->value("DBserver"); + if (!DBserver.empty()) { + // if DBserver property present, remove auth header + return std::make_pair(DBserver, true); + } + + return {std::make_pair(StaticStrings::Empty, false)}; } //////////////////////////////////////////////////////////////////////////////// @@ -712,8 +803,16 @@ void RestReplicationHandler::handleCommandClusterInventory() { vocbase->toVelocyPack(resultBuilder); } + auto& exec = ExecContext::current(); + ExecContextSuperuserScope escope(exec.isAdminUser()); + resultBuilder.add("collections", VPackValue(VPackValueType::Array)); for (std::shared_ptr const& c : cols) { + if (!exec.isAdminUser() && + !exec.canUseCollection(vocbase->name(), c->name(), auth::Level::RO)) { + continue; + } + // We want to check if the collection is usable and all followers // are in sync: std::shared_ptr shardMap = c->shardIds(); @@ -784,7 +883,7 @@ void RestReplicationHandler::handleCommandRestoreCollection() { bool force = _request->parsedValue("force", false); bool ignoreDistributeShardsLikeErrors = _request->parsedValue("ignoreDistributeShardsLikeErrors", false); - + Result res; if (ServerState::instance()->isCoordinator()) { res = processRestoreCollectionCoordinator(slice, overwrite, force, @@ -1080,7 +1179,7 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( } // now re-create the collection - + // Build up new information that we need to merge with the given one VPackBuilder toMerge; toMerge.openObject(); @@ -1094,9 +1193,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( // force one shard, and force distributeShardsLike to be "_graphs" toMerge.add(StaticStrings::NumberOfShards, VPackValue(1)); if (!_vocbase.IsSystemName(name)) { - // system-collections will be sharded normally. only user collections will get - // the forced sharding - toMerge.add(StaticStrings::DistributeShardsLike, VPackValue(_vocbase.shardingPrototypeName())); + // system-collections will be sharded normally. only user collections will + // get the forced sharding + toMerge.add(StaticStrings::DistributeShardsLike, + VPackValue(_vocbase.shardingPrototypeName())); } } else { size_t numberOfShards = 1; @@ -1116,10 +1216,10 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( if (_vocbase.sharding() == "single" && parameters.get(StaticStrings::DistributeShardsLike).isNone() && - !_vocbase.IsSystemName(name) && - numberOfShards <= 1) { + !_vocbase.IsSystemName(name) && numberOfShards <= 1) { // shard like _graphs - toMerge.add(StaticStrings::DistributeShardsLike, VPackValue(_vocbase.shardingPrototypeName())); + toMerge.add(StaticStrings::DistributeShardsLike, + VPackValue(_vocbase.shardingPrototypeName())); } } @@ -1128,13 +1228,14 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( // not an error: for historical reasons the write concern is read from the // variable "minReplicationFactor" VPackSlice writeConcernSlice = parameters.get(StaticStrings::WriteConcern); - if (writeConcernSlice.isNone()) { // minReplicationFactor is deprecated in 3.6 + if (writeConcernSlice.isNone()) { // minReplicationFactor is deprecated in 3.6 writeConcernSlice = parameters.get(StaticStrings::MinReplicationFactor); } - bool isValidReplicationFactorSlice = replicationFactorSlice.isInteger() || - (replicationFactorSlice.isString() && - replicationFactorSlice.isEqualString(StaticStrings::Satellite)); + bool isValidReplicationFactorSlice = + replicationFactorSlice.isInteger() || + (replicationFactorSlice.isString() && + replicationFactorSlice.isEqualString(StaticStrings::Satellite)); bool isValidWriteConcernSlice = replicationFactorSlice.isInteger() && writeConcernSlice.isInteger() && @@ -1142,7 +1243,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( writeConcernSlice.getInt() > 0; if (!isValidReplicationFactorSlice) { - size_t replicationFactor = _vocbase.server().getFeature().defaultReplicationFactor(); + size_t replicationFactor = + _vocbase.server().getFeature().defaultReplicationFactor(); if (replicationFactor == 0) { replicationFactor = 1; } @@ -1255,7 +1357,8 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( // not desired, so it is hardcoded to false auto cols = ClusterMethods::createCollectionOnCoordinator(_vocbase, merged, ignoreDistributeShardsLikeErrors, - createWaitsForSyncReplication, false, false, nullptr); + createWaitsForSyncReplication, + false, false, nullptr); ExecContext const& exec = ExecContext::current(); TRI_ASSERT(cols.size() == 1); if (name[0] != '_' && !exec.isSuperuser()) { @@ -1402,7 +1505,7 @@ Result RestReplicationHandler::processRestoreUsersBatch(std::string const& colle VPackSlice allMarkersSlice = allMarkers.slice(); std::string aql( - "FOR u IN @restored UPSERT {name: u.name} INSERT u REPLACE u " + "FOR u IN @restored UPSERT {user: u.user} INSERT u REPLACE u " "INTO @@collection OPTIONS {ignoreErrors: true, silent: true, " "waitForSync: false, isRestore: true}"); @@ -1455,8 +1558,7 @@ Result RestReplicationHandler::processRestoreUsersBatch(std::string const& colle AuthenticationFeature* af = AuthenticationFeature::instance(); TRI_ASSERT(af->userManager() != nullptr); if (af->userManager() != nullptr) { - af->userManager()->triggerLocalReload(); - af->userManager()->triggerGlobalReload(); + af->userManager()->triggerCacheRevalidation(); } return queryResult.result; @@ -1917,8 +2019,8 @@ void RestReplicationHandler::handleCommandRestoreView() { if (!overwrite) { generateError(GeneralResponse::responseCode(TRI_ERROR_ARANGO_DUPLICATE_NAME), TRI_ERROR_ARANGO_DUPLICATE_NAME, - std::string("unable to restore view '") + nameSlice.copyString() + ": " + - TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME)); + std::string("unable to restore view '") + nameSlice.copyString() + + ": " + TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME)); return; } @@ -2524,7 +2626,7 @@ void RestReplicationHandler::handleCommandHoldReadLockCollection() { "'ttl' and 'id'"); return; } - + VPackSlice collection = body.get("collection"); VPackSlice ttlSlice = body.get("ttl"); VPackSlice idSlice = body.get("id"); @@ -2535,8 +2637,9 @@ void RestReplicationHandler::handleCommandHoldReadLockCollection() { rebootId = RebootId(body.get(StaticStrings::RebootId).getNumber()); serverId = body.get("serverId").copyString(); } else { - generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "'rebootId' must be accompanied by string attribute 'serverId'"); + generateError( + rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, + "'rebootId' must be accompanied by string attribute 'serverId'"); return; } } else { @@ -2903,7 +3006,6 @@ uint64_t RestReplicationHandler::determineChunkSize() const { return chunkSize; } - ReplicationApplier* RestReplicationHandler::getApplier(bool& global) { global = _request->parsedValue("global", false); @@ -2922,11 +3024,9 @@ ReplicationApplier* RestReplicationHandler::getApplier(bool& global) { } } -Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id, - LogicalCollection& col, double ttl, - AccessMode::Type access, - RebootId const& rebootId, - std::string const& serverId) { +Result RestReplicationHandler::createBlockingTransaction( + aql::QueryId id, LogicalCollection& col, double ttl, AccessMode::Type access, + RebootId const& rebootId, std::string const& serverId) { // This is a constant JSON structure for Queries. // we actually do not need a plan, as we only want the query registry to have // a hold of our transaction @@ -2963,25 +3063,25 @@ Result RestReplicationHandler::createBlockingTransaction(aql::QueryId id, std::string vn = _vocbase.name(); try { - std::function f = - [=]() { - try { - // Code does not matter, read only access, so we can roll back. - QueryRegistryFeature::registry()->destroy(vn, id, TRI_ERROR_QUERY_KILLED, false); - } catch (...) { - // All errors that show up here can only be - // triggered if the query is destroyed in between. - } - }; + std::function f = [=]() { + try { + // Code does not matter, read only access, so we can roll back. + QueryRegistryFeature::registry()->destroy(vn, id, TRI_ERROR_QUERY_KILLED, false); + } catch (...) { + // All errors that show up here can only be + // triggered if the query is destroyed in between. + } + }; std::string comment = std::string("SynchronizeShard from ") + serverId + - " for " + col.name() + " access mode " + AccessMode::typeString(access); + " for " + col.name() + " access mode " + + AccessMode::typeString(access); auto rGuard = std::make_unique( - ci.rebootTracker().callMeOnChange( - RebootTracker::PeerState(serverId, rebootId), f, comment)); + ci.rebootTracker().callMeOnChange(RebootTracker::PeerState(serverId, rebootId), + f, comment)); queryRegistry->insert(id, query.get(), ttl, true, true, std::move(rGuard)); - + } catch (...) { // For compatibility we only return this error return {TRI_ERROR_TRANSACTION_INTERNAL, "cannot begin read transaction"}; @@ -3150,8 +3250,8 @@ void RestReplicationHandler::registerTombstone(aql::QueryId id) const { std::string key = IdToTombstoneKey(_vocbase, id); { WRITE_LOCKER(writeLocker, RestReplicationHandler::_tombLock); - RestReplicationHandler::_tombstones.try_emplace(key, std::chrono::steady_clock::now() + - RestReplicationHandler::_tombstoneTimeout); + RestReplicationHandler::_tombstones.try_emplace( + key, std::chrono::steady_clock::now() + RestReplicationHandler::_tombstoneTimeout); } timeoutTombstones(); } diff --git a/arangod/RestHandler/RestReplicationHandler.h b/arangod/RestHandler/RestReplicationHandler.h index d8b9984c3d..28367cecda 100644 --- a/arangod/RestHandler/RestReplicationHandler.h +++ b/arangod/RestHandler/RestReplicationHandler.h @@ -28,8 +28,8 @@ #include "Aql/types.h" #include "Basics/Common.h" #include "Basics/Result.h" -#include "Cluster/ResultT.h" #include "Cluster/ClusterTypes.h" +#include "Cluster/ResultT.h" #include "Replication/Syncer.h" #include "Replication/common-defines.h" #include "RestHandler/RestVocbaseBaseHandler.h" @@ -107,7 +107,7 @@ class RestReplicationHandler : public RestVocbaseBaseHandler { static std::string const HoldReadLockCollection; protected: - std::string forwardingTarget() override final; + ResultT> forwardingTarget() override final; ////////////////////////////////////////////////////////////////////////////// /// @brief creates an error if called on a coordinator server @@ -309,8 +309,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler { /// @brief restores the structure of a collection, coordinator case ////////////////////////////////////////////////////////////////////////////// - Result processRestoreCollectionCoordinator(VPackSlice const&, bool overwrite, - bool force, + Result processRestoreCollectionCoordinator(VPackSlice const& collection, + bool dropExisting, bool force, bool ignoreDistributeShardsLikeErrors); ////////////////////////////////////////////////////////////////////////////// @@ -474,9 +474,8 @@ class RestReplicationHandler : public RestVocbaseBaseHandler { /// It will be registered with the given id, and it will have /// the given time to live. ////////////////////////////////////////////////////////////////////////////// - Result createBlockingTransaction(aql::QueryId id, LogicalCollection& col, - double ttl, AccessMode::Type access, - RebootId const& rebootId, + Result createBlockingTransaction(aql::QueryId id, LogicalCollection& col, double ttl, + AccessMode::Type access, RebootId const& rebootId, std::string const& serverId); ////////////////////////////////////////////////////////////////////////////// @@ -504,6 +503,13 @@ class RestReplicationHandler : public RestVocbaseBaseHandler { ////////////////////////////////////////////////////////////////////////////// ResultT cancelBlockingTransaction(aql::QueryId id) const; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Validate that the requesting user has access rights to this route + /// Will return TRI_ERROR_NO_ERROR if user has access + /// Will return error code otherwise. + ////////////////////////////////////////////////////////////////////////////// + Result testPermissions(); }; } // namespace arangodb #endif diff --git a/arangod/RestHandler/RestTasksHandler.cpp b/arangod/RestHandler/RestTasksHandler.cpp index 3b179fcce2..a5514f64af 100644 --- a/arangod/RestHandler/RestTasksHandler.cpp +++ b/arangod/RestHandler/RestTasksHandler.cpp @@ -74,26 +74,26 @@ RestStatus RestTasksHandler::execute() { } /// @brief returns the short id of the server which should handle this request -std::string RestTasksHandler::forwardingTarget() { +ResultT> RestTasksHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::POST && type != rest::RequestType::PUT && type != rest::RequestType::GET && type != rest::RequestType::DELETE_REQ) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } void RestTasksHandler::getTasks() { diff --git a/arangod/RestHandler/RestTasksHandler.h b/arangod/RestHandler/RestTasksHandler.h index 31dc12d938..ad07606811 100644 --- a/arangod/RestHandler/RestTasksHandler.h +++ b/arangod/RestHandler/RestTasksHandler.h @@ -38,7 +38,7 @@ class RestTasksHandler : public arangodb::RestVocbaseBaseHandler { RestStatus execute() override; protected: - virtual std::string forwardingTarget() override; + virtual ResultT> forwardingTarget() override; private: void getTasks(); diff --git a/arangod/RestHandler/RestTransactionHandler.cpp b/arangod/RestHandler/RestTransactionHandler.cpp index 1287de840a..e9e33bf5a0 100644 --- a/arangod/RestHandler/RestTransactionHandler.cpp +++ b/arangod/RestHandler/RestTransactionHandler.cpp @@ -341,24 +341,24 @@ void RestTransactionHandler::cancel() { } /// @brief returns the short id of the server which should handle this request -std::string RestTransactionHandler::forwardingTarget() { +ResultT> RestTransactionHandler::forwardingTarget() { rest::RequestType const type = _request->requestType(); if (type != rest::RequestType::GET && type != rest::RequestType::PUT && type != rest::RequestType::DELETE_REQ) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } std::vector const& suffixes = _request->suffixes(); if (suffixes.size() < 1) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } uint64_t tick = arangodb::basics::StringUtils::uint64(suffixes[0]); uint32_t sourceServer = TRI_ExtractServerIdFromTick(tick); if (sourceServer == ServerState::instance()->getShortId()) { - return ""; + return {std::make_pair(StaticStrings::Empty, false)}; } auto& ci = server().getFeature().clusterInfo(); - return ci.getCoordinatorByShortID(sourceServer); + return {std::make_pair(ci.getCoordinatorByShortID(sourceServer), false)}; } diff --git a/arangod/RestHandler/RestTransactionHandler.h b/arangod/RestHandler/RestTransactionHandler.h index c94d001e4d..f3851215a7 100644 --- a/arangod/RestHandler/RestTransactionHandler.h +++ b/arangod/RestHandler/RestTransactionHandler.h @@ -48,7 +48,7 @@ class RestTransactionHandler : public arangodb::RestVocbaseBaseHandler { void cancel() override final; protected: - virtual std::string forwardingTarget() override; + virtual ResultT> forwardingTarget() override; private: void executeGetState(); diff --git a/arangosh/Restore/RestoreFeature.cpp b/arangosh/Restore/RestoreFeature.cpp index 34fb15fd10..e2c59236eb 100644 --- a/arangosh/Restore/RestoreFeature.cpp +++ b/arangosh/Restore/RestoreFeature.cpp @@ -1141,24 +1141,42 @@ arangodb::Result processInputDirectory( arangodb::Result processJob(arangodb::httpclient::SimpleHttpClient& httpClient, arangodb::RestoreFeature::JobData& jobData) { arangodb::Result result; - if (jobData.options.indexesFirst && jobData.options.importStructure) { - // restore indexes first if we are using rocksdb + + VPackSlice const parameters = jobData.collection.get("parameters"); + std::string const cname = + arangodb::basics::VelocyPackHelper::getStringValue(parameters, "name", ""); + + if (cname == "_users") { + // special case: never restore data in the _users collection first as it could + // potentially change user permissions. In that case index creation will fail. result = ::restoreIndexes(httpClient, jobData); if (result.fail()) { return result; } - } - if (jobData.options.importData) { result = ::restoreData(httpClient, jobData); if (result.fail()) { return result; } - } - if (!jobData.options.indexesFirst && jobData.options.importStructure) { - // restore indexes second if we are using mmfiles - result = ::restoreIndexes(httpClient, jobData); - if (result.fail()) { - return result; + } else { + if (jobData.options.indexesFirst && jobData.options.importStructure) { + // restore indexes first if we are using rocksdb + result = ::restoreIndexes(httpClient, jobData); + if (result.fail()) { + return result; + } + } + if (jobData.options.importData) { + result = ::restoreData(httpClient, jobData); + if (result.fail()) { + return result; + } + } + if (!jobData.options.indexesFirst && jobData.options.importStructure) { + // restore indexes second if we are using mmfiles + result = ::restoreIndexes(httpClient, jobData); + if (result.fail()) { + return result; + } } } diff --git a/js/client/modules/@arangodb/testsuites/dump.js b/js/client/modules/@arangodb/testsuites/dump.js index a2621d08a0..a9f1cae42f 100644 --- a/js/client/modules/@arangodb/testsuites/dump.js +++ b/js/client/modules/@arangodb/testsuites/dump.js @@ -113,6 +113,13 @@ class DumpRestoreHelper { print(CYAN + Date() + ': ' + this.which + ' and Restore - ' + s + RESET); } + adjustRestoreToDump() + { + this.restoreOptions = this.dumpOptions; + this.restoreConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo); + this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.restoreOptions, this.instanceInfo.rootDir, this.options.coreCheck); + } + isAlive() { return pu.arangod.check.instanceAlive(this.instanceInfo, this.options); } @@ -157,8 +164,14 @@ class DumpRestoreHelper { return this.validate(this.results.setup); } - dumpFrom(database) { + dumpFrom(database, separateDir = false) { this.print('dump'); + if (separateDir) { + if (!fs.exists(fs.join(this.instanceInfo.rootDir, 'dump'))) { + fs.makeDirectory(fs.join(this.instanceInfo.rootDir, 'dump')); + } + this.dumpConfig.setOutputDirectory('dump' + fs.pathSeparator + database); + } if (!this.dumpConfig.haveSetAllDatabases()) { this.dumpConfig.setDatabase(database); } @@ -166,8 +179,19 @@ class DumpRestoreHelper { return this.validate(this.results.dump); } - restoreTo(database) { + restoreTo(database, options = { separate: false, fromDir: '' }) { this.print('restore'); + + if (options.hasOwnProperty('separate') && options.separate === true) { + if (!options.hasOwnProperty('fromDir') || typeof options.fromDir !== 'string') { + options.fromDir = database; + } + if (!fs.exists(fs.join(this.instanceInfo.rootDir, 'dump'))) { + fs.makeDirectory(fs.join(this.instanceInfo.rootDir, 'dump')); + } + this.restoreConfig.setInputDirectory('dump' + fs.pathSeparator + options.fromDir, true); + } + if (!this.restoreConfig.haveSetAllDatabases()) { this.restoreConfig.setDatabase(database); } @@ -213,6 +237,7 @@ class DumpRestoreHelper { restoreFoxxComplete(database) { this.print('Foxx Apps with full restore'); this.restoreConfig.setDatabase(database); + this.restoreConfig.setIncludeSystem(true); this.results.restoreFoxxComplete = this.arangorestore(); return this.validate(this.results.restoreFoxxComplete); } @@ -343,14 +368,28 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore const cleanupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpCleanup)); const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain)); const tearDownFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpTearDown)); - if ( - !helper.runSetupSuite(setupFile) || - !helper.dumpFrom('UnitTestsDumpSrc') || - !helper.runCleanupSuite(cleanupFile) || - !helper.restoreTo('UnitTestsDumpDst') || - !helper.runTests(testFile,'UnitTestsDumpDst') || - !helper.tearDown(tearDownFile)) { - return helper.extractResults(); + + if (options.hasOwnProperty("multipleDumps") && options.multipleDumps) { + if (!helper.runSetupSuite(setupFile) || + !helper.dumpFrom('_system', true) || + !helper.dumpFrom('UnitTestsDumpSrc', true) || + !helper.runCleanupSuite(cleanupFile) || + !helper.restoreTo('UnitTestsDumpDst', { separate: true, fromDir: 'UnitTestsDumpSrc'}) || + !helper.restoreTo('_system', { separate: true }) || + !helper.runTests(testFile,'UnitTestsDumpDst') || + !helper.tearDown(tearDownFile)) { + return helper.extractResults(); + } + } + else { + if (!helper.runSetupSuite(setupFile) || + !helper.dumpFrom('UnitTestsDumpSrc') || + !helper.runCleanupSuite(cleanupFile) || + !helper.restoreTo('UnitTestsDumpDst') || + !helper.runTests(testFile,'UnitTestsDumpDst') || + !helper.tearDown(tearDownFile)) { + return helper.extractResults(); + } } if (tstFiles.hasOwnProperty("dumpCheckGraph")) { @@ -365,6 +404,10 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restore if (tstFiles.hasOwnProperty("foxxTest")) { const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest)); + if (options.hasOwnProperty("multipleDumps") && options.multipleDumps) { + helper.adjustRestoreToDump(); + helper.restoreConfig.setInputDirectory(fs.join('dump','UnitTestsDumpSrc'), true); + } if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') || !helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') || !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') || @@ -410,20 +453,6 @@ function dumpMultiple (options) { } function dumpAuthentication (options) { - if (options.cluster) { - if (options.extremeVerbosity) { - print(CYAN + 'Skipped because of cluster.' + RESET); - } - - return { - 'dump_authentication': { - 'status': true, - 'message': 'skipped because of cluster', - 'skipped': true - } - }; - } - const clientAuth = { 'server.authentication': 'true' }; @@ -438,16 +467,26 @@ function dumpAuthentication (options) { password: 'foobarpasswd' }; + let restoreAuthOpts = { + username: 'foobaruser', + password: 'pinus' + }; + _.defaults(dumpAuthOpts, options); + _.defaults(restoreAuthOpts, options); + let tstFiles = { dumpSetup: 'dump-authentication-setup.js', - dumpCleanup: 'cleanup-nothing.js', + dumpCleanup: 'cleanup-alter-user.js', dumpAgain: 'dump-authentication.js', dumpTearDown: 'dump-teardown.js', foxxTest: 'check-foxx.js' }; - return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, dumpAuthOpts, 'dump_authentication', tstFiles, function(){}); + options.multipleDumps = true; + options['server.jwt-secret'] = 'haxxmann'; + + return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, restoreAuthOpts, 'dump_authentication', tstFiles, function(){}); } function dumpEncrypted (options) { diff --git a/lib/Rest/GeneralRequest.h b/lib/Rest/GeneralRequest.h index e99a2c50bb..f4d36cad6b 100644 --- a/lib/Rest/GeneralRequest.h +++ b/lib/Rest/GeneralRequest.h @@ -167,6 +167,10 @@ class GeneralRequest { return _headers; } + void removeHeader(std::string key) { + _headers.erase(key); + } + #ifdef ARANGODB_USE_GOOGLE_TESTS void addHeader(std::string key, std::string value) { _headers.try_emplace(std::move(key), std::move(value)); diff --git a/tests/js/server/dump/cleanup-alter-user.js b/tests/js/server/dump/cleanup-alter-user.js new file mode 100644 index 0000000000..3c542a9f99 --- /dev/null +++ b/tests/js/server/dump/cleanup-alter-user.js @@ -0,0 +1,41 @@ +/*jshint globalstrict:false, strict:false */ +/* global db */ + +//////////////////////////////////////////////////////////////////////////////// +/// @brief teardown for dump/reload tests +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2010-2012 triagens GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB Inc, Cologne, Germany +/// +/// @author Wilfried Goesgens +/// @author Copyright 2019, ArangoDB Inc, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +(function () { + 'use strict'; + + var users = require("@arangodb/users"); + users.update("foobaruser", "pinus", true); +})(); + +return { + status: true +}; + diff --git a/tests/js/server/dump/dump-authentication.js b/tests/js/server/dump/dump-authentication.js index 13d97eb17c..989a7264fd 100644 --- a/tests/js/server/dump/dump-authentication.js +++ b/tests/js/server/dump/dump-authentication.js @@ -163,6 +163,8 @@ function dumpTestSuite () { assertEqual(users.permission(uName, "_system"), 'rw'); assertEqual(users.permission(uName, "UnitTestsDumpSrc"), 'rw'); assertEqual(users.permission(uName, "UnitTestsDumpEmpty"), 'rw'); + + assertTrue(users.isValid("foobaruser", "foobarpasswd")); } };