From c067e17bb6d148c353e870dfadbbe0ece002a38e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Neunh=C3=B6ffer?= Date: Tue, 1 Oct 2019 15:45:28 +0200 Subject: [PATCH] Port backup size to 3.5. (#10117) * Port backup size to 3.5. * Lars' fixes from devel. * Fix compilation one more time. * Lars' datetime fix from devel. --- .../post_admin_backup_download.md | 12 +++- .../BackupRestore/post_admin_backup_upload.md | 12 +++- arangod/Cluster/ClusterMethods.cpp | 59 ++++++++++++++++--- arangod/StorageEngine/HotBackupCommon.h | 34 ++++++++++- arangod/Utils/ExecContext.h | 19 ++++++ arangosh/Backup/BackupFeature.cpp | 20 ++++++- 6 files changed, 140 insertions(+), 16 deletions(-) diff --git a/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_download.md b/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_download.md index 8c0fc983ee..f793badf7d 100644 --- a/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_download.md +++ b/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_download.md @@ -5,7 +5,8 @@ @RESTDESCRIPTION Download a specific local backup from a remote repository, or query -progress on a previously scheduled download operation. +progress on a previously scheduled download operation, or abort +a running download operation. @RESTBODYPARAM{id,string,optional,string} The identifier for this backup. This is required when a download @@ -24,8 +25,13 @@ attribute. See the description of the _arangobackup_ program in the manual for a description of the `config` object. @RESTBODYPARAM{downloadId,string,optional,string} -Download ID to specify for which download operation progress is queried. -If you specify this, leave out all other body parameters. +Download ID to specify for which download operation progress is queried, or +the download operation to abort. +If you specify this, leave out all the above body parameters. + +@RESTBODYPARAM{abort,boolean,optional,boolean} +Set this to `true` if a running download operation should be aborted. In +this case, the only other body parameter which is needed is `downloadId`. @RESTRETURNCODES diff --git a/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_upload.md b/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_upload.md index ae0091d79c..c71147f219 100644 --- a/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_upload.md +++ b/Documentation/DocuBlocks/Rest/BackupRestore/post_admin_backup_upload.md @@ -5,7 +5,8 @@ @RESTDESCRIPTION Upload a specific local backup to a remote repository, or query -progress on a previously scheduled upload operation. +progress on a previously scheduled upload operation, or abort +a running upload operation. @RESTBODYPARAM{id,string,optional,string} The identifier for this backup. This is required when an upload @@ -24,8 +25,13 @@ attribute. See the description of the _arangobackup_ program in the manual for a description of the `config` object. @RESTBODYPARAM{uploadId,string,optional,string} -Upload ID to specify for which upload operation progress is queried. -If you specify this, leave out all other body parameters. +Upload ID to specify for which upload operation progress is queried or +the upload operation to abort. +If you specify this, leave out all the above body parameters. + +@RESTBODYPARAM{abort,boolean,optional,boolean} +Set this to `true` if a running upload operation should be aborted. In +this case, the only other body parameter which is needed is `uploadId`. @RESTRETURNCODES diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index d364e253d8..7d25365591 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -3285,6 +3285,8 @@ arangodb::Result hotBackupList(std::vector const& dbServers, VPackSlic // check here that the backups are all made with the same version std::string version; + size_t totalSize = 0; + size_t totalFiles = 0; for (BackupMeta const& meta : i.second) { if (version.empty()) { @@ -3299,10 +3301,15 @@ arangodb::Result hotBackupList(std::vector const& dbServers, VPackSlic break; } } + totalSize += meta._sizeInBytes; + totalFiles += meta._nrFiles; } if (valid) { BackupMeta& front = i.second.front(); + front._sizeInBytes = totalSize; + front._nrFiles = totalFiles; + front._serverId = ""; // makes no sense for whole cluster hotBackups.insert(std::make_pair(front._id, front)); } } @@ -3859,7 +3866,8 @@ std::vector idPath{"result", "id"}; arangodb::Result hotBackupDBServers(std::string const& backupId, std::string const& timeStamp, std::vector dbServers, - VPackSlice agencyDump, bool force) { + VPackSlice agencyDump, bool force, + BackupMeta& meta) { auto cc = ClusterComm::instance(); if (cc == nullptr) { // nullptr happens only during controlled shutdown @@ -3873,6 +3881,7 @@ arangodb::Result hotBackupDBServers(std::string const& backupId, std::string con builder.add("agency-dump", agencyDump); builder.add("timestamp", VPackValue(timeStamp)); builder.add("allowInconsistent", VPackValue(force)); + builder.add("nrDBServers", VPackValue(dbServers.size())); } auto body = std::make_shared(builder.toJson()); @@ -3889,6 +3898,10 @@ arangodb::Result hotBackupDBServers(std::string const& backupId, std::string con LOG_TOPIC("478ef", DEBUG, Logger::BACKUP) << "Inquiring about backup " << backupId; // Now listen to the results: + size_t totalSize = 0; + size_t totalFiles = 0; + std::string version; + bool sizeValid = true; for (auto const& req : requests) { auto res = req.result; int commError = handleGeneralCommErrors(&res); @@ -3899,14 +3912,16 @@ arangodb::Result hotBackupDBServers(std::string const& backupId, std::string con TRI_ASSERT(res.answer != nullptr); auto resBody = res.answer->toVelocyPackBuilderPtrNoUniquenessChecks(); VPackSlice resSlice = resBody->slice(); - if (!resSlice.isObject()) { + if (!resSlice.isObject() || !resSlice.hasKey("result")) { // Response has invalid format return arangodb::Result(TRI_ERROR_HTTP_CORRUPTED_JSON, std::string("result to take snapshot on ") + - req.destination + " not an object"); + req.destination + " not an object or has no 'result' attribute"); } + resSlice = resSlice.get("result"); - if (!resSlice.hasKey(idPath) || !resSlice.get(idPath).isString()) { + if (!resSlice.hasKey(BackupMeta::ID) || + !resSlice.get(BackupMeta::ID).isString()) { LOG_TOPIC("6240a", ERR, Logger::BACKUP) << "DB server " << req.destination << "is missing backup " << backupId; return arangodb::Result(TRI_ERROR_FILE_NOT_FOUND, @@ -3914,10 +3929,35 @@ arangodb::Result hotBackupDBServers(std::string const& backupId, std::string con " on server " + req.destination); } + if (resSlice.hasKey(BackupMeta::SIZEINBYTES)) { + totalSize += VelocyPackHelper::getNumericValue(resSlice, BackupMeta::SIZEINBYTES, 0); + } else { + sizeValid = false; + } + if (resSlice.hasKey(BackupMeta::NRFILES)) { + totalFiles += VelocyPackHelper::getNumericValue(resSlice, BackupMeta::NRFILES, 0); + } else { + sizeValid = false; + } + if (version.empty() && resSlice.hasKey(BackupMeta::VERSION)) { + VPackSlice verSlice = resSlice.get(BackupMeta::VERSION); + if (verSlice.isString()) { + version = verSlice.copyString(); + } + } + LOG_TOPIC("b370d", DEBUG, Logger::BACKUP) << req.destination << " created local backup " - << resSlice.get(idPath).copyString(); + << resSlice.get(BackupMeta::ID).copyString(); } + if (sizeValid) { + meta = BackupMeta(backupId, version, timeStamp, totalSize, totalFiles, static_cast(dbServers.size()), "", force); + } else { + meta = BackupMeta(backupId, version, timeStamp, 0, 0, static_cast(dbServers.size()), "", force); + LOG_TOPIC("54265", WARN, Logger::BACKUP) + << "Could not determine total size of backup with id '" << backupId + << "'!"; + } LOG_TOPIC("5c5e9", DEBUG, Logger::BACKUP) << "Have created backup " << backupId; return arangodb::Result(); @@ -4141,8 +4181,9 @@ arangodb::Result hotBackupCoordinator(VPackSlice const payload, VPackBuilder& re return result; } + BackupMeta meta(backupId, "", timeStamp, 0, 0, static_cast(dbServers.size()), "", !gotLocks); // Temporary result = hotBackupDBServers(backupId, timeStamp, dbServers, agency->slice(), - /* force */ !gotLocks); + /* force */ !gotLocks, meta); if (!result.ok()) { unlockDBServerTransactions(backupId, dbServers); ci->agencyHotBackupUnlock(backupId, timeout, supervisionOff); @@ -4190,6 +4231,10 @@ arangodb::Result hotBackupCoordinator(VPackSlice const payload, VPackBuilder& re { VPackObjectBuilder o(&report); report.add("id", VPackValue(timeStamp + "_" + backupId)); + report.add("sizeInBytes", VPackValue(meta._sizeInBytes)); + report.add("nrFiles", VPackValue(meta._nrFiles)); + report.add("nrDBServers", VPackValue(meta._nrDBServers)); + report.add("datetime", VPackValue(meta._datetime)); if (!gotLocks) { report.add("potentiallyInconsistent", VPackValue(true)); } @@ -4200,7 +4245,7 @@ arangodb::Result hotBackupCoordinator(VPackSlice const payload, VPackBuilder& re } catch (std::exception const& e) { return arangodb::Result( TRI_ERROR_HOT_BACKUP_INTERNAL, - std::string("caught exception cretaing cluster backup: ") + e.what()); + std::string("caught exception creating cluster backup: ") + e.what()); } } diff --git a/arangod/StorageEngine/HotBackupCommon.h b/arangod/StorageEngine/HotBackupCommon.h index 547e78d545..0ff08ffb57 100644 --- a/arangod/StorageEngine/HotBackupCommon.h +++ b/arangod/StorageEngine/HotBackupCommon.h @@ -29,6 +29,9 @@ #include #include +#include "Basics/VelocyPackHelper.h" +#include "Cluster/ServerState.h" + namespace arangodb { constexpr char const* BAD_PARAMS_CREATE = "backup payload must be an object " @@ -43,10 +46,20 @@ struct BackupMeta { std::string _id; std::string _version; std::string _datetime; + size_t _sizeInBytes; + size_t _nrFiles; + unsigned int _nrDBServers; + std::string _serverId; + bool _potentiallyInconsistent; static constexpr const char *ID = "id"; static constexpr const char *VERSION = "version"; static constexpr const char *DATETIME = "datetime"; + static constexpr const char *SIZEINBYTES = "sizeInBytes"; + static constexpr const char *NRFILES = "nrFiles"; + static constexpr const char *NRDBSERVERS = "nrDBServers"; + static constexpr const char *SERVERID = "serverId"; + static constexpr const char *POTENTIALLYINCONSISTENT = "potentiallyInconsistent"; void toVelocyPack(VPackBuilder &builder) const { { @@ -54,6 +67,13 @@ struct BackupMeta { builder.add(ID, VPackValue(_id)); builder.add(VERSION, VPackValue(_version)); builder.add(DATETIME, VPackValue(_datetime)); + builder.add(SIZEINBYTES, VPackValue(_sizeInBytes)); + builder.add(NRFILES, VPackValue(_nrFiles)); + builder.add(NRDBSERVERS, VPackValue(_nrDBServers)); + if (ServerState::instance()->isDBServer()) { + builder.add(SERVERID, VPackValue(_serverId)); + } + builder.add(POTENTIALLYINCONSISTENT, VPackValue(_potentiallyInconsistent)); } } @@ -63,14 +83,24 @@ struct BackupMeta { meta._id = slice.get(ID).copyString(); meta._version = slice.get(VERSION).copyString(); meta._datetime = slice.get(DATETIME).copyString(); + meta._sizeInBytes = basics::VelocyPackHelper::getNumericValue( + slice, SIZEINBYTES, 0); + meta._nrFiles = basics::VelocyPackHelper::getNumericValue( + slice, NRFILES, 0); + meta._nrDBServers = basics::VelocyPackHelper::getNumericValue( + slice, NRDBSERVERS, 1); + meta._serverId = basics::VelocyPackHelper::getStringValue(slice, SERVERID, ""); + meta._potentiallyInconsistent = basics::VelocyPackHelper::getBooleanValue(slice, POTENTIALLYINCONSISTENT, false); return meta; } catch (std::exception const& e) { return ResultT::error(TRI_ERROR_BAD_PARAMETER, e.what()); } } - BackupMeta(std::string const& id, std::string const& version, std::string const& datetime) : - _id(id), _version(version), _datetime(datetime) {} + BackupMeta(std::string const& id, std::string const& version, std::string const& datetime, size_t sizeInBytes, size_t nrFiles, unsigned int nrDBServers, std::string const& serverId, bool potentiallyInconsistent) : + _id(id), _version(version), _datetime(datetime), + _sizeInBytes(sizeInBytes), _nrFiles(nrFiles), _nrDBServers(nrDBServers), + _serverId(serverId), _potentiallyInconsistent(potentiallyInconsistent) {} private: BackupMeta() {} diff --git a/arangod/Utils/ExecContext.h b/arangod/Utils/ExecContext.h index f00bbe5b97..b865286658 100644 --- a/arangod/Utils/ExecContext.h +++ b/arangod/Utils/ExecContext.h @@ -158,6 +158,25 @@ struct ExecContextScope { private: ExecContext const* _old; }; + +struct ExecContextSuperuserScope { + explicit ExecContextSuperuserScope() + : _old(ExecContext::CURRENT) { + ExecContext::CURRENT = ExecContext::superuser(); + } + + explicit ExecContextSuperuserScope(bool cond) : _old(ExecContext::CURRENT) { + if (cond) { + ExecContext::CURRENT = ExecContext::superuser(); + } + } + + ~ExecContextSuperuserScope() { ExecContext::CURRENT = _old; } + +private: + ExecContext const* _old; +}; + } // namespace arangodb #endif diff --git a/arangosh/Backup/BackupFeature.cpp b/arangosh/Backup/BackupFeature.cpp index 77481f8b79..9c089919be 100644 --- a/arangosh/Backup/BackupFeature.cpp +++ b/arangosh/Backup/BackupFeature.cpp @@ -242,6 +242,17 @@ arangodb::Result executeList(arangodb::httpclient::SimpleHttpClient& client, if (meta.ok()) { LOG_TOPIC("0f208", INFO, arangodb::Logger::BACKUP) << " version: " << meta.get()._version; LOG_TOPIC("55af7", INFO, arangodb::Logger::BACKUP) << " date/time: " << meta.get()._datetime; + LOG_TOPIC("43522", INFO, arangodb::Logger::BACKUP) << " size in bytes: " << meta.get()._sizeInBytes; + LOG_TOPIC("12532", INFO, arangodb::Logger::BACKUP) << " number of files: " << meta.get()._nrFiles; + LOG_TOPIC("43212", INFO, arangodb::Logger::BACKUP) << " number of DBServers: " << meta.get()._nrDBServers; + if (!meta.get()._serverId.empty()) { + LOG_TOPIC("11112", INFO, arangodb::Logger::BACKUP) << " serverId: " << meta.get()._serverId; + } + if (meta.get()._potentiallyInconsistent) { + LOG_TOPIC("56241", INFO, arangodb::Logger::BACKUP) << " potentiallyInconsistent: true"; + } else { + LOG_TOPIC("56242", INFO, arangodb::Logger::BACKUP) << " potentiallyInconsistent: false"; + } } } } @@ -315,7 +326,14 @@ arangodb::Result executeCreate(arangodb::httpclient::SimpleHttpClient& client, LOG_TOPIC("c4d37", INFO, arangodb::Logger::BACKUP) << "Backup succeeded. Generated identifier '" << identifier.copyString() << "'"; - + VPackSlice sizeInBytes = resultObject.get("sizeInBytes"); + VPackSlice nrFiles = resultObject.get("nrFiles"); + if (sizeInBytes.isInteger() && nrFiles.isInteger()) { + uint64_t size = sizeInBytes.getNumber(); + uint64_t nr = nrFiles.getNumber(); + LOG_TOPIC("ce423", INFO, arangodb::Logger::BACKUP) + << "Total size of backup: " << size << ", number of files: " << nr; + } return result; }