diff --git a/CHANGELOG b/CHANGELOG index 3dfce8f2a4..4657141379 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -106,6 +106,25 @@ v1.5.x (XXXX-XX-XX) v1.4.x (XXXX-XX-XX) ------------------- +* added `--force` option for arangorestore + this option allows continuing a restore operation even if the server reports errors + in the middle of the restore operation + +* better error reporting for arangorestore + in case the server returned an HTTP error, arangorestore previously reported this + error as `internal error` without any details only. Now server-side errors are + reported by arangorestore with the server's error message + +* include more system collections in dumps produced by arangodump + previously some system collections were intentionally excluded from dumps, even if the + dump was run with `--include-system-collections`. for example, the collections `_aal`, + `_modules`, `_routing`, and `_users` were excluded. This makes sense in a replication + context but not always in a dump context. + When specifying `--include-system-collections`, arangodump will now include the above- + mentioned collections in the dump, too. Some other system collections are still excluded + even when the dump is run with `--include-system-collections`, for example `_replication` + and `_trx`. + * fixed issue #701: ArangoStatement undefined in arangosh * fixed typos in configuration files diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 335f3c1d34..a989eb5321 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -36,6 +36,9 @@ #include "HttpServer/HttpServer.h" #include "Replication/InitialSyncer.h" #include "Rest/HttpRequest.h" +#include "Utils/EmbeddableTransaction.h" +#include "Utils/RestTransactionContext.h" +#include "Utils/SingleCollectionWriteTransaction.h" #include "VocBase/compactor.h" #include "VocBase/replication-applier.h" #include "VocBase/replication-dump.h" @@ -292,18 +295,29 @@ bool RestReplicationHandler::filterCollection (TRI_vocbase_col_t* collection, // invalid type return false; } - - if (TRI_ExcludeCollectionReplication(collection->_name)) { - // collection is excluded - return false; - } - + bool includeSystem = *((bool*) data); if (! includeSystem && collection->_name[0] == '_') { // exclude all system collections return false; } + if (TRI_ExcludeCollectionReplication(collection->_name)) { + // collection is excluded from replication + + if (includeSystem) { + // now check if we still should include it in the dump + if (! TRI_EqualString(collection->_name, TRI_COL_NAME_USERS) && + ! TRI_EqualString(collection->_name, "_aal") && + ! TRI_EqualString(collection->_name, "_modules") && + ! TRI_EqualString(collection->_name, "_routing")) { + return false; + } + } + // _aal, _modules, _routing should be included + } + + // all other cases should be included return true; } @@ -1601,10 +1615,16 @@ void RestReplicationHandler::handleCommandRestoreCollection () { if (found) { recycleIds = StringUtils::boolean(value); } + + bool force = false; + value = _request->value("force", found); + if (found) { + force = StringUtils::boolean(value); + } TRI_server_id_t remoteServerId = 0; // TODO string errorMsg; - int res = processRestoreCollection(json, overwrite, recycleIds, remoteServerId, errorMsg); + int res = processRestoreCollection(json, overwrite, recycleIds, force, remoteServerId, errorMsg); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); @@ -1634,10 +1654,17 @@ void RestReplicationHandler::handleCommandRestoreIndexes () { "invalid JSON"); return; } + + bool found; + bool force = false; + char const* value = _request->value("force", found); + if (found) { + force = StringUtils::boolean(value); + } TRI_server_id_t remoteServerId = 0; // TODO string errorMsg; - int res = processRestoreIndexes(json, remoteServerId, errorMsg); + int res = processRestoreIndexes(json, force, remoteServerId, errorMsg); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); @@ -1661,6 +1688,7 @@ void RestReplicationHandler::handleCommandRestoreIndexes () { int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collection, bool dropExisting, bool reuseId, + bool force, TRI_server_id_t remoteServerId, string& errorMsg) { if (! JsonHelper::isArray(collection)) { @@ -1725,9 +1753,34 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti if (col != 0) { if (dropExisting) { int res = TRI_DropCollectionVocBase(_vocbase, col, remoteServerId); + + if (res == TRI_ERROR_FORBIDDEN) { + // some collections must not be dropped + + // instead, truncate them + CollectionNameResolver resolver(_vocbase); + SingleCollectionWriteTransaction, UINT64_MAX> trx(_vocbase, resolver, col->_cid); + + res = trx.begin(); + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + TRI_barrier_t* barrier = TRI_CreateBarrierElement(&(trx.primaryCollection()->_barrierList)); + + if (barrier == 0) { + return TRI_ERROR_INTERNAL; + } + + res = trx.truncate(false); + res = trx.finish(res); + + TRI_FreeBarrier(barrier); + + return res; + } if (res != TRI_ERROR_NO_ERROR) { - errorMsg = "unable to drop collection: " + string(TRI_errno_string(res)); + errorMsg = "unable to drop collection '" + name + "': " + string(TRI_errno_string(res)); return res; } @@ -1735,7 +1788,7 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti else { int res = TRI_ERROR_ARANGO_DUPLICATE_NAME; - errorMsg = "unable to drop collection: " + string(TRI_errno_string(res)); + errorMsg = "unable to create collection '" + name + "': " + string(TRI_errno_string(res)); return res; } @@ -1758,6 +1811,7 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti //////////////////////////////////////////////////////////////////////////////// int RestReplicationHandler::processRestoreIndexes (TRI_json_t* const collection, + bool force, TRI_server_id_t remoteServerId, string& errorMsg) { if (! JsonHelper::isArray(collection)) { @@ -1978,6 +2032,7 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons TRI_transaction_collection_t* trxCollection, TRI_server_id_t generatingServer, bool useRevision, + bool force, std::string& errorMsg) { const string invalidMsg = "received invalid JSON data for collection " + StringUtils::itoa(trxCollection->_cid); @@ -2066,7 +2121,7 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons TRI_FreeJson(TRI_CORE_MEM_ZONE, json); - if (res != TRI_ERROR_NO_ERROR) { + if (res != TRI_ERROR_NO_ERROR && ! force) { return res; } } @@ -2085,6 +2140,7 @@ int RestReplicationHandler::processRestoreData (CollectionNameResolver const& re TRI_voc_cid_t cid, TRI_server_id_t generatingServer, bool useRevision, + bool force, string& errorMsg) { TRI_transaction_t* trx = TRI_CreateTransaction(_vocbase, @@ -2128,7 +2184,7 @@ int RestReplicationHandler::processRestoreData (CollectionNameResolver const& re // TODO: waitForSync disabled here. use for initial replication, too // sync at end of trx trxCollection->_waitForSync = false; - res = processRestoreDataBatch(resolver, trxCollection, generatingServer, useRevision, errorMsg); + res = processRestoreDataBatch(resolver, trxCollection, generatingServer, useRevision, force, errorMsg); } if (res == TRI_ERROR_NO_ERROR) { @@ -2170,11 +2226,17 @@ void RestReplicationHandler::handleCommandRestoreData () { if (value != 0) { recycleIds = StringUtils::boolean(value); } + + bool force = false; + value = _request->value("force"); + if (value != 0) { + force = StringUtils::boolean(value); + } TRI_server_id_t remoteServerId = 0; // TODO string errorMsg; - int res = processRestoreData(resolver, cid, remoteServerId, recycleIds, errorMsg); + int res = processRestoreData(resolver, cid, remoteServerId, recycleIds, force, errorMsg); if (res != TRI_ERROR_NO_ERROR) { generateError(HttpResponse::SERVER_ERROR, res); diff --git a/arangod/RestHandler/RestReplicationHandler.h b/arangod/RestHandler/RestReplicationHandler.h index 8f0da2aae2..3e7816a219 100644 --- a/arangod/RestHandler/RestReplicationHandler.h +++ b/arangod/RestHandler/RestReplicationHandler.h @@ -247,6 +247,7 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// int processRestoreCollection (struct TRI_json_s* const, + bool, bool, bool, TRI_server_id_t, @@ -257,6 +258,7 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// int processRestoreIndexes (struct TRI_json_s* const, + bool, TRI_server_id_t, std::string&); @@ -280,6 +282,7 @@ namespace triagens { struct TRI_transaction_collection_s*, TRI_server_id_t, bool, + bool, std::string&); //////////////////////////////////////////////////////////////////////////////// @@ -290,6 +293,7 @@ namespace triagens { TRI_voc_cid_t, TRI_server_id_t, bool, + bool, std::string&); //////////////////////////////////////////////////////////////////////////////// diff --git a/arangosh/V8Client/arangorestore.cpp b/arangosh/V8Client/arangorestore.cpp index b114c35cde..672b69fcdb 100644 --- a/arangosh/V8Client/arangorestore.cpp +++ b/arangosh/V8Client/arangorestore.cpp @@ -136,6 +136,12 @@ static bool Overwrite = true; static bool RecycleIds = false; +//////////////////////////////////////////////////////////////////////////////// +/// @brief continue restore even in the face of errors +//////////////////////////////////////////////////////////////////////////////// + +static bool Force = false; + //////////////////////////////////////////////////////////////////////////////// /// @brief statistics //////////////////////////////////////////////////////////////////////////////// @@ -172,6 +178,7 @@ static void ParseProgramOptions (int argc, char* argv[]) { ("batch-size", &ChunkSize, "maximum size for individual data batches (in bytes)") ("import-data", &ImportData, "import data into collection") ("recycle-ids", &RecycleIds, "recycle collection and revision ids from dump") + ("force", &Force, "continue restore even in the face of some server-side errors") ("create-collection", &ImportStructure, "create collection structure") ("include-system-collections", &IncludeSystemCollections, "include system collections") ("input-directory", &InputDirectory, "input directory") @@ -366,7 +373,8 @@ static int SendRestoreCollection (TRI_json_t const* json, const string url = "/_api/replication/restore-collection" "?overwrite=" + string(Overwrite ? "true" : "false") + - "&recycleIds=" + string(RecycleIds ? "true" : "false"); + "&recycleIds=" + string(RecycleIds ? "true" : "false") + + "&force=" + string(Force ? "true" : "false"); const string body = JsonHelper::toString(json); @@ -406,7 +414,7 @@ static int SendRestoreIndexes (TRI_json_t const* json, string& errorMsg) { map headers; - const string url = "/_api/replication/restore-indexes"; + const string url = "/_api/replication/restore-indexes?force=" + string(Force ? "true" : "false"); const string body = JsonHelper::toString(json); SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT, @@ -450,7 +458,8 @@ static int SendRestoreData (string const& cid, const string url = "/_api/replication/restore-data?collection=" + StringUtils::urlEncode(cname) + - "&recycleIds=" + (RecycleIds ? "true" : "false"); + "&recycleIds=" + (RecycleIds ? "true" : "false") + + "&force=" + (Force ? "true" : "false"); SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT, url, @@ -629,12 +638,22 @@ static int ProcessInputDirectory (string& errorMsg) { if (ImportStructure) { // re-create collection if (Progress) { - cout << "Creating collection '" << cname << "'..." << endl; + if (Overwrite) { + cout << "Re-creating collection '" << cname << "'..." << endl; + } + else { + cout << "Creating collection '" << cname << "'..." << endl; + } } int res = SendRestoreCollection(json, errorMsg); if (res != TRI_ERROR_NO_ERROR) { + if (Force) { + cerr << errorMsg << endl; + continue; + } + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections); return TRI_ERROR_INTERNAL; @@ -727,9 +746,19 @@ static int ProcessInputDirectory (string& errorMsg) { if (res != TRI_ERROR_NO_ERROR) { TRI_CLOSE(fd); - errorMsg = string(TRI_errno_string(res)); - TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections); + if (errorMsg.empty()) { + errorMsg = string(TRI_errno_string(res)); + } + else { + errorMsg = string(TRI_errno_string(res)) + ": " + errorMsg; + } + if (Force) { + cerr << errorMsg << endl; + continue; + } + + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections); return res; } @@ -759,6 +788,11 @@ static int ProcessInputDirectory (string& errorMsg) { int res = SendRestoreIndexes(json, errorMsg); if (res != TRI_ERROR_NO_ERROR) { + if (Force) { + cerr << errorMsg << endl; + continue; + } + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections); return TRI_ERROR_INTERNAL;