Merge branch 'devel' of https://github.com/triAGENS/ArangoDB into devel

CHANGELOG

@@ -106,6 +106,25 @@ v1.5.x (XXXX-XX-XX)
 v1.4.x (XXXX-XX-XX)
 -------------------
 
+* added `--force` option for arangorestore
+  this option allows continuing a restore operation even if the server reports errors
+  in the middle of the restore operation
+
+* better error reporting for arangorestore
+  in case the server returned an HTTP error, arangorestore previously reported this
+  error as `internal error` without any details only. Now server-side errors are
+  reported by arangorestore with the server's error message
+
+* include more system collections in dumps produced by arangodump
+  previously some system collections were intentionally excluded from dumps, even if the
+  dump was run with `--include-system-collections`. for example, the collections `_aal`,
+  `_modules`, `_routing`, and `_users` were excluded. This makes sense in a replication
+  context but not always in a dump context.
+  When specifying `--include-system-collections`, arangodump will now include the above-
+  mentioned collections in the dump, too. Some other system collections are still excluded
+  even when the dump is run with `--include-system-collections`, for example `_replication`
+  and `_trx`.
+
 * fixed issue #701: ArangoStatement undefined in arangosh
 
 * fixed typos in configuration files

arangod/RestHandler/RestReplicationHandler.cpp

@@ -36,6 +36,9 @@
 #include "HttpServer/HttpServer.h"
 #include "Replication/InitialSyncer.h"
 #include "Rest/HttpRequest.h"
+#include "Utils/EmbeddableTransaction.h"
+#include "Utils/RestTransactionContext.h"
+#include "Utils/SingleCollectionWriteTransaction.h"
 #include "VocBase/compactor.h"
 #include "VocBase/replication-applier.h"
 #include "VocBase/replication-dump.h"
@@ -293,17 +296,28 @@ bool RestReplicationHandler::filterCollection (TRI_vocbase_col_t* collection,
     return false;
   }
   
-  if (TRI_ExcludeCollectionReplication(collection->_name)) {
-    // collection is excluded
-    return false;
-  }
-
   bool includeSystem = *((bool*) data);
   if (! includeSystem && collection->_name[0] == '_') {
     // exclude all system collections
     return false;
   }
 
+  if (TRI_ExcludeCollectionReplication(collection->_name)) {
+    // collection is excluded from replication
+
+    if (includeSystem) {
+      // now check if we still should include it in the dump
+      if (! TRI_EqualString(collection->_name, TRI_COL_NAME_USERS) &&
+          ! TRI_EqualString(collection->_name, "_aal") &&
+          ! TRI_EqualString(collection->_name, "_modules") &&
+          ! TRI_EqualString(collection->_name, "_routing")) {
+        return false;
+      }
+    }
+    // _aal, _modules, _routing should be included
+  }
+
+
   // all other cases should be included
   return true;
 }
@@ -1602,9 +1616,15 @@ void RestReplicationHandler::handleCommandRestoreCollection () {
     recycleIds = StringUtils::boolean(value);
   }
   
+  bool force = false;
+  value = _request->value("force", found);
+  if (found) {
+    force = StringUtils::boolean(value);
+  }
+
   TRI_server_id_t remoteServerId = 0; // TODO
   string errorMsg;
-  int res = processRestoreCollection(json, overwrite, recycleIds, remoteServerId, errorMsg);
+  int res = processRestoreCollection(json, overwrite, recycleIds, force, remoteServerId, errorMsg);
 
   TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
   
@@ -1635,9 +1655,16 @@ void RestReplicationHandler::handleCommandRestoreIndexes () {
     return;
   }
   
+  bool found;
+  bool force = false;
+  char const* value = _request->value("force", found);
+  if (found) {
+    force = StringUtils::boolean(value);
+  }
+
   TRI_server_id_t remoteServerId = 0; // TODO
   string errorMsg;
-  int res = processRestoreIndexes(json, remoteServerId, errorMsg);
+  int res = processRestoreIndexes(json, force, remoteServerId, errorMsg);
 
   TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
   
@@ -1661,6 +1688,7 @@ void RestReplicationHandler::handleCommandRestoreIndexes () {
 int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collection,
                                                       bool dropExisting,
                                                       bool reuseId,
+                                                      bool force,
                                                       TRI_server_id_t remoteServerId,
                                                       string& errorMsg) {
   if (! JsonHelper::isArray(collection)) {
@@ -1726,8 +1754,33 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti
     if (dropExisting) {
       int res = TRI_DropCollectionVocBase(_vocbase, col, remoteServerId);
 
+      if (res == TRI_ERROR_FORBIDDEN) {
+        // some collections must not be dropped
+
+        // instead, truncate them
+        CollectionNameResolver resolver(_vocbase);
+        SingleCollectionWriteTransaction<EmbeddableTransaction<RestTransactionContext>, UINT64_MAX> trx(_vocbase, resolver, col->_cid);
+
+        res = trx.begin();
         if (res != TRI_ERROR_NO_ERROR) {
-        errorMsg = "unable to drop collection: " + string(TRI_errno_string(res));
+          return res;
+        }
+        TRI_barrier_t* barrier = TRI_CreateBarrierElement(&(trx.primaryCollection()->_barrierList));
+
+        if (barrier == 0) {
+          return TRI_ERROR_INTERNAL;
+        }
+
+        res = trx.truncate(false);
+        res = trx.finish(res);
+
+        TRI_FreeBarrier(barrier);
+
+        return res;
+      }
+ 
+      if (res != TRI_ERROR_NO_ERROR) {
+        errorMsg = "unable to drop collection '" + name + "': " + string(TRI_errno_string(res));
 
         return res;
       }
@@ -1735,7 +1788,7 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti
     else {
       int res = TRI_ERROR_ARANGO_DUPLICATE_NAME;
 
-      errorMsg = "unable to drop collection: " + string(TRI_errno_string(res));
+      errorMsg = "unable to create collection '" + name + "': " + string(TRI_errno_string(res));
 
       return res;
     }
@@ -1758,6 +1811,7 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti
 ////////////////////////////////////////////////////////////////////////////////
 
 int RestReplicationHandler::processRestoreIndexes (TRI_json_t* const collection,
+                                                   bool force,
                                                    TRI_server_id_t remoteServerId,
                                                    string& errorMsg) {
   if (! JsonHelper::isArray(collection)) {
@@ -1978,6 +2032,7 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons
                                                      TRI_transaction_collection_t* trxCollection,
                                                      TRI_server_id_t generatingServer,
                                                      bool useRevision,
+                                                     bool force,
                                                      std::string& errorMsg) {
   const string invalidMsg = "received invalid JSON data for collection " + 
                             StringUtils::itoa(trxCollection->_cid);
@@ -2066,7 +2121,7 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons
       
       TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
 
-      if (res != TRI_ERROR_NO_ERROR) {
+      if (res != TRI_ERROR_NO_ERROR && ! force) {
         return res;
       }
     }
@@ -2085,6 +2140,7 @@ int RestReplicationHandler::processRestoreData (CollectionNameResolver const& re
                                                 TRI_voc_cid_t cid,
                                                 TRI_server_id_t generatingServer,
                                                 bool useRevision,
+                                                bool force,
                                                 string& errorMsg) {
 
   TRI_transaction_t* trx = TRI_CreateTransaction(_vocbase, 
@@ -2128,7 +2184,7 @@ int RestReplicationHandler::processRestoreData (CollectionNameResolver const& re
     // TODO: waitForSync disabled here. use for initial replication, too
     // sync at end of trx
     trxCollection->_waitForSync = false;
-    res = processRestoreDataBatch(resolver, trxCollection, generatingServer, useRevision, errorMsg);
+    res = processRestoreDataBatch(resolver, trxCollection, generatingServer, useRevision, force, errorMsg);
   }
       
   if (res == TRI_ERROR_NO_ERROR) {
@@ -2171,10 +2227,16 @@ void RestReplicationHandler::handleCommandRestoreData () {
     recycleIds = StringUtils::boolean(value);
   }
   
+  bool force = false;
+  value = _request->value("force");
+  if (value != 0) {
+    force = StringUtils::boolean(value);
+  }
+    
   TRI_server_id_t remoteServerId = 0; // TODO
   string errorMsg;
 
-  int res = processRestoreData(resolver, cid, remoteServerId, recycleIds, errorMsg);
+  int res = processRestoreData(resolver, cid, remoteServerId, recycleIds, force, errorMsg);
 
   if (res != TRI_ERROR_NO_ERROR) {
     generateError(HttpResponse::SERVER_ERROR, res);

arangod/RestHandler/RestReplicationHandler.h

@@ -247,6 +247,7 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         int processRestoreCollection (struct TRI_json_s* const,
+                                      bool,
                                       bool,
                                       bool,
                                       TRI_server_id_t,
@@ -257,6 +258,7 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
         int processRestoreIndexes (struct TRI_json_s* const,
+                                   bool,
                                    TRI_server_id_t,
                                    std::string&);
 
@@ -280,6 +282,7 @@ namespace triagens {
                                      struct TRI_transaction_collection_s*,
                                      TRI_server_id_t,
                                      bool,
+                                     bool,
                                      std::string&);
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -290,6 +293,7 @@ namespace triagens {
                                 TRI_voc_cid_t,
                                 TRI_server_id_t,
                                 bool,
+                                bool,
                                 std::string&);
 
 ////////////////////////////////////////////////////////////////////////////////

arangosh/V8Client/arangorestore.cpp

@@ -136,6 +136,12 @@ static bool Overwrite = true;
 
 static bool RecycleIds = false;
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief continue restore even in the face of errors
+////////////////////////////////////////////////////////////////////////////////
+
+static bool Force = false;
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief statistics
 ////////////////////////////////////////////////////////////////////////////////
@@ -172,6 +178,7 @@ static void ParseProgramOptions (int argc, char* argv[]) {
     ("batch-size", &ChunkSize, "maximum size for individual data batches (in bytes)")
     ("import-data", &ImportData, "import data into collection")
     ("recycle-ids", &RecycleIds, "recycle collection and revision ids from dump")
+    ("force", &Force, "continue restore even in the face of some server-side errors")
     ("create-collection", &ImportStructure, "create collection structure")
     ("include-system-collections", &IncludeSystemCollections, "include system collections")
     ("input-directory", &InputDirectory, "input directory")
@@ -366,7 +373,8 @@ static int SendRestoreCollection (TRI_json_t const* json,
 
   const string url = "/_api/replication/restore-collection"
                      "?overwrite=" + string(Overwrite ? "true" : "false") +
-                     "&recycleIds=" + string(RecycleIds ? "true" : "false");
+                     "&recycleIds=" + string(RecycleIds ? "true" : "false") + 
+                     "&force=" + string(Force ? "true" : "false");
 
   const string body = JsonHelper::toString(json);
 
@@ -406,7 +414,7 @@ static int SendRestoreIndexes (TRI_json_t const* json,
                                string& errorMsg) {
   map<string, string> headers;
 
-  const string url = "/_api/replication/restore-indexes"; 
+  const string url = "/_api/replication/restore-indexes?force=" + string(Force ? "true" : "false");
   const string body = JsonHelper::toString(json);
 
   SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT, 
@@ -450,7 +458,8 @@ static int SendRestoreData (string const& cid,
 
   const string url = "/_api/replication/restore-data?collection=" + 
                      StringUtils::urlEncode(cname) +
-                     "&recycleIds=" + (RecycleIds ? "true" : "false");
+                     "&recycleIds=" + (RecycleIds ? "true" : "false") + 
+                     "&force=" + (Force ? "true" : "false");
 
   SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT, 
                                                url,
@@ -629,12 +638,22 @@ static int ProcessInputDirectory (string& errorMsg) {
       if (ImportStructure) {
         // re-create collection
         if (Progress) {
+          if (Overwrite) {
+            cout << "Re-creating collection '" << cname << "'..." << endl;
+          }
+          else {
             cout << "Creating collection '" << cname << "'..." << endl;
           }
+        }
 
         int res = SendRestoreCollection(json, errorMsg);
 
         if (res != TRI_ERROR_NO_ERROR) {
+          if (Force) {
+            cerr << errorMsg << endl;
+            continue;
+          }
+
           TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
 
           return TRI_ERROR_INTERNAL;
@@ -727,9 +746,19 @@ static int ProcessInputDirectory (string& errorMsg) {
 
               if (res != TRI_ERROR_NO_ERROR) {
                 TRI_CLOSE(fd);
+                if (errorMsg.empty()) {
                   errorMsg = string(TRI_errno_string(res));
-                TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
+                }
+                else {
+                  errorMsg = string(TRI_errno_string(res)) + ": " + errorMsg;
+                }
 
+                if (Force) {
+                  cerr << errorMsg << endl;
+                  continue;
+                }
+
+                TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
                 return res;
               }
 
@@ -759,6 +788,11 @@ static int ProcessInputDirectory (string& errorMsg) {
           int res = SendRestoreIndexes(json, errorMsg);
 
           if (res != TRI_ERROR_NO_ERROR) {
+            if (Force) {
+              cerr << errorMsg << endl;
+              continue;
+            }
+
             TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
 
             return TRI_ERROR_INTERNAL;