Merge branch 'JHMH' of https://github.com/arangodb/arangodb into JHMH

Documentation/Books/Makefile

@@ -40,10 +40,10 @@ book-check-mdpp-leftovers:
 	fi
 
 ppbook-check-html-link:
-	@if test "`grep -r '\[.*\]\(.*\)' ppbooks/$(NAME) |grep '\.md:' |grep html |grep -v http://|grep -v https:// |grep -v header.css |wc -l`" -gt 0; then \
+	@if test "`egrep -r '\[.*\]\(.*\)' ppbooks/$(NAME) |grep '\.md:' |grep html |grep -v http://|grep -v https:// |grep -v header.css |wc -l`" -gt 0; then \
 		echo "Found links to .html files inside of the document! use <foo>.md instead!"; \
 		echo; \
-		grep -r '\[.*\]\(.*\)' ppbooks/$(NAME)  | grep '\.md:' | grep html |grep -v http://|grep -v https:// |grep -v header.css ; \
+		egrep -r '\[.*\]\(.*\)' ppbooks/$(NAME)  | grep '\.md:' | grep html |grep -v http://|grep -v https:// |grep -v header.css ; \
 		exit 1; \
 	fi
 

Documentation/Books/Users/Replication/ExampleSetup.mdpp

@@ -24,8 +24,11 @@ require("org/arangodb/replication").applier.stop();
 
 The *stop* operation will terminate any replication activity in the _system database on the slave.
 
-After that, do an initial sync of the slave with data from the master. Execute the following
+
-commands on the slave:
+!SECTION Initial synchronization
+
+After that, we perform an initial sync of the slave with data from the master. To do this,
+execute the following commands on the slave:
 
 ```js
 db._useDatabase("_system");
@@ -56,6 +59,8 @@ assume we got the following last log tick:
 }
 ```
 
+!SECTION Continuous synchronization
+
 Now, we could start the replication applier in the slave database using the last log tick.
 However, there is one thing to consider: replication on the slave will be running until the 
 slave gets shut down. When the slave server gets restarted, replication will be turned off again. 
@@ -141,3 +146,29 @@ a write lock on the collections involved in the transaction.
 You may also want to check the master and slave states via the HTTP APIs 
 (see [HTTP Interface for Replication](../HttpReplications/README.md)).
 
+
+!SECTION Initial synchronization from the ArangoShell
+
+The *sync* may take a long time to complete. If it's called from the ArangoShell, the connection
+may time out, which will effectively discard the result of the *sync* operation. Therefore in the
+ArangoShell, the optional *async* attribute can be used to start the synchronization as a background 
+process on the slave. If the *async* attribute is set to *true*, the call to *sync* will return 
+almost instantly with an id string. Using this id string, the status of the sync job on the slave 
+can be queried using the *getSyncResult* function as follows:
+
+```js
+db._useDatabase("_system");
+var replication = require("org/arangodb/replication");
+var id = replication.sync({
+  endpoint: "tcp://master.domain.org:8529",
+  username: "myuser",
+  password: "mypasswd",
+  async: true
+});
+
+print(replication.getSyncResult(id));
+```
+
+*getSyncResult* will return *false* as long as the synchronization is not complete, and return the
+synchronization result otherwise.
+

Documentation/Books/Users/Replication/SyncingCollections.mdpp

@@ -38,8 +38,33 @@ the correct server, in the correct database!
 Setting the optional *incremental* attribute in the call to *syncCollection* will start an
 incremental transfer of data. This may be useful in case when the slave already has
 parts or almost all of the data in the collection and only the differences need to be
-synchronized.
+synchronized. Note that to compute the differences the incremental transfer will build a sorted
+list of all document keys in the collection on both the slave and the master, which may still be
+expensive for huge collections in terms of memory usage and runtime. During building the list
+of keys the collection will be read-locked on the master.
 
 The *initialSyncMaxWaitTime* attribute in the call to *syncCollection* controls how long the
 slave will wait for a master's response. This wait time can be used to control after what time 
 the synchronization will give up and fail. 
+
+When *syncCollection* is called from the ArangoShell, the optional *async* attribute can be used
+to start the synchronization as a background process on the slave. If *async* is set to *true*,
+the call to *syncCollection* will return almost instantly with an id string. Using this id string,
+the status of the sync job on the slave can be queried using the *getSyncResult* function as follows:
+
+
+```js
+db._useDatabase("_system");
+var replication = require("org/arangodb/replication");
+var id = replication.syncCollection("test", {
+  endpoint: "tcp://master.domain.org:8529",
+  username: "myuser",
+  password: "mypasswd",
+  async: true
+});
+
+print(replication.getSyncResult(id));
+```
+
+*getSyncResult* will return *false* as long as the synchronization is not complete, and return the
+synchronization result otherwise.

arangod/VocBase/document-collection.cpp

@@ -1336,7 +1336,6 @@ typedef struct open_iterator_state_s {
   uint64_t                   _documents;
   int64_t                    _initialCount;
   uint32_t                   _trxCollections;
-  uint32_t                   _numOps;
   bool                       _trxPrepared;
 }
 open_iterator_state_t;

js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js

@@ -171,9 +171,19 @@ var sync = function (config) {
   var db = internal.db;
 
   var body = JSON.stringify(config || { });
-  var requestResult = db._connection.PUT("/_api/replication/sync", body);
+  var requestResult;
+  if (config.async) {
+    var headers = { "X-Arango-Async" : "store" };
+    requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers);
+  }
+  else {
+    requestResult = db._connection.PUT("/_api/replication/sync", body);
+  }
 
   arangosh.checkRequestResult(requestResult);
+  if (config.async) {
+    return requestResult.headers["x-arango-async-id"];
+  }
 
   return requestResult;
 };
@@ -191,14 +201,36 @@ var syncCollection = function (collection, config) {
   config.restrictCollections = [ collection ];
   config.includeSystem = true;
   var body = JSON.stringify(config);
-
+  var requestResult;
-  var requestResult = db._connection.PUT("/_api/replication/sync", body);
+  if (config.async) {
+    var headers = { "X-Arango-Async" : "store" };
+    requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers);
+  }
+  else {
+    requestResult = db._connection.PUT("/_api/replication/sync", body);
+  }
 
   arangosh.checkRequestResult(requestResult);
+  if (config.async) {
+    return requestResult.headers["x-arango-async-id"];
+  }
 
   return requestResult;
 };
 
+var getSyncResult = function (id) {
+  var db = internal.db;
+
+  var requestResult = db._connection.PUT_RAW("/_api/job/" + encodeURIComponent(id), "");
+  arangosh.checkRequestResult(requestResult);
+
+  if (requestResult.headers.hasOwnProperty("x-arango-async-id")) {
+    return JSON.parse(requestResult.body);
+  }
+
+  return false;
+};
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief fetches a server's id
 ////////////////////////////////////////////////////////////////////////////////
@@ -221,6 +253,7 @@ exports.logger         = logger;
 exports.applier         = applier;
 exports.sync            = sync;
 exports.syncCollection  = syncCollection;
+exports.getSyncResult   = getSyncResult;
 exports.serverId        = serverId;
 
 // -----------------------------------------------------------------------------

js/client/modules/org/arangodb/replication.js

@@ -170,9 +170,19 @@ var sync = function (config) {
   var db = internal.db;
 
   var body = JSON.stringify(config || { });
-  var requestResult = db._connection.PUT("/_api/replication/sync", body);
+  var requestResult;
+  if (config.async) {
+    var headers = { "X-Arango-Async" : "store" };
+    requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers);
+  }
+  else {
+    requestResult = db._connection.PUT("/_api/replication/sync", body);
+  }
 
   arangosh.checkRequestResult(requestResult);
+  if (config.async) {
+    return requestResult.headers["x-arango-async-id"];
+  }
 
   return requestResult;
 };
@@ -190,14 +200,36 @@ var syncCollection = function (collection, config) {
   config.restrictCollections = [ collection ];
   config.includeSystem = true;
   var body = JSON.stringify(config);
-
+  var requestResult;
-  var requestResult = db._connection.PUT("/_api/replication/sync", body);
+  if (config.async) {
+    var headers = { "X-Arango-Async" : "store" };
+    requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers);
+  }
+  else {
+    requestResult = db._connection.PUT("/_api/replication/sync", body);
+  }
 
   arangosh.checkRequestResult(requestResult);
+  if (config.async) {
+    return requestResult.headers["x-arango-async-id"];
+  }
 
   return requestResult;
 };
 
+var getSyncResult = function (id) {
+  var db = internal.db;
+
+  var requestResult = db._connection.PUT_RAW("/_api/job/" + encodeURIComponent(id), "");
+  arangosh.checkRequestResult(requestResult);
+
+  if (requestResult.headers.hasOwnProperty("x-arango-async-id")) {
+    return JSON.parse(requestResult.body);
+  }
+
+  return false;
+};
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief fetches a server's id
 ////////////////////////////////////////////////////////////////////////////////
@@ -220,6 +252,7 @@ exports.logger         = logger;
 exports.applier         = applier;
 exports.sync            = sync;
 exports.syncCollection  = syncCollection;
+exports.getSyncResult   = getSyncResult;
 exports.serverId        = serverId;
 
 // -----------------------------------------------------------------------------

js/common/modules/org/arangodb-common.js

@@ -519,8 +519,9 @@ exports.checkAvailableVersions = function (version) {
   }
 
   try {
-    var u = "https://www.arangodb.com/repositories/versions.php?version=";
+    var u = "https://www.arangodb.com/repositories/versions.php?version=" + version +
-    var d = internal.download(u + version, "", {timeout: 300});
+            "&os=" + internal.platform;
+    var d = internal.download(u, "", {timeout: 300});
     var v = JSON.parse(d.body);
 
     if (v.hasOwnProperty("bugfix")) {

js/server/modules/org/arangodb/testing.js

@@ -74,8 +74,8 @@ var optionsDocumentation = [
   '     of a small local cluster',
   '   - `clusterNodes`: number of DB-Servers to use',
   '   - valgrindHosts  - configure which clustercomponents to run using valgrintd',
-  '        Coordinator - run Coordinator with valgrind',
+  '        Coordinator - flag to run Coordinator with valgrind',
-  '        DBServer    - run DBServers with valgrind',
+  '        DBServer    - flag to run DBServers with valgrind',
   '   - `test`: path to single test to execute for "single" test target',
   '   - `cleanup`: if set to true (the default), the cluster data files',
   '     and logs are removed after termination of the test.',
@@ -327,7 +327,7 @@ function startInstance (protocol, options, addArgs, testname, tmpDir) {
     }
 
     if (options.valgrindHosts.DBServer === true) {
-      valgrindHosts += 'DBServer';
+      valgrindHosts += 'DBserver';
     }
     
   }