From ae8904a5b6b1876498013c707de427befb14853a Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 1 Dec 2015 12:04:02 +0100 Subject: [PATCH 1/7] added async flag for sync() and syncCollection() --- arangosh/V8Client/V8ClientConnection.h | 8 +-- .../client/org/arangodb/replication.js | 49 ++++++++++++++++--- js/client/modules/org/arangodb/replication.js | 49 ++++++++++++++++--- 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/arangosh/V8Client/V8ClientConnection.h b/arangosh/V8Client/V8ClientConnection.h index 6c151d6967..dfb6d93a53 100644 --- a/arangosh/V8Client/V8ClientConnection.h +++ b/arangosh/V8Client/V8ClientConnection.h @@ -299,10 +299,10 @@ namespace triagens { v8::Handle requestData (v8::Isolate* isolate, rest::HttpRequest::HttpRequestType method, - std::string const& location, - const char* body, - const size_t bodySize, - std::map const& headerFields); + std::string const& location, + const char* body, + const size_t bodySize, + std::map const& headerFields); //////////////////////////////////////////////////////////////////////////////// /// @brief executes a request diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js b/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js index c9ed3b25c5..889b042ca5 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js @@ -171,9 +171,19 @@ var sync = function (config) { var db = internal.db; var body = JSON.stringify(config || { }); - var requestResult = db._connection.PUT("/_api/replication/sync", body); + var requestResult; + if (config.async) { + var headers = { "X-Arango-Async" : "store" }; + requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers); + } + else { + requestResult = db._connection.PUT("/_api/replication/sync", body); + } arangosh.checkRequestResult(requestResult); + if (config.async) { + return requestResult.headers["x-arango-async-id"]; + } return requestResult; }; @@ -191,14 +201,36 @@ var syncCollection = function (collection, config) { config.restrictCollections = [ collection ]; config.includeSystem = true; var body = JSON.stringify(config); - - var requestResult = db._connection.PUT("/_api/replication/sync", body); + var requestResult; + if (config.async) { + var headers = { "X-Arango-Async" : "store" }; + requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers); + } + else { + requestResult = db._connection.PUT("/_api/replication/sync", body); + } arangosh.checkRequestResult(requestResult); + if (config.async) { + return requestResult.headers["x-arango-async-id"]; + } return requestResult; }; +var checkSyncStatus = function (id) { + var db = internal.db; + + var requestResult = db._connection.PUT_RAW("/_api/job/" + encodeURIComponent(id), ""); + arangosh.checkRequestResult(requestResult); + + if (requestResult.headers.hasOwnProperty("x-arango-async-id")) { + return JSON.parse(requestResult.body); + } + + return false; +}; + //////////////////////////////////////////////////////////////////////////////// /// @brief fetches a server's id //////////////////////////////////////////////////////////////////////////////// @@ -217,11 +249,12 @@ var serverId = function () { // --SECTION-- module exports // ----------------------------------------------------------------------------- -exports.logger = logger; -exports.applier = applier; -exports.sync = sync; -exports.syncCollection = syncCollection; -exports.serverId = serverId; +exports.logger = logger; +exports.applier = applier; +exports.sync = sync; +exports.syncCollection = syncCollection; +exports.checkSyncStatus = checkSyncStatus; +exports.serverId = serverId; // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE diff --git a/js/client/modules/org/arangodb/replication.js b/js/client/modules/org/arangodb/replication.js index 3a49bf297e..a40013ad31 100644 --- a/js/client/modules/org/arangodb/replication.js +++ b/js/client/modules/org/arangodb/replication.js @@ -170,9 +170,19 @@ var sync = function (config) { var db = internal.db; var body = JSON.stringify(config || { }); - var requestResult = db._connection.PUT("/_api/replication/sync", body); + var requestResult; + if (config.async) { + var headers = { "X-Arango-Async" : "store" }; + requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers); + } + else { + requestResult = db._connection.PUT("/_api/replication/sync", body); + } arangosh.checkRequestResult(requestResult); + if (config.async) { + return requestResult.headers["x-arango-async-id"]; + } return requestResult; }; @@ -190,14 +200,36 @@ var syncCollection = function (collection, config) { config.restrictCollections = [ collection ]; config.includeSystem = true; var body = JSON.stringify(config); - - var requestResult = db._connection.PUT("/_api/replication/sync", body); + var requestResult; + if (config.async) { + var headers = { "X-Arango-Async" : "store" }; + requestResult = db._connection.PUT_RAW("/_api/replication/sync", body, headers); + } + else { + requestResult = db._connection.PUT("/_api/replication/sync", body); + } arangosh.checkRequestResult(requestResult); + if (config.async) { + return requestResult.headers["x-arango-async-id"]; + } return requestResult; }; +var checkSyncStatus = function (id) { + var db = internal.db; + + var requestResult = db._connection.PUT_RAW("/_api/job/" + encodeURIComponent(id), ""); + arangosh.checkRequestResult(requestResult); + + if (requestResult.headers.hasOwnProperty("x-arango-async-id")) { + return JSON.parse(requestResult.body); + } + + return false; +}; + //////////////////////////////////////////////////////////////////////////////// /// @brief fetches a server's id //////////////////////////////////////////////////////////////////////////////// @@ -216,11 +248,12 @@ var serverId = function () { // --SECTION-- module exports // ----------------------------------------------------------------------------- -exports.logger = logger; -exports.applier = applier; -exports.sync = sync; -exports.syncCollection = syncCollection; -exports.serverId = serverId; +exports.logger = logger; +exports.applier = applier; +exports.sync = sync; +exports.syncCollection = syncCollection; +exports.checkSyncStatus = checkSyncStatus; +exports.serverId = serverId; // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE From 69fe6bf257b63c8a18782fa048c69d5960a71f9e Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 1 Dec 2015 12:12:44 +0100 Subject: [PATCH 2/7] updated documentation --- .../Users/Replication/SyncingCollections.mdpp | 27 ++++++++++++++++++- .../client/org/arangodb/replication.js | 4 +-- js/client/modules/org/arangodb/replication.js | 4 +-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/Documentation/Books/Users/Replication/SyncingCollections.mdpp b/Documentation/Books/Users/Replication/SyncingCollections.mdpp index d3d83979b7..a2f4d3c2fc 100644 --- a/Documentation/Books/Users/Replication/SyncingCollections.mdpp +++ b/Documentation/Books/Users/Replication/SyncingCollections.mdpp @@ -38,8 +38,33 @@ the correct server, in the correct database! Setting the optional *incremental* attribute in the call to *syncCollection* will start an incremental transfer of data. This may be useful in case when the slave already has parts or almost all of the data in the collection and only the differences need to be -synchronized. +synchronized. Note that to compute the differences the incremental transfer will build a sorted +list of all document keys in the collection on both the slave and the master, which may still be +expensive for huge collections in terms of memory usage and runtime. During building the list +of keys the collection will be read-locked on the master. The *initialSyncMaxWaitTime* attribute in the call to *syncCollection* controls how long the slave will wait for a master's response. This wait time can be used to control after what time the synchronization will give up and fail. + +When *syncCollection* is called from the ArangoShell, the optional *async* attribute can be used +to trigger the synchronization as a background process on the slave. If *async* is set to *true*, +the call to *syncCollection* will return almost instantly with a id string. Using this id string, +the status of the sync job on the slave can be queried using the *getSyncResult* function as follows: + + +```js +db._useDatabase("_system"); +var replication = require("org/arangodb/replication"); +var id = replication.syncCollection("test", { + endpoint: "tcp://master.domain.org:8529", + username: "myuser", + password: "mypasswd", + async: true +}); + +print(replication.getSyncResult(id)); +``` + +*getSyncResult* will return false as long as the synchronization is not complete, and return the +synchronization result otherwise. diff --git a/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js b/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js index 889b042ca5..286650015e 100644 --- a/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js +++ b/js/apps/system/_admin/aardvark/APP/frontend/js/modules/client/org/arangodb/replication.js @@ -218,7 +218,7 @@ var syncCollection = function (collection, config) { return requestResult; }; -var checkSyncStatus = function (id) { +var getSyncResult = function (id) { var db = internal.db; var requestResult = db._connection.PUT_RAW("/_api/job/" + encodeURIComponent(id), ""); @@ -253,7 +253,7 @@ exports.logger = logger; exports.applier = applier; exports.sync = sync; exports.syncCollection = syncCollection; -exports.checkSyncStatus = checkSyncStatus; +exports.getSyncResult = getSyncResult; exports.serverId = serverId; // ----------------------------------------------------------------------------- diff --git a/js/client/modules/org/arangodb/replication.js b/js/client/modules/org/arangodb/replication.js index a40013ad31..b6238bb10b 100644 --- a/js/client/modules/org/arangodb/replication.js +++ b/js/client/modules/org/arangodb/replication.js @@ -217,7 +217,7 @@ var syncCollection = function (collection, config) { return requestResult; }; -var checkSyncStatus = function (id) { +var getSyncResult = function (id) { var db = internal.db; var requestResult = db._connection.PUT_RAW("/_api/job/" + encodeURIComponent(id), ""); @@ -252,7 +252,7 @@ exports.logger = logger; exports.applier = applier; exports.sync = sync; exports.syncCollection = syncCollection; -exports.checkSyncStatus = checkSyncStatus; +exports.getSyncResult = getSyncResult; exports.serverId = serverId; // ----------------------------------------------------------------------------- From 47fc98a43f4e4e3a8e17c34dc980208a0f17f410 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 1 Dec 2015 12:52:08 +0100 Subject: [PATCH 3/7] documentation --- .../Books/Users/Replication/ExampleSetup.mdpp | 35 +++++++++++++++++-- .../Users/Replication/SyncingCollections.mdpp | 6 ++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Documentation/Books/Users/Replication/ExampleSetup.mdpp b/Documentation/Books/Users/Replication/ExampleSetup.mdpp index 742cc9f337..11d374b2f5 100644 --- a/Documentation/Books/Users/Replication/ExampleSetup.mdpp +++ b/Documentation/Books/Users/Replication/ExampleSetup.mdpp @@ -24,8 +24,11 @@ require("org/arangodb/replication").applier.stop(); The *stop* operation will terminate any replication activity in the _system database on the slave. -After that, do an initial sync of the slave with data from the master. Execute the following -commands on the slave: + +!SECTION Initial synchronization + +After that, we perform an initial sync of the slave with data from the master. To do this, +execute the following commands on the slave: ```js db._useDatabase("_system"); @@ -56,6 +59,8 @@ assume we got the following last log tick: } ``` +!SECTION Continuous synchronization + Now, we could start the replication applier in the slave database using the last log tick. However, there is one thing to consider: replication on the slave will be running until the slave gets shut down. When the slave server gets restarted, replication will be turned off again. @@ -141,3 +146,29 @@ a write lock on the collections involved in the transaction. You may also want to check the master and slave states via the HTTP APIs (see [HTTP Interface for Replication](../HttpReplications/README.md)). + +!SECTION Initial synchronization from the ArangoShell + +The *sync* may take a long time to complete. If it's called from the ArangoShell, the connection +may time out, which will effectively discard the result of the *sync* operation. Therefore in the +ArangoShell, the optional *async* attribute can be used to start the synchronization as a background +process on the slave. If the *async* attribute is set to *true*, the call to *sync* will return +almost instantly with an id string. Using this id string, the status of the sync job on the slave +can be queried using the *getSyncResult* function as follows: + +```js +db._useDatabase("_system"); +var replication = require("org/arangodb/replication"); +var id = replication.sync({ + endpoint: "tcp://master.domain.org:8529", + username: "myuser", + password: "mypasswd", + async: true +}); + +print(replication.getSyncResult(id)); +``` + +*getSyncResult* will return *false* as long as the synchronization is not complete, and return the +synchronization result otherwise. + diff --git a/Documentation/Books/Users/Replication/SyncingCollections.mdpp b/Documentation/Books/Users/Replication/SyncingCollections.mdpp index a2f4d3c2fc..9eb0f9d345 100644 --- a/Documentation/Books/Users/Replication/SyncingCollections.mdpp +++ b/Documentation/Books/Users/Replication/SyncingCollections.mdpp @@ -48,8 +48,8 @@ slave will wait for a master's response. This wait time can be used to control a the synchronization will give up and fail. When *syncCollection* is called from the ArangoShell, the optional *async* attribute can be used -to trigger the synchronization as a background process on the slave. If *async* is set to *true*, -the call to *syncCollection* will return almost instantly with a id string. Using this id string, +to start the synchronization as a background process on the slave. If *async* is set to *true*, +the call to *syncCollection* will return almost instantly with an id string. Using this id string, the status of the sync job on the slave can be queried using the *getSyncResult* function as follows: @@ -66,5 +66,5 @@ var id = replication.syncCollection("test", { print(replication.getSyncResult(id)); ``` -*getSyncResult* will return false as long as the synchronization is not complete, and return the +*getSyncResult* will return *false* as long as the synchronization is not complete, and return the synchronization result otherwise. From 9c0d7420f4e5d06f91ddff7be8bd7e0e27a1635e Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 1 Dec 2015 12:56:41 +0100 Subject: [PATCH 4/7] removed unused struct attribute --- arangod/VocBase/document-collection.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/arangod/VocBase/document-collection.cpp b/arangod/VocBase/document-collection.cpp index 1c1d9afad2..8b1f35c113 100644 --- a/arangod/VocBase/document-collection.cpp +++ b/arangod/VocBase/document-collection.cpp @@ -1336,7 +1336,6 @@ typedef struct open_iterator_state_s { uint64_t _documents; int64_t _initialCount; uint32_t _trxCollections; - uint32_t _numOps; bool _trxPrepared; } open_iterator_state_t; From dd2ba7a0becfdf124d42e71aacccef3f5c22e473 Mon Sep 17 00:00:00 2001 From: Frank Celler Date: Tue, 1 Dec 2015 12:55:16 +0100 Subject: [PATCH 5/7] added OS to version check to catch new packages --- js/common/modules/org/arangodb-common.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/js/common/modules/org/arangodb-common.js b/js/common/modules/org/arangodb-common.js index 4f86d9041f..8c8fe3e5cc 100644 --- a/js/common/modules/org/arangodb-common.js +++ b/js/common/modules/org/arangodb-common.js @@ -519,8 +519,9 @@ exports.checkAvailableVersions = function (version) { } try { - var u = "https://www.arangodb.com/repositories/versions.php?version="; - var d = internal.download(u + version, "", {timeout: 300}); + var u = "https://www.arangodb.com/repositories/versions.php?version=" + version + + "&os=" + internal.platform; + var d = internal.download(u, "", {timeout: 300}); var v = JSON.parse(d.body); if (v.hasOwnProperty("bugfix")) { From c16fead9415fbaee02f36230e064c670f5554729 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 1 Dec 2015 16:35:33 +0100 Subject: [PATCH 6/7] Use egrep here too. --- Documentation/Books/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Books/Makefile b/Documentation/Books/Makefile index df3bc8bc87..9a8ffc9a33 100644 --- a/Documentation/Books/Makefile +++ b/Documentation/Books/Makefile @@ -40,10 +40,10 @@ book-check-mdpp-leftovers: fi ppbook-check-html-link: - @if test "`grep -r '\[.*\]\(.*\)' ppbooks/$(NAME) |grep '\.md:' |grep html |grep -v http://|grep -v https:// |grep -v header.css |wc -l`" -gt 0; then \ + @if test "`egrep -r '\[.*\]\(.*\)' ppbooks/$(NAME) |grep '\.md:' |grep html |grep -v http://|grep -v https:// |grep -v header.css |wc -l`" -gt 0; then \ echo "Found links to .html files inside of the document! use .md instead!"; \ echo; \ - grep -r '\[.*\]\(.*\)' ppbooks/$(NAME) | grep '\.md:' | grep html |grep -v http://|grep -v https:// |grep -v header.css ; \ + egrep -r '\[.*\]\(.*\)' ppbooks/$(NAME) | grep '\.md:' | grep html |grep -v http://|grep -v https:// |grep -v header.css ; \ exit 1; \ fi From 60819f9e80751bfe05a6c08192f7fbd3f9c47284 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 1 Dec 2015 16:36:23 +0100 Subject: [PATCH 7/7] Fix spelling for starting valgrind db servers --- js/server/modules/org/arangodb/testing.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/server/modules/org/arangodb/testing.js b/js/server/modules/org/arangodb/testing.js index 08b35179a6..6cc3b89193 100644 --- a/js/server/modules/org/arangodb/testing.js +++ b/js/server/modules/org/arangodb/testing.js @@ -74,8 +74,8 @@ var optionsDocumentation = [ ' of a small local cluster', ' - `clusterNodes`: number of DB-Servers to use', ' - valgrindHosts - configure which clustercomponents to run using valgrintd', - ' Coordinator - run Coordinator with valgrind', - ' DBServer - run DBServers with valgrind', + ' Coordinator - flag to run Coordinator with valgrind', + ' DBServer - flag to run DBServers with valgrind', ' - `test`: path to single test to execute for "single" test target', ' - `cleanup`: if set to true (the default), the cluster data files', ' and logs are removed after termination of the test.', @@ -327,7 +327,7 @@ function startInstance (protocol, options, addArgs, testname, tmpDir) { } if (options.valgrindHosts.DBServer === true) { - valgrindHosts += 'DBServer'; + valgrindHosts += 'DBserver'; } }