diff --git a/js/server/tests/shell/shell-synchronous-replication-cluster.js b/js/server/tests/shell/shell-synchronous-replication-cluster.js index ef723a4b02..ab39181665 100644 --- a/js/server/tests/shell/shell-synchronous-replication-cluster.js +++ b/js/server/tests/shell/shell-synchronous-replication-cluster.js @@ -1,5 +1,5 @@ /*jshint globalstrict:false, strict:false */ -/*global assertTrue, assertEqual */ +/*global assertTrue, assertEqual, fail */ //////////////////////////////////////////////////////////////////////////////// /// @brief test synchronous replication in the cluster @@ -28,13 +28,16 @@ /// @author Copyright 2016, ArangoDB GmbH, Cologne, Germany //////////////////////////////////////////////////////////////////////////////// -var jsunity = require("jsunity"); +const jsunity = require("jsunity"); -var arangodb = require("@arangodb"); -var db = arangodb.db; -var _ = require("lodash"); -var print = require("internal").print; -var wait = require("internal").wait; +const arangodb = require("@arangodb"); +const db = arangodb.db; +const ERRORS = arangodb.errors; +const _ = require("lodash"); +const print = require("internal").print; +const wait = require("internal").wait; +const suspendExternal = require("internal").suspendExternal; +const continueExternal = require("internal").continueExternal; //////////////////////////////////////////////////////////////////////////////// @@ -62,7 +65,6 @@ function SynchronousReplicationSuite () { s => global.ArangoClusterInfo.getCollectionInfoCurrent("_system", cn, s) ); let replicas = ccinfo.map(s => s.servers.length); - print("Replicas:", replicas); if (_.all(replicas, x => x === 2)) { print("Replication up and running!"); return true; @@ -72,6 +74,193 @@ function SynchronousReplicationSuite () { return false; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail the follower +//////////////////////////////////////////////////////////////////////////////// + + function failFollower() { + var follower = cinfo.shards[shards[0]][1]; + var endpoint = global.ArangoClusterInfo.getServerEndpoint(follower); + // Now look for instanceInfo: + var pos = _.findIndex(global.instanceInfo.arangods, + x => x.endpoint === endpoint); + assertTrue(pos >= 0); + assertTrue(suspendExternal(global.instanceInfo.arangods[pos].pid)); + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief heal the follower +//////////////////////////////////////////////////////////////////////////////// + + function healFollower() { + var follower = cinfo.shards[shards[0]][1]; + var endpoint = global.ArangoClusterInfo.getServerEndpoint(follower); + // Now look for instanceInfo: + var pos = _.findIndex(global.instanceInfo.arangods, + x => x.endpoint === endpoint); + assertTrue(pos >= 0); + assertTrue(continueExternal(global.instanceInfo.arangods[pos].pid)); + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief produce failure +//////////////////////////////////////////////////////////////////////////////// + + function makeFailure(failure) { + if (failure.follower) { + failFollower(); + } else { + failLeader(); + } + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief heal failure +//////////////////////////////////////////////////////////////////////////////// + + function healFailure(failure) { + if (failure.follower) { + healFollower(); + } else { + healLeader(); + } + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief basic operations, with various failure modes: +//////////////////////////////////////////////////////////////////////////////// + + function runBasicOperations(failure, healing) { + if (failure.place === 1) { makeFailure(failure); } + + // Insert with check: + var id = c.insert({Hallo:12}); + assertEqual(1, c.count()); + + if (healing.place === 1) { healFailure(healing); } + if (failure.place === 2) { makeFailure(failure); } + + var doc = c.document(id._key); + assertEqual(12, doc.Hallo); + + if (healing.place === 2) { healFailure(healing); } + if (failure.place === 3) { makeFailure(failure); } + + var ids = c.insert([{Hallo:13}, {Hallo:14}]); + assertEqual(3, c.count()); + assertEqual(2, ids.length); + + if (healing.place === 3) { healFailure(healing); } + if (failure.place === 4) { makeFailure(failure); } + + var docs = c.document([ids[0]._key, ids[1]._key]); + assertEqual(2, docs.length); + assertEqual(13, docs[0].Hallo); + assertEqual(14, docs[1].Hallo); + + if (healing.place === 4) { healFailure(healing); } + if (failure.place === 5) { makeFailure(failure); } + + // Replace with check: + c.replace(id._key, {"Hallo": 100}); + + if (healing.place === 5) { healFailure(healing); } + if (failure.place === 6) { makeFailure(failure); } + + doc = c.document(id._key); + assertEqual(100, doc.Hallo); + + if (healing.place === 6) { healFailure(healing); } + if (failure.place === 7) { makeFailure(failure); } + + c.replace([ids[0]._key, ids[1]._key], [{Hallo:101}, {Hallo:102}]); + + if (healing.place === 7) { healFailure(healing); } + if (failure.place === 8) { makeFailure(failure); } + + docs = c.document([ids[0]._key, ids[1]._key]); + assertEqual(2, docs.length); + assertEqual(101, docs[0].Hallo); + assertEqual(102, docs[1].Hallo); + + if (healing.place === 8) { healFailure(healing); } + if (failure.place === 9) { makeFailure(failure); } + + // Update with check: + c.update(id._key, {"Hallox": 105}); + + if (healing.place === 9) { healFailure(healing); } + if (failure.place === 10) { makeFailure(failure); } + + doc = c.document(id._key); + assertEqual(100, doc.Hallo); + assertEqual(105, doc.Hallox); + + if (healing.place === 10) { healFailure(healing); } + if (failure.place === 11) { makeFailure(failure); } + + c.update([ids[0]._key, ids[1]._key], [{Hallox:106}, {Hallox:107}]); + + if (healing.place === 11) { healFailure(healing); } + if (failure.place === 12) { makeFailure(failure); } + + docs = c.document([ids[0]._key, ids[1]._key]); + assertEqual(2, docs.length); + assertEqual(101, docs[0].Hallo); + assertEqual(102, docs[1].Hallo); + assertEqual(106, docs[0].Hallox); + assertEqual(107, docs[1].Hallox); + + if (healing.place === 12) { healFailure(healing); } + if (failure.place === 13) { makeFailure(failure); } + + // AQL: + var q = db._query(`FOR x IN @@cn + FILTER x.Hallo > 0 + SORT x.Hallo + RETURN {"Hallo": x.Hallo}`, {"@cn": cn}); + docs = q.toArray(); + assertEqual(3, docs.length); + assertEqual([{Hallo:100}, {Hallo:101}, {Hallo:102}], docs); + + if (healing.place === 13) { healFailure(healing); } + if (failure.place === 14) { makeFailure(failure); } + + // Remove with check: + c.remove(id._key); + + if (healing.place === 14) { healFailure(healing); } + if (failure.place === 15) { makeFailure(failure); } + + try { + doc = c.document(id._key); + fail(); + } + catch (e1) { + assertEqual(ERRORS.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code, e1.errorNum); + } + assertEqual(2, c.count()); + + if (healing.place === 15) { healFailure(healing); } + if (failure.place === 16) { makeFailure(failure); } + + c.remove([ids[0]._key, ids[1]._key]); + + if (healing.place === 16) { healFailure(healing); } + if (failure.place === 17) { makeFailure(failure); } + + docs = c.document([ids[0]._key, ids[1]._key]); + assertEqual(2, docs.length); + assertTrue(docs[0].error); + assertTrue(docs[1].error); + + if (healing.place === 17) { healFailure(healing); } + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief the actual tests +//////////////////////////////////////////////////////////////////////////////// + return { //////////////////////////////////////////////////////////////////////////////// @@ -80,7 +269,7 @@ function SynchronousReplicationSuite () { setUp : function () { db._drop(cn); - c = db._create(cn, {numberOfShards: 2, replicationFactor: 2}); + c = db._create(cn, {numberOfShards: 1, replicationFactor: 2}); }, //////////////////////////////////////////////////////////////////////////////// @@ -96,7 +285,6 @@ function SynchronousReplicationSuite () { //////////////////////////////////////////////////////////////////////////////// testCheckInstanceInfo : function () { - require("internal").print("InstanceInfo is:", global.instanceInfo); assertTrue(global.instanceInfo !== undefined); }, @@ -106,7 +294,197 @@ function SynchronousReplicationSuite () { testSetup : function () { assertTrue(waitForSynchronousReplication()); - assertEqual(12, 12); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief run a standard check without failures: +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperations : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({}, {}); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief run a standard check with failures: +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFailureFollower : function () { + assertTrue(waitForSynchronousReplication()); + failFollower(); + runBasicOperations({}, {}); + healFollower(); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 1 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail1 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:1, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 2 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail2 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:2, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 3 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail3 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:3, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 4 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail4 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:4, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 5 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail5 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:5, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 6 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail6 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:6, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 7 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail7 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:7, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 8 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail8 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:8, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 9 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail9 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:9, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 10 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail10 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:10, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 11 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail11 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:11, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 12 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail12 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:12, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 13 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail13 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:13, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 14 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail14 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:14, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 15 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail15 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:15, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 16 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail16 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:16, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fail in place 17 +//////////////////////////////////////////////////////////////////////////////// + + testBasicOperationsFollowerFail17 : function () { + assertTrue(waitForSynchronousReplication()); + runBasicOperations({place:17, follower:true}, {place:17, follower: true}); + assertTrue(waitForSynchronousReplication()); }, //////////////////////////////////////////////////////////////////////////////// @@ -114,6 +492,8 @@ function SynchronousReplicationSuite () { //////////////////////////////////////////////////////////////////////////////// testDummy : function () { + assertEqual(12, 12); + wait(15); } };