From c4aad1e0cd0568334145ea40bd2cb2a564f09e64 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 24 Jun 2014 15:11:19 +0200 Subject: [PATCH] added tests --- UnitTests/Makefile.unittests | 1 + arangod/V8Server/v8-vocbase.cpp | 15 +++ arangod/VocBase/transaction.cpp | 2 +- arangod/Wal/CollectorThread.cpp | 4 + arangod/Wal/LogfileManager.h | 24 +++++ .../aardvark/frontend/js/bootstrap/errors.js | 1 + js/common/bootstrap/errors.js | 1 + js/server/tests/recovery/write-throttling.js | 81 ++++++++++++++ js/server/tests/shell-wal-noncluster.js | 100 ++++++++++++++++++ lib/BasicsC/debugging.c | 10 +- lib/BasicsC/errors.dat | 3 +- lib/BasicsC/voc-errors.c | 18 ++-- lib/BasicsC/voc-errors.h | 37 +++++-- 13 files changed, 274 insertions(+), 23 deletions(-) create mode 100644 js/server/tests/recovery/write-throttling.js diff --git a/UnitTests/Makefile.unittests b/UnitTests/Makefile.unittests index 5b0542a6b9..e3782bdb72 100755 --- a/UnitTests/Makefile.unittests +++ b/UnitTests/Makefile.unittests @@ -210,6 +210,7 @@ unittests-recovery: @echo "================================================================================" @echo + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="write-throttling" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="collector-oom" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="transaction-no-abort" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="transaction-no-commit" diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index 68072b69f8..8f3fa7adde 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -3796,6 +3796,21 @@ static v8::Handle JS_AdjustWal (v8::Arguments const& argv) { triagens::wal::LogfileManager::instance()->reserveLogfiles(logfiles); } + if (object->Has(TRI_V8_STRING("historicLogfiles"))) { + uint32_t logfiles = static_cast(TRI_ObjectToUInt64(object->Get(TRI_V8_STRING("historicLogfiles")), true)); + triagens::wal::LogfileManager::instance()->historicLogfiles(logfiles); + } + + if (object->Has(TRI_V8_STRING("throttleWait"))) { + uint64_t value = TRI_ObjectToUInt64(object->Get(TRI_V8_STRING("throttleWait")), true); + triagens::wal::LogfileManager::instance()->maxThrottleWait(value); + } + + if (object->Has(TRI_V8_STRING("throttleWhenPending"))) { + uint64_t value = TRI_ObjectToUInt64(object->Get(TRI_V8_STRING("throttleWhenPending")), true); + triagens::wal::LogfileManager::instance()->throttleWhenPending(value); + } + return scope.Close(v8::True()); } diff --git a/arangod/VocBase/transaction.cpp b/arangod/VocBase/transaction.cpp index 9713f8f156..92f573aedc 100644 --- a/arangod/VocBase/transaction.cpp +++ b/arangod/VocBase/transaction.cpp @@ -1025,7 +1025,7 @@ int TRI_BeginTransaction (TRI_transaction_t* trx, while (triagens::wal::LogfileManager::instance()->isThrottled()) { if (++iterations == maxIterations) { - return TRI_ERROR_LOCK_TIMEOUT; + return TRI_ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT; } usleep(WaitTime); diff --git a/arangod/Wal/CollectorThread.cpp b/arangod/Wal/CollectorThread.cpp index b35ac2731b..d1f1a43ebd 100644 --- a/arangod/Wal/CollectorThread.cpp +++ b/arangod/Wal/CollectorThread.cpp @@ -416,6 +416,10 @@ bool CollectorThread::collectLogfiles () { //////////////////////////////////////////////////////////////////////////////// bool CollectorThread::processQueuedOperations () { + TRI_IF_FAILURE("CollectorThreadProcessQueuedOperations") { + return false; + } + MUTEX_LOCKER(_operationsQueueLock); if (_operationsQueue.empty()) { diff --git a/arangod/Wal/LogfileManager.h b/arangod/Wal/LogfileManager.h index 97d7e45eda..b8713985fb 100644 --- a/arangod/Wal/LogfileManager.h +++ b/arangod/Wal/LogfileManager.h @@ -221,6 +221,14 @@ namespace triagens { return _historicLogfiles; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief set the number of historic logfiles +//////////////////////////////////////////////////////////////////////////////// + + inline void historicLogfiles (uint32_t value) { + _historicLogfiles = value; + } + //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not there was a SHUTDOWN file with a tick value /// at server start @@ -254,6 +262,14 @@ namespace triagens { return _maxThrottleWait; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief maximum wait time when write-throttled (in milliseconds) +//////////////////////////////////////////////////////////////////////////////// + + inline void maxThrottleWait (uint64_t value) { + _maxThrottleWait = value; + } + //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not write-throttling is currently enabled //////////////////////////////////////////////////////////////////////////////// @@ -294,6 +310,14 @@ namespace triagens { return _throttleWhenPending; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief set the value of --wal.throttle-when-pending +//////////////////////////////////////////////////////////////////////////////// + + inline void throttleWhenPending (uint64_t value) { + _throttleWhenPending = value; + } + //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not we are in the recovery mode //////////////////////////////////////////////////////////////////////////////// diff --git a/js/apps/system/aardvark/frontend/js/bootstrap/errors.js b/js/apps/system/aardvark/frontend/js/bootstrap/errors.js index 7c8e7723dc..0b71ca59df 100644 --- a/js/apps/system/aardvark/frontend/js/bootstrap/errors.js +++ b/js/apps/system/aardvark/frontend/js/bootstrap/errors.js @@ -103,6 +103,7 @@ "ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE" : { "code" : 1233, "message" : "edge attribute missing" }, "ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING" : { "code" : 1234, "message" : "index insertion warning - attribute missing in document" }, "ERROR_ARANGO_INDEX_CREATION_FAILED" : { "code" : 1235, "message" : "index creation failed" }, + "ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT" : { "code" : 1236, "message" : "write-throttling timeout" }, "ERROR_ARANGO_DATAFILE_FULL" : { "code" : 1300, "message" : "datafile full" }, "ERROR_ARANGO_EMPTY_DATADIR" : { "code" : 1301, "message" : "server database directory is empty" }, "ERROR_REPLICATION_NO_RESPONSE" : { "code" : 1400, "message" : "no response" }, diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index 7c8e7723dc..0b71ca59df 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -103,6 +103,7 @@ "ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE" : { "code" : 1233, "message" : "edge attribute missing" }, "ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING" : { "code" : 1234, "message" : "index insertion warning - attribute missing in document" }, "ERROR_ARANGO_INDEX_CREATION_FAILED" : { "code" : 1235, "message" : "index creation failed" }, + "ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT" : { "code" : 1236, "message" : "write-throttling timeout" }, "ERROR_ARANGO_DATAFILE_FULL" : { "code" : 1300, "message" : "datafile full" }, "ERROR_ARANGO_EMPTY_DATADIR" : { "code" : 1301, "message" : "server database directory is empty" }, "ERROR_REPLICATION_NO_RESPONSE" : { "code" : 1400, "message" : "no response" }, diff --git a/js/server/tests/recovery/write-throttling.js b/js/server/tests/recovery/write-throttling.js new file mode 100644 index 0000000000..273022a25e --- /dev/null +++ b/js/server/tests/recovery/write-throttling.js @@ -0,0 +1,81 @@ + +var db = require("org/arangodb").db; +var internal = require("internal"); +var jsunity = require("jsunity"); + + +function runSetup () { + internal.debugClearFailAt(); + + db._drop("UnitTestsRecovery"); + var c = db._create("UnitTestsRecovery"), i; + internal.flushWal(true, true); + internal.adjustWal({ throttleWait: 1000, throttleWhenPending: 1000 }); + + for (i = 0; i < 10000; ++i) { + c.save({ _key: "test" + i, value1: "test" + i, value2: i }); + } + + internal.debugSetFailAt("CollectorThreadProcessQueuedOperations"); + internal.flushWal(true, false); + + // now let the write throttling become active + internal.wait(5); + try { + c.save({ _key: "foo" }); + } + catch (err) { + } + + internal.debugSegfault("crashing server"); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite +//////////////////////////////////////////////////////////////////////////////// + +function recoverySuite () { + jsunity.jsUnity.attachAssertions(); + + return { + setUp: function () { + }, + tearDown: function () { + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test whether we can restore the data +//////////////////////////////////////////////////////////////////////////////// + + testWriteThrottling : function () { + var i, c = db._collection("UnitTestsRecovery"); + + assertEqual(10000, c.count()); + for (i = 0; i < 10000; ++i) { + var doc = c.document("test" + i); + + assertEqual("test" + i, doc.value1); + assertEqual(i, doc.value2); + } + + assertFalse(c.exists("foo")); + } + + }; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief executes the test suite +//////////////////////////////////////////////////////////////////////////////// + +function main (argv) { + if (argv[1] === "setup") { + runSetup(); + return 0; + } + else { + jsunity.run(recoverySuite); + return jsunity.done() ? 0 : 1; + } +} + diff --git a/js/server/tests/shell-wal-noncluster.js b/js/server/tests/shell-wal-noncluster.js index 372fb9a7f2..a24cf440d4 100644 --- a/js/server/tests/shell-wal-noncluster.js +++ b/js/server/tests/shell-wal-noncluster.js @@ -52,6 +52,7 @@ function walFailureSuite () { }, tearDown: function () { + internal.debugClearFailAt(); db._drop(cn); c = null; }, @@ -80,6 +81,105 @@ function walFailureSuite () { testHelper.waitUnload(c); assertEqual(1000, c.count()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test no more available logfiles +//////////////////////////////////////////////////////////////////////////////// + + testNoMoreLogfiles : function () { + internal.debugClearFailAt(); + + db._drop(cn); + c = db._create(cn); + + var i; + for (i = 0; i < 100; ++i) { + c.save({ _key: "test" + i, a: i }); + } + assertEqual(100, c.count()); + + internal.flushWal(true, true); + + internal.debugSetFailAt("LogfileManagerGetWriteableLogfile"); + try { + for (i = 0; i < 200000; ++i) { + c.save({ _key: "foo" + i, value: "the quick brown foxx jumped over the lazy dog" }); + } + + // we don't know the exact point where the above operation failed, however, + // it must fail somewhere + fail(); + } + catch (err) { + assertEqual(internal.errors.ERROR_ARANGO_NO_JOURNAL.code, err.errorNum); + } + + internal.debugClearFailAt(); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test write throttling +//////////////////////////////////////////////////////////////////////////////// + + testWriteNoThrottling : function () { + internal.debugClearFailAt(); + + db._drop(cn); + c = db._create(cn); + + internal.flushWal(true, true); + + internal.debugSetFailAt("CollectorThreadProcessQueuedOperations"); + internal.adjustWal({ throttleWait: 1000, throttleWhenPending: 1000 }); + + var i; + for (i = 0; i < 10; ++i) { + c.save({ _key: "test" + i, a: i }); + } + + internal.flushWal(true, false); + + c.save({ _key: "foo" }); + assertEqual("foo", c.document("foo")._key); + assertEqual(11, c.count()); + + internal.debugClearFailAt(); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test write throttling +//////////////////////////////////////////////////////////////////////////////// + + testWriteThrottling : function () { + internal.debugClearFailAt(); + + db._drop(cn); + c = db._create(cn); + + internal.flushWal(true, true); + + internal.debugSetFailAt("CollectorThreadProcessQueuedOperations"); + internal.adjustWal({ throttleWait: 1000, throttleWhenPending: 1000 }); + + var i; + for (i = 0; i < 1005; ++i) { + c.save({ _key: "test" + i, a: i }); + } + + internal.flushWal(true, false); + internal.wait(3); + + try { + c.save({ _key: "foo" }); + fail(); + } + catch (err) { + assertEqual(internal.errors.ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT.code, err.errorNum); + } + + internal.debugClearFailAt(); + assertEqual(1005, c.count()); } }; diff --git a/lib/BasicsC/debugging.c b/lib/BasicsC/debugging.c index 397fc79e42..19f2bb1bd0 100644 --- a/lib/BasicsC/debugging.c +++ b/lib/BasicsC/debugging.c @@ -85,11 +85,12 @@ static char* MakeValue (char const* value) { //////////////////////////////////////////////////////////////////////////////// /// @brief cause a segmentation violation +/// this is used for crash and recovery tests //////////////////////////////////////////////////////////////////////////////// void TRI_SegfaultDebugging (char const* message) { - LOG_WARNING("causing intentional segfault: %s", message); - // make sure the log message is flushed + LOG_WARNING("SUMMON BAAL: %s", message); + // make sure the latest log messages are flushed TRI_ShutdownLogging(true); // and now crash @@ -103,6 +104,11 @@ void TRI_SegfaultDebugging (char const* message) { bool TRI_ShouldFailDebugging (char const* value) { char* found = NULL; + // try without the lock first (to speed things up) + if (FailurePoints == NULL) { + return false; + } + TRI_ReadLockReadWriteLock(&FailurePointsLock); if (FailurePoints != NULL) { diff --git a/lib/BasicsC/errors.dat b/lib/BasicsC/errors.dat index 7355ac59de..a32d56fd0f 100755 --- a/lib/BasicsC/errors.dat +++ b/lib/BasicsC/errors.dat @@ -121,6 +121,7 @@ ERROR_ARANGO_INVALID_KEY_GENERATOR,1232,"invalid key generator","Will be raised ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE,1233,"edge attribute missing","will be raised when the _from or _to values of an edge are undefined or contain an invalid value." ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING,1234,"index insertion warning - attribute missing in document","Will be raised when an attempt to insert a document into an index is caused by in the document not having one or more attributes which the index is built on." ERROR_ARANGO_INDEX_CREATION_FAILED,1235,"index creation failed","Will be raised when an attempt to create an index has failed." +ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT,1236,"write-throttling timeout","Will be raised when the server is write-throttled and a write operation has waited too long for the server to process queued operations." ################################################################################ ## ArangoDB storage errors @@ -352,4 +353,4 @@ RESULT_ELEMENT_NOT_FOUND,10003,"element not found in structure","Will be returne ################################################################################ ## foxx app update via github ################################################################################ -ERROR_APP_ALREADY_EXISTS,20000,"newest version of app already installed","newest version of app already installed" \ No newline at end of file +ERROR_APP_ALREADY_EXISTS,20000,"newest version of app already installed","newest version of app already installed" diff --git a/lib/BasicsC/voc-errors.c b/lib/BasicsC/voc-errors.c index bdd4885a48..d0d45edfcf 100644 --- a/lib/BasicsC/voc-errors.c +++ b/lib/BasicsC/voc-errors.c @@ -2,9 +2,14 @@ /// @brief auto-generated file generated from errors.dat //////////////////////////////////////////////////////////////////////////////// -#include "BasicsC/common.h" +#include #include "./lib/BasicsC/voc-errors.h" +//////////////////////////////////////////////////////////////////////////////// +/// @addtogroup VocError +/// @{ +//////////////////////////////////////////////////////////////////////////////// + void TRI_InitialiseErrorMessages (void) { REG_ERROR(ERROR_NO_ERROR, "no error"); REG_ERROR(ERROR_FAILED, "failed"); @@ -94,6 +99,7 @@ void TRI_InitialiseErrorMessages (void) { REG_ERROR(ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE, "edge attribute missing"); REG_ERROR(ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING, "index insertion warning - attribute missing in document"); REG_ERROR(ERROR_ARANGO_INDEX_CREATION_FAILED, "index creation failed"); + REG_ERROR(ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT, "write-throttling timeout"); REG_ERROR(ERROR_ARANGO_DATAFILE_FULL, "datafile full"); REG_ERROR(ERROR_ARANGO_EMPTY_DATADIR, "server database directory is empty"); REG_ERROR(ERROR_REPLICATION_NO_RESPONSE, "no response"); @@ -239,11 +245,7 @@ void TRI_InitialiseErrorMessages (void) { REG_ERROR(ERROR_APP_ALREADY_EXISTS, "newest version of app already installed"); } -// ----------------------------------------------------------------------------- -// --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @} +//////////////////////////////////////////////////////////////////////////////// -// Local Variables: -// mode: outline-minor -// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" -// End: diff --git a/lib/BasicsC/voc-errors.h b/lib/BasicsC/voc-errors.h index 313889918a..abe4719150 100644 --- a/lib/BasicsC/voc-errors.h +++ b/lib/BasicsC/voc-errors.h @@ -211,6 +211,9 @@ extern "C" { /// index is built on. /// - 1235: @LIT{index creation failed} /// Will be raised when an attempt to create an index has failed. +/// - 1236: @LIT{write-throttling timeout} +/// Will be raised when the server is write-throttled and a write operation +/// has waited too long for the server to process queued operations. /// - 1300: @LIT{datafile full} /// Will be raised when the datafile reaches its limit. /// - 1301: @LIT{server database directory is empty} @@ -525,7 +528,7 @@ extern "C" { /// - 1934: @LIT{Invalid example type. Has to be Array or Object} /// Invalid example type. Has to be Array or Object. /// - 1935: @LIT{Invalid number of arguments. Expected: } -/// Invalid number of arguments. Expected: +/// Invalid number of arguments. Expected: /// - 1936: @LIT{Invalid parameter type.} /// Invalid parameter type. /// - 1937: @LIT{Invalid id} @@ -572,6 +575,11 @@ extern "C" { /// newest version of app already installed //////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +/// @addtogroup VocError +/// @{ +//////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// /// @brief helper macro to define an error string //////////////////////////////////////////////////////////////////////////////// @@ -1484,6 +1492,17 @@ void TRI_InitialiseErrorMessages (void); #define TRI_ERROR_ARANGO_INDEX_CREATION_FAILED (1235) +//////////////////////////////////////////////////////////////////////////////// +/// @brief 1236: ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT +/// +/// write-throttling timeout +/// +/// Will be raised when the server is write-throttled and a write operation has +/// waited too long for the server to process queued operations. +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT (1236) + //////////////////////////////////////////////////////////////////////////////// /// @brief 1300: ERROR_ARANGO_DATAFILE_FULL /// @@ -2784,9 +2803,9 @@ void TRI_InitialiseErrorMessages (void); //////////////////////////////////////////////////////////////////////////////// /// @brief 1935: ERROR_GRAPH_INVALID_NUMBER_OF_ARGUMENTS /// -/// Invalid number of arguments. Expected: +/// Invalid number of arguments. Expected: /// -/// Invalid number of arguments. Expected: +/// Invalid number of arguments. Expected: //////////////////////////////////////////////////////////////////////////////// #define TRI_ERROR_GRAPH_INVALID_NUMBER_OF_ARGUMENTS (1935) @@ -2983,17 +3002,13 @@ void TRI_InitialiseErrorMessages (void); #define TRI_ERROR_APP_ALREADY_EXISTS (20000) +//////////////////////////////////////////////////////////////////////////////// +/// @} +//////////////////////////////////////////////////////////////////////////////// + #ifdef __cplusplus } #endif #endif -// ----------------------------------------------------------------------------- -// --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- - -// Local Variables: -// mode: outline-minor -// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" -// End: