From 1d24a4e14a84ae711692f36496247a2b7faf3200 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 2 Jan 2018 13:20:07 +0100 Subject: [PATCH] temporarily break flush thread feature in order to prevent some loss (#4159) --- arangod/MMFiles/MMFilesCollectorThread.cpp | 2 +- arangod/MMFiles/MMFilesLogfileManager.cpp | 2 +- arangod/MMFiles/MMFilesWalLogfile.h | 7 +- js/server/tests/recovery/documents.js | 81 ++++++++++++++++++++++ js/server/tests/recovery/many-documents.js | 80 +++++++++++++++++++++ 5 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 js/server/tests/recovery/documents.js create mode 100644 js/server/tests/recovery/many-documents.js diff --git a/arangod/MMFiles/MMFilesCollectorThread.cpp b/arangod/MMFiles/MMFilesCollectorThread.cpp index 145e270f5a..8068e73924 100644 --- a/arangod/MMFiles/MMFilesCollectorThread.cpp +++ b/arangod/MMFiles/MMFilesCollectorThread.cpp @@ -404,7 +404,7 @@ int MMFilesCollectorThread::collectLogfiles(bool& worked) { try { int res = collect(logfile); - // LOG_TOPIC(TRACE, Logger::COLLECTOR) << "collected logfile: " << logfile->id() << ". result: " << res; + LOG_TOPIC(TRACE, Logger::COLLECTOR) << "collected logfile: " << logfile->id() << ". result: " << res; if (res == TRI_ERROR_NO_ERROR) { // reset collector status diff --git a/arangod/MMFiles/MMFilesLogfileManager.cpp b/arangod/MMFiles/MMFilesLogfileManager.cpp index 02f3f0525a..4e45a29594 100644 --- a/arangod/MMFiles/MMFilesLogfileManager.cpp +++ b/arangod/MMFiles/MMFilesLogfileManager.cpp @@ -1426,7 +1426,7 @@ MMFilesWalLogfile* MMFilesLogfileManager::getCollectableLogfile() { return logfile; } - if (logfile->id() > minId) { + if (logfile->id() > minId || !logfile->hasBeenReleased(released)) { // abort early break; } diff --git a/arangod/MMFiles/MMFilesWalLogfile.h b/arangod/MMFiles/MMFilesWalLogfile.h index 0e42f7d869..aeabea27e1 100644 --- a/arangod/MMFiles/MMFilesWalLogfile.h +++ b/arangod/MMFiles/MMFilesWalLogfile.h @@ -145,7 +145,7 @@ class MMFilesWalLogfile { /// @brief whether or not the logfile can be collected inline bool canBeCollected(TRI_voc_tick_t releasedTick) const { - if (releasedTick <= df()->maxTick()) { + if (releasedTick < df()->maxTick()) { return false; } @@ -153,6 +153,11 @@ class MMFilesWalLogfile { _status == StatusType::COLLECTION_REQUESTED); } + /// @brief whether or not the logfile can be collected + inline bool hasBeenReleased(TRI_voc_tick_t releasedTick) const { + return (releasedTick >= df()->maxTick()); + } + /// @brief whether or not the logfile can be removed inline bool canBeRemoved() const { return (_status == StatusType::COLLECTED && _collectQueueSize == 0 && diff --git a/js/server/tests/recovery/documents.js b/js/server/tests/recovery/documents.js new file mode 100644 index 0000000000..5a32454701 --- /dev/null +++ b/js/server/tests/recovery/documents.js @@ -0,0 +1,81 @@ +/* jshint globalstrict:false, strict:false, unused : false */ +/* global assertEqual, assertTrue */ +// ////////////////////////////////////////////////////////////////////////////// +// / @brief tests for recovery +// / +// / @file +// / +// / DISCLAIMER +// / +// / Copyright 2010-2012 triagens GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License") +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is triAGENS GmbH, Cologne, Germany +// / +// / @author Jan Steemann +// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany +// ////////////////////////////////////////////////////////////////////////////// + +var db = require('@arangodb').db; +var internal = require('internal'); +var jsunity = require('jsunity'); + +function runSetup () { + 'use strict'; + internal.debugClearFailAt(); + + db._drop('UnitTestsRecovery'); + var c = db._create('UnitTestsRecovery'), i; + + for (i = 0; i < 1000; ++i) { + c.save({ _key: 'test' + i, value: i }); + } + // shut down normally! no crashing! +} + +function recoverySuite () { + 'use strict'; + jsunity.jsUnity.attachAssertions(); + + return { + setUp: function () {}, + tearDown: function () {}, + + testDocuments: function () { + var i, c = db._collection('UnitTestsRecovery'); + + assertEqual(1000, c.count()); + for (i = 0; i < 1000; ++i) { + assertTrue(c.exists('test' + i)); + assertEqual(i, c.document('test' + i).value); + } + } + + }; +} + +// ////////////////////////////////////////////////////////////////////////////// +// / @brief executes the test suite +// ////////////////////////////////////////////////////////////////////////////// + +function main (argv) { + 'use strict'; + if (argv[1] === 'setup') { + runSetup(); + return 0; + } else { + jsunity.run(recoverySuite); + return jsunity.done().status ? 0 : 1; + } +} diff --git a/js/server/tests/recovery/many-documents.js b/js/server/tests/recovery/many-documents.js new file mode 100644 index 0000000000..0638fc3808 --- /dev/null +++ b/js/server/tests/recovery/many-documents.js @@ -0,0 +1,80 @@ +/* jshint globalstrict:false, strict:false, unused : false */ +/* global assertEqual, assertTrue */ +// ////////////////////////////////////////////////////////////////////////////// +// / @brief tests for recovery +// / +// / @file +// / +// / DISCLAIMER +// / +// / Copyright 2010-2012 triagens GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License") +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is triAGENS GmbH, Cologne, Germany +// / +// / @author Jan Steemann +// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany +// ////////////////////////////////////////////////////////////////////////////// + +var db = require('@arangodb').db; +var internal = require('internal'); +var jsunity = require('jsunity'); + +function runSetup () { + 'use strict'; + internal.debugClearFailAt(); + + db._drop('UnitTestsRecovery'); + var c = db._create('UnitTestsRecovery'), i; + + for (i = 0; i < 100000; ++i) { + c.save({ _key: 'test' + i, value: i }); + } + // shut down normally! no crashing! +} + +function recoverySuite () { + 'use strict'; + jsunity.jsUnity.attachAssertions(); + + return { + setUp: function () {}, + tearDown: function () {}, + + testManyDocuments: function () { + var i, c = db._collection('UnitTestsRecovery'); + + assertEqual(100000, c.count()); + for (i = 0; i < 100000; ++i) { + assertEqual(i, c.document('test' + i).value); + } + } + + }; +} + +// ////////////////////////////////////////////////////////////////////////////// +// / @brief executes the test suite +// ////////////////////////////////////////////////////////////////////////////// + +function main (argv) { + 'use strict'; + if (argv[1] === 'setup') { + runSetup(); + return 0; + } else { + jsunity.run(recoverySuite); + return jsunity.done().status ? 0 : 1; + } +}