From a667f633d577cf06f452d0b08111bf7d013fa397 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 4 Sep 2012 22:30:44 +0200 Subject: [PATCH] issue #150 --- arangod/MRServer/ApplicationMR.cpp | 74 +++++++++++++++++++++++++--- arangod/MRServer/ApplicationMR.h | 28 ++++++++++- arangod/V8Server/ApplicationV8.cpp | 77 ++++++++++++++++++++++++++---- arangod/V8Server/ApplicationV8.h | 24 ++++++++++ lib/Basics/ConditionLocker.cpp | 8 ++++ lib/Basics/ConditionLocker.h | 6 +++ 6 files changed, 200 insertions(+), 17 deletions(-) diff --git a/arangod/MRServer/ApplicationMR.cpp b/arangod/MRServer/ApplicationMR.cpp index 52a9362c1d..8acb303c99 100644 --- a/arangod/MRServer/ApplicationMR.cpp +++ b/arangod/MRServer/ApplicationMR.cpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -/// @brief MR enigne configuration +/// @brief MR engine configuration /// /// @file /// @@ -28,6 +28,8 @@ #include "ApplicationMR.h" #include "Basics/ConditionLocker.h" +#include "Basics/ReadLocker.h" +#include "Basics/WriteLocker.h" #include "Logger/Logger.h" #include "MRServer/mr-actions.h" #include "VocBase/vocbase.h" @@ -58,16 +60,42 @@ namespace { public: MRGcThread (ApplicationMR* applicationMR) : Thread("mr-gc"), - _applicationMR(applicationMR) { + _applicationMR(applicationMR), + _lock(), + _lastGcStamp(TRI_microtime()) { } public: + +//////////////////////////////////////////////////////////////////////////////// +/// @brief collect garbage in an endless loop (main functon of GC thread) +//////////////////////////////////////////////////////////////////////////////// + void run () { _applicationMR->collectGarbage(); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the timestamp of the last GC +//////////////////////////////////////////////////////////////////////////////// + + double getLastGcStamp () { + READ_LOCKER(_lock); + return _lastGcStamp; + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief set the global GC timestamp +//////////////////////////////////////////////////////////////////////////////// + + void updateGcStamp (double value) { + WRITE_LOCKER(_lock); + } + private: ApplicationMR* _applicationMR; + ReadWriteLock _lock; + double _lastGcStamp; }; } @@ -99,6 +127,7 @@ ApplicationMR::ApplicationMR (string const& binaryPath) _startupModules(), _actionPath(), _gcInterval(1000), + _gcFrequency(10.0), _startupLoader(), _actionLoader(), _vocbase(0), @@ -171,16 +200,25 @@ ApplicationMR::MRContext* ApplicationMR::enterContext () { //////////////////////////////////////////////////////////////////////////////// void ApplicationMR::exitContext (MRContext* context) { + MRGcThread* gc = dynamic_cast(_gcThread); + assert(gc != 0); + double lastGc = gc->getLastGcStamp(); + ++context->_dirt; { CONDITION_LOCKER(guard, _contextCondition); - if (context->_dirt < _gcInterval) { - _freeContexts.push_back(context); + if (context->_lastGcStamp + _gcFrequency < lastGc) { + LOGGER_TRACE << "periodic gc interval reached"; + _dirtyContexts.push_back(context); + } + else if (context->_dirt >= _gcInterval) { + LOGGER_TRACE << "maximum number of requests reached"; + _dirtyContexts.push_back(context); } else { - _dirtyContexts.push_back(context); + _freeContexts.push_back(context); } guard.broadcast(); @@ -194,28 +232,47 @@ void ApplicationMR::exitContext (MRContext* context) { //////////////////////////////////////////////////////////////////////////////// void ApplicationMR::collectGarbage () { + MRGcThread* gc = dynamic_cast(_gcThread); + assert(gc != 0); + uint64_t waitTime = (uint64_t) (_gcFrequency * 1000.0 * 1000.0); + while (_stopping == 0) { MRContext* context = 0; + bool gotSignal = false; { CONDITION_LOCKER(guard, _contextCondition); if (_dirtyContexts.empty()) { - guard.wait(); + gotSignal = guard.wait(waitTime); } if (! _dirtyContexts.empty()) { context = _dirtyContexts.back(); _dirtyContexts.pop_back(); } + + if (context == 0 && ! gotSignal) { + // we did not find a dirty context + // so we'll pop one of the free contexts and clean it up + context = _freeContexts.back(); + if (context != 0) { + _freeContexts.pop_back(); + } + } } + // update last gc time + double lastGc = TRI_microtime(); + gc->updateGcStamp(lastGc); + if (context != 0) { LOGGER_TRACE << "collecting MR garbage"; // TODO context->_dirt = 0; + context->_lastGcStamp = lastGc; { CONDITION_LOCKER(guard, _contextCondition); @@ -254,7 +311,8 @@ void ApplicationMR::disableActions () { void ApplicationMR::setupOptions (map& options) { options["RUBY Options:help-admin"] - ("ruby.gc-interval", &_gcInterval, "Ruby garbage collection interval (each x requests)") + ("ruby.gc-interval", &_gcInterval, "Ruby request-based garbage collection interval (each x requests)") + ("ruby.gc-frequency", &_gcFrequency, "Ruby time-based garbage collection frequency (each x seconds)") ; options["RUBY Options:help-admin"] @@ -407,6 +465,8 @@ bool ApplicationMR::prepareMRInstance (size_t i) { } } + context->_lastGcStamp = TRI_microtime(); + // and return from the context LOGGER_TRACE << "initialised MR context #" << i; diff --git a/arangod/MRServer/ApplicationMR.h b/arangod/MRServer/ApplicationMR.h index 12e83fd5a7..dae0ef7e3a 100644 --- a/arangod/MRServer/ApplicationMR.h +++ b/arangod/MRServer/ApplicationMR.h @@ -89,7 +89,19 @@ namespace triagens { struct MRContext { MR_state_t* _mrs; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief number of requests since last GC of the context +//////////////////////////////////////////////////////////////////////////////// + size_t _dirt; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief timestamp of last GC for the context +//////////////////////////////////////////////////////////////////////////////// + + double _lastGcStamp; + }; //////////////////////////////////////////////////////////////////////////////// @@ -287,16 +299,28 @@ namespace triagens { string _actionPath; //////////////////////////////////////////////////////////////////////////////// -/// @brief JavaScript garbage collection interval (each x requests) +/// @brief MRuby garbage collection interval (each x requests) /// /// @CMDOPT{--ruby.gc-interval @CA{interval}} /// /// Specifies the interval (approximately in number of requests) that the -/// garbage collection for JavaScript objects will be run in each thread. +/// garbage collection for MRuby objects will be run in each thread. //////////////////////////////////////////////////////////////////////////////// uint64_t _gcInterval; +//////////////////////////////////////////////////////////////////////////////// +/// @brief MRuby garbage collection frequency (each x seconds) +/// +/// @CMDOPT{--ruby.gc-frequency @CA{frequency}} +/// +/// Specifies the frequency in seconds for the automatic garbage collection of +/// MRuby objects. This setting is useful to have the garbage collection +/// still work in periods with no or little numbers of requests. +//////////////////////////////////////////////////////////////////////////////// + + double _gcFrequency; + //////////////////////////////////////////////////////////////////////////////// /// @brief MR startup loader //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/V8Server/ApplicationV8.cpp b/arangod/V8Server/ApplicationV8.cpp index e1b246daf5..8c6d9aeca2 100644 --- a/arangod/V8Server/ApplicationV8.cpp +++ b/arangod/V8Server/ApplicationV8.cpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////////// -/// @brief V8 enigne configuration +/// @brief V8 engine configuration /// /// @file /// @@ -28,6 +28,8 @@ #include "ApplicationV8.h" #include "Basics/ConditionLocker.h" +#include "Basics/ReadLocker.h" +#include "Basics/WriteLocker.h" #include "Logger/Logger.h" #include "V8/v8-conv.h" #include "V8/v8-shell.h" @@ -65,16 +67,43 @@ namespace { public: V8GcThread (ApplicationV8* applicationV8) : Thread("v8-gc"), - _applicationV8(applicationV8) { + _applicationV8(applicationV8), + _lock(), + _lastGcStamp(TRI_microtime()) { } public: + +//////////////////////////////////////////////////////////////////////////////// +/// @brief collect garbage in an endless loop (main functon of GC thread) +//////////////////////////////////////////////////////////////////////////////// + void run () { _applicationV8->collectGarbage(); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the timestamp of the last GC +//////////////////////////////////////////////////////////////////////////////// + + double getLastGcStamp () { + READ_LOCKER(_lock); + return _lastGcStamp; + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief set the global GC timestamp +//////////////////////////////////////////////////////////////////////////////// + + void updateGcStamp (double value) { + WRITE_LOCKER(_lock); + _lastGcStamp = value; + } + private: ApplicationV8* _applicationV8; + ReadWriteLock _lock; + double _lastGcStamp; }; } @@ -107,6 +136,7 @@ ApplicationV8::ApplicationV8 (string const& binaryPath) _actionPath(), _useActions(true), _gcInterval(1000), + _gcFrequency(10.0), _startupLoader(), _actionLoader(), _vocbase(0), @@ -183,20 +213,29 @@ ApplicationV8::V8Context* ApplicationV8::enterContext () { //////////////////////////////////////////////////////////////////////////////// void ApplicationV8::exitContext (V8Context* context) { + V8GcThread* gc = dynamic_cast(_gcThread); + assert(gc != 0); + double lastGc = gc->getLastGcStamp(); + context->_context->Exit(); context->_isolate->Exit(); delete context->_locker; - + ++context->_dirt; { CONDITION_LOCKER(guard, _contextCondition); - if (context->_dirt < _gcInterval) { - _freeContexts.push_back(context); + if (context->_lastGcStamp + _gcFrequency < lastGc) { + LOGGER_TRACE << "periodic gc interval reached"; + _dirtyContexts.push_back(context); + } + else if (context->_dirt >= _gcInterval) { + LOGGER_TRACE << "maximum number of requests reached"; + _dirtyContexts.push_back(context); } else { - _dirtyContexts.push_back(context); + _freeContexts.push_back(context); } guard.broadcast(); @@ -210,21 +249,39 @@ void ApplicationV8::exitContext (V8Context* context) { //////////////////////////////////////////////////////////////////////////////// void ApplicationV8::collectGarbage () { + V8GcThread* gc = dynamic_cast(_gcThread); + assert(gc != 0); + uint64_t waitTime = (uint64_t) (_gcFrequency * 1000.0 * 1000.0); + while (_stopping == 0) { V8Context* context = 0; + bool gotSignal = false; { CONDITION_LOCKER(guard, _contextCondition); if (_dirtyContexts.empty()) { - guard.wait(); + gotSignal = guard.wait(waitTime); } if (! _dirtyContexts.empty()) { context = _dirtyContexts.back(); _dirtyContexts.pop_back(); } + + if (context == 0 && ! gotSignal) { + // we did not find a dirty context + // so we'll pop one of the free contexts and clean it up + context = _freeContexts.back(); + if (context != 0) { + _freeContexts.pop_back(); + } + } } + + // update last gc time + double lastGc = TRI_microtime(); + gc->updateGcStamp(lastGc); if (context != 0) { LOGGER_TRACE << "collecting V8 garbage"; @@ -241,6 +298,7 @@ void ApplicationV8::collectGarbage () { delete context->_locker; context->_dirt = 0; + context->_lastGcStamp = lastGc; { CONDITION_LOCKER(guard, _contextCondition); @@ -279,7 +337,8 @@ void ApplicationV8::disableActions () { void ApplicationV8::setupOptions (map& options) { options["JAVASCRIPT Options:help-admin"] - ("javascript.gc-interval", &_gcInterval, "JavaScript garbage collection interval (each x requests)") + ("javascript.gc-interval", &_gcInterval, "JavaScript request-based garbage collection interval (each x requests)") + ("javascript.gc-frequency", &_gcFrequency, "JavaScript time-based garbage collection frequency (each x seconds)") ; options["JAVASCRIPT Options:help-admin"] @@ -475,6 +534,8 @@ bool ApplicationV8::prepareV8Instance (size_t i) { context->_context->Exit(); context->_isolate->Exit(); delete context->_locker; + + context->_lastGcStamp = TRI_microtime(); LOGGER_TRACE << "initialised V8 context #" << i; diff --git a/arangod/V8Server/ApplicationV8.h b/arangod/V8Server/ApplicationV8.h index a4fa6bd067..e91304d2b8 100644 --- a/arangod/V8Server/ApplicationV8.h +++ b/arangod/V8Server/ApplicationV8.h @@ -91,7 +91,19 @@ namespace triagens { v8::Persistent _context; v8::Isolate* _isolate; v8::Locker* _locker; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief number of requests since last GC of the context +//////////////////////////////////////////////////////////////////////////////// + size_t _dirt; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief timestamp of last GC for the context +//////////////////////////////////////////////////////////////////////////////// + + double _lastGcStamp; + }; //////////////////////////////////////////////////////////////////////////////// @@ -305,6 +317,18 @@ namespace triagens { uint64_t _gcInterval; +//////////////////////////////////////////////////////////////////////////////// +/// @brief JavaScript garbage collection frequency (each x seconds) +/// +/// @CMDOPT{--javascript.gc-frequency @CA{frequency}} +/// +/// Specifies the frequency in seconds for the automatic garbage collection of +/// JavaScript objects. This setting is useful to have the garbage collection +/// still work in periods with no or little numbers of requests. +//////////////////////////////////////////////////////////////////////////////// + + double _gcFrequency; + //////////////////////////////////////////////////////////////////////////////// /// @brief V8 startup loader //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/Basics/ConditionLocker.cpp b/lib/Basics/ConditionLocker.cpp index e9469e60f5..097ef7befe 100644 --- a/lib/Basics/ConditionLocker.cpp +++ b/lib/Basics/ConditionLocker.cpp @@ -95,6 +95,14 @@ void ConditionLocker::wait () { _conditionVariable->wait(); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief waits for an event to occur, with a timeout +//////////////////////////////////////////////////////////////////////////////// + +bool ConditionLocker::wait (uint64_t delay) { + return _conditionVariable->wait(delay); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief broadcasts an event //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/Basics/ConditionLocker.h b/lib/Basics/ConditionLocker.h index 4114fe3f06..543f389141 100644 --- a/lib/Basics/ConditionLocker.h +++ b/lib/Basics/ConditionLocker.h @@ -138,6 +138,12 @@ namespace triagens { void wait (); +//////////////////////////////////////////////////////////////////////////////// +/// @brief waits for an event to occur, using a timeout +//////////////////////////////////////////////////////////////////////////////// + + bool wait (uint64_t); + //////////////////////////////////////////////////////////////////////////////// /// @brief broadcasts an event ////////////////////////////////////////////////////////////////////////////////