diff --git a/CHANGELOG b/CHANGELOG index cfdcd6b602..9b78f22fe2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,7 @@ v2.1.0 (XXXX-XX-XX) ------------------- * make ArangoDB not send back a `WWW-Authenticate` header to a client in case the - client sends the `X-Requested-With` header with a value of `XMLHttpRequest` + client sends the `X-Omit-WWW-Authenticate` HTTP header. This is done to prevent browsers from showing their built-in HTTP authentication dialog for AJAX requests that require authentication. diff --git a/Documentation/ImplementorManual/Communication.md b/Documentation/ImplementorManual/Communication.md index 9ca16d760e..b7123bd6aa 100644 --- a/Documentation/ImplementorManual/Communication.md +++ b/Documentation/ImplementorManual/Communication.md @@ -130,9 +130,10 @@ password if supported. If the client is a browser, then sending back this header normally trigger the display of the browser-side HTTP authentication dialog. As showing the browser HTTP authentication dialog is undesired in AJAX requests, ArangoDB can be told to not send the `WWW-Authenticate` header back to the client. -Whenever a client sends the `X-Requested-With` HTTP header with a value of `XMLHttpRequest` +Whenever a client sends the `X-Omit-WWW-Authenticate` HTTP header (with an arbitrary value) to ArangoDB, ArangoDB will only send status code 401, but no `WWW-Authenticate` header. -This allows clients to implement their own credentials mechanism. +This allows clients to implement credentials handling and bypassing the browser's +built-in dialog. Error Handling {#CommunicationErrors} ===================================== diff --git a/arangod/RestServer/ArangoServer.cpp b/arangod/RestServer/ArangoServer.cpp index b2c09c925e..4dc1a8615f 100644 --- a/arangod/RestServer/ArangoServer.cpp +++ b/arangod/RestServer/ArangoServer.cpp @@ -388,7 +388,7 @@ void ArangoServer::buildApplicationServer () { _applicationServer->setUserConfigFile(".arango" + string(1, TRI_DIR_SEPARATOR_CHAR) + string(conf)); /* - _logfileManager = new wal::LogfileManager(); + _logfileManager = new wal::LogfileManager(&_databasePath); _applicationServer->addFeature(_logfileManager); */ // ............................................................................. diff --git a/arangod/Wal/AllocatorThread.cpp b/arangod/Wal/AllocatorThread.cpp index 6f40a69ebd..852fd98f33 100644 --- a/arangod/Wal/AllocatorThread.cpp +++ b/arangod/Wal/AllocatorThread.cpp @@ -40,7 +40,7 @@ using namespace triagens::wal; /// @brief wait interval for the allocator thread when idle //////////////////////////////////////////////////////////////////////////////// -const uint64_t AllocatorThread::Interval = 1000000; +const uint64_t AllocatorThread::Interval = 500000; // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors @@ -54,7 +54,7 @@ AllocatorThread::AllocatorThread (LogfileManager* logfileManager) : Thread("WalAllocator"), _logfileManager(logfileManager), _condition(), - _createRequests(0), + _requestedSize(0), _stop(0) { allowAsynchronousCancelation(); @@ -92,21 +92,24 @@ void AllocatorThread::stop () { /// @brief signal the creation of a new logfile //////////////////////////////////////////////////////////////////////////////// -void AllocatorThread::signalLogfileCreation () { +void AllocatorThread::signal (uint32_t size) { + assert(size > 0); + CONDITION_LOCKER(guard, _condition); - if (_createRequests == 0) { - ++_createRequests; - guard.signal(); + if (_requestedSize == 0 || size > _requestedSize) { + _requestedSize = size; } + + guard.signal(); } //////////////////////////////////////////////////////////////////////////////// /// @brief creates a new reserve logfile //////////////////////////////////////////////////////////////////////////////// -bool AllocatorThread::createReserveLogfile () { - int res = _logfileManager->createReserveLogfile(); +bool AllocatorThread::createReserveLogfile (uint32_t size) { + int res = _logfileManager->createReserveLogfile(size); return (res == TRI_ERROR_NO_ERROR); } @@ -121,25 +124,23 @@ bool AllocatorThread::createReserveLogfile () { void AllocatorThread::run () { while (_stop == 0) { - uint32_t createRequests = 0; + uint32_t requestedSize = 0; { CONDITION_LOCKER(guard, _condition); - createRequests = _createRequests; + requestedSize = _requestedSize; + _requestedSize = 0; } - if (createRequests == 0 && ! _logfileManager->hasReserveLogfiles()) { - if (createReserveLogfile()) { + if (requestedSize == 0 && ! _logfileManager->hasReserveLogfiles()) { + if (createReserveLogfile(0)) { continue; } LOG_ERROR("unable to create new wal reserve logfile"); } - else if (createRequests > 0) { - if (createReserveLogfile()) { - CONDITION_LOCKER(guard, _condition); - --_createRequests; - + else if (requestedSize > 0 && _logfileManager->logfileCreationAllowed(requestedSize)) { + if (createReserveLogfile(requestedSize)) { continue; } diff --git a/arangod/Wal/AllocatorThread.h b/arangod/Wal/AllocatorThread.h index ccfe2aa531..5eeed87d79 100644 --- a/arangod/Wal/AllocatorThread.h +++ b/arangod/Wal/AllocatorThread.h @@ -85,13 +85,13 @@ namespace triagens { /// @brief signal the creation of a new logfile //////////////////////////////////////////////////////////////////////////////// - void signalLogfileCreation (); + void signal (uint32_t); //////////////////////////////////////////////////////////////////////////////// /// @brief creates a new reserve logfile //////////////////////////////////////////////////////////////////////////////// - bool createReserveLogfile (); + bool createReserveLogfile (uint32_t); // ----------------------------------------------------------------------------- // --SECTION-- Thread methods @@ -130,10 +130,10 @@ namespace triagens { basics::ConditionVariable _condition; //////////////////////////////////////////////////////////////////////////////// -/// @brief number of pending logfile creation requests +/// @brief requested logfile size //////////////////////////////////////////////////////////////////////////////// - uint32_t _createRequests; + uint32_t _requestedSize; //////////////////////////////////////////////////////////////////////////////// /// @brief stop flag diff --git a/arangod/Wal/CollectorThread.cpp b/arangod/Wal/CollectorThread.cpp index dd6404abbb..65e6b7675b 100644 --- a/arangod/Wal/CollectorThread.cpp +++ b/arangod/Wal/CollectorThread.cpp @@ -88,6 +88,15 @@ void CollectorThread::stop () { } } +//////////////////////////////////////////////////////////////////////////////// +/// @brief signal the thread that there is something to do +//////////////////////////////////////////////////////////////////////////////// + +void CollectorThread::signal () { + CONDITION_LOCKER(guard, _condition); + guard.signal(); +} + // ----------------------------------------------------------------------------- // --SECTION-- Thread methods // ----------------------------------------------------------------------------- @@ -99,10 +108,10 @@ void CollectorThread::stop () { void CollectorThread::run () { while (_stop == 0) { // collect a logfile if any qualifies - bool worked = this->collectLogfile(); + bool worked = this->collectLogfiles(); // delete a logfile if any qualifies - worked |= this->removeLogfile(); + worked |= this->removeLogfiles(); CONDITION_LOCKER(guard, _condition); if (! worked) { @@ -122,7 +131,7 @@ void CollectorThread::run () { /// @brief perform collection of a logfile (if any) //////////////////////////////////////////////////////////////////////////////// -bool CollectorThread::collectLogfile () { +bool CollectorThread::collectLogfiles () { Logfile* logfile = _logfileManager->getCollectableLogfile(); if (logfile == nullptr) { @@ -130,9 +139,10 @@ bool CollectorThread::collectLogfile () { } _logfileManager->setCollectionRequested(logfile); - // TODO: implement collection + + // TODO: implement the actual logfile collection - LOG_INFO("collecting logfile %llu", (unsigned long long) logfile->id()); + LOG_TRACE("collecting logfile %llu", (unsigned long long) logfile->id()); _logfileManager->setCollectionDone(logfile); return true; @@ -142,14 +152,14 @@ bool CollectorThread::collectLogfile () { /// @brief perform removal of a logfile (if any) //////////////////////////////////////////////////////////////////////////////// -bool CollectorThread::removeLogfile () { +bool CollectorThread::removeLogfiles () { Logfile* logfile = _logfileManager->getRemovableLogfile(); if (logfile == nullptr) { return false; } - _logfileManager->removeLogfile(logfile); + _logfileManager->removeLogfile(logfile, true); return true; } diff --git a/arangod/Wal/CollectorThread.h b/arangod/Wal/CollectorThread.h index 097513e7d2..0fa878a37f 100644 --- a/arangod/Wal/CollectorThread.h +++ b/arangod/Wal/CollectorThread.h @@ -81,6 +81,12 @@ namespace triagens { void stop (); +//////////////////////////////////////////////////////////////////////////////// +/// @brief signal the thread that there is something to do +//////////////////////////////////////////////////////////////////////////////// + + void signal (); + // ----------------------------------------------------------------------------- // --SECTION-- Thread methods // ----------------------------------------------------------------------------- @@ -103,13 +109,13 @@ namespace triagens { /// @brief perform collection of a logfile (if any) //////////////////////////////////////////////////////////////////////////////// - bool collectLogfile (); + bool collectLogfiles (); //////////////////////////////////////////////////////////////////////////////// /// @brief perform removal of a logfile (if any) //////////////////////////////////////////////////////////////////////////////// - bool removeLogfile (); + bool removeLogfiles (); // ----------------------------------------------------------------------------- // --SECTION-- private variables diff --git a/arangod/Wal/Logfile.cpp b/arangod/Wal/Logfile.cpp index a3fdad7e95..c4336692d6 100644 --- a/arangod/Wal/Logfile.cpp +++ b/arangod/Wal/Logfile.cpp @@ -80,7 +80,7 @@ Logfile* Logfile::create (std::string const& filename, } } - Logfile* logfile = new Logfile(id, df, StatusType::OPEN); + Logfile* logfile = new Logfile(id, df, StatusType::EMPTY); return logfile; } @@ -125,7 +125,7 @@ int Logfile::seal () { _df->_state = TRI_DF_STATE_READ; if (res == TRI_ERROR_NO_ERROR) { - LOG_INFO("sealed logfile %llu", (unsigned long long) id()); + LOG_TRACE("sealed logfile %llu", (unsigned long long) id()); setStatus(StatusType::SEALED); } diff --git a/arangod/Wal/Logfile.h b/arangod/Wal/Logfile.h index 847a391c26..9e645f6cef 100644 --- a/arangod/Wal/Logfile.h +++ b/arangod/Wal/Logfile.h @@ -139,6 +139,14 @@ namespace triagens { return _id; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the logfile status +//////////////////////////////////////////////////////////////////////////////// + + inline Logfile::StatusType status () const { + return _status; + } + //////////////////////////////////////////////////////////////////////////////// /// @brief return the allocated size of the logfile //////////////////////////////////////////////////////////////////////////////// @@ -156,8 +164,7 @@ namespace triagens { return 0; } - // TODO: decide whether we need _footerSize - return static_cast(allocatedSize() - _df->_footerSize - _df->_currentSize); + return static_cast(allocatedSize() - _df->_currentSize); } //////////////////////////////////////////////////////////////////////////////// @@ -210,6 +217,14 @@ namespace triagens { return (_status == StatusType::COLLECTED); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the logfile overhead +//////////////////////////////////////////////////////////////////////////////// + + static inline uint32_t overhead () { + return TRI_JOURNAL_OVERHEAD; + } + //////////////////////////////////////////////////////////////////////////////// /// @brief return the logfile status as a string //////////////////////////////////////////////////////////////////////////////// @@ -265,10 +280,10 @@ namespace triagens { break; } - LOG_INFO("changing logfile status from %s to %s for logfile %llu", - statusText(_status).c_str(), - statusText(status).c_str(), - (unsigned long long) id()); + LOG_TRACE("changing logfile status from %s to %s for logfile %llu", + statusText(_status).c_str(), + statusText(status).c_str(), + (unsigned long long) id()); _status = status; } diff --git a/arangod/Wal/LogfileManager.cpp b/arangod/Wal/LogfileManager.cpp index 3a44e344cf..18dc6de9df 100644 --- a/arangod/Wal/LogfileManager.cpp +++ b/arangod/Wal/LogfileManager.cpp @@ -44,6 +44,16 @@ using namespace triagens::wal; +// ----------------------------------------------------------------------------- +// --SECTION-- class LogfileManager +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief minimum logfile size +//////////////////////////////////////////////////////////////////////////////// + +const uint32_t LogfileManager::MinFilesize = 8 * 1024 * 1024; + // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- @@ -52,12 +62,15 @@ using namespace triagens::wal; /// @brief create the logfile manager //////////////////////////////////////////////////////////////////////////////// -LogfileManager::LogfileManager () +LogfileManager::LogfileManager (std::string* databasePath) : ApplicationFeature("logfile-manager"), + _databasePath(databasePath), _directory(), _filesize(32 * 1024 * 1024), - _reserveLogfiles(3), + _reserveLogfiles(4), _historicLogfiles(10), + _maxOpenLogfiles(10), + _allowOversizeEntries(true), _slots(nullptr), _synchroniserThread(nullptr), _allocatorThread(nullptr), @@ -68,15 +81,13 @@ LogfileManager::LogfileManager () _regex(), _shutdown(0) { - LOG_INFO("creating wal logfile manager"); + LOG_TRACE("creating wal logfile manager"); int res = regcomp(&_regex, "^logfile-([0-9][0-9]*)\\.db$", REG_EXTENDED); if (res != 0) { THROW_INTERNAL_ERROR("could not compile regex"); } - - _slots = new Slots(this, 1048576, 0); } //////////////////////////////////////////////////////////////////////////////// @@ -84,7 +95,7 @@ LogfileManager::LogfileManager () //////////////////////////////////////////////////////////////////////////////// LogfileManager::~LogfileManager () { - LOG_INFO("shutting down wal logfile manager"); + LOG_TRACE("shutting down wal logfile manager"); regfree(&_regex); @@ -103,9 +114,11 @@ LogfileManager::~LogfileManager () { void LogfileManager::setupOptions (std::map& options) { options["Write-ahead log options:help-wal"] + ("wal.allow-oversize-entries", &_allowOversizeEntries, "allow entries that are bigger than --wal.logfile-size") ("wal.logfile-size", &_filesize, "size of each logfile") - ("wal.historic-logfiles", &_historicLogfiles, "number of historic logfiles to keep after collection") - ("wal.reserve-logfiles", &_reserveLogfiles, "number of reserve logfiles to maintain") + ("wal.historic-logfiles", &_historicLogfiles, "maximum number of historic logfiles to keep after collection") + ("wal.reserve-logfiles", &_reserveLogfiles, "maximum number of reserve logfiles to maintain") + ("wal.open-logfiles", &_maxOpenLogfiles, "maximum number of parallel open logfiles") ("wal.directory", &_directory, "logfile directory") ; } @@ -115,6 +128,18 @@ void LogfileManager::setupOptions (std::maplastAssignedTick(), - (unsigned long long) _lastCollectedId); + LOG_TRACE("logfile manager last tick: %llu, last collected: %llu", + (unsigned long long) _slots->lastAssignedTick(), + (unsigned long long) _lastCollectedId); } res = openLogfiles(); @@ -194,10 +228,10 @@ bool LogfileManager::start () { } } - LOG_INFO("wal logfile manager configuration: historic logfiles: %lu, reserve logfiles: %lu, filesize: %lu", - (unsigned long) _historicLogfiles, - (unsigned long) _reserveLogfiles, - (unsigned long) _filesize); + LOG_TRACE("wal logfile manager configuration: historic logfiles: %lu, reserve logfiles: %lu, filesize: %lu", + (unsigned long) _historicLogfiles, + (unsigned long) _reserveLogfiles, + (unsigned long) _filesize); return true; } @@ -208,11 +242,18 @@ bool LogfileManager::start () { bool LogfileManager::open () { for (size_t i = 0; i < 50 * 1024 * 1024; ++i) { - void* p = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, 64, true); + size_t s; + + s = 64; +/* if (i % 10000000 == 9999999) { + s = 1024 * 1024 * 8; + } + */ + void* p = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, s, true); TRI_df_marker_t* marker = static_cast(p); marker->_type = TRI_DF_MARKER_HEADER; - marker->_size = 64; + marker->_size = s; marker->_crc = 0; marker->_tick = 0; @@ -220,7 +261,7 @@ if (i % 500000 == 0) { LOG_INFO("now at: %d", (int) i); } memcpy(static_cast(p) + sizeof(TRI_df_marker_t), "the fox is brown\0", strlen("the fox is brown") + 1); - this->allocateAndWrite(p, static_cast(64), false); + this->allocateAndWrite(p, static_cast(s), false); TRI_Free(TRI_UNKNOWN_MEM_ZONE, p); } @@ -248,20 +289,19 @@ void LogfileManager::stop () { _shutdown = 1; - LOG_INFO("stopping collector thread"); // stop threads + + LOG_TRACE("stopping collector thread"); stopCollectorThread(); - LOG_INFO("stopping allocator thread"); + LOG_TRACE("stopping allocator thread"); stopAllocatorThread(); - LOG_INFO("stopping synchroniser thread"); + LOG_TRACE("stopping synchroniser thread"); stopSynchroniserThread(); - LOG_INFO("closing logfiles"); - sleep(1); - // close all open logfiles + LOG_TRACE("closing logfiles"); closeLogfiles(); int res = writeShutdownInfo(); @@ -275,6 +315,38 @@ void LogfileManager::stop () { // --SECTION-- public methods // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief whether or not it is currently allowed to create an additional +/// logfile +//////////////////////////////////////////////////////////////////////////////// + +bool LogfileManager::logfileCreationAllowed (uint32_t size) { + if (size + Logfile::overhead() > filesize()) { + // oversize entry. this is always allowed because otherwise everything would + // lock + return true; + } + + uint32_t numberOfLogfiles = 0; + + // note: this information could also be cached instead of being recalculated + // everytime + READ_LOCKER(_logfilesLock); + + for (auto it = _logfiles.begin(); it != _logfiles.end(); ++it) { + Logfile* logfile = (*it).second; + + assert(logfile != nullptr); + + if (logfile->status() == Logfile::StatusType::OPEN || + logfile->status() == Logfile::StatusType::SEAL_REQUESTED) { + ++numberOfLogfiles; + } + } + + return (numberOfLogfiles <= _maxOpenLogfiles); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not there are reserve logfiles //////////////////////////////////////////////////////////////////////////////// @@ -285,11 +357,14 @@ bool LogfileManager::hasReserveLogfiles () { // note: this information could also be cached instead of being recalculated // everytime READ_LOCKER(_logfilesLock); - - for (auto it = _logfiles.begin(); it != _logfiles.end(); ++it) { + + // reverse-scan the logfiles map + for (auto it = _logfiles.rbegin(); it != _logfiles.rend(); ++it) { Logfile* logfile = (*it).second; + + assert(logfile != nullptr); - if (logfile != nullptr && logfile->freeSize() > 0 && ! logfile->isSealed()) { + if (logfile->freeSize() > 0 && ! logfile->isSealed()) { if (++numberOfLogfiles >= reserveLogfiles()) { return true; } @@ -307,40 +382,6 @@ void LogfileManager::signalSync () { _synchroniserThread->signalSync(); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief seal logfiles that require sealing -//////////////////////////////////////////////////////////////////////////////// - -void LogfileManager::sealLogfiles () { - std::vector logfiles; - - // create a copy of all logfiles that can be sealed - { - READ_LOCKER(_logfilesLock); - for (auto it = _logfiles.begin(); it != _logfiles.end(); ++it) { - Logfile* logfile = (*it).second; - - if (logfile != nullptr && logfile->canBeSealed()) { - logfiles.push_back(logfile); - } - } - } - - // now seal them - for (auto it = logfiles.begin(); it != logfiles.end(); ++it) { - // remove the logfile from the list of logfiles temporarily - // this is required so any concurrent operations on the logfile are not - // affect - Logfile* logfile = (*it); - unlinkLogfile(logfile); - - // TODO: handle return value - logfile->seal(); - - relinkLogfile(logfile); - } -} - //////////////////////////////////////////////////////////////////////////////// /// @brief allocate space in a logfile for later writing //////////////////////////////////////////////////////////////////////////////// @@ -351,6 +392,11 @@ SlotInfo LogfileManager::allocate (uint32_t size) { return SlotInfo(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE); } + if (size > _filesize && ! _allowOversizeEntries) { + // entry is too big for a logfile + return SlotInfo(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE); + } + return _slots->nextUnused(size); } @@ -414,47 +460,85 @@ void LogfileManager::relinkLogfile (Logfile* logfile) { /// @brief remove a logfile from the inventory only //////////////////////////////////////////////////////////////////////////////// -void LogfileManager::unlinkLogfile (Logfile* logfile) { +bool LogfileManager::unlinkLogfile (Logfile* logfile) { Logfile::IdType const id = logfile->id(); - { - WRITE_LOCKER(_logfilesLock); - auto it = _logfiles.find(id); + WRITE_LOCKER(_logfilesLock); + auto it = _logfiles.find(id); - if (it == _logfiles.end()) { - return; - } - - _logfiles.erase(it); + if (it == _logfiles.end()) { + return false; } + + _logfiles.erase(it); + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief remove a logfile from the inventory only +//////////////////////////////////////////////////////////////////////////////// + +Logfile* LogfileManager::unlinkLogfile (Logfile::IdType id) { + WRITE_LOCKER(_logfilesLock); + auto it = _logfiles.find(id); + + if (it == _logfiles.end()) { + return nullptr; + } + + _logfiles.erase(it); + + return (*it).second; } //////////////////////////////////////////////////////////////////////////////// /// @brief remove a logfile from the inventory and in the file system //////////////////////////////////////////////////////////////////////////////// -void LogfileManager::removeLogfile (Logfile* logfile) { - unlinkLogfile(logfile); - +void LogfileManager::removeLogfile (Logfile* logfile, + bool unlink) { + if (unlink) { + unlinkLogfile(logfile); + } + // old filename Logfile::IdType const id = logfile->id(); std::string const filename = logfileName(id); + LOG_TRACE("removing logfile '%s'", filename.c_str()); LOG_INFO("removing logfile '%s'", filename.c_str()); - + // now close the logfile delete logfile; int res = TRI_ERROR_NO_ERROR; // now physically remove the file - basics::FileUtils::remove(filename, &res); - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("unable to remove logfile '%s': %s", filename.c_str(), TRI_errno_string(res)); + if (! basics::FileUtils::remove(filename, &res)) { + LOG_ERROR("unable to remove logfile '%s': %s", + filename.c_str(), + TRI_errno_string(res)); return; } } +//////////////////////////////////////////////////////////////////////////////// +/// @brief seal a logfile +//////////////////////////////////////////////////////////////////////////////// + +int LogfileManager::sealLogfile (Logfile::IdType id) { + LOG_TRACE("sealing logfile '%llu", (unsigned long long) id); + + Logfile* logfile = unlinkLogfile(id); + assert(logfile != nullptr); + + logfile->seal(); + relinkLogfile(logfile); + + return TRI_ERROR_NO_ERROR; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief request sealing of a logfile //////////////////////////////////////////////////////////////////////////////// @@ -471,6 +555,20 @@ int LogfileManager::requestSealing (Logfile* logfile) { return TRI_ERROR_NO_ERROR; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the status of a logfile +//////////////////////////////////////////////////////////////////////////////// + +Logfile::StatusType LogfileManager::getLogfileStatus (Logfile::IdType id) { + READ_LOCKER(_logfilesLock); + auto it = _logfiles.find(id); + + if (it == _logfiles.end()) { + return Logfile::StatusType::UNKNOWN; + } + return (*it).second->status(); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief return the file descriptor of a logfile //////////////////////////////////////////////////////////////////////////////// @@ -481,6 +579,7 @@ int LogfileManager::getLogfileDescriptor (Logfile::IdType id) { if (it == _logfiles.end()) { // error + LOG_ERROR("could not find logfile %llu", (unsigned long long) id); return -1; } @@ -500,18 +599,42 @@ Logfile* LogfileManager::getWriteableLogfile (uint32_t size) { while (++iterations < 1000) { { WRITE_LOCKER(_logfilesLock); + auto it = _logfiles.begin(); - for (auto it = _logfiles.begin(); it != _logfiles.end(); ++it) { + while (it != _logfiles.end()) { Logfile* logfile = (*it).second; - if (logfile != nullptr && logfile->isWriteable(size)) { + assert(logfile != nullptr); + + if (logfile->isWriteable(size)) { + // found a logfile + if (logfile->status() == Logfile::StatusType::EMPTY) { + logfile->setStatus(Logfile::StatusType::OPEN); + } + return logfile; } + + if (logfile->status() == Logfile::StatusType::EMPTY && + ! logfile->isWriteable(size)) { + // we found an empty logfile, but the entry won't fit + + // delete the logfile from the sequence of logfiles + _logfiles.erase(it++); + + // and physically remove the file + // note: this will also delete the logfile object! + removeLogfile(logfile, false); + + } + else { + ++it; + } } } // signal & sleep outside the lock - _allocatorThread->signalLogfileCreation(); + _allocatorThread->signal(size); usleep(10000); } @@ -571,8 +694,12 @@ Logfile* LogfileManager::getRemovableLogfile () { void LogfileManager::setCollectionRequested (Logfile* logfile) { assert(logfile != nullptr); - WRITE_LOCKER(_logfilesLock); - logfile->setStatus(Logfile::StatusType::COLLECTION_REQUESTED); + { + WRITE_LOCKER(_logfilesLock); + logfile->setStatus(Logfile::StatusType::COLLECTION_REQUESTED); + } + + _collectorThread->signal(); } //////////////////////////////////////////////////////////////////////////////// @@ -582,8 +709,12 @@ void LogfileManager::setCollectionRequested (Logfile* logfile) { void LogfileManager::setCollectionDone (Logfile* logfile) { assert(logfile != nullptr); - WRITE_LOCKER(_logfilesLock); - logfile->setStatus(Logfile::StatusType::COLLECTED); + { + WRITE_LOCKER(_logfilesLock); + logfile->setStatus(Logfile::StatusType::COLLECTED); + } + + _collectorThread->signal(); } // ----------------------------------------------------------------------------- @@ -844,12 +975,25 @@ int LogfileManager::openLogfiles () { /// @brief allocates a new reserve logfile //////////////////////////////////////////////////////////////////////////////// -int LogfileManager::createReserveLogfile () { +int LogfileManager::createReserveLogfile (uint32_t size) { Logfile::IdType const id = nextId(); std::string const filename = logfileName(id); - LOG_INFO("creating empty logfile '%s'", filename.c_str()); - Logfile* logfile = Logfile::create(filename.c_str(), id, filesize()); + LOG_TRACE("creating empty logfile '%s' with size %lu", + filename.c_str(), + (unsigned long) size); + + uint32_t realsize; + if (size > 0 && size > filesize()) { + // create a logfile with the requested size + realsize = size + Logfile::overhead(); + } + else { + // create a logfile with default size + realsize = filesize(); + } + + Logfile* logfile = Logfile::create(filename.c_str(), id, realsize); if (logfile == nullptr) { int res = TRI_errno(); diff --git a/arangod/Wal/LogfileManager.h b/arangod/Wal/LogfileManager.h index 6aa1d8cd43..b75fd45016 100644 --- a/arangod/Wal/LogfileManager.h +++ b/arangod/Wal/LogfileManager.h @@ -68,7 +68,7 @@ namespace triagens { public: - LogfileManager (); + LogfileManager (std::string*); //////////////////////////////////////////////////////////////////////////////// /// @brief destroy the logfile manager @@ -125,12 +125,11 @@ namespace triagens { public: //////////////////////////////////////////////////////////////////////////////// -/// @brief get the maximum size of an entry +/// @brief get the maximum size of a logfile entry //////////////////////////////////////////////////////////////////////////////// inline uint32_t maxEntrySize () const { - // TODO: account for datafile overhead - return _filesize; + return 2 << 30; // 2 GB } //////////////////////////////////////////////////////////////////////////////// @@ -173,6 +172,13 @@ namespace triagens { return _slots; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief whether or not it is currently allowed to create an additional +/// logfile +//////////////////////////////////////////////////////////////////////////////// + + bool logfileCreationAllowed (uint32_t); + //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not there are reserve logfiles //////////////////////////////////////////////////////////////////////////////// @@ -185,12 +191,6 @@ namespace triagens { void signalSync (); -//////////////////////////////////////////////////////////////////////////////// -/// @brief seal logfiles that require sealing -//////////////////////////////////////////////////////////////////////////////// - - void sealLogfiles (); - //////////////////////////////////////////////////////////////////////////////// /// @brief reserve space in a logfile //////////////////////////////////////////////////////////////////////////////// @@ -222,13 +222,26 @@ namespace triagens { /// @brief remove a logfile from the inventory only //////////////////////////////////////////////////////////////////////////////// - void unlinkLogfile (Logfile*); + bool unlinkLogfile (Logfile*); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief remove a logfile from the inventory only +//////////////////////////////////////////////////////////////////////////////// + + Logfile* unlinkLogfile (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief remove a logfile from the inventory and in the file system //////////////////////////////////////////////////////////////////////////////// - void removeLogfile (Logfile*); + void removeLogfile (Logfile*, + bool); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief seal a logfile +//////////////////////////////////////////////////////////////////////////////// + + int sealLogfile (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief request sealing of a logfile @@ -236,6 +249,12 @@ namespace triagens { int requestSealing (Logfile*); +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the status of a logfile +//////////////////////////////////////////////////////////////////////////////// + + Logfile::StatusType getLogfileStatus (Logfile::IdType); + //////////////////////////////////////////////////////////////////////////////// /// @brief return the file descriptor of a logfile //////////////////////////////////////////////////////////////////////////////// @@ -356,7 +375,7 @@ namespace triagens { /// @brief allocate a new reserve logfile //////////////////////////////////////////////////////////////////////////////// - int createReserveLogfile (); + int createReserveLogfile (uint32_t); //////////////////////////////////////////////////////////////////////////////// /// @brief get an id for the next logfile @@ -388,6 +407,12 @@ namespace triagens { private: +//////////////////////////////////////////////////////////////////////////////// +/// @brief the arangod config variable containing the database path +//////////////////////////////////////////////////////////////////////////////// + + std::string* _databasePath; + //////////////////////////////////////////////////////////////////////////////// /// @brief the logfile directory //////////////////////////////////////////////////////////////////////////////// @@ -401,17 +426,29 @@ namespace triagens { uint32_t _filesize; //////////////////////////////////////////////////////////////////////////////// -/// @brief the target number of reserve logfiles +/// @brief maximum number of reserve logfiles //////////////////////////////////////////////////////////////////////////////// uint32_t _reserveLogfiles; //////////////////////////////////////////////////////////////////////////////// -/// @brief the target number of historic logfiles +/// @brief maximum number of historic logfiles //////////////////////////////////////////////////////////////////////////////// uint32_t _historicLogfiles; +//////////////////////////////////////////////////////////////////////////////// +/// @brief maximum number of parallel open logfiles +//////////////////////////////////////////////////////////////////////////////// + + uint32_t _maxOpenLogfiles; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief allow entries that are bigger than a single logfile +//////////////////////////////////////////////////////////////////////////////// + + bool _allowOversizeEntries; + //////////////////////////////////////////////////////////////////////////////// /// @brief the slots manager //////////////////////////////////////////////////////////////////////////////// @@ -461,11 +498,16 @@ namespace triagens { regex_t _regex; //////////////////////////////////////////////////////////////////////////////// -/// @brief whether or not we have been shutdown already +/// @brief whether or not we have been shut down already //////////////////////////////////////////////////////////////////////////////// volatile sig_atomic_t _shutdown; +//////////////////////////////////////////////////////////////////////////////// +/// @brief minimum logfile size +//////////////////////////////////////////////////////////////////////////////// + + static const uint32_t MinFilesize; }; } diff --git a/arangod/Wal/Slot.cpp b/arangod/Wal/Slot.cpp index a6d58669a1..5e3c3517ff 100644 --- a/arangod/Wal/Slot.cpp +++ b/arangod/Wal/Slot.cpp @@ -42,9 +42,7 @@ Slot::Slot () _logfileId(0), _mem(nullptr), _size(0), - _status(StatusType::UNUSED), - _waitForSync(false) { - + _status(StatusType::UNUSED) { } //////////////////////////////////////////////////////////////////////////////// @@ -70,6 +68,8 @@ std::string Slot::statusText () const { return "used"; case StatusType::RETURNED: return "returned"; + case StatusType::RETURNED_WFS: + return "returned (wfs)"; } // listen, damn compilers!! @@ -95,7 +95,6 @@ void Slot::setUnused () { _mem = nullptr; _size = 0; _status = StatusType::UNUSED; - _waitForSync = false; } //////////////////////////////////////////////////////////////////////////////// @@ -120,8 +119,12 @@ void Slot::setUsed (void* mem, void Slot::setReturned (bool waitForSync) { assert(isUsed()); - _status = StatusType::RETURNED; - _waitForSync = waitForSync; + if (waitForSync) { + _status = StatusType::RETURNED_WFS; + } + else { + _status = StatusType::RETURNED; + } } // Local Variables: diff --git a/arangod/Wal/Slot.h b/arangod/Wal/Slot.h index 6c1345970a..2ad5f53847 100644 --- a/arangod/Wal/Slot.h +++ b/arangod/Wal/Slot.h @@ -61,9 +61,10 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// enum class StatusType : uint32_t { - UNUSED = 0, - USED = 1, - RETURNED = 2 + UNUSED = 0, + USED = 1, + RETURNED = 2, + RETURNED_WFS = 3 }; // ----------------------------------------------------------------------------- @@ -155,7 +156,8 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// inline bool isReturned () const { - return _status == StatusType::RETURNED; + return (_status == StatusType::RETURNED || + _status == StatusType::RETURNED_WFS); } //////////////////////////////////////////////////////////////////////////////// @@ -163,7 +165,7 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// inline bool waitForSync () const { - return _waitForSync; + return (_status == StatusType::RETURNED_WFS); } //////////////////////////////////////////////////////////////////////////////// @@ -223,12 +225,6 @@ namespace triagens { StatusType _status; -//////////////////////////////////////////////////////////////////////////////// -/// @brief whether or not a sync was requested explicitly -//////////////////////////////////////////////////////////////////////////////// - - bool _waitForSync; - }; } diff --git a/arangod/Wal/Slots.cpp b/arangod/Wal/Slots.cpp index 75414e0ed4..c0cd263997 100644 --- a/arangod/Wal/Slots.cpp +++ b/arangod/Wal/Slots.cpp @@ -281,8 +281,6 @@ SyncRegion Slots::getSyncRegion () { break; } } - - // LOG_INFO("returning region: %d - %d, %d", (int) region.firstSlotIndex, (int) region.lastSlotIndex, (int) region.logfileId); return region; } diff --git a/arangod/Wal/SynchroniserThread.cpp b/arangod/Wal/SynchroniserThread.cpp index 11f037784e..a4e599006a 100644 --- a/arangod/Wal/SynchroniserThread.cpp +++ b/arangod/Wal/SynchroniserThread.cpp @@ -123,12 +123,14 @@ void SynchroniserThread::run () { if (waiting > 0) { // get region to sync SyncRegion region = _logfileManager->slots()->getSyncRegion(); + Logfile::IdType const id = region.logfileId; - if (region.logfileId != 0) { + if (id != 0) { // now perform the actual syncing + Logfile::StatusType status = _logfileManager->getLogfileStatus(id); + assert(status == Logfile::StatusType::OPEN || status == Logfile::StatusType::SEAL_REQUESTED); // get the logfile's file descriptor - // TODO: we might cache the file descriptor here int fd = getLogfileDescriptor(region); if (fd < 0) { @@ -139,19 +141,26 @@ void SynchroniserThread::run () { void** mmHandle = NULL; bool res = TRI_MSync(fd, mmHandle, region.mem, region.mem + region.size); - // LOG_INFO("Syncing from %p to %p, length: %d", region.mem, region.mem + region.size, (int) region.size); + LOG_TRACE("syncing logfile %llu, region %p - %p, length: %lu, wfs: %s", + (unsigned long long) id, + region.mem, + region.mem + region.size, + (unsigned long) region.size, + region.waitForSync ? "true" : "false"); if (! res) { LOG_ERROR("unable to sync wal logfile region"); // TODO: how to recover from this state? } + + if (status == Logfile::StatusType::SEAL_REQUESTED) { + // additionally seal the logfile + _logfileManager->sealLogfile(id); + } } _logfileManager->slots()->returnSyncRegion(region); } - - // seal any logfiles that require sealing - _logfileManager->sealLogfiles(); } // now wait until we are woken up or there is something to do @@ -161,7 +170,9 @@ void SynchroniserThread::run () { assert(_waiting >= waiting); _waiting -= waiting; } + if (_waiting == 0) { + // sleep if nothing to do guard.wait(Interval); } }