//////////////////////////////////////////////////////////////////////////////// /// @brief Write-ahead log file manager /// /// @file /// /// DISCLAIMER /// /// Copyright 2014 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. /// You may obtain a copy of the License at /// /// http://www.apache.org/licenses/LICENSE-2.0 /// /// Unless required by applicable law or agreed to in writing, software /// distributed under the License is distributed on an "AS IS" BASIS, /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. /// See the License for the specific language governing permissions and /// limitations under the License. /// /// Copyright holder is ArangoDB GmbH, Cologne, Germany /// /// @author Jan Steemann /// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany /// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany //////////////////////////////////////////////////////////////////////////////// #ifndef ARANGODB_WAL_LOGFILE_MANAGER_H #define ARANGODB_WAL_LOGFILE_MANAGER_H 1 #include "Basics/Common.h" #include "Basics/Mutex.h" #include "Basics/ReadWriteLock.h" #include "ApplicationServer/ApplicationFeature.h" #include "VocBase/voc-types.h" #include "Wal/Logfile.h" #include "Wal/Marker.h" #include "Wal/Slots.h" #include struct TRI_server_t; namespace triagens { namespace wal { class AllocatorThread; class CollectorThread; struct RecoverState; class RemoverThread; class Slot; class SynchronizerThread; struct LogfileRange { LogfileRange (Logfile::IdType id, std::string const& filename, std::string const& state, TRI_voc_tick_t tickMin, TRI_voc_tick_t tickMax) : id(id), filename(filename), state(state), tickMin(tickMin), tickMax(tickMax) { } Logfile::IdType id; std::string filename; std::string state; TRI_voc_tick_t tickMin; TRI_voc_tick_t tickMax; }; typedef std::vector LogfileRanges; // ----------------------------------------------------------------------------- // --SECTION-- LogfileManagerState // ----------------------------------------------------------------------------- struct LogfileManagerState { TRI_voc_tick_t lastTick; TRI_voc_tick_t lastDataTick; uint64_t numEvents; std::string timeString; }; // ----------------------------------------------------------------------------- // --SECTION-- class LogfileManager // ----------------------------------------------------------------------------- class LogfileManager : public rest::ApplicationFeature { friend class AllocatorThread; friend class CollectorThread; //////////////////////////////////////////////////////////////////////////////// /// @brief LogfileManager //////////////////////////////////////////////////////////////////////////////// private: LogfileManager (LogfileManager const&) = delete; LogfileManager& operator= (LogfileManager const&) = delete; // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- public: LogfileManager (TRI_server_t*, std::string*); //////////////////////////////////////////////////////////////////////////////// /// @brief destroy the logfile manager //////////////////////////////////////////////////////////////////////////////// ~LogfileManager (); //////////////////////////////////////////////////////////////////////////////// /// @brief get the logfile manager instance //////////////////////////////////////////////////////////////////////////////// static LogfileManager* instance (); //////////////////////////////////////////////////////////////////////////////// /// @brief initialize the logfile manager instance //////////////////////////////////////////////////////////////////////////////// static void initialize (std::string*, TRI_server_t*); // ----------------------------------------------------------------------------- // --SECTION-- ApplicationFeature methods // ----------------------------------------------------------------------------- public: //////////////////////////////////////////////////////////////////////////////// /// {@inheritDoc} //////////////////////////////////////////////////////////////////////////////// void setupOptions (std::map&); //////////////////////////////////////////////////////////////////////////////// /// {@inheritDoc} //////////////////////////////////////////////////////////////////////////////// bool prepare (); //////////////////////////////////////////////////////////////////////////////// /// {@inheritDoc} //////////////////////////////////////////////////////////////////////////////// bool open (); //////////////////////////////////////////////////////////////////////////////// /// {@inheritDoc} //////////////////////////////////////////////////////////////////////////////// bool start (); //////////////////////////////////////////////////////////////////////////////// /// {@inheritDoc} //////////////////////////////////////////////////////////////////////////////// void close (); //////////////////////////////////////////////////////////////////////////////// /// {@inheritDoc} //////////////////////////////////////////////////////////////////////////////// void stop (); // ----------------------------------------------------------------------------- // --SECTION-- public methods // ----------------------------------------------------------------------------- public: //////////////////////////////////////////////////////////////////////////////// /// @brief get the logfile directory //////////////////////////////////////////////////////////////////////////////// inline std::string directory () const { return _directory; } //////////////////////////////////////////////////////////////////////////////// /// @brief get the logfile size //////////////////////////////////////////////////////////////////////////////// inline uint32_t filesize () const { return _filesize; } //////////////////////////////////////////////////////////////////////////////// /// @brief set the logfile size //////////////////////////////////////////////////////////////////////////////// inline void filesize (uint32_t value) { _filesize = value; } //////////////////////////////////////////////////////////////////////////////// /// @brief get the sync interval //////////////////////////////////////////////////////////////////////////////// inline uint64_t syncInterval () const { return _syncInterval / 1000; } //////////////////////////////////////////////////////////////////////////////// /// @brief set the sync interval //////////////////////////////////////////////////////////////////////////////// inline void syncInterval (uint64_t value) { _syncInterval = value * 1000; } //////////////////////////////////////////////////////////////////////////////// /// @brief get the number of reserve logfiles //////////////////////////////////////////////////////////////////////////////// inline uint32_t reserveLogfiles () const { return _reserveLogfiles; } //////////////////////////////////////////////////////////////////////////////// /// @brief set the number of reserve logfiles //////////////////////////////////////////////////////////////////////////////// inline void reserveLogfiles (uint32_t value) { _reserveLogfiles = value; } //////////////////////////////////////////////////////////////////////////////// /// @brief get the number of historic logfiles to keep //////////////////////////////////////////////////////////////////////////////// inline uint32_t historicLogfiles () const { return _historicLogfiles; } //////////////////////////////////////////////////////////////////////////////// /// @brief set the number of historic logfiles //////////////////////////////////////////////////////////////////////////////// inline void historicLogfiles (uint32_t value) { _historicLogfiles = value; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not shape information should be suppress when writing /// markers into the write-ahead log //////////////////////////////////////////////////////////////////////////////// inline bool suppressShapeInformation () const { return _suppressShapeInformation; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not there was a SHUTDOWN file with a tick value /// at server start //////////////////////////////////////////////////////////////////////////////// inline bool hasFoundLastTick () const { return _hasFoundLastTick; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not we are in the recovery phase //////////////////////////////////////////////////////////////////////////////// inline bool isInRecovery () const { return _inRecovery; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not we are in the shutdown phase //////////////////////////////////////////////////////////////////////////////// inline bool isInShutdown () const { return (_shutdown != 0); } //////////////////////////////////////////////////////////////////////////////// /// @brief return the slots manager //////////////////////////////////////////////////////////////////////////////// Slots* slots () { return _slots; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not oversize entries are allowed //////////////////////////////////////////////////////////////////////////////// inline bool allowOversizeEntries () const { return _allowOversizeEntries; } //////////////////////////////////////////////////////////////////////////////// /// @brief sets the "allowOversizeEntries" value //////////////////////////////////////////////////////////////////////////////// inline void allowOversizeEntries (bool value) { _allowOversizeEntries = value; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not write-throttling can be enabled //////////////////////////////////////////////////////////////////////////////// inline bool canBeThrottled () const { return (_throttleWhenPending > 0); } //////////////////////////////////////////////////////////////////////////////// /// @brief maximum wait time when write-throttled (in milliseconds) //////////////////////////////////////////////////////////////////////////////// inline uint64_t maxThrottleWait () const { return _maxThrottleWait; } //////////////////////////////////////////////////////////////////////////////// /// @brief maximum wait time when write-throttled (in milliseconds) //////////////////////////////////////////////////////////////////////////////// inline void maxThrottleWait (uint64_t value) { _maxThrottleWait = value; } //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not write-throttling is currently enabled //////////////////////////////////////////////////////////////////////////////// inline bool isThrottled () { return (_writeThrottled != 0); } //////////////////////////////////////////////////////////////////////////////// /// @brief activate write-throttling //////////////////////////////////////////////////////////////////////////////// void activateWriteThrottling () { _writeThrottled = 1; } //////////////////////////////////////////////////////////////////////////////// /// @brief deactivate write-throttling //////////////////////////////////////////////////////////////////////////////// void deactivateWriteThrottling () { _writeThrottled = 0; } //////////////////////////////////////////////////////////////////////////////// /// @brief allow or disallow writes to the WAL //////////////////////////////////////////////////////////////////////////////// inline void allowWrites (bool value) { _allowWrites = value; } //////////////////////////////////////////////////////////////////////////////// /// @brief get the value of --wal.throttle-when-pending //////////////////////////////////////////////////////////////////////////////// inline uint64_t throttleWhenPending () const { return _throttleWhenPending; } //////////////////////////////////////////////////////////////////////////////// /// @brief set the value of --wal.throttle-when-pending //////////////////////////////////////////////////////////////////////////////// inline void throttleWhenPending (uint64_t value) { _throttleWhenPending = value; if (_throttleWhenPending == 0) { deactivateWriteThrottling(); } } //////////////////////////////////////////////////////////////////////////////// /// @brief registers a transaction //////////////////////////////////////////////////////////////////////////////// int registerTransaction (TRI_voc_tid_t); //////////////////////////////////////////////////////////////////////////////// /// @brief unregisters a transaction //////////////////////////////////////////////////////////////////////////////// void unregisterTransaction (TRI_voc_tid_t, bool); //////////////////////////////////////////////////////////////////////////////// /// @brief return the set of failed transactions //////////////////////////////////////////////////////////////////////////////// std::unordered_set getFailedTransactions (); //////////////////////////////////////////////////////////////////////////////// /// @brief return the set of dropped collections /// this is used during recovery and not used afterwards //////////////////////////////////////////////////////////////////////////////// std::unordered_set getDroppedCollections (); //////////////////////////////////////////////////////////////////////////////// /// @brief return the set of dropped databases /// this is used during recovery and not used afterwards //////////////////////////////////////////////////////////////////////////////// std::unordered_set getDroppedDatabases (); //////////////////////////////////////////////////////////////////////////////// /// @brief unregister a list of failed transactions //////////////////////////////////////////////////////////////////////////////// void unregisterFailedTransactions (std::unordered_set const&); //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not it is currently allowed to create an additional /// logfile //////////////////////////////////////////////////////////////////////////////// bool logfileCreationAllowed (uint32_t); //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not there are reserve logfiles //////////////////////////////////////////////////////////////////////////////// bool hasReserveLogfiles (); //////////////////////////////////////////////////////////////////////////////// /// @brief signal that a sync operation is required //////////////////////////////////////////////////////////////////////////////// void signalSync (); //////////////////////////////////////////////////////////////////////////////// /// @brief reserve space in a logfile //////////////////////////////////////////////////////////////////////////////// SlotInfo allocate (uint32_t); //////////////////////////////////////////////////////////////////////////////// /// @brief reserve space in a logfile, version for legends //////////////////////////////////////////////////////////////////////////////// SlotInfo allocate (uint32_t, TRI_voc_cid_t cid, TRI_shape_sid_t sid, uint32_t legendOffset, void*& oldLegend); //////////////////////////////////////////////////////////////////////////////// /// @brief finalize a log entry //////////////////////////////////////////////////////////////////////////////// void finalize (SlotInfo&, bool); //////////////////////////////////////////////////////////////////////////////// /// @brief write data into the logfile /// this is a convenience function that combines allocate, memcpy and finalize //////////////////////////////////////////////////////////////////////////////// SlotInfoCopy allocateAndWrite (void*, uint32_t, bool); //////////////////////////////////////////////////////////////////////////////// /// @brief write data into the logfile /// this is a convenience function that combines allocate, memcpy and finalize, /// this version is for markers with legends //////////////////////////////////////////////////////////////////////////////// SlotInfoCopy allocateAndWrite (void*, uint32_t, bool, TRI_voc_cid_t, TRI_shape_sid_t, uint32_t, void*&); //////////////////////////////////////////////////////////////////////////////// /// @brief write data into the logfile /// this is a convenience function that combines allocate, memcpy and finalize //////////////////////////////////////////////////////////////////////////////// SlotInfoCopy allocateAndWrite (Marker const&, bool); //////////////////////////////////////////////////////////////////////////////// /// @brief wait for the collector queue to get cleared for the given collection //////////////////////////////////////////////////////////////////////////////// int waitForCollectorQueue (TRI_voc_cid_t, double); //////////////////////////////////////////////////////////////////////////////// /// @brief finalize and seal the currently open logfile /// this is useful to ensure that any open writes up to this point have made /// it into a logfile //////////////////////////////////////////////////////////////////////////////// int flush (bool, bool, bool); //////////////////////////////////////////////////////////////////////////////// /// @brief re-inserts a logfile back into the inventory only //////////////////////////////////////////////////////////////////////////////// void relinkLogfile (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief remove a logfile from the inventory only //////////////////////////////////////////////////////////////////////////////// bool unlinkLogfile (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief remove a logfile from the inventory only //////////////////////////////////////////////////////////////////////////////// Logfile* unlinkLogfile (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief removes logfiles that are allowed to be removed //////////////////////////////////////////////////////////////////////////////// bool removeLogfiles (); //////////////////////////////////////////////////////////////////////////////// /// @brief sets the status of a logfile to open //////////////////////////////////////////////////////////////////////////////// void setLogfileOpen (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief sets the status of a logfile to seal-requested //////////////////////////////////////////////////////////////////////////////// void setLogfileSealRequested (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief sets the status of a logfile to sealed //////////////////////////////////////////////////////////////////////////////// void setLogfileSealed (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief sets the status of a logfile to sealed //////////////////////////////////////////////////////////////////////////////// void setLogfileSealed (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief return the status of a logfile //////////////////////////////////////////////////////////////////////////////// Logfile::StatusType getLogfileStatus (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief return the file descriptor of a logfile //////////////////////////////////////////////////////////////////////////////// int getLogfileDescriptor (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief get the current open region of a logfile /// this uses the slots lock //////////////////////////////////////////////////////////////////////////////// void getActiveLogfileRegion (Logfile*, char const*&, char const*&); //////////////////////////////////////////////////////////////////////////////// /// @brief get logfiles for a tick range //////////////////////////////////////////////////////////////////////////////// std::vector getLogfilesForTickRange (TRI_voc_tick_t, TRI_voc_tick_t, bool& minTickIncluded); //////////////////////////////////////////////////////////////////////////////// /// @brief return logfiles for a tick range //////////////////////////////////////////////////////////////////////////////// void returnLogfiles (std::vector const&); //////////////////////////////////////////////////////////////////////////////// /// @brief get a logfile by id //////////////////////////////////////////////////////////////////////////////// Logfile* getLogfile (Logfile::IdType); //////////////////////////////////////////////////////////////////////////////// /// @brief get a logfile and its status by id //////////////////////////////////////////////////////////////////////////////// Logfile* getLogfile (Logfile::IdType, Logfile::StatusType&); //////////////////////////////////////////////////////////////////////////////// /// @brief get a logfile for writing. this may return nullptr //////////////////////////////////////////////////////////////////////////////// int getWriteableLogfile (uint32_t, Logfile::StatusType&, Logfile*&); //////////////////////////////////////////////////////////////////////////////// /// @brief get a logfile to collect. this may return nullptr //////////////////////////////////////////////////////////////////////////////// Logfile* getCollectableLogfile (); //////////////////////////////////////////////////////////////////////////////// /// @brief get a logfile to remove. this may return nullptr /// if it returns a logfile, the logfile is removed from the list of available /// logfiles //////////////////////////////////////////////////////////////////////////////// Logfile* getRemovableLogfile (); //////////////////////////////////////////////////////////////////////////////// /// @brief increase the number of collect operations for a logfile //////////////////////////////////////////////////////////////////////////////// void increaseCollectQueueSize (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief decrease the number of collect operations for a logfile //////////////////////////////////////////////////////////////////////////////// void decreaseCollectQueueSize (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief mark a file as being requested for collection //////////////////////////////////////////////////////////////////////////////// void setCollectionRequested (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief mark a file as being done with collection //////////////////////////////////////////////////////////////////////////////// void setCollectionDone (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief force the status of a specific logfile //////////////////////////////////////////////////////////////////////////////// void forceStatus (Logfile*, Logfile::StatusType); //////////////////////////////////////////////////////////////////////////////// /// @brief return the current state //////////////////////////////////////////////////////////////////////////////// LogfileManagerState state (); //////////////////////////////////////////////////////////////////////////////// /// @brief return the current available logfile ranges //////////////////////////////////////////////////////////////////////////////// LogfileRanges ranges (); //////////////////////////////////////////////////////////////////////////////// /// @brief get information about running transactions //////////////////////////////////////////////////////////////////////////////// std::tuple runningTransactions (); // ----------------------------------------------------------------------------- // --SECTION-- private methods // ----------------------------------------------------------------------------- private: //////////////////////////////////////////////////////////////////////////////// /// @brief remove a logfile in the file system //////////////////////////////////////////////////////////////////////////////// void removeLogfile (Logfile*); //////////////////////////////////////////////////////////////////////////////// /// @brief wait for the collector thread to collect a specific logfile //////////////////////////////////////////////////////////////////////////////// int waitForCollector (Logfile::IdType, double); //////////////////////////////////////////////////////////////////////////////// /// @brief run the recovery procedure /// this is called after the logfiles have been scanned completely and /// recovery state has been build. additionally, all databases have been /// opened already so we can use collections //////////////////////////////////////////////////////////////////////////////// int runRecovery (); //////////////////////////////////////////////////////////////////////////////// /// @brief closes all logfiles //////////////////////////////////////////////////////////////////////////////// void closeLogfiles (); //////////////////////////////////////////////////////////////////////////////// /// @brief reads the shutdown information //////////////////////////////////////////////////////////////////////////////// int readShutdownInfo (); //////////////////////////////////////////////////////////////////////////////// /// @brief writes the shutdown information //////////////////////////////////////////////////////////////////////////////// int writeShutdownInfo (bool); //////////////////////////////////////////////////////////////////////////////// /// @brief start the synchronizer thread //////////////////////////////////////////////////////////////////////////////// int startSynchronizerThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief stop the synchronizer thread //////////////////////////////////////////////////////////////////////////////// void stopSynchronizerThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief start the allocator thread //////////////////////////////////////////////////////////////////////////////// int startAllocatorThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief stop the allocator thread //////////////////////////////////////////////////////////////////////////////// void stopAllocatorThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief start the collector thread //////////////////////////////////////////////////////////////////////////////// int startCollectorThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief stop the collector thread //////////////////////////////////////////////////////////////////////////////// void stopCollectorThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief start the remover thread //////////////////////////////////////////////////////////////////////////////// int startRemoverThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief stop the remover thread //////////////////////////////////////////////////////////////////////////////// void stopRemoverThread (); //////////////////////////////////////////////////////////////////////////////// /// @brief check which logfiles are present in the log directory //////////////////////////////////////////////////////////////////////////////// int inventory (); //////////////////////////////////////////////////////////////////////////////// /// @brief inspect all found WAL logfiles /// this searches for the max tick in the logfiles and builds up the initial /// transaction state //////////////////////////////////////////////////////////////////////////////// int inspectLogfiles (); //////////////////////////////////////////////////////////////////////////////// /// @brief allocate a new reserve logfile //////////////////////////////////////////////////////////////////////////////// int createReserveLogfile (uint32_t); //////////////////////////////////////////////////////////////////////////////// /// @brief get an id for the next logfile //////////////////////////////////////////////////////////////////////////////// Logfile::IdType nextId (); //////////////////////////////////////////////////////////////////////////////// /// @brief ensure the wal logfiles directory is actually there //////////////////////////////////////////////////////////////////////////////// int ensureDirectory (); //////////////////////////////////////////////////////////////////////////////// /// @brief return the absolute name of the shutdown file //////////////////////////////////////////////////////////////////////////////// std::string shutdownFilename () const; //////////////////////////////////////////////////////////////////////////////// /// @brief return an absolute filename for a logfile id //////////////////////////////////////////////////////////////////////////////// std::string logfileName (Logfile::IdType) const; //////////////////////////////////////////////////////////////////////////////// /// @brief return the current time as a string //////////////////////////////////////////////////////////////////////////////// static std::string getTimeString (); // ----------------------------------------------------------------------------- // --SECTION-- private variables // ----------------------------------------------------------------------------- private: //////////////////////////////////////////////////////////////////////////////// /// @brief pointer to the server //////////////////////////////////////////////////////////////////////////////// TRI_server_t* _server; //////////////////////////////////////////////////////////////////////////////// /// @brief the arangod config variable containing the database path //////////////////////////////////////////////////////////////////////////////// std::string* _databasePath; //////////////////////////////////////////////////////////////////////////////// /// @brief the WAL logfiles directory /// @startDocuBlock WalLogfileDirectory /// `--wal.directory` /// /// Specifies the directory in which the write-ahead logfiles should be /// stored. If this option is not specified, it defaults to the subdirectory /// *journals* in the server's global database directory. If the directory is /// not present, it will be created. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// std::string _directory; //////////////////////////////////////////////////////////////////////////////// /// @brief state during recovery //////////////////////////////////////////////////////////////////////////////// RecoverState* _recoverState; //////////////////////////////////////////////////////////////////////////////// /// @brief the size of each WAL logfile /// @startDocuBlock WalLogfileSize /// `--wal.logfile-size` /// /// Specifies the filesize (in bytes) for each write-ahead logfile. The logfile /// size should be chosen so that each logfile can store a considerable amount of /// documents. The bigger the logfile size is chosen, the longer it will take /// to fill up a single logfile, which also influences the delay until the data /// in a logfile will be garbage-collected and written to collection journals /// and datafiles. It also affects how long logfile recovery will take at /// server start. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// uint32_t _filesize; //////////////////////////////////////////////////////////////////////////////// /// @brief maximum number of reserve logfiles /// @startDocuBlock WalLogfileReserveLogfiles /// `--wal.reserve-logfiles` /// /// The maximum number of reserve logfiles that ArangoDB will create in a /// background process. Reserve logfiles are useful in the situation when an /// operation needs to be written to a logfile but the reserve space in the /// logfile is too low for storing the operation. In this case, a new logfile /// needs to be created to store the operation. Creating new logfiles is /// normally slow, so ArangoDB will try to pre-create logfiles in a background /// process so there are always reserve logfiles when the active logfile gets /// full. The number of reserve logfiles that ArangoDB keeps in the background /// is configurable with this option. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// uint32_t _reserveLogfiles; //////////////////////////////////////////////////////////////////////////////// /// @brief maximum number of historic logfiles /// @startDocuBlock WalLogfileHistoricLogfiles /// `--wal.historic-logfiles` /// /// The maximum number of historic logfiles that ArangoDB will keep after they /// have been garbage-collected. If no replication is used, there is no need /// to keep historic logfiles except for having a local changelog. /// /// In a replication setup, the number of historic logfiles affects the amount /// of data a slave can fetch from the master's logs. The more historic /// logfiles, the more historic data is available for a slave, which is useful /// if the connection between master and slave is unstable or slow. Not having /// enough historic logfiles available might lead to logfile data being deleted /// on the master already before a slave has fetched it. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// uint32_t _historicLogfiles; //////////////////////////////////////////////////////////////////////////////// /// @brief maximum number of parallel open logfiles //////////////////////////////////////////////////////////////////////////////// uint32_t _maxOpenLogfiles; //////////////////////////////////////////////////////////////////////////////// /// @brief maximum number of slots to be used in parallel /// @startDocuBlock WalLogfileSlots /// `--wal.slots` /// /// Configures the amount of write slots the write-ahead log can give to write /// operations in parallel. Any write operation will lease a slot and return it /// to the write-ahead log when it is finished writing the data. A slot will /// remain blocked until the data in it was synchronized to disk. After that, /// a slot becomes reusable by following operations. The required number of /// slots is thus determined by the parallelity of write operations and the /// disk synchronization speed. Slow disks probably need higher values, and fast /// disks may only require a value lower than the default. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// uint32_t _numberOfSlots; //////////////////////////////////////////////////////////////////////////////// /// @brief interval for automatic, non-requested disk syncs /// @startDocuBlock WalLogfileSyncInterval /// `--wal.sync-interval` /// /// The interval (in milliseconds) that ArangoDB will use to automatically /// synchronize data in its write-ahead logs to disk. Automatic syncs will only /// be performed for not-yet synchronized data, and only for operations that /// have been executed without the *waitForSync* attribute. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// uint64_t _syncInterval; //////////////////////////////////////////////////////////////////////////////// /// @brief maximum wait time for write-throttling //////////////////////////////////////////////////////////////////////////////// uint64_t _maxThrottleWait; //////////////////////////////////////////////////////////////////////////////// /// @brief throttle writes to WAL when at least such many operations are /// waiting for garbage collection /// @startDocuBlock WalLogfileThrottling /// `--wal.throttle-when-pending` /// /// The maximum value for the number of write-ahead log garbage-collection queue /// elements. If set to *0*, the queue size is unbounded, and no /// writtle-throttling will occur. If set to a non-zero value, writte-throttling /// will automatically kick in when the garbage-collection queue contains at /// least as many elements as specified by this option. /// While write-throttling is active, data-modification operations will /// intentionally be delayed by a configurable amount of time. This is to /// ensure the write-ahead log garbage collector can catch up with the /// operations executed. /// Write-throttling will stay active until the garbage-collection queue size /// goes down below the specified value. /// Write-throttling is turned off by default. /// /// `--wal.throttle-wait` /// /// This option determines the maximum wait time (in milliseconds) for /// operations that are write-throttled. If write-throttling is active and a /// new write operation is to be executed, it will wait for at most the /// specified amount of time for the write-ahead log garbage-collection queue /// size to fall below the throttling threshold. If the queue size decreases /// before the maximum wait time is over, the operation will be executed /// normally. If the queue size does not decrease before the wait time is over, /// the operation will be aborted with an error. /// This option only has an effect if `--wal.throttle-when-pending` has a /// non-zero value, which is not the default. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// uint64_t _throttleWhenPending; //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not oversize entries are allowed /// @startDocuBlock WalLogfileAllowOversizeEntries /// `--wal.allow-oversize-entries` /// /// Whether or not it is allowed to store individual documents that are bigger /// than would fit into a single logfile. Setting the option to false will make /// such operations fail with an error. Setting the option to true will make /// such operations succeed, but with a high potential performance impact. /// The reason is that for each oversize operation, an individual oversize /// logfile needs to be created which may also block other operations. /// The option should be set to *false* if it is certain that documents will /// always have a size smaller than a single logfile. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// bool _allowOversizeEntries; //////////////////////////////////////////////////////////////////////////////// /// @brief ignore logfile errors when opening logfiles /// @startDocuBlock WalLogfileIgnoreLogfileErrors /// `--wal.ignore-logfile-errors` /// /// Ignores any recovery errors caused by corrupted logfiles on startup. When /// set to *false*, the recovery procedure on startup will fail with an error /// whenever it encounters a corrupted (that includes only half-written) /// logfile. This is a security precaution to prevent data loss in case of disk /// errors etc. When the recovery procedure aborts because of corruption, any /// corrupted files can be inspected and fixed (or removed) manually and the /// server can be restarted afterwards. /// /// Setting the option to *true* will make the server continue with the recovery /// procedure even in case it detects corrupt logfile entries. In this case it /// will stop at the first corrupted logfile entry and ignore all others, which /// might cause data loss. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// bool _ignoreLogfileErrors; //////////////////////////////////////////////////////////////////////////////// /// @brief ignore recovery errors /// @startDocuBlock WalLogfileIgnoreRecoveryErrors /// `--wal.ignore-recovery-errors` /// /// Ignores any recovery errors not caused by corrupted logfiles but by logical /// errors. Logical errors can occur if logfiles or any other server datafiles /// have been manually edited or the server is somehow misconfigured. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// bool _ignoreRecoveryErrors; //////////////////////////////////////////////////////////////////////////////// /// @brief suppress shape information /// @startDocuBlock WalLogfileSuppressShapeInformation /// `--wal.suppress-shape-information` /// /// Setting this variable to *true* will lead to no shape information being /// written into the write-ahead logfiles for documents or edges. While this is /// a good optimization for a single server to save memory (and disk space), it /// it will effectively disable using the write-ahead log as a reliable source /// for replicating changes to other servers. A master server with this option /// set to *true* will not be able to fully reproduce the structure of saved /// documents after a collection has been deleted. In case a replication client /// requests a document for which the collection is already deleted, the master /// will return an empty document. Note that this only affects replication and /// not normal operation on the master. /// /// **Do not set this variable to *true* on a server that you plan to use as a /// replication master** /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// bool _suppressShapeInformation; //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not writes to the WAL are allowed //////////////////////////////////////////////////////////////////////////////// bool _allowWrites; //////////////////////////////////////////////////////////////////////////////// /// @brief this is true if there was a SHUTDOWN file with a last tick at /// server start //////////////////////////////////////////////////////////////////////////////// bool _hasFoundLastTick; //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not the recovery procedure is running //////////////////////////////////////////////////////////////////////////////// bool _inRecovery; //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not the logfile manager was properly initialized and /// started //////////////////////////////////////////////////////////////////////////////// bool _startCalled; //////////////////////////////////////////////////////////////////////////////// /// @brief a lock protecting the _logfiles map and the logfiles' statuses //////////////////////////////////////////////////////////////////////////////// basics::ReadWriteLock _logfilesLock; //////////////////////////////////////////////////////////////////////////////// /// @brief the logfiles //////////////////////////////////////////////////////////////////////////////// std::map _logfiles; //////////////////////////////////////////////////////////////////////////////// /// @brief the slots manager //////////////////////////////////////////////////////////////////////////////// Slots* _slots; //////////////////////////////////////////////////////////////////////////////// /// @brief the synchronizer thread //////////////////////////////////////////////////////////////////////////////// SynchronizerThread* _synchronizerThread; //////////////////////////////////////////////////////////////////////////////// /// @brief the allocator thread //////////////////////////////////////////////////////////////////////////////// AllocatorThread* _allocatorThread; //////////////////////////////////////////////////////////////////////////////// /// @brief the collector thread //////////////////////////////////////////////////////////////////////////////// CollectorThread* _collectorThread; //////////////////////////////////////////////////////////////////////////////// /// @brief the logfile remover thread //////////////////////////////////////////////////////////////////////////////// RemoverThread* _removerThread; //////////////////////////////////////////////////////////////////////////////// /// @brief last opened logfile id. note: writing to this variable is protected /// by the _idLock //////////////////////////////////////////////////////////////////////////////// std::atomic _lastOpenedId; //////////////////////////////////////////////////////////////////////////////// /// @brief last fully collected logfile id. note: writing to this variable is /// protected by the_idLock //////////////////////////////////////////////////////////////////////////////// std::atomic _lastCollectedId; //////////////////////////////////////////////////////////////////////////////// /// @brief last fully sealed logfile id. note: writing to this variable is /// protected by the _idLock //////////////////////////////////////////////////////////////////////////////// std::atomic _lastSealedId; //////////////////////////////////////////////////////////////////////////////// /// @brief a lock protecting the shutdown file //////////////////////////////////////////////////////////////////////////////// basics::Mutex _shutdownFileLock; //////////////////////////////////////////////////////////////////////////////// /// @brief a lock protecting _transactions and _failedTransactions //////////////////////////////////////////////////////////////////////////////// basics::ReadWriteLock _transactionsLock; //////////////////////////////////////////////////////////////////////////////// /// @brief currently ongoing transactions //////////////////////////////////////////////////////////////////////////////// std::unordered_map> _transactions; //////////////////////////////////////////////////////////////////////////////// /// @brief set of failed transactions //////////////////////////////////////////////////////////////////////////////// std::unordered_set _failedTransactions; //////////////////////////////////////////////////////////////////////////////// /// @brief set of dropped collections /// this is populated during recovery and not used afterwards //////////////////////////////////////////////////////////////////////////////// std::unordered_set _droppedCollections; //////////////////////////////////////////////////////////////////////////////// /// @brief set of dropped databases /// this is populated during recovery and not used afterwards //////////////////////////////////////////////////////////////////////////////// std::unordered_set _droppedDatabases; //////////////////////////////////////////////////////////////////////////////// /// @brief a lock protecting the updates of _lastCollectedId, _lastSealedId, /// and _lastOpenedId //////////////////////////////////////////////////////////////////////////////// basics::Mutex _idLock; //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not write-throttling is currently enabled //////////////////////////////////////////////////////////////////////////////// int _writeThrottled; //////////////////////////////////////////////////////////////////////////////// /// @brief regex to match logfiles //////////////////////////////////////////////////////////////////////////////// regex_t _filenameRegex; //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not we have been shut down already //////////////////////////////////////////////////////////////////////////////// volatile sig_atomic_t _shutdown; }; } } #endif // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // ----------------------------------------------------------------------------- // Local Variables: // mode: outline-minor // outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" // End: