Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
arangodb/arangod/Wal/LogfileManager.h

1230 lines
51 KiB
C++
Raw Blame History

////////////////////////////////////////////////////////////////////////////////
/// @brief Write-ahead log file manager
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_WAL_LOGFILE_MANAGER_H
#define ARANGODB_WAL_LOGFILE_MANAGER_H 1
#include "Basics/Common.h"
#include "Basics/Mutex.h"
#include "Basics/ReadWriteLock.h"
#include "ApplicationServer/ApplicationFeature.h"
#include "VocBase/voc-types.h"
#include "Wal/Logfile.h"
#include "Wal/Marker.h"
#include "Wal/Slots.h"
#include <regex.h>
struct TRI_server_t;
namespace triagens {
namespace wal {
class AllocatorThread;
class CollectorThread;
struct RecoverState;
class RemoverThread;
class Slot;
class SynchroniserThread;
struct LogfileRange {
LogfileRange (Logfile::IdType id,
std::string const& filename,
std::string const& state,
TRI_voc_tick_t tickMin,
TRI_voc_tick_t tickMax)
: id(id),
filename(filename),
state(state),
tickMin(tickMin),
tickMax(tickMax) {
}
Logfile::IdType id;
std::string filename;
std::string state;
TRI_voc_tick_t tickMin;
TRI_voc_tick_t tickMax;
};
typedef std::vector<LogfileRange> LogfileRanges;
// -----------------------------------------------------------------------------
// --SECTION-- LogfileManagerState
// -----------------------------------------------------------------------------
struct LogfileManagerState {
TRI_voc_tick_t lastTick;
TRI_voc_tick_t lastDataTick;
uint64_t numEvents;
std::string timeString;
};
// -----------------------------------------------------------------------------
// --SECTION-- class LogfileManager
// -----------------------------------------------------------------------------
class LogfileManager : public rest::ApplicationFeature {
friend class AllocatorThread;
friend class CollectorThread;
////////////////////////////////////////////////////////////////////////////////
/// @brief LogfileManager
////////////////////////////////////////////////////////////////////////////////
private:
LogfileManager (LogfileManager const&) = delete;
LogfileManager& operator= (LogfileManager const&) = delete;
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
public:
LogfileManager (TRI_server_t*,
std::string*);
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy the logfile manager
////////////////////////////////////////////////////////////////////////////////
~LogfileManager ();
////////////////////////////////////////////////////////////////////////////////
/// @brief get the logfile manager instance
////////////////////////////////////////////////////////////////////////////////
static LogfileManager* instance ();
////////////////////////////////////////////////////////////////////////////////
/// @brief initialize the logfile manager instance
////////////////////////////////////////////////////////////////////////////////
static void initialize (std::string*,
TRI_server_t*);
// -----------------------------------------------------------------------------
// --SECTION-- ApplicationFeature methods
// -----------------------------------------------------------------------------
public:
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////
void setupOptions (std::map<std::string, triagens::basics::ProgramOptionsDescription>&);
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////
bool prepare ();
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////
bool open ();
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////
bool start ();
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////
void close ();
////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////
void stop ();
// -----------------------------------------------------------------------------
// --SECTION-- public methods
// -----------------------------------------------------------------------------
public:
////////////////////////////////////////////////////////////////////////////////
/// @brief get the logfile directory
////////////////////////////////////////////////////////////////////////////////
inline std::string directory () const {
return _directory;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the logfile size
////////////////////////////////////////////////////////////////////////////////
inline uint32_t filesize () const {
return _filesize;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief set the logfile size
////////////////////////////////////////////////////////////////////////////////
inline void filesize (uint32_t value) {
_filesize = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the sync interval
////////////////////////////////////////////////////////////////////////////////
inline uint64_t syncInterval () const {
return _syncInterval / 1000;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief set the sync interval
////////////////////////////////////////////////////////////////////////////////
inline void syncInterval (uint64_t value) {
_syncInterval = value * 1000;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the number of reserve logfiles
////////////////////////////////////////////////////////////////////////////////
inline uint32_t reserveLogfiles () const {
return _reserveLogfiles;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief set the number of reserve logfiles
////////////////////////////////////////////////////////////////////////////////
inline void reserveLogfiles (uint32_t value) {
_reserveLogfiles = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the number of historic logfiles to keep
////////////////////////////////////////////////////////////////////////////////
inline uint32_t historicLogfiles () const {
return _historicLogfiles;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief set the number of historic logfiles
////////////////////////////////////////////////////////////////////////////////
inline void historicLogfiles (uint32_t value) {
_historicLogfiles = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not shape information should be suppress when writing
/// markers into the write-ahead log
////////////////////////////////////////////////////////////////////////////////
inline bool suppressShapeInformation () const {
return _suppressShapeInformation;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not there was a SHUTDOWN file with a tick value
/// at server start
////////////////////////////////////////////////////////////////////////////////
inline bool hasFoundLastTick () const {
return _hasFoundLastTick;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not we are in the recovery phase
////////////////////////////////////////////////////////////////////////////////
inline bool isInRecovery () const {
return _inRecovery;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not we are in the shutdown phase
////////////////////////////////////////////////////////////////////////////////
inline bool isInShutdown () const {
return (_shutdown != 0);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return the slots manager
////////////////////////////////////////////////////////////////////////////////
Slots* slots () {
return _slots;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not oversize entries are allowed
////////////////////////////////////////////////////////////////////////////////
inline bool allowOversizeEntries () const {
return _allowOversizeEntries;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief sets the "allowOversizeEntries" value
////////////////////////////////////////////////////////////////////////////////
inline void allowOversizeEntries (bool value) {
_allowOversizeEntries = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not write-throttling can be enabled
////////////////////////////////////////////////////////////////////////////////
inline bool canBeThrottled () const {
return (_throttleWhenPending > 0);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum wait time when write-throttled (in milliseconds)
////////////////////////////////////////////////////////////////////////////////
inline uint64_t maxThrottleWait () const {
return _maxThrottleWait;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum wait time when write-throttled (in milliseconds)
////////////////////////////////////////////////////////////////////////////////
inline void maxThrottleWait (uint64_t value) {
_maxThrottleWait = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not write-throttling is currently enabled
////////////////////////////////////////////////////////////////////////////////
inline bool isThrottled () {
return (_writeThrottled != 0);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief activate write-throttling
////////////////////////////////////////////////////////////////////////////////
void activateWriteThrottling () {
_writeThrottled = 1;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief deactivate write-throttling
////////////////////////////////////////////////////////////////////////////////
void deactivateWriteThrottling () {
_writeThrottled = 0;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief allow or disallow writes to the WAL
////////////////////////////////////////////////////////////////////////////////
inline void allowWrites (bool value) {
_allowWrites = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the value of --wal.throttle-when-pending
////////////////////////////////////////////////////////////////////////////////
inline uint64_t throttleWhenPending () const {
return _throttleWhenPending;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief set the value of --wal.throttle-when-pending
////////////////////////////////////////////////////////////////////////////////
inline void throttleWhenPending (uint64_t value) {
_throttleWhenPending = value;
if (_throttleWhenPending == 0) {
deactivateWriteThrottling();
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief registers a transaction
////////////////////////////////////////////////////////////////////////////////
int registerTransaction (TRI_voc_tid_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief unregisters a transaction
////////////////////////////////////////////////////////////////////////////////
void unregisterTransaction (TRI_voc_tid_t,
bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief return the set of failed transactions
////////////////////////////////////////////////////////////////////////////////
std::unordered_set<TRI_voc_tid_t> getFailedTransactions ();
////////////////////////////////////////////////////////////////////////////////
/// @brief return the set of dropped collections
/// this is used during recovery and not used afterwards
////////////////////////////////////////////////////////////////////////////////
std::unordered_set<TRI_voc_cid_t> getDroppedCollections ();
////////////////////////////////////////////////////////////////////////////////
/// @brief return the set of dropped databases
/// this is used during recovery and not used afterwards
////////////////////////////////////////////////////////////////////////////////
std::unordered_set<TRI_voc_tick_t> getDroppedDatabases ();
////////////////////////////////////////////////////////////////////////////////
/// @brief unregister a list of failed transactions
////////////////////////////////////////////////////////////////////////////////
void unregisterFailedTransactions (std::unordered_set<TRI_voc_tid_t> const&);
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not it is currently allowed to create an additional
/// logfile
////////////////////////////////////////////////////////////////////////////////
bool logfileCreationAllowed (uint32_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not there are reserve logfiles
////////////////////////////////////////////////////////////////////////////////
bool hasReserveLogfiles ();
////////////////////////////////////////////////////////////////////////////////
/// @brief signal that a sync operation is required
////////////////////////////////////////////////////////////////////////////////
void signalSync ();
////////////////////////////////////////////////////////////////////////////////
/// @brief reserve space in a logfile
////////////////////////////////////////////////////////////////////////////////
SlotInfo allocate (uint32_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief reserve space in a logfile, version for legends
////////////////////////////////////////////////////////////////////////////////
SlotInfo allocate (uint32_t,
TRI_voc_cid_t cid,
TRI_shape_sid_t sid,
uint32_t legendOffset,
void*& oldLegend);
////////////////////////////////////////////////////////////////////////////////
/// @brief finalise a log entry
////////////////////////////////////////////////////////////////////////////////
void finalise (SlotInfo&, bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief write data into the logfile
/// this is a convenience function that combines allocate, memcpy and finalise
////////////////////////////////////////////////////////////////////////////////
SlotInfoCopy allocateAndWrite (void*,
uint32_t,
bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief write data into the logfile
/// this is a convenience function that combines allocate, memcpy and finalise,
/// this version is for markers with legends
////////////////////////////////////////////////////////////////////////////////
SlotInfoCopy allocateAndWrite (void*,
uint32_t,
bool,
TRI_voc_cid_t,
TRI_shape_sid_t,
uint32_t,
void*&);
////////////////////////////////////////////////////////////////////////////////
/// @brief write data into the logfile
/// this is a convenience function that combines allocate, memcpy and finalise
////////////////////////////////////////////////////////////////////////////////
SlotInfoCopy allocateAndWrite (Marker const&,
bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief finalise and seal the currently open logfile
/// this is useful to ensure that any open writes up to this point have made
/// it into a logfile
////////////////////////////////////////////////////////////////////////////////
int flush (bool,
bool,
bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief re-inserts a logfile back into the inventory only
////////////////////////////////////////////////////////////////////////////////
void relinkLogfile (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief remove a logfile from the inventory only
////////////////////////////////////////////////////////////////////////////////
bool unlinkLogfile (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief remove a logfile from the inventory only
////////////////////////////////////////////////////////////////////////////////
Logfile* unlinkLogfile (Logfile::IdType);
////////////////////////////////////////////////////////////////////////////////
/// @brief removes logfiles that are allowed to be removed
////////////////////////////////////////////////////////////////////////////////
bool removeLogfiles ();
////////////////////////////////////////////////////////////////////////////////
/// @brief sets the status of a logfile to open
////////////////////////////////////////////////////////////////////////////////
void setLogfileOpen (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief sets the status of a logfile to seal-requested
////////////////////////////////////////////////////////////////////////////////
void setLogfileSealRequested (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief sets the status of a logfile to sealed
////////////////////////////////////////////////////////////////////////////////
void setLogfileSealed (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief sets the status of a logfile to sealed
////////////////////////////////////////////////////////////////////////////////
void setLogfileSealed (Logfile::IdType);
////////////////////////////////////////////////////////////////////////////////
/// @brief return the status of a logfile
////////////////////////////////////////////////////////////////////////////////
Logfile::StatusType getLogfileStatus (Logfile::IdType);
////////////////////////////////////////////////////////////////////////////////
/// @brief return the file descriptor of a logfile
////////////////////////////////////////////////////////////////////////////////
int getLogfileDescriptor (Logfile::IdType);
////////////////////////////////////////////////////////////////////////////////
/// @brief get the current open region of a logfile
/// this uses the slots lock
////////////////////////////////////////////////////////////////////////////////
void getActiveLogfileRegion (Logfile*,
char const*&,
char const*&);
////////////////////////////////////////////////////////////////////////////////
/// @brief get logfiles for a tick range
////////////////////////////////////////////////////////////////////////////////
std::vector<Logfile*> getLogfilesForTickRange (TRI_voc_tick_t,
TRI_voc_tick_t,
bool& minTickIncluded);
////////////////////////////////////////////////////////////////////////////////
/// @brief return logfiles for a tick range
////////////////////////////////////////////////////////////////////////////////
void returnLogfiles (std::vector<Logfile*> const&);
////////////////////////////////////////////////////////////////////////////////
/// @brief get a logfile by id
////////////////////////////////////////////////////////////////////////////////
Logfile* getLogfile (Logfile::IdType);
////////////////////////////////////////////////////////////////////////////////
/// @brief get a logfile and its status by id
////////////////////////////////////////////////////////////////////////////////
Logfile* getLogfile (Logfile::IdType,
Logfile::StatusType&);
////////////////////////////////////////////////////////////////////////////////
/// @brief get a logfile for writing. this may return nullptr
////////////////////////////////////////////////////////////////////////////////
Logfile* getWriteableLogfile (uint32_t,
Logfile::StatusType&);
////////////////////////////////////////////////////////////////////////////////
/// @brief get a logfile to collect. this may return nullptr
////////////////////////////////////////////////////////////////////////////////
Logfile* getCollectableLogfile ();
////////////////////////////////////////////////////////////////////////////////
/// @brief get a logfile to remove. this may return nullptr
/// if it returns a logfile, the logfile is removed from the list of available
/// logfiles
////////////////////////////////////////////////////////////////////////////////
Logfile* getRemovableLogfile ();
////////////////////////////////////////////////////////////////////////////////
/// @brief increase the number of collect operations for a logfile
////////////////////////////////////////////////////////////////////////////////
void increaseCollectQueueSize (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief decrease the number of collect operations for a logfile
////////////////////////////////////////////////////////////////////////////////
void decreaseCollectQueueSize (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief mark a file as being requested for collection
////////////////////////////////////////////////////////////////////////////////
void setCollectionRequested (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief mark a file as being done with collection
////////////////////////////////////////////////////////////////////////////////
void setCollectionDone (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief force the status of a specific logfile
////////////////////////////////////////////////////////////////////////////////
void forceStatus (Logfile*,
Logfile::StatusType);
////////////////////////////////////////////////////////////////////////////////
/// @brief return the current state
////////////////////////////////////////////////////////////////////////////////
LogfileManagerState state ();
////////////////////////////////////////////////////////////////////////////////
/// @brief return the current available logfile ranges
////////////////////////////////////////////////////////////////////////////////
LogfileRanges ranges ();
// -----------------------------------------------------------------------------
// --SECTION-- private methods
// -----------------------------------------------------------------------------
private:
////////////////////////////////////////////////////////////////////////////////
/// @brief remove a logfile in the file system
////////////////////////////////////////////////////////////////////////////////
void removeLogfile (Logfile*);
////////////////////////////////////////////////////////////////////////////////
/// @brief wait for the collector thread to collect a specific logfile
////////////////////////////////////////////////////////////////////////////////
void waitForCollector (Logfile::IdType,
double);
////////////////////////////////////////////////////////////////////////////////
/// @brief run the recovery procedure
/// this is called after the logfiles have been scanned completely and
/// recovery state has been build. additionally, all databases have been
/// opened already so we can use collections
////////////////////////////////////////////////////////////////////////////////
int runRecovery ();
////////////////////////////////////////////////////////////////////////////////
/// @brief closes all logfiles
////////////////////////////////////////////////////////////////////////////////
void closeLogfiles ();
////////////////////////////////////////////////////////////////////////////////
/// @brief reads the shutdown information
////////////////////////////////////////////////////////////////////////////////
int readShutdownInfo ();
////////////////////////////////////////////////////////////////////////////////
/// @brief writes the shutdown information
////////////////////////////////////////////////////////////////////////////////
int writeShutdownInfo (bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief start the synchroniser thread
////////////////////////////////////////////////////////////////////////////////
int startSynchroniserThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief stop the synchroniser thread
////////////////////////////////////////////////////////////////////////////////
void stopSynchroniserThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief start the allocator thread
////////////////////////////////////////////////////////////////////////////////
int startAllocatorThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief stop the allocator thread
////////////////////////////////////////////////////////////////////////////////
void stopAllocatorThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief start the collector thread
////////////////////////////////////////////////////////////////////////////////
int startCollectorThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief stop the collector thread
////////////////////////////////////////////////////////////////////////////////
void stopCollectorThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief start the remover thread
////////////////////////////////////////////////////////////////////////////////
int startRemoverThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief stop the remover thread
////////////////////////////////////////////////////////////////////////////////
void stopRemoverThread ();
////////////////////////////////////////////////////////////////////////////////
/// @brief check which logfiles are present in the log directory
////////////////////////////////////////////////////////////////////////////////
int inventory ();
////////////////////////////////////////////////////////////////////////////////
/// @brief inspect all found WAL logfiles
/// this searches for the max tick in the logfiles and builds up the initial
/// transaction state
////////////////////////////////////////////////////////////////////////////////
int inspectLogfiles ();
////////////////////////////////////////////////////////////////////////////////
/// @brief allocate a new reserve logfile
////////////////////////////////////////////////////////////////////////////////
int createReserveLogfile (uint32_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief get an id for the next logfile
////////////////////////////////////////////////////////////////////////////////
Logfile::IdType nextId ();
////////////////////////////////////////////////////////////////////////////////
/// @brief ensure the wal logfiles directory is actually there
////////////////////////////////////////////////////////////////////////////////
int ensureDirectory ();
////////////////////////////////////////////////////////////////////////////////
/// @brief return the absolute name of the shutdown file
////////////////////////////////////////////////////////////////////////////////
std::string shutdownFilename () const;
////////////////////////////////////////////////////////////////////////////////
/// @brief return an absolute filename for a logfile id
////////////////////////////////////////////////////////////////////////////////
std::string logfileName (Logfile::IdType) const;
////////////////////////////////////////////////////////////////////////////////
/// @brief return the current time as a string
////////////////////////////////////////////////////////////////////////////////
static std::string getTimeString ();
// -----------------------------------------------------------------------------
// --SECTION-- private variables
// -----------------------------------------------------------------------------
private:
////////////////////////////////////////////////////////////////////////////////
/// @brief pointer to the server
////////////////////////////////////////////////////////////////////////////////
TRI_server_t* _server;
////////////////////////////////////////////////////////////////////////////////
/// @brief the arangod config variable containing the database path
////////////////////////////////////////////////////////////////////////////////
std::string* _databasePath;
////////////////////////////////////////////////////////////////////////////////
/// @brief the WAL logfiles directory
/// @startDocuBlock WalLogfileDirectory
/// `--wal.directory`
///
/// Specifies the directory in which the write-ahead logfiles should be
/// stored. If this option is not specified, it defaults to the subdirectory
/// *journals* in the server's global database directory. If the directory is
/// not present, it will be created.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
std::string _directory;
////////////////////////////////////////////////////////////////////////////////
/// @brief state during recovery
////////////////////////////////////////////////////////////////////////////////
RecoverState* _recoverState;
////////////////////////////////////////////////////////////////////////////////
/// @brief the size of each WAL logfile
/// @startDocuBlock WalLogfileSize
/// `--wal.logfile-size`
///
/// Specifies the filesize (in bytes) for each write-ahead logfile. The logfile
/// size should be chosen so that each logfile can store a considerable amount of
/// documents. The bigger the logfile size is chosen, the longer it will take
/// to fill up a single logfile, which also influences the delay until the data
/// in a logfile will be garbage-collected and written to collection journals
/// and datafiles. It also affects how long logfile recovery will take at
/// server start.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
uint32_t _filesize;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum number of reserve logfiles
/// @startDocuBlock WalLogfileReserveLogfiles
/// `--wal.reserve-logfiles`
///
/// The maximum number of reserve logfiles that ArangoDB will create in a
/// background process. Reserve logfiles are useful in the situation when an
/// operation needs to be written to a logfile but the reserve space in the
/// logfile is too low for storing the operation. In this case, a new logfile
/// needs to be created to store the operation. Creating new logfiles is
/// normally slow, so ArangoDB will try to pre-create logfiles in a background
/// process so there are always reserve logfiles when the active logfile gets
/// full. The number of reserve logfiles that ArangoDB keeps in the background
/// is configurable with this option.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
uint32_t _reserveLogfiles;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum number of historic logfiles
/// @startDocuBlock WalLogfileHistoricLogfiles
/// `--wal.historic-logfiles`
///
/// The maximum number of historic logfiles that ArangoDB will keep after they
/// have been garbage-collected. If no replication is used, there is no need
/// to keep historic logfiles except for having a local changelog.
///
/// In a replication setup, the number of historic logfiles affects the amount
/// of data a slave can fetch from the master's logs. The more historic
/// logfiles, the more historic data is available for a slave, which is useful
/// if the connection between master and slave is unstable or slow. Not having
/// enough historic logfiles available might lead to logfile data being deleted
/// on the master already before a slave has fetched it.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
uint32_t _historicLogfiles;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum number of parallel open logfiles
////////////////////////////////////////////////////////////////////////////////
uint32_t _maxOpenLogfiles;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum number of slots to be used in parallel
/// @startDocuBlock WalLogfileSlots
/// `--wal.slots`
///
/// Configures the amount of write slots the write-ahead log can give to write
/// operations in parallel. Any write operation will lease a slot and return it
/// to the write-ahead log when it is finished writing the data. A slot will
/// remain blocked until the data in it was synchronized to disk. After that,
/// a slot becomes reusable by following operations. The required number of
/// slots is thus determined by the parallelity of write operations and the
/// disk synchronization speed. Slow disks probably need higher values, and fast
/// disks may only require a value lower than the default.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
uint32_t _numberOfSlots;
////////////////////////////////////////////////////////////////////////////////
/// @brief interval for automatic, non-requested disk syncs
/// @startDocuBlock WalLogfileSyncInterval
/// `--wal.sync-interval`
///
/// The interval (in milliseconds) that ArangoDB will use to automatically
/// synchronize data in its write-ahead logs to disk. Automatic syncs will only
/// be performed for not-yet synchronized data, and only for operations that
/// have been executed without the *waitForSync* attribute.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
uint64_t _syncInterval;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum wait time for write-throttling
////////////////////////////////////////////////////////////////////////////////
uint64_t _maxThrottleWait;
////////////////////////////////////////////////////////////////////////////////
/// @brief throttle writes to WAL when at least such many operations are
/// waiting for garbage collection
/// @startDocuBlock WalLogfileThrottling
/// `--wal.throttle-when-pending`
///
/// The maximum value for the number of write-ahead log garbage-collection queue
/// elements. If set to *0*, the queue size is unbounded, and no
/// writtle-throttling will occur. If set to a non-zero value, writte-throttling
/// will automatically kick in when the garbage-collection queue contains at
/// least as many elements as specified by this option.
/// While write-throttling is active, data-modification operations will
/// intentionally be delayed by a configurable amount of time. This is to
/// ensure the write-ahead log garbage collector can catch up with the
/// operations executed.
/// Write-throttling will stay active until the garbage-collection queue size
/// goes down below the specified value.
/// Write-throttling is turned off by default.
///
/// `--wal.throttle-wait`
///
/// This option determines the maximum wait time (in milliseconds) for
/// operations that are write-throttled. If write-throttling is active and a
/// new write operation is to be executed, it will wait for at most the
/// specified amount of time for the write-ahead log garbage-collection queue
/// size to fall below the throttling threshold. If the queue size decreases
/// before the maximum wait time is over, the operation will be executed
/// normally. If the queue size does not decrease before the wait time is over,
/// the operation will be aborted with an error.
/// This option only has an effect if `--wal.throttle-when-pending` has a
/// non-zero value, which is not the default.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
uint64_t _throttleWhenPending;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not oversize entries are allowed
/// @startDocuBlock WalLogfileAllowOversizeEntries
/// `--wal.allow-oversize-entries`
///
/// Whether or not it is allowed to store individual documents that are bigger
/// than would fit into a single logfile. Setting the option to false will make
/// such operations fail with an error. Setting the option to true will make
/// such operations succeed, but with a high potential performance impact.
/// The reason is that for each oversize operation, an individual oversize
/// logfile needs to be created which may also block other operations.
/// The option should be set to *false* if it is certain that documents will
/// always have a size smaller than a single logfile.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
bool _allowOversizeEntries;
////////////////////////////////////////////////////////////////////////////////
/// @brief ignore logfile errors when opening logfiles
/// @startDocuBlock WalLogfileIgnoreLogfileErrors
/// `--wal.ignore-logfile-errors`
///
/// Ignores any recovery errors caused by corrupted logfiles on startup. When
/// set to *false*, the recovery procedure on startup will fail with an error
/// whenever it encounters a corrupted (that includes only half-written)
/// logfile. This is a security precaution to prevent data loss in case of disk
/// errors etc. When the recovery procedure aborts because of corruption, any
/// corrupted files can be inspected and fixed (or removed) manually and the
/// server can be restarted afterwards.
///
/// Setting the option to *true* will make the server continue with the recovery
/// procedure even in case it detects corrupt logfile entries. In this case it
/// will stop at the first corrupted logfile entry and ignore all others, which
/// might cause data loss.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
bool _ignoreLogfileErrors;
////////////////////////////////////////////////////////////////////////////////
/// @brief ignore recovery errors
/// @startDocuBlock WalLogfileIgnoreRecoveryErrors
/// `--wal.ignore-recovery-errors`
///
/// Ignores any recovery errors not caused by corrupted logfiles but by logical
/// errors. Logical errors can occur if logfiles or any other server datafiles
/// have been manually edited or the server is somehow misconfigured.
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
bool _ignoreRecoveryErrors;
////////////////////////////////////////////////////////////////////////////////
/// @brief suppress shape information
/// @startDocuBlock WalLogfileSuppressShapeInformation
/// `--wal.suppress-shape-information`
///
/// Setting this variable to *true* will lead to no shape information being
/// written into the write-ahead logfiles for documents or edges. While this is
/// a good optimization for a single server to save memory (and disk space), it
/// it will effectively disable using the write-ahead log as a reliable source
/// for replicating changes to other servers. A master server with this option
/// set to *true* will not be able to fully reproduce the structure of saved
/// documents after a collection has been deleted. In case a replication client
/// requests a document for which the collection is already deleted, the master
/// will return an empty document. Note that this only affects replication and
/// not normal operation on the master.
///
/// **Do not set this variable to *true* on a server that you plan to use as a
/// replication master**
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
bool _suppressShapeInformation;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not writes to the WAL are allowed
////////////////////////////////////////////////////////////////////////////////
bool _allowWrites;
////////////////////////////////////////////////////////////////////////////////
/// @brief this is true if there was a SHUTDOWN file with a last tick at
/// server start
////////////////////////////////////////////////////////////////////////////////
bool _hasFoundLastTick;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not the recovery procedure is running
////////////////////////////////////////////////////////////////////////////////
bool _inRecovery;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not the logfile manager was properly initialized and
/// started
////////////////////////////////////////////////////////////////////////////////
bool _startCalled;
////////////////////////////////////////////////////////////////////////////////
/// @brief a lock protecting the _logfiles map and the logfiles' statuses
////////////////////////////////////////////////////////////////////////////////
basics::ReadWriteLock _logfilesLock;
////////////////////////////////////////////////////////////////////////////////
/// @brief the logfiles
////////////////////////////////////////////////////////////////////////////////
std::map<Logfile::IdType, Logfile*> _logfiles;
////////////////////////////////////////////////////////////////////////////////
/// @brief the slots manager
////////////////////////////////////////////////////////////////////////////////
Slots* _slots;
////////////////////////////////////////////////////////////////////////////////
/// @brief the synchroniser thread
////////////////////////////////////////////////////////////////////////////////
SynchroniserThread* _synchroniserThread;
////////////////////////////////////////////////////////////////////////////////
/// @brief the allocator thread
////////////////////////////////////////////////////////////////////////////////
AllocatorThread* _allocatorThread;
////////////////////////////////////////////////////////////////////////////////
/// @brief the collector thread
////////////////////////////////////////////////////////////////////////////////
CollectorThread* _collectorThread;
////////////////////////////////////////////////////////////////////////////////
/// @brief the logfile remover thread
////////////////////////////////////////////////////////////////////////////////
RemoverThread* _removerThread;
////////////////////////////////////////////////////////////////////////////////
/// @brief last opened logfile id. note: writing to this variable is protected
/// by the _idLock
////////////////////////////////////////////////////////////////////////////////
std::atomic<Logfile::IdType> _lastOpenedId;
////////////////////////////////////////////////////////////////////////////////
/// @brief last fully collected logfile id. note: writing to this variable is
/// protected by the_idLock
////////////////////////////////////////////////////////////////////////////////
std::atomic<Logfile::IdType> _lastCollectedId;
////////////////////////////////////////////////////////////////////////////////
/// @brief last fully sealed logfile id. note: writing to this variable is
/// protected by the _idLock
////////////////////////////////////////////////////////////////////////////////
std::atomic<Logfile::IdType> _lastSealedId;
////////////////////////////////////////////////////////////////////////////////
/// @brief a lock protecting the shutdown file
////////////////////////////////////////////////////////////////////////////////
basics::Mutex _shutdownFileLock;
////////////////////////////////////////////////////////////////////////////////
/// @brief a lock protecting _transactions and _failedTransactions
////////////////////////////////////////////////////////////////////////////////
basics::ReadWriteLock _transactionsLock;
////////////////////////////////////////////////////////////////////////////////
/// @brief currently ongoing transactions
////////////////////////////////////////////////////////////////////////////////
std::unordered_map<TRI_voc_tid_t, std::pair<Logfile::IdType, Logfile::IdType>> _transactions;
////////////////////////////////////////////////////////////////////////////////
/// @brief set of failed transactions
////////////////////////////////////////////////////////////////////////////////
std::unordered_set<TRI_voc_tid_t> _failedTransactions;
////////////////////////////////////////////////////////////////////////////////
/// @brief set of dropped collections
/// this is populated during recovery and not used afterwards
////////////////////////////////////////////////////////////////////////////////
std::unordered_set<TRI_voc_cid_t> _droppedCollections;
////////////////////////////////////////////////////////////////////////////////
/// @brief set of dropped databases
/// this is populated during recovery and not used afterwards
////////////////////////////////////////////////////////////////////////////////
std::unordered_set<TRI_voc_tick_t> _droppedDatabases;
////////////////////////////////////////////////////////////////////////////////
/// @brief a lock protecting the updates of _lastCollectedId, _lastSealedId,
/// and _lastOpenedId
////////////////////////////////////////////////////////////////////////////////
basics::Mutex _idLock;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not write-throttling is currently enabled
////////////////////////////////////////////////////////////////////////////////
int _writeThrottled;
////////////////////////////////////////////////////////////////////////////////
/// @brief regex to match logfiles
////////////////////////////////////////////////////////////////////////////////
regex_t _filenameRegex;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not we have been shut down already
////////////////////////////////////////////////////////////////////////////////
volatile sig_atomic_t _shutdown;
};
}
}
#endif
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:
Reference in New Issue View Git Blame Copy Permalink