diff --git a/CHANGELOG b/CHANGELOG index 1531b22dda..13aefd5a82 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -48,6 +48,8 @@ devel v3.0.6 (XXXX-XX-XX) ------------------- +* slightly better error diagnostics for AQL query compilation and replication + * fixed issue #2018 * fixed issue #2015 diff --git a/CMakeLists.txt b/CMakeLists.txt index ed18c9fa4a..67f2a9910d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,7 +232,7 @@ else () set(MAKE make) endif () -find_package(PythonInterp 2 REQUIRED) +find_package(PythonInterp 2 EXACT REQUIRED) get_filename_component(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}" REALPATH) ################################################################################ diff --git a/arangod/Aql/Executor.cpp b/arangod/Aql/Executor.cpp index 8468bd54d7..5a3c9fe87c 100644 --- a/arangod/Aql/Executor.cpp +++ b/arangod/Aql/Executor.cpp @@ -1058,6 +1058,10 @@ void Executor::generateCodeNode(AstNode const* node) { arangodb::basics::StringBuffer* Executor::initializeBuffer() { if (_buffer == nullptr) { _buffer = new arangodb::basics::StringBuffer(TRI_UNKNOWN_MEM_ZONE, 512, false); + + if (_buffer->stringBuffer()->_buffer == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + } } else { _buffer->clear(); } diff --git a/arangod/StorageEngine/MMFilesCollection.cpp b/arangod/StorageEngine/MMFilesCollection.cpp index 4b7be1e871..52a3d4d72d 100644 --- a/arangod/StorageEngine/MMFilesCollection.cpp +++ b/arangod/StorageEngine/MMFilesCollection.cpp @@ -24,6 +24,7 @@ #include "MMFilesCollection.h" #include "Basics/FileUtils.h" #include "Basics/ReadLocker.h" +#include "Basics/StaticStrings.h" #include "Basics/WriteLocker.h" #include "Basics/memory-map.h" #include "Logger/Logger.h" @@ -57,13 +58,13 @@ MMFilesCollection::~MMFilesCollection() { close(); for (auto& it : _datafiles) { - TRI_FreeDatafile(it); + delete it; } for (auto& it : _journals) { - TRI_FreeDatafile(it); + delete it; } for (auto& it : _compactors) { - TRI_FreeDatafile(it); + delete it; } } @@ -97,15 +98,15 @@ int MMFilesCollection::close() { /// @brief seal a datafile int MMFilesCollection::sealDatafile(TRI_datafile_t* datafile, bool isCompactor) { - int res = TRI_SealDatafile(datafile); + int res = datafile->seal(); if (res != TRI_ERROR_NO_ERROR) { - LOG(ERR) << "failed to seal journal '" << datafile->getName(datafile) + LOG(ERR) << "failed to seal journal '" << datafile->getName() << "': " << TRI_errno_string(res); return res; } - if (!isCompactor && datafile->isPhysical(datafile)) { + if (!isCompactor && datafile->isPhysical()) { // rename the file std::string dname("datafile-" + std::to_string(datafile->_fid) + ".db"); std::string filename = arangodb::basics::FileUtils::buildFilename(_logicalCollection->path(), dname); @@ -113,9 +114,9 @@ int MMFilesCollection::sealDatafile(TRI_datafile_t* datafile, bool isCompactor) res = TRI_RenameDatafile(datafile, filename.c_str()); if (res == TRI_ERROR_NO_ERROR) { - LOG(TRACE) << "closed file '" << datafile->getName(datafile) << "'"; + LOG(TRACE) << "closed file '" << datafile->getName() << "'"; } else { - LOG(ERR) << "failed to rename datafile '" << datafile->getName(datafile) + LOG(ERR) << "failed to rename datafile '" << datafile->getName() << "' to '" << filename << "': " << TRI_errno_string(res); } } @@ -177,12 +178,12 @@ int MMFilesCollection::syncActiveJournal() { // we only need to care about physical datafiles // anonymous regions do not need to be synced - if (datafile->isPhysical(datafile)) { + if (datafile->isPhysical()) { char const* synced = datafile->_synced; char* written = datafile->_written; if (synced < written) { - bool ok = datafile->sync(datafile, synced, written); + bool ok = datafile->sync(synced, written); if (ok) { LOG_TOPIC(TRACE, Logger::COLLECTOR) << "msync succeeded " @@ -281,7 +282,7 @@ int MMFilesCollection::reserveJournalSpace(TRI_voc_tick_t tick, // TRI_ERROR_ARANGO_DATAFILE_FULL... // journal is full, close it and sync LOG_TOPIC(DEBUG, Logger::COLLECTOR) << "closing full journal '" - << datafile->getName(datafile) << "'"; + << datafile->getName() << "'"; // make sure we have enough room in the target vector before we go on _datafiles.reserve(_datafiles.size() + 1); @@ -394,7 +395,7 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, if (_logicalCollection->isVolatile()) { // in-memory collection - datafile = TRI_CreateDatafile(nullptr, fid, journalSize, true); + datafile = TRI_CreateDatafile(StaticStrings::Empty, fid, journalSize, true); } else { // construct a suitable filename (which may be temporary at the beginning) std::string jname; @@ -420,7 +421,7 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, TRI_UnlinkFile(filename.c_str()); } - datafile = TRI_CreateDatafile(filename.c_str(), fid, journalSize, true); + datafile = TRI_CreateDatafile(filename, fid, journalSize, true); } if (datafile == nullptr) { @@ -437,10 +438,10 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, TRI_ASSERT(datafile != nullptr); if (isCompactor) { - LOG(TRACE) << "created new compactor '" << datafile->getName(datafile) + LOG(TRACE) << "created new compactor '" << datafile->getName() << "'"; } else { - LOG(TRACE) << "created new journal '" << datafile->getName(datafile) << "'"; + LOG(TRACE) << "created new journal '" << datafile->getName() << "'"; } // create a collection header, still in the temporary file @@ -454,12 +455,12 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, if (res != TRI_ERROR_NO_ERROR) { LOG(ERR) << "cannot create collection header in file '" - << datafile->getName(datafile) << "': " << TRI_errno_string(res); + << datafile->getName() << "': " << TRI_errno_string(res); // close the journal and remove it - TRI_CloseDatafile(datafile); - TRI_UnlinkFile(datafile->getName(datafile)); - TRI_FreeDatafile(datafile); + std::string temp(datafile->getName()); + delete datafile; + TRI_UnlinkFile(temp.c_str()); EnsureErrorCode(res); @@ -481,12 +482,12 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, if (res != TRI_ERROR_NO_ERROR) { int res = datafile->_lastError; LOG(ERR) << "cannot create collection header in file '" - << datafile->getName(datafile) << "': " << TRI_last_error(); + << datafile->getName() << "': " << TRI_last_error(); // close the datafile and remove it - TRI_CloseDatafile(datafile); - TRI_UnlinkFile(datafile->getName(datafile)); - TRI_FreeDatafile(datafile); + std::string temp(datafile->getName()); + delete datafile; + TRI_UnlinkFile(temp.c_str()); EnsureErrorCode(res); @@ -497,7 +498,7 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, // if a physical file, we can rename it from the temporary name to the correct // name - if (!isCompactor && datafile->isPhysical(datafile)) { + if (!isCompactor && datafile->isPhysical()) { // and use the correct name std::string jname("journal-" + std::to_string(datafile->_fid) + ".db"); std::string filename = arangodb::basics::FileUtils::buildFilename(_logicalCollection->path(), jname); @@ -505,19 +506,19 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid, int res = TRI_RenameDatafile(datafile, filename.c_str()); if (res != TRI_ERROR_NO_ERROR) { - LOG(ERR) << "failed to rename journal '" << datafile->getName(datafile) + LOG(ERR) << "failed to rename journal '" << datafile->getName() << "' to '" << filename << "': " << TRI_errno_string(res); - TRI_CloseDatafile(datafile); - TRI_UnlinkFile(datafile->getName(datafile)); - TRI_FreeDatafile(datafile); + std::string temp(datafile->getName()); + delete datafile; + TRI_UnlinkFile(temp.c_str()); EnsureErrorCode(res); return nullptr; } - LOG(TRACE) << "renamed journal from '" << datafile->getName(datafile) + LOG(TRACE) << "renamed journal from '" << datafile->getName() << "' to '" << filename << "'"; } @@ -574,7 +575,7 @@ bool MMFilesCollection::iterateDatafilesVector(std::vector cons return false; } - if (datafile->isPhysical(datafile) && datafile->_isSealed) { + if (datafile->isPhysical() && datafile->_isSealed) { TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize, TRI_MADVISE_RANDOM); } @@ -593,7 +594,7 @@ bool MMFilesCollection::closeDatafiles(std::vector const& files continue; } - int res = TRI_CloseDatafile(datafile); + int res = datafile->close(); if (res != TRI_ERROR_NO_ERROR) { result = false; @@ -637,7 +638,7 @@ void MMFilesCollection::figures(std::shared_ptr& /// @brief iterate over a vector of datafiles and pick those with a specific /// data range -std::vector MMFilesCollection::datafilesInRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax) { +std::vector MMFilesCollection::datafilesInRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax) { std::vector result; auto apply = [&dataMin, &dataMax, &result](TRI_datafile_t const* datafile, bool isJournal) { @@ -768,4 +769,34 @@ int MMFilesCollection::applyForTickRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t return false; // hasMore = false } + +/// @brief disallow compaction of the collection +void MMFilesCollection::preventCompaction() { + _compactionLock.readLock(); +} + +/// @brief try disallowing compaction of the collection +bool MMFilesCollection::tryPreventCompaction() { + return _compactionLock.tryReadLock(); +} + +/// @brief re-allow compaction of the collection +void MMFilesCollection::allowCompaction() { + _compactionLock.unlock(); +} + +/// @brief exclusively lock the collection for compaction +void MMFilesCollection::lockForCompaction() { + _compactionLock.writeLock(); +} + +/// @brief try to exclusively lock the collection for compaction +bool MMFilesCollection::tryLockForCompaction() { + return _compactionLock.tryWriteLock(); +} + +/// @brief signal that compaction is finished +void MMFilesCollection::finishCompaction() { + _compactionLock.unlock(); +} diff --git a/arangod/StorageEngine/MMFilesCollection.h b/arangod/StorageEngine/MMFilesCollection.h index 289782f674..3076ba2bf9 100644 --- a/arangod/StorageEngine/MMFilesCollection.h +++ b/arangod/StorageEngine/MMFilesCollection.h @@ -38,6 +38,14 @@ class MMFilesCollection final : public PhysicalCollection { friend class MMFilesCompactorThread; friend class MMFilesEngine; + struct DatafileDescription { + TRI_datafile_t const* _data; + TRI_voc_tick_t _dataMin; + TRI_voc_tick_t _dataMax; + TRI_voc_tick_t _tickMax; + bool _isJournal; + }; + public: explicit MMFilesCollection(LogicalCollection*); ~MMFilesCollection(); @@ -86,6 +94,13 @@ class MMFilesCollection final : public PhysicalCollection { /// @brief iterates over a collection bool iterateDatafiles(std::function const& cb) override; + void preventCompaction() override; + bool tryPreventCompaction() override; + void allowCompaction() override; + void lockForCompaction() override; + bool tryLockForCompaction() override; + void finishCompaction() override; + private: /// @brief creates a datafile TRI_datafile_t* createDatafile(TRI_voc_fid_t fid, @@ -107,6 +122,8 @@ class MMFilesCollection final : public PhysicalCollection { std::vector _datafiles; // all datafiles std::vector _journals; // all journals std::vector _compactors; // all compactor files + + arangodb::basics::ReadWriteLock _compactionLock; }; } diff --git a/arangod/StorageEngine/MMFilesCompactorThread.cpp b/arangod/StorageEngine/MMFilesCompactorThread.cpp index 2cb9f17b06..e9c199bf6d 100644 --- a/arangod/StorageEngine/MMFilesCompactorThread.cpp +++ b/arangod/StorageEngine/MMFilesCompactorThread.cpp @@ -36,9 +36,10 @@ #include "StorageEngine/StorageEngine.h" #include "Utils/SingleCollectionTransaction.h" #include "Utils/StandaloneTransactionContext.h" +#include "VocBase/CompactionLocker.h" #include "VocBase/DatafileHelper.h" -#include "VocBase/collection.h" #include "VocBase/LogicalCollection.h" +#include "VocBase/collection.h" #include "VocBase/ticks.h" #include "VocBase/vocbase.h" @@ -84,7 +85,7 @@ void MMFilesCompactorThread::DropDatafileCallback(TRI_datafile_t* datafile, Logi std::string name("deleted-" + std::to_string(fid) + ".db"); std::string filename = arangodb::basics::FileUtils::buildFilename(collection->path(), name); - if (datafile->isPhysical(datafile)) { + if (datafile->isPhysical()) { // copy the current filename copy = datafile->_filename; @@ -95,19 +96,19 @@ void MMFilesCompactorThread::DropDatafileCallback(TRI_datafile_t* datafile, Logi } } - LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "finished compacting datafile '" << datafile->getName(datafile) << "'"; + LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "finished compacting datafile '" << datafile->getName() << "'"; - int res = TRI_CloseDatafile(datafile); + int res = datafile->close(); if (res != TRI_ERROR_NO_ERROR) { - LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot close obsolete datafile '" << datafile->getName(datafile) << "': " << TRI_errno_string(res); - } else if (datafile->isPhysical(datafile)) { + LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot close obsolete datafile '" << datafile->getName() << "': " << TRI_errno_string(res); + } else if (datafile->isPhysical()) { LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "wiping compacted datafile from disk"; res = TRI_UnlinkFile(filename.c_str()); if (res != TRI_ERROR_NO_ERROR) { - LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot wipe obsolete datafile '" << datafile->getName(datafile) << "': " << TRI_errno_string(res); + LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot wipe obsolete datafile '" << datafile->getName() << "': " << TRI_errno_string(res); } // check for .dead files @@ -122,7 +123,7 @@ void MMFilesCompactorThread::DropDatafileCallback(TRI_datafile_t* datafile, Logi } } - TRI_FreeDatafile(datafile); + delete datafile; } //////////////////////////////////////////////////////////////////////////////// @@ -146,7 +147,7 @@ void MMFilesCompactorThread::RenameDatafileCallback(TRI_datafile_t* datafile, vo bool ok = false; TRI_ASSERT(datafile->_fid == compactor->_fid); - if (datafile->isPhysical(datafile)) { + if (datafile->isPhysical()) { // construct a suitable tempname std::string jname("temp-" + std::to_string(datafile->_fid) + ".db"); std::string tempFilename = arangodb::basics::FileUtils::buildFilename(collection->path(), jname); @@ -155,12 +156,12 @@ void MMFilesCompactorThread::RenameDatafileCallback(TRI_datafile_t* datafile, vo int res = TRI_RenameDatafile(datafile, tempFilename.c_str()); if (res != TRI_ERROR_NO_ERROR) { - LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename datafile '" << datafile->getName(datafile) << "' to '" << tempFilename << "': " << TRI_errno_string(res); + LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename datafile '" << datafile->getName() << "' to '" << tempFilename << "': " << TRI_errno_string(res); } else { res = TRI_RenameDatafile(compactor, realName.c_str()); if (res != TRI_ERROR_NO_ERROR) { - LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename compaction file '" << compactor->getName(compactor) << "' to '" << realName << "': " << TRI_errno_string(res); + LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename compaction file '" << compactor->getName() << "' to '" << realName << "': " << TRI_errno_string(res); } } @@ -187,7 +188,7 @@ void MMFilesCompactorThread::RenameDatafileCallback(TRI_datafile_t* datafile, vo /// @brief remove an empty compactor file int MMFilesCompactorThread::removeCompactor(LogicalCollection* collection, TRI_datafile_t* compactor) { - LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty compaction file '" << compactor->getName(compactor) << "'"; + LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty compaction file '" << compactor->getName() << "'"; // remove the compactor from the list of compactors bool ok = static_cast(collection->getPhysical())->removeCompactor(compactor); @@ -199,15 +200,12 @@ int MMFilesCompactorThread::removeCompactor(LogicalCollection* collection, } // close the file & remove it - if (compactor->isPhysical(compactor)) { - std::string filename = compactor->getName(compactor); - TRI_CloseDatafile(compactor); - TRI_FreeDatafile(compactor); - + if (compactor->isPhysical()) { + std::string filename = compactor->getName(); + delete compactor; TRI_UnlinkFile(filename.c_str()); } else { - TRI_CloseDatafile(compactor); - TRI_FreeDatafile(compactor); + delete compactor; } return TRI_ERROR_NO_ERROR; @@ -216,7 +214,7 @@ int MMFilesCompactorThread::removeCompactor(LogicalCollection* collection, /// @brief remove an empty datafile int MMFilesCompactorThread::removeDatafile(LogicalCollection* collection, TRI_datafile_t* df) { - LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty datafile '" << df->getName(df) << "'"; + LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty datafile '" << df->getName() << "'"; bool ok = static_cast(collection->getPhysical())->removeDatafile(df); @@ -251,7 +249,7 @@ MMFilesCompactorThread::compaction_initial_context_t MMFilesCompactorThread::get TRI_datafile_t* df = compaction._datafile; // We will sequentially scan the logfile for collection: - if (df->isPhysical(df)) { + if (df->isPhysical()) { TRI_MMFileAdvise(df->_data, df->_maximalSize, TRI_MADVISE_SEQUENTIAL); TRI_MMFileAdvise(df->_data, df->_maximalSize, TRI_MADVISE_WILLNEED); } @@ -317,7 +315,7 @@ MMFilesCompactorThread::compaction_initial_context_t MMFilesCompactorThread::get } } - if (df->isPhysical(df)) { + if (df->isPhysical()) { TRI_MMFileAdvise(df->_data, df->_maximalSize, TRI_MADVISE_RANDOM); } @@ -425,6 +423,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection, trx.addHint(TRI_TRANSACTION_HINT_NO_BEGIN_MARKER, true); trx.addHint(TRI_TRANSACTION_HINT_NO_ABORT_MARKER, true); trx.addHint(TRI_TRANSACTION_HINT_NO_COMPACTION_LOCK, true); + trx.addHint(TRI_TRANSACTION_HINT_NO_THROTTLING, true); compaction_initial_context_t initial = getCompactionContext(&trx, collection, toCompact); @@ -446,7 +445,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection, return; } - LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "created new compactor file '" << compactor->getName(compactor) << "'"; + LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "created new compactor file '" << compactor->getName() << "'"; // these attributes remain the same for all datafiles we collect context._collection = collection; @@ -465,7 +464,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection, auto compaction = toCompact[i]; TRI_datafile_t* df = compaction._datafile; - LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "compacting datafile '" << df->getName(df) << "' into '" << compactor->getName(compactor) << "', number: " << i << ", keep deletions: " << compaction._keepDeletions; + LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "compacting datafile '" << df->getName() << "' into '" << compactor->getName() << "', number: " << i << ", keep deletions: " << compaction._keepDeletions; // if this is the first datafile in the list of datafiles, we can also // collect @@ -476,7 +475,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection, bool ok = TRI_IterateDatafile(df, compactifier); if (!ok) { - LOG_TOPIC(WARN, Logger::COMPACTOR) << "failed to compact datafile '" << df->getName(df) << "'"; + LOG_TOPIC(WARN, Logger::COMPACTOR) << "failed to compact datafile '" << df->getName() << "'"; // compactor file does not need to be removed now. will be removed on next // startup // TODO: Remove file @@ -510,8 +509,8 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection, auto compaction = toCompact[i]; TRI_datafile_t* datafile = compaction._datafile; - if (datafile->isPhysical(datafile)) { - std::string filename(datafile->getName(datafile)); + if (datafile->isPhysical()) { + std::string filename(datafile->getName()); filename.append(".dead"); TRI_WriteFile(filename.c_str(), "", 0); @@ -544,8 +543,8 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection, auto compaction = toCompact[i]; TRI_datafile_t* datafile = compaction._datafile; - if (datafile->isPhysical(datafile)) { - std::string filename(datafile->getName(datafile)); + if (datafile->isPhysical()) { + std::string filename(datafile->getName()); filename.append(".dead"); TRI_WriteFile(filename.c_str(), "", 0); @@ -877,9 +876,9 @@ void MMFilesCompactorThread::run() { // check whether someone else holds a read-lock on the compaction // lock - TRY_WRITE_LOCKER(locker, document->_compactionLock); + TryCompactionLocker compactionLocker(collection); - if (!locker.isLocked()) { + if (!compactionLocker.isLocked()) { // someone else is holding the compactor lock, we'll not compact continue; } @@ -958,6 +957,7 @@ uint64_t MMFilesCompactorThread::getNumberOfDocuments(LogicalCollection* collect // only try to acquire the lock here // if lock acquisition fails, we go on and report an (arbitrary) positive number trx.addHint(TRI_TRANSACTION_HINT_TRY_LOCK, false); + trx.addHint(TRI_TRANSACTION_HINT_NO_THROTTLING, true); int res = trx.begin(); diff --git a/arangod/StorageEngine/MMFilesEngine.cpp b/arangod/StorageEngine/MMFilesEngine.cpp index 7668310ece..dfa79636df 100644 --- a/arangod/StorageEngine/MMFilesEngine.cpp +++ b/arangod/StorageEngine/MMFilesEngine.cpp @@ -90,7 +90,7 @@ static uint64_t getNumericFilenamePartFromDatabase(std::string const& filename) } static uint64_t getNumericFilenamePartFromDatafile(TRI_datafile_t const* datafile) { - return getNumericFilenamePartFromDatafile(datafile->getName(datafile)); + return getNumericFilenamePartFromDatafile(datafile->getName()); } @@ -1306,12 +1306,11 @@ bool MMFilesEngine::iterateFiles(std::vector const& files) { for (auto const& filename : files) { LOG(DEBUG) << "iterating over collection journal file '" << filename << "'"; - TRI_datafile_t* datafile = TRI_OpenDatafile(filename.c_str(), true); + TRI_datafile_t* datafile = TRI_OpenDatafile(filename, true); if (datafile != nullptr) { TRI_IterateDatafile(datafile, cb); - TRI_CloseDatafile(datafile); - TRI_FreeDatafile(datafile); + delete datafile; } } @@ -1858,7 +1857,7 @@ int MMFilesEngine::openCollection(TRI_vocbase_t* vocbase, LogicalCollection* col } TRI_datafile_t* datafile = - TRI_OpenDatafile(filename.c_str(), ignoreErrors); + TRI_OpenDatafile(filename, ignoreErrors); if (datafile == nullptr) { LOG_TOPIC(ERR, Logger::DATAFILES) << "cannot open datafile '" @@ -1963,10 +1962,8 @@ int MMFilesEngine::openCollection(TRI_vocbase_t* vocbase, LogicalCollection* col // stop if necessary if (stop) { for (auto& datafile : all) { - LOG(TRACE) << "closing datafile '" << datafile->_filename << "'"; - - TRI_CloseDatafile(datafile); - TRI_FreeDatafile(datafile); + LOG(TRACE) << "closing datafile '" << datafile->getName() << "'"; + delete datafile; } return TRI_ERROR_INTERNAL; diff --git a/arangod/Utils/TransactionContext.cpp b/arangod/Utils/TransactionContext.cpp index f29e515f4c..8de8f08e57 100644 --- a/arangod/Utils/TransactionContext.cpp +++ b/arangod/Utils/TransactionContext.cpp @@ -38,7 +38,7 @@ using namespace arangodb; // custom type value handler, used for deciphering the _id attribute -struct CustomTypeHandler : public VPackCustomTypeHandler { +struct CustomTypeHandler final : public VPackCustomTypeHandler { CustomTypeHandler(TRI_vocbase_t* vocbase, CollectionNameResolver const* resolver) : vocbase(vocbase), resolver(resolver) {} diff --git a/arangod/V8Server/v8-collection.cpp b/arangod/V8Server/v8-collection.cpp index 7944292781..c6981eb51e 100644 --- a/arangod/V8Server/v8-collection.cpp +++ b/arangod/V8Server/v8-collection.cpp @@ -2332,7 +2332,7 @@ static void JS_TruncateDatafileVocbaseCol( TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_COLLECTION_NOT_UNLOADED); } - int res = TRI_TruncateDatafile(path.c_str(), (TRI_voc_size_t)size); + int res = TRI_TruncateDatafile(path, (TRI_voc_size_t)size); if (res != TRI_ERROR_NO_ERROR) { TRI_V8_THROW_EXCEPTION_MESSAGE(res, "cannot truncate datafile"); diff --git a/arangod/VocBase/CompactionLocker.h b/arangod/VocBase/CompactionLocker.h new file mode 100644 index 0000000000..69bffa9f0a --- /dev/null +++ b/arangod/VocBase/CompactionLocker.h @@ -0,0 +1,102 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_VOCBASE_COMPACTION_LOCKER_H +#define ARANGOD_VOCBASE_COMPACTION_LOCKER_H 1 + +#include "Basics/Common.h" +#include "VocBase/LogicalCollection.h" + +namespace arangodb { + +class CompactionPreventer { + public: + explicit CompactionPreventer(LogicalCollection* collection) + : _collection(collection) { + _collection->preventCompaction(); + } + + ~CompactionPreventer() { _collection->allowCompaction(); } + + private: + LogicalCollection* _collection; +}; + +class TryCompactionPreventer { + public: + explicit TryCompactionPreventer(LogicalCollection* collection) + : _collection(collection), _isLocked(false) { + _isLocked = _collection->tryPreventCompaction(); + } + + ~TryCompactionPreventer() { + if (_isLocked) { + _collection->allowCompaction(); + } + } + + bool isLocked() const { return _isLocked; } + + private: + LogicalCollection* _collection; + bool _isLocked; +}; + +class CompactionLocker { + public: + explicit CompactionLocker(LogicalCollection* collection) + : _collection(collection) { + _collection->lockForCompaction(); + } + + ~CompactionLocker() { + _collection->finishCompaction(); + } + + private: + LogicalCollection* _collection; +}; + +class TryCompactionLocker { + public: + explicit TryCompactionLocker(LogicalCollection* collection) + : _collection(collection), _isLocked(false) { + _isLocked = _collection->tryLockForCompaction(); + } + + ~TryCompactionLocker() { + if (_isLocked) { + _collection->finishCompaction(); + } + } + + bool isLocked() const { return _isLocked; } + + private: + LogicalCollection* _collection; + bool _isLocked; +}; + +} // namespace arangodb + +#endif diff --git a/arangod/VocBase/DatafileDescription.h b/arangod/VocBase/DatafileDescription.h deleted file mode 100644 index 0fecf7eb46..0000000000 --- a/arangod/VocBase/DatafileDescription.h +++ /dev/null @@ -1,44 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany -/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -//////////////////////////////////////////////////////////////////////////////// - -#ifndef ARANGOD_VOCBASE_DATAFILE_DESCRIPTION_H -#define ARANGOD_VOCBASE_DATAFILE_DESCRIPTION_H 1 - -#include "Basics/Common.h" -#include "VocBase/voc-types.h" - -struct TRI_datafile_t; - -namespace arangodb { - -struct DatafileDescription { - TRI_datafile_t const* _data; - TRI_voc_tick_t _dataMin; - TRI_voc_tick_t _dataMax; - TRI_voc_tick_t _tickMax; - bool _isJournal; -}; - -} // namespace arangodb - -#endif diff --git a/arangod/VocBase/DatafileStatistics.h b/arangod/VocBase/DatafileStatistics.h index 87dd73497e..d2e1c5eaf5 100644 --- a/arangod/VocBase/DatafileStatistics.h +++ b/arangod/VocBase/DatafileStatistics.h @@ -28,8 +28,6 @@ #include "Basics/ReadWriteLock.h" #include "VocBase/voc-types.h" -struct TRI_datafile_t; - namespace arangodb { /// @brief datafile statistics diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index 4be0a4f164..b516ce7eb5 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -25,7 +25,6 @@ #define ARANGOD_VOCBASE_LOGICAL_COLLECTION_H 1 #include "Basics/Common.h" -#include "VocBase/DatafileDescription.h" #include "VocBase/MasterPointers.h" #include "VocBase/PhysicalCollection.h" #include "VocBase/voc-types.h" @@ -199,6 +198,17 @@ class LogicalCollection { return getPhysical()->applyForTickRange(dataMin, dataMax, callback); } + /// @brief disallow starting the compaction of the collection + void preventCompaction() { getPhysical()->preventCompaction(); } + bool tryPreventCompaction() { return getPhysical()->tryPreventCompaction(); } + /// @brief re-allow starting the compaction of the collection + void allowCompaction() { getPhysical()->allowCompaction(); } + + /// @brief compaction finished + void lockForCompaction() { getPhysical()->lockForCompaction(); } + bool tryLockForCompaction() { return getPhysical()->tryLockForCompaction(); } + void finishCompaction() { getPhysical()->finishCompaction(); } + PhysicalCollection* getPhysical() const { TRI_ASSERT(_physical != nullptr); @@ -444,6 +454,7 @@ class LogicalCollection { mutable arangodb::basics::ReadWriteLock _idxLock; // lock protecting the indexes }; + } // namespace arangodb #endif diff --git a/arangod/VocBase/PhysicalCollection.h b/arangod/VocBase/PhysicalCollection.h index 94190c53a4..08eb62b4fa 100644 --- a/arangod/VocBase/PhysicalCollection.h +++ b/arangod/VocBase/PhysicalCollection.h @@ -25,7 +25,6 @@ #define ARANGOD_VOCBASE_PHYSICAL_COLLECTION_H 1 #include "Basics/Common.h" -#include "VocBase/DatafileDescription.h" #include "VocBase/voc-types.h" #include @@ -63,6 +62,27 @@ class PhysicalCollection { /// @brief iterates over a collection virtual bool iterateDatafiles(std::function const& cb) = 0; + /// @brief disallow compaction of the collection + /// after this call it is guaranteed that no compaction will be started until allowCompaction() is called + virtual void preventCompaction() = 0; + + /// @brief try disallowing compaction of the collection + /// returns true if compaction is disallowed, and false if not + virtual bool tryPreventCompaction() = 0; + + /// @brief re-allow compaction of the collection + virtual void allowCompaction() = 0; + + /// @brief exclusively lock the collection for compaction + virtual void lockForCompaction() = 0; + + /// @brief try to exclusively lock the collection for compaction + /// after this call it is guaranteed that no compaction will be started until allowCompaction() is called + virtual bool tryLockForCompaction() = 0; + + /// @brief signal that compaction is finished + virtual void finishCompaction() = 0; + protected: LogicalCollection* _logicalCollection; }; diff --git a/arangod/VocBase/collection.h b/arangod/VocBase/collection.h index f7bc90268f..e56fa644ff 100644 --- a/arangod/VocBase/collection.h +++ b/arangod/VocBase/collection.h @@ -86,7 +86,7 @@ class VocbaseCollectionInfo { int64_t _initialCount; // initial count, used when loading a collection uint32_t _indexBuckets; // number of buckets used in hash tables for indexes - char _name[TRI_COL_PATH_LENGTH]; // name of the collection + char _name[512]; // name of the collection std::shared_ptr const> _keyOptions; // options for key creation @@ -292,7 +292,6 @@ struct TRI_collection_t { std::unique_ptr _keyGenerator; std::atomic _uncollectedLogfileEntries; - arangodb::basics::ReadWriteLock _compactionLock; private: mutable arangodb::Ditches _ditches; diff --git a/arangod/VocBase/datafile.cpp b/arangod/VocBase/datafile.cpp index e4665d62d0..410e31dde2 100644 --- a/arangod/VocBase/datafile.cpp +++ b/arangod/VocBase/datafile.cpp @@ -41,10 +41,9 @@ using namespace arangodb; using namespace arangodb::basics; -//////////////////////////////////////////////////////////////////////////////// -/// @brief check if a marker appears to be created by ArangoDB 28 -//////////////////////////////////////////////////////////////////////////////// +namespace { +/// @brief check if a marker appears to be created by ArangoDB 28 static TRI_voc_crc_t Crc28(TRI_voc_crc_t crc, void const* data, size_t length) { static TRI_voc_crc_t const CrcPolynomial = 0xEDB88320; unsigned char* current = (unsigned char*) data; @@ -91,82 +90,7 @@ static bool IsMarker28 (void const* marker) { return crc == m->_crc; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief return whether the datafile is a physical file (true) or an -/// anonymous mapped region (false) -//////////////////////////////////////////////////////////////////////////////// - -static bool IsPhysicalDatafile(TRI_datafile_t const* datafile) { - return datafile->_filename != nullptr; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief return the name of a datafile -//////////////////////////////////////////////////////////////////////////////// - -static char const* GetNameDatafile(TRI_datafile_t const* datafile) { - if (datafile->_filename == nullptr) { - // anonymous regions do not have a filename - return "anonymous region"; - } - - // return name of the physical file - return datafile->_filename; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief close a datafile -//////////////////////////////////////////////////////////////////////////////// - -static void CloseDatafile(TRI_datafile_t* datafile) { - TRI_ASSERT(datafile->_state != TRI_DF_STATE_CLOSED); - - if (datafile->isPhysical(datafile)) { - int res = TRI_CLOSE(datafile->_fd); - - if (res != TRI_ERROR_NO_ERROR) { - LOG(ERR) << "unable to close datafile '" << datafile->getName(datafile) << "': " << res; - } - } - - datafile->_state = TRI_DF_STATE_CLOSED; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief destroy a datafile -//////////////////////////////////////////////////////////////////////////////// - -static void DestroyDatafile(TRI_datafile_t* datafile) { - if (datafile->_filename != nullptr) { - TRI_FreeString(TRI_CORE_MEM_ZONE, datafile->_filename); - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief sync the data of a datafile -//////////////////////////////////////////////////////////////////////////////// - -static bool SyncDatafile(TRI_datafile_t* datafile, char const* begin, - char const* end) { - if (datafile->_filename == nullptr) { - // anonymous regions do not need to be synced - return true; - } - - TRI_ASSERT(datafile->_fd >= 0); - - if (begin == end) { - // no need to sync - return true; - } - - return TRI_MSync(datafile->_fd, begin, end); -} - -//////////////////////////////////////////////////////////////////////////////// /// @brief calculates the actual CRC of a marker, without bounds checks -//////////////////////////////////////////////////////////////////////////////// - static TRI_voc_crc_t CalculateCrcValue(TRI_df_marker_t const* marker) { TRI_voc_size_t zero = 0; off_t o = marker->offsetOfCrc(); @@ -185,10 +109,7 @@ static TRI_voc_crc_t CalculateCrcValue(TRI_df_marker_t const* marker) { return crc; } -//////////////////////////////////////////////////////////////////////////////// /// @brief checks a CRC of a marker, with bounds checks -//////////////////////////////////////////////////////////////////////////////// - static bool CheckCrcMarker(TRI_df_marker_t const* marker, char const* end) { TRI_voc_size_t const size = marker->getSize(); @@ -204,17 +125,19 @@ static bool CheckCrcMarker(TRI_df_marker_t const* marker, char const* end) { return marker->getCrc() == expected; } +} + //////////////////////////////////////////////////////////////////////////////// /// @brief creates a new datafile /// /// returns the file descriptor or -1 if the file cannot be created //////////////////////////////////////////////////////////////////////////////// -static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) { +static int CreateDatafile(std::string const& filename, TRI_voc_size_t maximalSize) { TRI_ERRORBUF; // open the file - int fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, + int fd = TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); TRI_IF_FAILURE("CreateDatafile1") { @@ -265,7 +188,7 @@ static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) { } TRI_CLOSE(fd); - TRI_UnlinkFile(filename); + TRI_UnlinkFile(filename.c_str()); return -1; } @@ -282,7 +205,7 @@ static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) { TRI_CLOSE(fd); // remove empty file - TRI_UnlinkFile(filename); + TRI_UnlinkFile(filename.c_str()); LOG(ERR) << "cannot seek in datafile '" << filename << "': '" << TRI_GET_ERRORBUF << "'"; return -1; @@ -291,59 +214,6 @@ static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) { return fd; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief initializes a datafile -//////////////////////////////////////////////////////////////////////////////// - -static void InitDatafile(TRI_datafile_t* datafile, char* filename, int fd, - void* mmHandle, TRI_voc_size_t maximalSize, - TRI_voc_size_t currentSize, TRI_voc_fid_t fid, - char* data) { - // filename is a string for physical datafiles, and NULL for anonymous regions - // fd is a positive value for physical datafiles, and -1 for anonymous regions - if (filename == nullptr) { - TRI_ASSERT(fd == -1); - } else { - TRI_ASSERT(fd >= 0); - } - - datafile->_state = TRI_DF_STATE_READ; - datafile->_fid = fid; - - datafile->_filename = filename; - datafile->_fd = fd; - datafile->_mmHandle = mmHandle; - - datafile->_initSize = maximalSize; - datafile->_maximalSize = maximalSize; - datafile->_currentSize = currentSize; - datafile->_footerSize = sizeof(TRI_df_footer_marker_t); - - datafile->_isSealed = false; - datafile->_lastError = TRI_ERROR_NO_ERROR; - - datafile->_full = false; - - datafile->_data = data; - datafile->_next = data + currentSize; - - datafile->_synced = data; - datafile->_written = nullptr; - - // reset tick aggregates - datafile->_tickMin = 0; - datafile->_tickMax = 0; - datafile->_dataMin = 0; - datafile->_dataMax = 0; - - // initialize function pointers - datafile->isPhysical = &IsPhysicalDatafile; - datafile->getName = &GetNameDatafile; - datafile->close = &CloseDatafile; - datafile->destroy = &DestroyDatafile; - datafile->sync = &SyncDatafile; -} - //////////////////////////////////////////////////////////////////////////////// /// @brief truncates a datafile /// @@ -357,7 +227,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile, void* mmHandle; // this function must not be called for non-physical datafiles - TRI_ASSERT(datafile->isPhysical(datafile)); + TRI_ASSERT(datafile->isPhysical()); size_t pageSize = PageSizeFeature::getPageSize(); // use multiples of page-size @@ -368,7 +238,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile, // sanity check if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) > maximalSize) { - LOG(ERR) << "cannot create datafile '" << datafile->getName(datafile) << "', maximal size " << (unsigned int)maximalSize << " is too small"; + LOG(ERR) << "cannot create datafile '" << datafile->getName() << "', maximal size " << maximalSize << " is too small"; return TRI_set_errno(TRI_ERROR_ARANGO_MAXIMAL_SIZE_TOO_SMALL); } @@ -477,13 +347,13 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile, // rename files std::string oldname = arangodb::basics::FileUtils::buildFilename(datafile->_filename, ".corrupted"); - res = TRI_RenameFile(datafile->_filename, oldname.c_str()); + res = TRI_RenameFile(datafile->_filename.c_str(), oldname.c_str()); if (res != TRI_ERROR_NO_ERROR) { return res; } - res = TRI_RenameFile(filename.c_str(), datafile->_filename); + res = TRI_RenameFile(filename.c_str(), datafile->_filename.c_str()); if (res != TRI_ERROR_NO_ERROR) { return res; @@ -493,7 +363,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile, // call will return an error datafile->_state = TRI_DF_STATE_WRITE; - return TRI_SealDatafile(datafile); + return datafile->seal(); } //////////////////////////////////////////////////////////////////////////////// @@ -502,7 +372,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile, static bool TryRepairDatafile(TRI_datafile_t* datafile) { // this function must not be called for non-physical datafiles - TRI_ASSERT(datafile->isPhysical(datafile)); + TRI_ASSERT(datafile->isPhysical()); char* ptr = datafile->_data; char* end = datafile->_data + datafile->_currentSize; @@ -552,7 +422,7 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) { if (isFollowedByNullBytes) { // only last marker in datafile was corrupt. fix the datafile in // place - LOG(INFO) << "truncating datafile '" << datafile->getName(datafile) << "' at position " << currentSize; + LOG(INFO) << "truncating datafile '" << datafile->getName() << "' at position " << currentSize; int res = TruncateAndSealDatafile(datafile, currentSize); return (res == TRI_ERROR_NO_ERROR); } @@ -572,32 +442,26 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) { // next marker looks good. // create a temporary buffer - auto buffer = - TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, size, false); - - if (buffer == nullptr) { - return false; - } + auto buffer = std::unique_ptr(new char[size]); // create a new marker in the temporary buffer - auto temp = reinterpret_cast(buffer); + auto temp = reinterpret_cast(buffer.get()); DatafileHelper::InitMarker( - reinterpret_cast(buffer), TRI_DF_MARKER_BLANK, + reinterpret_cast(buffer.get()), TRI_DF_MARKER_BLANK, static_cast(size)); temp->setCrc(CalculateCrcValue(temp)); // all done. now copy back the marker into the file - memcpy(static_cast(ptr), buffer, + memcpy(static_cast(ptr), buffer.get(), static_cast(size)); - TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer); - - bool ok = datafile->sync(datafile, ptr, (ptr + size)); + buffer.reset(); // don't need the buffer anymore + bool ok = datafile->sync(ptr, (ptr + size)); if (ok) { - LOG(INFO) << "zeroed single invalid marker in datafile '" << datafile->getName(datafile) << "' at position " << currentSize; + LOG(INFO) << "zeroed single invalid marker in datafile '" << datafile->getName() << "' at position " << currentSize; } else { - LOG(ERR) << "could not zero single invalid marker in datafile '" << datafile->getName(datafile) << "' at position " << currentSize; + LOG(ERR) << "could not zero single invalid marker in datafile '" << datafile->getName() << "' at position " << currentSize; return false; } } else { @@ -628,9 +492,9 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) { //////////////////////////////////////////////////////////////////////////////// static bool FixDatafile(TRI_datafile_t* datafile, TRI_voc_size_t currentSize) { - LOG(WARN) << "datafile '" << datafile->getName(datafile) << "' is corrupted at position " << currentSize; + LOG(WARN) << "datafile '" << datafile->getName() << "' is corrupted at position " << currentSize; - LOG(WARN) << "setting datafile '" << datafile->getName(datafile) << "' to read-only and ignoring all data from this file beyond this position"; + LOG(WARN) << "setting datafile '" << datafile->getName() << "' to read-only and ignoring all data from this file beyond this position"; datafile->_currentSize = currentSize; TRI_ASSERT(datafile->_initSize == datafile->_maximalSize); @@ -649,14 +513,14 @@ static bool FixDatafile(TRI_datafile_t* datafile, TRI_voc_size_t currentSize) { static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { // this function must not be called for non-physical datafiles - TRI_ASSERT(datafile->isPhysical(datafile)); + TRI_ASSERT(datafile->isPhysical()); char* ptr = datafile->_data; char* end = datafile->_data + datafile->_currentSize; TRI_voc_size_t currentSize = 0; if (datafile->_currentSize == 0) { - LOG(WARN) << "current size is 0 in read-only datafile '" << datafile->getName(datafile) << "', trying to fix"; + LOG(WARN) << "current size is 0 in read-only datafile '" << datafile->getName() << "', trying to fix"; end = datafile->_data + datafile->_maximalSize; } @@ -677,7 +541,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { #endif if (size == 0) { - LOG(DEBUG) << "reached end of datafile '" << datafile->getName(datafile) << "' data, current size " << currentSize; + LOG(DEBUG) << "reached end of datafile '" << datafile->getName() << "' data, current size " << currentSize; datafile->_currentSize = currentSize; datafile->_next = datafile->_data + datafile->_currentSize; @@ -698,7 +562,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { datafile->_next = datafile->_data + datafile->_currentSize; datafile->_state = TRI_DF_STATE_OPEN_ERROR; - LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' too small, size " << size << ", should be at least " << sizeof(TRI_df_marker_t); + LOG(WARN) << "marker in datafile '" << datafile->getName() << "' too small, size " << size << ", should be at least " << sizeof(TRI_df_marker_t); updateTick(maxTick); @@ -717,7 +581,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { datafile->_next = datafile->_data + datafile->_currentSize; datafile->_state = TRI_DF_STATE_OPEN_ERROR; - LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' points with size " << size << " beyond end of file"; + LOG(WARN) << "marker in datafile '" << datafile->getName() << "' points with size " << size << " beyond end of file"; updateTick(maxTick); @@ -730,7 +594,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { if (!TRI_IsValidMarkerDatafile(marker)) { if (type == 0 && size < 128) { // ignore markers with type 0 and a small size - LOG(WARN) << "ignoring suspicious marker in datafile '" << datafile->getName(datafile) << "': type: " << type << ", size: " << size; + LOG(WARN) << "ignoring suspicious marker in datafile '" << datafile->getName() << "': type: " << type << ", size: " << size; } else { if (ignoreFailures) { return FixDatafile(datafile, currentSize); @@ -742,7 +606,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { datafile->_next = datafile->_data + datafile->_currentSize; datafile->_state = TRI_DF_STATE_OPEN_ERROR; - LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' is corrupt: type: " << type << ", size: " << size; + LOG(WARN) << "marker in datafile '" << datafile->getName() << "' is corrupt: type: " << type << ", size: " << size; updateTick(maxTick); @@ -775,7 +639,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { if (isFollowedByNullBytes) { // only last marker in datafile was corrupt. fix the datafile in // place - LOG(WARN) << "datafile '" << datafile->getName(datafile) << "' automatically truncated at last marker"; + LOG(WARN) << "datafile '" << datafile->getName() << "' automatically truncated at last marker"; ignoreFailures = true; } else { // there is some other stuff following. now inspect it... @@ -812,7 +676,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { datafile->_next = datafile->_data + datafile->_currentSize; datafile->_state = TRI_DF_STATE_OPEN_ERROR; - LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); + LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName() << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); if (nextMarkerOk) { LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it"; @@ -834,7 +698,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { currentSize += static_cast(alignedSize); if (marker->getType() == TRI_DF_MARKER_FOOTER) { - LOG(DEBUG) << "found footer, reached end of datafile '" << datafile->getName(datafile) << "', current size " << currentSize; + LOG(DEBUG) << "found footer, reached end of datafile '" << datafile->getName() << "', current size " << currentSize; datafile->_isSealed = true; datafile->_currentSize = currentSize; @@ -945,21 +809,7 @@ static TRI_datafile_t* CreateAnonymousDatafile(TRI_voc_fid_t fid, return nullptr; } - // create datafile structure - TRI_datafile_t* datafile = static_cast( - TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false)); - - if (datafile == nullptr) { - TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY); - - LOG(ERR) << "out of memory"; - return nullptr; - } - - InitDatafile(datafile, nullptr, fd, mmHandle, maximalSize, 0, fid, - static_cast(data)); - - return datafile; + return new TRI_datafile_t(StaticStrings::Empty, fd, mmHandle, maximalSize, 0, fid, static_cast(data)); } #endif @@ -968,10 +818,10 @@ static TRI_datafile_t* CreateAnonymousDatafile(TRI_voc_fid_t fid, /// @brief creates a new physical datafile //////////////////////////////////////////////////////////////////////////////// -static TRI_datafile_t* CreatePhysicalDatafile(char const* filename, +static TRI_datafile_t* CreatePhysicalDatafile(std::string const& filename, TRI_voc_fid_t fid, TRI_voc_size_t maximalSize) { - TRI_ASSERT(filename != nullptr); + TRI_ASSERT(!filename.empty()); int fd = CreateDatafile(filename, maximalSize); @@ -996,7 +846,7 @@ static TRI_datafile_t* CreatePhysicalDatafile(char const* filename, TRI_CLOSE(fd); // remove empty file - TRI_UnlinkFile(filename); + TRI_UnlinkFile(filename.c_str()); LOG(ERR) << "cannot memory map file '" << filename << "': '" << TRI_errno_string((int)res) << "'"; LOG(ERR) << "The database directory might reside on a shared folder " @@ -1006,47 +856,33 @@ static TRI_datafile_t* CreatePhysicalDatafile(char const* filename, } // create datafile structure - auto datafile = static_cast( - TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false)); - - if (datafile == nullptr) { - TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY); + try { + return new TRI_datafile_t(filename, fd, mmHandle, maximalSize, 0, fid, static_cast(data)); + } catch (...) { TRI_CLOSE(fd); - - LOG(ERR) << "out of memory"; return nullptr; } - - InitDatafile(datafile, TRI_DuplicateString(filename), fd, mmHandle, - maximalSize, 0, fid, static_cast(data)); - - // Advise OS that sequential access is going to happen: - TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize, - TRI_MADVISE_SEQUENTIAL); - - return datafile; } //////////////////////////////////////////////////////////////////////////////// /// @brief opens a datafile //////////////////////////////////////////////////////////////////////////////// -static TRI_datafile_t* OpenDatafile(char const* filename, bool ignoreErrors) { +static TRI_datafile_t* OpenDatafile(std::string const& filename, bool ignoreErrors) { TRI_ERRORBUF; void* data; TRI_stat_t status; void* mmHandle; // this function must not be called for non-physical datafiles - TRI_ASSERT(filename != nullptr); - - TRI_voc_fid_t fid = GetNumericFilenamePart(filename); + TRI_ASSERT(!filename.empty()); + TRI_voc_fid_t fid = GetNumericFilenamePart(filename.c_str()); // .......................................................................... // attempt to open a datafile file // .......................................................................... - int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC); + int fd = TRI_OPEN(filename.c_str(), O_RDWR | TRI_O_CLOEXEC); if (fd < 0) { TRI_SYSTEM_ERROR(); @@ -1163,27 +999,21 @@ static TRI_datafile_t* OpenDatafile(char const* filename, bool ignoreErrors) { } // create datafile structure - TRI_datafile_t* datafile = static_cast( - TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false)); - - if (datafile == nullptr) { + try { + return new TRI_datafile_t(filename, fd, mmHandle, size, size, fid, static_cast(data)); + } catch (...) { TRI_UNMMFile(data, size, fd, &mmHandle); TRI_CLOSE(fd); return nullptr; } - - InitDatafile(datafile, TRI_DuplicateString(filename), fd, mmHandle, size, - size, fid, static_cast(data)); - - return datafile; } //////////////////////////////////////////////////////////////////////////////// /// @brief creates either an anonymous or a physical datafile //////////////////////////////////////////////////////////////////////////////// -TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid, +TRI_datafile_t* TRI_CreateDatafile(std::string const& filename, TRI_voc_fid_t fid, TRI_voc_size_t maximalSize, bool withInitialMarkers) { TRI_datafile_t* datafile; @@ -1205,7 +1035,7 @@ TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid, } // create either an anonymous or a physical datafile - if (filename == nullptr) { + if (filename.empty()) { #ifdef TRI_HAVE_ANONYMOUS_MMAP datafile = CreateAnonymousDatafile(fid, maximalSize); #else @@ -1227,40 +1057,22 @@ TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid, int res = WriteInitialHeaderMarker(datafile, fid, maximalSize); if (res != TRI_ERROR_NO_ERROR) { - LOG(ERR) << "cannot write header to datafile '" << datafile->getName(datafile) << "'"; + LOG(ERR) << "cannot write header to datafile '" << datafile->getName() << "'"; TRI_UNMMFile(datafile->_data, datafile->_maximalSize, datafile->_fd, &datafile->_mmHandle); - datafile->close(datafile); - datafile->destroy(datafile); - TRI_Free(TRI_UNKNOWN_MEM_ZONE, datafile); + datafile->close(); + delete datafile; return nullptr; } } - LOG(DEBUG) << "created datafile '" << datafile->getName(datafile) << "' of size " << maximalSize << " and page-size " << pageSize; + LOG(DEBUG) << "created datafile '" << datafile->getName() << "' of size " << maximalSize << " and page-size " << pageSize; return datafile; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief frees the memory allocated, but does not free the pointer -//////////////////////////////////////////////////////////////////////////////// - -void TRI_DestroyDatafile(TRI_datafile_t* datafile) { - datafile->destroy(datafile); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief frees the memory allocated and but frees the pointer -//////////////////////////////////////////////////////////////////////////////// - -void TRI_FreeDatafile(TRI_datafile_t* datafile) { - TRI_DestroyDatafile(datafile); - TRI_Free(TRI_UNKNOWN_MEM_ZONE, datafile); -} - //////////////////////////////////////////////////////////////////////////////// /// @brief returns the name for a marker //////////////////////////////////////////////////////////////////////////////// @@ -1436,15 +1248,15 @@ int TRI_WriteElementDatafile(TRI_datafile_t* datafile, void* position, // out of bounds check for writing into a datafile if (position == nullptr || position < (void*)datafile->_data || position >= (void*)(datafile->_data + datafile->_maximalSize)) { - LOG(ERR) << "logic error. writing out of bounds of datafile '" << datafile->getName(datafile) << "'"; + LOG(ERR) << "logic error. writing out of bounds of datafile '" << datafile->getName() << "'"; return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE); } memcpy(position, marker, static_cast(marker->getSize())); if (forceSync) { - bool ok = datafile->sync(datafile, static_cast(position), - ((char*)position) + marker->getSize()); + bool ok = datafile->sync(static_cast(position), + reinterpret_cast(position) + marker->getSize()); if (!ok) { datafile->_state = TRI_DF_STATE_WRITE_ERROR; @@ -1506,7 +1318,7 @@ int TRI_WriteCrcElementDatafile(TRI_datafile_t* datafile, void* position, TRI_df_marker_t* marker, bool forceSync) { TRI_ASSERT(marker->getTick() != 0); - if (datafile->isPhysical(datafile)) { + if (datafile->isPhysical()) { TRI_voc_crc_t crc = TRI_InitialCrc32(); crc = TRI_BlockCrc32(crc, (char const*)marker, marker->getSize()); @@ -1528,7 +1340,7 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile, void* data) { TRI_ASSERT(iterator != nullptr); - LOG(TRACE) << "iterating over datafile '" << datafile->getName(datafile) << "', fid: " << datafile->_fid; + LOG(TRACE) << "iterating over datafile '" << datafile->getName() << "', fid: " << datafile->_fid; char const* ptr = datafile->_data; char const* end = datafile->_data + datafile->_currentSize; @@ -1563,7 +1375,7 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile, /// also may set datafile's min/max tick values bool TRI_IterateDatafile(TRI_datafile_t* datafile, std::function const& cb) { - LOG(TRACE) << "iterating over datafile '" << datafile->getName(datafile) << "', fid: " << datafile->_fid; + LOG(TRACE) << "iterating over datafile '" << datafile->getName() << "', fid: " << datafile->_fid; char const* ptr = datafile->_data; char const* end = datafile->_data + datafile->_currentSize; @@ -1600,9 +1412,9 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile, /// The datafile will be opened read-only if a footer is found //////////////////////////////////////////////////////////////////////////////// -TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) { +TRI_datafile_t* TRI_OpenDatafile(std::string const& filename, bool ignoreFailures) { // this function must not be called for non-physical datafiles - TRI_ASSERT(filename != nullptr); + TRI_ASSERT(!filename.empty()); TRI_datafile_t* datafile = OpenDatafile(filename, false); @@ -1618,9 +1430,9 @@ TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) { &datafile->_mmHandle); TRI_CLOSE(datafile->_fd); - LOG(ERR) << "datafile '" << datafile->getName(datafile) << "' is corrupt"; + LOG(ERR) << "datafile '" << datafile->getName() << "' is corrupt"; // must free datafile here - TRI_FreeDatafile(datafile); + delete datafile; return nullptr; } @@ -1642,47 +1454,13 @@ TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) { return datafile; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief closes a datafile and all memory regions -//////////////////////////////////////////////////////////////////////////////// - -int TRI_CloseDatafile(TRI_datafile_t* datafile) { - if (datafile->_state == TRI_DF_STATE_READ || - datafile->_state == TRI_DF_STATE_WRITE) { - int res = TRI_UNMMFile(datafile->_data, datafile->_initSize, datafile->_fd, - &datafile->_mmHandle); - - if (res != TRI_ERROR_NO_ERROR) { - LOG(ERR) << "munmap failed with: " << res; - datafile->_state = TRI_DF_STATE_WRITE_ERROR; - datafile->_lastError = res; - return res; - } - - datafile->close(datafile); - datafile->_data = nullptr; - datafile->_next = nullptr; - datafile->_fd = -1; - - return TRI_ERROR_NO_ERROR; - } - - if (datafile->_state == TRI_DF_STATE_CLOSED) { - LOG(WARN) << "closing an already closed datafile '" << datafile->getName(datafile) << "'"; - return TRI_ERROR_NO_ERROR; - } - - TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE); - return TRI_ERROR_ARANGO_ILLEGAL_STATE; -} - //////////////////////////////////////////////////////////////////////////////// /// @brief renames a datafile //////////////////////////////////////////////////////////////////////////////// int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) { // this function must not be called for non-physical datafiles - TRI_ASSERT(datafile->isPhysical(datafile)); + TRI_ASSERT(datafile->isPhysical()); TRI_ASSERT(filename != nullptr); if (TRI_ExistsFile(filename)) { @@ -1693,7 +1471,7 @@ int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) { return TRI_ERROR_ARANGO_DATAFILE_ALREADY_EXISTS; } - int res = TRI_RenameFile(datafile->_filename, filename); + int res = TRI_RenameFile(datafile->_filename.c_str(), filename); if (res != TRI_ERROR_NO_ERROR) { datafile->_state = TRI_DF_STATE_RENAME_ERROR; @@ -1702,47 +1480,43 @@ int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) { return res; } - TRI_FreeString(TRI_CORE_MEM_ZONE, datafile->_filename); - datafile->_filename = TRI_DuplicateString(filename); + datafile->_filename = filename; return TRI_ERROR_NO_ERROR; } -//////////////////////////////////////////////////////////////////////////////// /// @brief seals a datafile, writes a footer, sets it to read-only -//////////////////////////////////////////////////////////////////////////////// - -int TRI_SealDatafile(TRI_datafile_t* datafile) { - if (datafile->_state == TRI_DF_STATE_READ) { - return TRI_set_errno(TRI_ERROR_ARANGO_READ_ONLY); +int TRI_datafile_t::seal() { + if (_state == TRI_DF_STATE_READ) { + return TRI_ERROR_ARANGO_READ_ONLY; } - if (datafile->_state != TRI_DF_STATE_WRITE) { - return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE); + if (_state != TRI_DF_STATE_WRITE) { + return TRI_ERROR_ARANGO_ILLEGAL_STATE; } - if (datafile->_isSealed) { - return TRI_set_errno(TRI_ERROR_ARANGO_DATAFILE_SEALED); + if (_isSealed) { + return TRI_ERROR_ARANGO_DATAFILE_SEALED; } // set a proper tick value - if (datafile->_tickMax == 0) { - datafile->_tickMax = TRI_NewTickServer(); + if (_tickMax == 0) { + _tickMax = TRI_NewTickServer(); } // create the footer - TRI_df_footer_marker_t footer = DatafileHelper::CreateFooterMarker(datafile->_tickMax); + TRI_df_footer_marker_t footer = DatafileHelper::CreateFooterMarker(_tickMax); // reserve space and write footer to file - datafile->_footerSize = 0; + _footerSize = 0; TRI_df_marker_t* position; int res = - TRI_ReserveElementDatafile(datafile, footer.base.getSize(), &position, 0); + TRI_ReserveElementDatafile(this, footer.base.getSize(), &position, 0); if (res == TRI_ERROR_NO_ERROR) { TRI_ASSERT(position != nullptr); - res = TRI_WriteCrcElementDatafile(datafile, position, &footer.base, false); + res = TRI_WriteCrcElementDatafile(this, position, &footer.base, false); } if (res != TRI_ERROR_NO_ERROR) { @@ -1750,44 +1524,42 @@ int TRI_SealDatafile(TRI_datafile_t* datafile) { } // sync file - bool ok = datafile->sync(datafile, datafile->_synced, - ((char*)datafile->_data) + datafile->_currentSize); + bool ok = sync(_synced, reinterpret_cast(_data) + _currentSize); if (!ok) { - datafile->_state = TRI_DF_STATE_WRITE_ERROR; + _state = TRI_DF_STATE_WRITE_ERROR; if (errno == ENOSPC) { - datafile->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL); + _lastError = TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL); } else { - datafile->_lastError = TRI_errno(); + _lastError = TRI_errno(); } LOG(ERR) << "msync failed with: " << TRI_last_error(); } // everything is now synced - datafile->_synced = datafile->_written; + _synced = _written; - TRI_ProtectMMFile(datafile->_data, datafile->_maximalSize, PROT_READ, - datafile->_fd, &datafile->_mmHandle); + TRI_ProtectMMFile(_data, _maximalSize, PROT_READ, + _fd, &_mmHandle); // seal datafile if (ok) { - datafile->_isSealed = true; - datafile->_state = TRI_DF_STATE_READ; + _isSealed = true; + _state = TRI_DF_STATE_READ; // note: _initSize must remain constant - TRI_ASSERT(datafile->_initSize == datafile->_maximalSize); - datafile->_maximalSize = datafile->_currentSize; + TRI_ASSERT(_initSize == _maximalSize); + _maximalSize = _currentSize; } if (!ok) { - return datafile->_lastError; + return _lastError; } - if (datafile->isPhysical(datafile)) { + if (isPhysical()) { // From now on we predict random access (until collection or compaction): - TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize, - TRI_MADVISE_RANDOM); + TRI_MMFileAdvise(_data, _maximalSize, TRI_MADVISE_RANDOM); } return TRI_ERROR_NO_ERROR; @@ -1798,9 +1570,9 @@ int TRI_SealDatafile(TRI_datafile_t* datafile) { /// this is called from the recovery procedure only //////////////////////////////////////////////////////////////////////////////// -int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position) { +int TRI_TruncateDatafile(std::string const& path, TRI_voc_size_t position) { // this function must not be called for non-physical datafiles - TRI_ASSERT(path != nullptr); + TRI_ASSERT(!path.empty()); TRI_datafile_t* datafile = OpenDatafile(path, true); @@ -1809,8 +1581,7 @@ int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position) { } int res = TruncateAndSealDatafile(datafile, position); - TRI_CloseDatafile(datafile); - TRI_FreeDatafile(datafile); + delete datafile; return res; } @@ -1835,8 +1606,7 @@ bool TRI_TryRepairDatafile(char const* path) { &datafile->_mmHandle); bool result = TryRepairDatafile(datafile); - TRI_CloseDatafile(datafile); - TRI_FreeDatafile(datafile); + delete datafile; return result; } @@ -1909,7 +1679,7 @@ static std::string DiagnoseMarker(TRI_df_marker_t const* marker, static DatafileScan ScanDatafile(TRI_datafile_t const* datafile) { // this function must not be called for non-physical datafiles - TRI_ASSERT(datafile->isPhysical(datafile)); + TRI_ASSERT(datafile->isPhysical()); char* ptr = datafile->_data; char* end = datafile->_data + datafile->_currentSize; @@ -2027,8 +1797,7 @@ DatafileScan TRI_ScanDatafile(char const* path) { if (datafile != nullptr) { scan = ScanDatafile(datafile); - TRI_CloseDatafile(datafile); - TRI_FreeDatafile(datafile); + delete datafile; } else { scan.currentSize = 0; scan.maximalSize = 0; @@ -2041,3 +1810,112 @@ DatafileScan TRI_ScanDatafile(char const* path) { return scan; } + + +TRI_datafile_t::TRI_datafile_t(std::string const& filename, int fd, void* mmHandle, TRI_voc_size_t maximalSize, + TRI_voc_size_t currentSize, TRI_voc_fid_t fid, char* data) + : _fid(fid), + _state(TRI_DF_STATE_READ), + _fd(fd), + _mmHandle(mmHandle), + _initSize(maximalSize), + _maximalSize(maximalSize), + _currentSize(currentSize), + _footerSize(sizeof(TRI_df_footer_marker_t)), + _data(data), + _next(data + currentSize), + _tickMin(0), + _tickMax(0), + _dataMin(0), + _dataMax(0), + _filename(filename), + _lastError(TRI_ERROR_NO_ERROR), + _full(false), + _isSealed(false), + _synced(data), + _written(nullptr) { + // filename is a string for physical datafiles, and NULL for anonymous regions + // fd is a positive value for physical datafiles, and -1 for anonymous regions + if (filename.empty()) { + TRI_ASSERT(fd == -1); + } else { + TRI_ASSERT(fd >= 0); + + // Advise OS that sequential access is going to happen: + TRI_MMFileAdvise(_data, _maximalSize, TRI_MADVISE_SEQUENTIAL); + } +} + +TRI_datafile_t::~TRI_datafile_t() { + try { + this->close(); + } catch (...) { + // silently continue as this is the destructor + } +} + +/// @brief return the name of a datafile +std::string TRI_datafile_t::getName() const { + if (_filename.empty()) { + // anonymous regions do not have a filename + return "anonymous region"; + } + + // return name of the physical file + return _filename; +} + +/// @brief close a datafile +int TRI_datafile_t::close() { + if (_state == TRI_DF_STATE_READ || + _state == TRI_DF_STATE_WRITE) { + int res = TRI_UNMMFile(_data, _initSize, _fd, &_mmHandle); + + if (res != TRI_ERROR_NO_ERROR) { + LOG(ERR) << "munmap failed with: " << res; + _state = TRI_DF_STATE_WRITE_ERROR; + _lastError = res; + return res; + } + + if (isPhysical()) { + int res = TRI_CLOSE(_fd); + + if (res != TRI_ERROR_NO_ERROR) { + LOG(ERR) << "unable to close datafile '" << getName() << "': " << res; + } + } + + _state = TRI_DF_STATE_CLOSED; + _data = nullptr; + _next = nullptr; + _fd = -1; + + return TRI_ERROR_NO_ERROR; + } + + if (_state == TRI_DF_STATE_CLOSED) { + LOG(WARN) << "closing an already closed datafile '" << getName() << "'"; + return TRI_ERROR_NO_ERROR; + } + + return TRI_ERROR_ARANGO_ILLEGAL_STATE; +} + +/// @brief sync the data of a datafile +bool TRI_datafile_t::sync(char const* begin, char const* end) { + if (!isPhysical()) { + // anonymous regions do not need to be synced + return true; + } + + TRI_ASSERT(_fd >= 0); + + if (begin == end) { + // no need to sync + return true; + } + + return TRI_MSync(_fd, begin, end); +} + diff --git a/arangod/VocBase/datafile.h b/arangod/VocBase/datafile.h index 4c9c020b1f..7a909d9c32 100644 --- a/arangod/VocBase/datafile.h +++ b/arangod/VocBase/datafile.h @@ -153,6 +153,30 @@ typedef uint32_t TRI_df_version_t; //////////////////////////////////////////////////////////////////////////////// struct TRI_datafile_t { + TRI_datafile_t(std::string const& filename, int fd, void* mmHandle, TRI_voc_size_t maximalSize, + TRI_voc_size_t currentsize, TRI_voc_fid_t fid, char* data); + ~TRI_datafile_t(); + + /// @brief return whether the datafile is a physical file (true) or an + /// anonymous mapped region (false) + inline bool isPhysical() const { return !_filename.empty(); } + + /// @brief return the name of a datafile + std::string getName() const; + + /// @brief close a datafile + int close(); + + /// @brief destroy a datafile + void destroy(); + + /// @brief sync the data of a datafile + bool sync(char const* begin, char const* end); + + /// @brief seals a datafile, writes a footer, sets it to read-only + int seal(); + + TRI_voc_fid_t _fid; // datafile identifier TRI_df_state_e _state; // state of the datafile (READ or WRITE) @@ -174,24 +198,12 @@ struct TRI_datafile_t { TRI_voc_tick_t _dataMin; // minimum tick value of document/edge marker TRI_voc_tick_t _dataMax; // maximum tick value of document/edge marker - char* _filename; // underlying filename - - // function pointers - bool (*isPhysical)(struct TRI_datafile_t const*); // returns true if - // the datafile is a - // physical file - char const* (*getName)( - struct TRI_datafile_t const*); // returns the name of a datafile - void (*close)(struct TRI_datafile_t*); // close the datafile - void (*destroy)(struct TRI_datafile_t*); // destroys the datafile - bool (*sync)(struct TRI_datafile_t*, char const*, - char const*); // syncs the datafile + std::string _filename; // underlying filename int _lastError; // last (critical) error bool _full; // at least one request was rejected because there is not enough // room bool _isSealed; // true, if footer has been written - // ............................................................................. // access to the following attributes must be protected by a _lock // ............................................................................. @@ -385,20 +397,8 @@ struct TRI_col_header_marker_t { /// ref TRI_CreatePhysicalDatafile, based on the first parameter //////////////////////////////////////////////////////////////////////////////// -TRI_datafile_t* TRI_CreateDatafile(char const*, TRI_voc_fid_t fid, - TRI_voc_size_t, bool); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief frees the memory allocated, but does not free the pointer -//////////////////////////////////////////////////////////////////////////////// - -void TRI_DestroyDatafile(TRI_datafile_t*); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief frees the memory allocated and but frees the pointer -//////////////////////////////////////////////////////////////////////////////// - -void TRI_FreeDatafile(TRI_datafile_t*); +TRI_datafile_t* TRI_CreateDatafile(std::string const& filename, TRI_voc_fid_t fid, + TRI_voc_size_t maximalSize, bool withInitialMarkers); //////////////////////////////////////////////////////////////////////////////// /// @brief returns the name for a marker @@ -460,19 +460,7 @@ bool TRI_IterateDatafile(TRI_datafile_t*, /// @brief opens an existing datafile read-only //////////////////////////////////////////////////////////////////////////////// -TRI_datafile_t* TRI_OpenDatafile(char const*, bool); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief closes a datafile and all memory regions -//////////////////////////////////////////////////////////////////////////////// - -int TRI_CloseDatafile(TRI_datafile_t* datafile); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief seals a database, writes a footer, sets it to read-only -//////////////////////////////////////////////////////////////////////////////// - -int TRI_SealDatafile(TRI_datafile_t* datafile) TRI_WARN_UNUSED_RESULT; +TRI_datafile_t* TRI_OpenDatafile(std::string const& filename, bool ignoreFailures); //////////////////////////////////////////////////////////////////////////////// /// @brief renames a datafile @@ -484,7 +472,7 @@ int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename); /// @brief truncates a datafile and seals it, only called by arango-dfdd //////////////////////////////////////////////////////////////////////////////// -int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position); +int TRI_TruncateDatafile(std::string const& path, TRI_voc_size_t position); //////////////////////////////////////////////////////////////////////////////// /// @brief try to repair a datafile, only called by arango-dfdd diff --git a/arangod/VocBase/replication-common.cpp b/arangod/VocBase/replication-common.cpp index 650cfe70c5..b7e55f6c93 100644 --- a/arangod/VocBase/replication-common.cpp +++ b/arangod/VocBase/replication-common.cpp @@ -22,11 +22,7 @@ //////////////////////////////////////////////////////////////////////////////// #include "replication-common.h" - -#include "Basics/files.h" #include "Basics/tri-strings.h" -#include "VocBase/collection.h" -#include "VocBase/vocbase.h" //////////////////////////////////////////////////////////////////////////////// /// @brief generate a timestamp string in a target buffer diff --git a/arangod/VocBase/replication-dump.cpp b/arangod/VocBase/replication-dump.cpp index ca7e4ceebe..0c94b3c82f 100644 --- a/arangod/VocBase/replication-dump.cpp +++ b/arangod/VocBase/replication-dump.cpp @@ -25,11 +25,12 @@ #include "Basics/ReadLocker.h" #include "Basics/VPackStringBufferAdapter.h" #include "Logger/Logger.h" -#include "VocBase/collection.h" +#include "VocBase/CompactionLocker.h" #include "VocBase/DatafileHelper.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/collection.h" #include "VocBase/datafile.h" #include "VocBase/collection.h" -#include "VocBase/LogicalCollection.h" #include "VocBase/vocbase.h" #include "Wal/Logfile.h" #include "Wal/LogfileManager.h" @@ -447,17 +448,17 @@ static int DumpCollection(TRI_replication_dump_t* dump, //////////////////////////////////////////////////////////////////////////////// int TRI_DumpCollectionReplication(TRI_replication_dump_t* dump, - arangodb::LogicalCollection* col, + arangodb::LogicalCollection* collection, TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax, bool withTicks) { - TRI_ASSERT(col != nullptr); - TRI_ASSERT(col->_collection != nullptr); + TRI_ASSERT(collection != nullptr); + TRI_ASSERT(collection->_collection != nullptr); // get a custom type handler auto customTypeHandler = dump->_transactionContext->orderCustomTypeHandler(); dump->_vpackOptions.customTypeHandler = customTypeHandler.get(); - TRI_collection_t* document = col->_collection; + TRI_collection_t* document = collection->_collection; TRI_ASSERT(document != nullptr); // create a barrier so the underlying collection is not unloaded @@ -470,10 +471,10 @@ int TRI_DumpCollectionReplication(TRI_replication_dump_t* dump, // block compaction int res; { - READ_LOCKER(locker, document->_compactionLock); + CompactionPreventer compactionPreventer(collection); try { - res = DumpCollection(dump, col, document->_vocbase->id(), document->_info.id(), dataMin, dataMax, withTicks); + res = DumpCollection(dump, collection, collection->vocbase()->id(), collection->cid(), dataMin, dataMax, withTicks); } catch (...) { res = TRI_ERROR_INTERNAL; } diff --git a/arangod/VocBase/transaction.cpp b/arangod/VocBase/transaction.cpp index 195f09839e..d32dd3929b 100644 --- a/arangod/VocBase/transaction.cpp +++ b/arangod/VocBase/transaction.cpp @@ -491,7 +491,7 @@ static int UseCollections(TRI_transaction_t* trx, int nestingLevel) { // read-lock the compaction lock if (!HasHint(trx, TRI_TRANSACTION_HINT_NO_COMPACTION_LOCK)) { if (!trxCollection->_compactionLocked) { - trxCollection->_collection->_collection->_compactionLock.readLock(); + trxCollection->_collection->preventCompaction(); trxCollection->_compactionLocked = true; } } @@ -548,7 +548,7 @@ static int UnuseCollections(TRI_transaction_t* trx, int nestingLevel) { if (trxCollection->_accessType == TRI_TRANSACTION_WRITE && trxCollection->_compactionLocked) { // read-unlock the compaction lock - trxCollection->_collection->_collection->_compactionLock.unlock(); + trxCollection->_collection->allowCompaction(); trxCollection->_compactionLocked = false; } } diff --git a/arangod/VocBase/vocbase.h b/arangod/VocBase/vocbase.h index 0d33f29ab0..2b26893937 100644 --- a/arangod/VocBase/vocbase.h +++ b/arangod/VocBase/vocbase.h @@ -63,12 +63,6 @@ class LogicalCollection; constexpr auto TRI_VOC_SYSTEM_DATABASE = "_system"; -//////////////////////////////////////////////////////////////////////////////// -/// @brief maximal path length -//////////////////////////////////////////////////////////////////////////////// - -constexpr size_t TRI_COL_PATH_LENGTH = 512; - //////////////////////////////////////////////////////////////////////////////// /// @brief maximal name length //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Wal/CollectorThread.cpp b/arangod/Wal/CollectorThread.cpp index 83507bf06a..6bf64d53d0 100644 --- a/arangod/Wal/CollectorThread.cpp +++ b/arangod/Wal/CollectorThread.cpp @@ -37,10 +37,11 @@ #include "Utils/DatabaseGuard.h" #include "Utils/SingleCollectionTransaction.h" #include "Utils/StandaloneTransactionContext.h" +#include "VocBase/CompactionLocker.h" #include "VocBase/DatafileHelper.h" #include "VocBase/DatafileStatistics.h" -#include "VocBase/collection.h" #include "VocBase/LogicalCollection.h" +#include "VocBase/collection.h" #include "Wal/Logfile.h" #include "Wal/LogfileManager.h" @@ -626,10 +627,9 @@ int CollectorThread::processCollectionOperations(CollectorCache* cache) { // first try to read-lock the compactor-lock, afterwards try to write-lock the // collection // if any locking attempt fails, release and try again next time - - TRY_READ_LOCKER(locker, document->_compactionLock); + TryCompactionPreventer compactionPreventer(collection); - if (!locker.isLocked()) { + if (!compactionPreventer.isLocked()) { return TRI_ERROR_LOCK_TIMEOUT; } diff --git a/arangod/Wal/CollectorThread.h b/arangod/Wal/CollectorThread.h index 56d3c02657..f50278b76f 100644 --- a/arangod/Wal/CollectorThread.h +++ b/arangod/Wal/CollectorThread.h @@ -37,8 +37,6 @@ #include "Wal/Logfile.h" struct TRI_collection_t; -struct TRI_datafile_t; -struct TRI_df_marker_t; namespace arangodb { class LogicalCollection; diff --git a/arangod/Wal/Logfile.cpp b/arangod/Wal/Logfile.cpp index f6e5075181..3e86fab25d 100644 --- a/arangod/Wal/Logfile.cpp +++ b/arangod/Wal/Logfile.cpp @@ -35,17 +35,13 @@ Logfile::Logfile(Logfile::IdType id, TRI_datafile_t* df, StatusType status) /// @brief destroy the logfile Logfile::~Logfile() { - if (_df != nullptr) { - TRI_CloseDatafile(_df); - TRI_FreeDatafile(_df); - } + delete _df; } /// @brief create a new logfile Logfile* Logfile::createNew(std::string const& filename, Logfile::IdType id, uint32_t size) { - TRI_datafile_t* df = TRI_CreateDatafile( - filename.c_str(), id, static_cast(size), false); + TRI_datafile_t* df = TRI_CreateDatafile(filename, id, static_cast(size), false); if (df == nullptr) { int res = TRI_errno(); @@ -62,7 +58,7 @@ Logfile* Logfile::createNew(std::string const& filename, Logfile::IdType id, /// @brief open an existing logfile Logfile* Logfile::openExisting(std::string const& filename, Logfile::IdType id, bool wasCollected, bool ignoreErrors) { - TRI_datafile_t* df = TRI_OpenDatafile(filename.c_str(), ignoreErrors); + TRI_datafile_t* df = TRI_OpenDatafile(filename, ignoreErrors); if (df == nullptr) { int res = TRI_errno(); diff --git a/arangod/Wal/Logfile.h b/arangod/Wal/Logfile.h index 5fbba9d3fd..b40e2f6e9a 100644 --- a/arangod/Wal/Logfile.h +++ b/arangod/Wal/Logfile.h @@ -76,7 +76,7 @@ class Logfile { if (_df == nullptr) { return ""; } - return _df->getName(_df); + return _df->getName(); } /// @brief return the datafile pointer diff --git a/lib/Basics/StringBuffer.h b/lib/Basics/StringBuffer.h index 617c671d80..97bfe3ce2b 100644 --- a/lib/Basics/StringBuffer.h +++ b/lib/Basics/StringBuffer.h @@ -210,22 +210,34 @@ int TRI_AppendString2StringBuffer(TRI_string_buffer_t* self, char const* str, //////////////////////////////////////////////////////////////////////////////// static inline void TRI_AppendCharUnsafeStringBuffer(TRI_string_buffer_t* self, char chr) { +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + TRI_ASSERT(self->_len - static_cast(self->_current - self->_buffer) > 0); +#endif *self->_current++ = chr; } static inline void TRI_AppendStringUnsafeStringBuffer(TRI_string_buffer_t* self, char const* str) { size_t len = strlen(str); +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + TRI_ASSERT(self->_len - static_cast(self->_current - self->_buffer) >= len); +#endif memcpy(self->_current, str, len); self->_current += len; } static inline void TRI_AppendStringUnsafeStringBuffer(TRI_string_buffer_t* self, char const* str, size_t len) { +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + TRI_ASSERT(self->_len - static_cast(self->_current - self->_buffer) >= len); +#endif memcpy(self->_current, str, len); self->_current += len; } static inline void TRI_AppendStringUnsafeStringBuffer(TRI_string_buffer_t* self, std::string const& str) { +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + TRI_ASSERT(self->_len - static_cast(self->_current - self->_buffer) >= str.size()); +#endif memcpy(self->_current, str.c_str(), str.size()); self->_current += str.size(); } diff --git a/lib/Basics/VelocyPackHelper.cpp b/lib/Basics/VelocyPackHelper.cpp index 72c88814b9..922858098f 100644 --- a/lib/Basics/VelocyPackHelper.cpp +++ b/lib/Basics/VelocyPackHelper.cpp @@ -47,13 +47,28 @@ unsigned long long XXH64(const void* input, size_t length, unsigned long long seed); } +using namespace arangodb; using VelocyPackHelper = arangodb::basics::VelocyPackHelper; static std::unique_ptr Translator; static std::unique_ptr ExcludeHandler; +static std::unique_ptr CustomTypeHandler; + +// a default custom type handler that prevents throwing exceptions when +// custom types are encountered during Slice.toJson() and family +struct DefaultCustomTypeHandler final : public VPackCustomTypeHandler { + void dump(VPackSlice const&, VPackDumper* dumper, VPackSlice const&) override { + LOG(WARN) << "DefaultCustomTypeHandler called"; + dumper->appendString("hello from CustomTypeHandler"); + } + std::string toString(VPackSlice const&, VPackOptions const*, VPackSlice const&) override { + LOG(WARN) << "DefaultCustomTypeHandler called"; + return "hello from CustomTypeHandler"; + } +}; // attribute exclude handler for skipping over system attributes -struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler { +struct SystemAttributeExcludeHandler final : public VPackAttributeExcludeHandler { bool shouldExclude(VPackSlice const& key, int nesting) override final { VPackValueLength keyLength; char const* p = key.getString(keyLength); @@ -81,10 +96,7 @@ struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler { } }; -//////////////////////////////////////////////////////////////////////////////// /// @brief static initializer for all VPack values -//////////////////////////////////////////////////////////////////////////////// - void VelocyPackHelper::initialize() { LOG(TRACE) << "initializing vpack"; @@ -104,6 +116,11 @@ void VelocyPackHelper::initialize() { VPackOptions::Defaults.attributeTranslator = Translator.get(); VPackOptions::Defaults.unsupportedTypeBehavior = VPackOptions::ConvertUnsupportedType; + + + CustomTypeHandler.reset(new DefaultCustomTypeHandler); + + VPackOptions::Defaults.customTypeHandler = CustomTypeHandler.get(); VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set // when converting to JSON for // HTTP xfer