Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'engine-vs-velocystream' of github.com:arangodb/arangodb into generic-col-types

This commit is contained in:
Michael Hackstein 2016-08-26 12:52:41 +02:00
parent 1c883a811c 8a6d054642
commit 0985e7fc88
27 changed files with 553 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG
View File

@ -48,6 +48,8 @@ devel
v3.0.6 (XXXX-XX-XX)
-------------------
* slightly better error diagnostics for AQL query compilation and replication
* fixed issue #2018
* fixed issue #2015

2
CMakeLists.txt
View File

@ -232,7 +232,7 @@ else ()
set(MAKE make)
endif ()
find_package(PythonInterp 2 REQUIRED)
find_package(PythonInterp 2 EXACT REQUIRED)
get_filename_component(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}" REALPATH)
################################################################################

4
arangod/Aql/Executor.cpp
View File

@ -1058,6 +1058,10 @@ void Executor::generateCodeNode(AstNode const* node) {
arangodb::basics::StringBuffer* Executor::initializeBuffer() {
if (_buffer == nullptr) {
_buffer = new arangodb::basics::StringBuffer(TRI_UNKNOWN_MEM_ZONE, 512, false);
if (_buffer->stringBuffer()->_buffer == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
} else {
_buffer->clear();
}

95
arangod/StorageEngine/MMFilesCollection.cpp
View File

@ -24,6 +24,7 @@
#include "MMFilesCollection.h"
#include "Basics/FileUtils.h"
#include "Basics/ReadLocker.h"
#include "Basics/StaticStrings.h"
#include "Basics/WriteLocker.h"
#include "Basics/memory-map.h"
#include "Logger/Logger.h"
@ -57,13 +58,13 @@ MMFilesCollection::~MMFilesCollection() {
close();
for (auto& it : _datafiles) {
TRI_FreeDatafile(it);
delete it;
}
for (auto& it : _journals) {
TRI_FreeDatafile(it);
delete it;
}
for (auto& it : _compactors) {
TRI_FreeDatafile(it);
delete it;
}
}
@ -97,15 +98,15 @@ int MMFilesCollection::close() {
/// @brief seal a datafile
int MMFilesCollection::sealDatafile(TRI_datafile_t* datafile, bool isCompactor) {
int res = TRI_SealDatafile(datafile);
int res = datafile->seal();
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "failed to seal journal '" << datafile->getName(datafile)
LOG(ERR) << "failed to seal journal '" << datafile->getName()
<< "': " << TRI_errno_string(res);
return res;
}
if (!isCompactor && datafile->isPhysical(datafile)) {
if (!isCompactor && datafile->isPhysical()) {
// rename the file
std::string dname("datafile-" + std::to_string(datafile->_fid) + ".db");
std::string filename = arangodb::basics::FileUtils::buildFilename(_logicalCollection->path(), dname);
@ -113,9 +114,9 @@ int MMFilesCollection::sealDatafile(TRI_datafile_t* datafile, bool isCompactor)
res = TRI_RenameDatafile(datafile, filename.c_str());
if (res == TRI_ERROR_NO_ERROR) {
LOG(TRACE) << "closed file '" << datafile->getName(datafile) << "'";
LOG(TRACE) << "closed file '" << datafile->getName() << "'";
} else {
LOG(ERR) << "failed to rename datafile '" << datafile->getName(datafile)
LOG(ERR) << "failed to rename datafile '" << datafile->getName()
<< "' to '" << filename << "': " << TRI_errno_string(res);
}
}
@ -177,12 +178,12 @@ int MMFilesCollection::syncActiveJournal() {
// we only need to care about physical datafiles
// anonymous regions do not need to be synced
if (datafile->isPhysical(datafile)) {
if (datafile->isPhysical()) {
char const* synced = datafile->_synced;
char* written = datafile->_written;
if (synced < written) {
bool ok = datafile->sync(datafile, synced, written);
bool ok = datafile->sync(synced, written);
if (ok) {
LOG_TOPIC(TRACE, Logger::COLLECTOR) << "msync succeeded "
@ -281,7 +282,7 @@ int MMFilesCollection::reserveJournalSpace(TRI_voc_tick_t tick,
// TRI_ERROR_ARANGO_DATAFILE_FULL...
// journal is full, close it and sync
LOG_TOPIC(DEBUG, Logger::COLLECTOR) << "closing full journal '"
<< datafile->getName(datafile) << "'";
<< datafile->getName() << "'";
// make sure we have enough room in the target vector before we go on
_datafiles.reserve(_datafiles.size() + 1);
@ -394,7 +395,7 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
if (_logicalCollection->isVolatile()) {
// in-memory collection
datafile = TRI_CreateDatafile(nullptr, fid, journalSize, true);
datafile = TRI_CreateDatafile(StaticStrings::Empty, fid, journalSize, true);
} else {
// construct a suitable filename (which may be temporary at the beginning)
std::string jname;
@ -420,7 +421,7 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
TRI_UnlinkFile(filename.c_str());
}
datafile = TRI_CreateDatafile(filename.c_str(), fid, journalSize, true);
datafile = TRI_CreateDatafile(filename, fid, journalSize, true);
}
if (datafile == nullptr) {
@ -437,10 +438,10 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
TRI_ASSERT(datafile != nullptr);
if (isCompactor) {
LOG(TRACE) << "created new compactor '" << datafile->getName(datafile)
LOG(TRACE) << "created new compactor '" << datafile->getName()
<< "'";
} else {
LOG(TRACE) << "created new journal '" << datafile->getName(datafile) << "'";
LOG(TRACE) << "created new journal '" << datafile->getName() << "'";
}
// create a collection header, still in the temporary file
@ -454,12 +455,12 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "cannot create collection header in file '"
<< datafile->getName(datafile) << "': " << TRI_errno_string(res);
<< datafile->getName() << "': " << TRI_errno_string(res);
// close the journal and remove it
TRI_CloseDatafile(datafile);
TRI_UnlinkFile(datafile->getName(datafile));
TRI_FreeDatafile(datafile);
std::string temp(datafile->getName());
delete datafile;
TRI_UnlinkFile(temp.c_str());
EnsureErrorCode(res);
@ -481,12 +482,12 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
if (res != TRI_ERROR_NO_ERROR) {
int res = datafile->_lastError;
LOG(ERR) << "cannot create collection header in file '"
<< datafile->getName(datafile) << "': " << TRI_last_error();
<< datafile->getName() << "': " << TRI_last_error();
// close the datafile and remove it
TRI_CloseDatafile(datafile);
TRI_UnlinkFile(datafile->getName(datafile));
TRI_FreeDatafile(datafile);
std::string temp(datafile->getName());
delete datafile;
TRI_UnlinkFile(temp.c_str());
EnsureErrorCode(res);
@ -497,7 +498,7 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
// if a physical file, we can rename it from the temporary name to the correct
// name
if (!isCompactor && datafile->isPhysical(datafile)) {
if (!isCompactor && datafile->isPhysical()) {
// and use the correct name
std::string jname("journal-" + std::to_string(datafile->_fid) + ".db");
std::string filename = arangodb::basics::FileUtils::buildFilename(_logicalCollection->path(), jname);
@ -505,19 +506,19 @@ TRI_datafile_t* MMFilesCollection::createDatafile(TRI_voc_fid_t fid,
int res = TRI_RenameDatafile(datafile, filename.c_str());
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "failed to rename journal '" << datafile->getName(datafile)
LOG(ERR) << "failed to rename journal '" << datafile->getName()
<< "' to '" << filename << "': " << TRI_errno_string(res);
TRI_CloseDatafile(datafile);
TRI_UnlinkFile(datafile->getName(datafile));
TRI_FreeDatafile(datafile);
std::string temp(datafile->getName());
delete datafile;
TRI_UnlinkFile(temp.c_str());
EnsureErrorCode(res);
return nullptr;
}
LOG(TRACE) << "renamed journal from '" << datafile->getName(datafile)
LOG(TRACE) << "renamed journal from '" << datafile->getName()
<< "' to '" << filename << "'";
}
@ -574,7 +575,7 @@ bool MMFilesCollection::iterateDatafilesVector(std::vector<TRI_datafile_t*> cons
return false;
}
if (datafile->isPhysical(datafile) && datafile->_isSealed) {
if (datafile->isPhysical() && datafile->_isSealed) {
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_RANDOM);
}
@ -593,7 +594,7 @@ bool MMFilesCollection::closeDatafiles(std::vector<TRI_datafile_t*> const& files
continue;
}
int res = TRI_CloseDatafile(datafile);
int res = datafile->close();
if (res != TRI_ERROR_NO_ERROR) {
result = false;
@ -637,7 +638,7 @@ void MMFilesCollection::figures(std::shared_ptr<arangodb::velocypack::Builder>&
/// @brief iterate over a vector of datafiles and pick those with a specific
/// data range
std::vector<DatafileDescription> MMFilesCollection::datafilesInRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax) {
std::vector<MMFilesCollection::DatafileDescription> MMFilesCollection::datafilesInRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax) {
std::vector<DatafileDescription> result;
auto apply = [&dataMin, &dataMax, &result](TRI_datafile_t const* datafile, bool isJournal) {
@ -768,4 +769,34 @@ int MMFilesCollection::applyForTickRange(TRI_voc_tick_t dataMin, TRI_voc_tick_t
return false; // hasMore = false
}
/// @brief disallow compaction of the collection
void MMFilesCollection::preventCompaction() {
_compactionLock.readLock();
}
/// @brief try disallowing compaction of the collection
bool MMFilesCollection::tryPreventCompaction() {
return _compactionLock.tryReadLock();
}
/// @brief re-allow compaction of the collection
void MMFilesCollection::allowCompaction() {
_compactionLock.unlock();
}
/// @brief exclusively lock the collection for compaction
void MMFilesCollection::lockForCompaction() {
_compactionLock.writeLock();
}
/// @brief try to exclusively lock the collection for compaction
bool MMFilesCollection::tryLockForCompaction() {
return _compactionLock.tryWriteLock();
}
/// @brief signal that compaction is finished
void MMFilesCollection::finishCompaction() {
_compactionLock.unlock();
}

17
arangod/StorageEngine/MMFilesCollection.h
View File

@ -38,6 +38,14 @@ class MMFilesCollection final : public PhysicalCollection {
friend class MMFilesCompactorThread;
friend class MMFilesEngine;
struct DatafileDescription {
TRI_datafile_t const* _data;
TRI_voc_tick_t _dataMin;
TRI_voc_tick_t _dataMax;
TRI_voc_tick_t _tickMax;
bool _isJournal;
};
public:
explicit MMFilesCollection(LogicalCollection*);
~MMFilesCollection();
@ -86,6 +94,13 @@ class MMFilesCollection final : public PhysicalCollection {
/// @brief iterates over a collection
bool iterateDatafiles(std::function<bool(TRI_df_marker_t const*, TRI_datafile_t*)> const& cb) override;
void preventCompaction() override;
bool tryPreventCompaction() override;
void allowCompaction() override;
void lockForCompaction() override;
bool tryLockForCompaction() override;
void finishCompaction() override;
private:
/// @brief creates a datafile
TRI_datafile_t* createDatafile(TRI_voc_fid_t fid,
@ -107,6 +122,8 @@ class MMFilesCollection final : public PhysicalCollection {
std::vector<TRI_datafile_t*> _datafiles; // all datafiles
std::vector<TRI_datafile_t*> _journals; // all journals
std::vector<TRI_datafile_t*> _compactors; // all compactor files
arangodb::basics::ReadWriteLock _compactionLock;
};
}

62
arangod/StorageEngine/MMFilesCompactorThread.cpp
View File

@ -36,9 +36,10 @@
#include "StorageEngine/StorageEngine.h"
#include "Utils/SingleCollectionTransaction.h"
#include "Utils/StandaloneTransactionContext.h"
#include "VocBase/CompactionLocker.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/collection.h"
#include "VocBase/LogicalCollection.h"
#include "VocBase/collection.h"
#include "VocBase/ticks.h"
#include "VocBase/vocbase.h"
@ -84,7 +85,7 @@ void MMFilesCompactorThread::DropDatafileCallback(TRI_datafile_t* datafile, Logi
std::string name("deleted-" + std::to_string(fid) + ".db");
std::string filename = arangodb::basics::FileUtils::buildFilename(collection->path(), name);
if (datafile->isPhysical(datafile)) {
if (datafile->isPhysical()) {
// copy the current filename
copy = datafile->_filename;
@ -95,19 +96,19 @@ void MMFilesCompactorThread::DropDatafileCallback(TRI_datafile_t* datafile, Logi
}
}
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "finished compacting datafile '" << datafile->getName(datafile) << "'";
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "finished compacting datafile '" << datafile->getName() << "'";
int res = TRI_CloseDatafile(datafile);
int res = datafile->close();
if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot close obsolete datafile '" << datafile->getName(datafile) << "': " << TRI_errno_string(res);
} else if (datafile->isPhysical(datafile)) {
LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot close obsolete datafile '" << datafile->getName() << "': " << TRI_errno_string(res);
} else if (datafile->isPhysical()) {
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "wiping compacted datafile from disk";
res = TRI_UnlinkFile(filename.c_str());
if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot wipe obsolete datafile '" << datafile->getName(datafile) << "': " << TRI_errno_string(res);
LOG_TOPIC(ERR, Logger::COMPACTOR) << "cannot wipe obsolete datafile '" << datafile->getName() << "': " << TRI_errno_string(res);
}
// check for .dead files
@ -122,7 +123,7 @@ void MMFilesCompactorThread::DropDatafileCallback(TRI_datafile_t* datafile, Logi
}
}
TRI_FreeDatafile(datafile);
delete datafile;
}
////////////////////////////////////////////////////////////////////////////////
@ -146,7 +147,7 @@ void MMFilesCompactorThread::RenameDatafileCallback(TRI_datafile_t* datafile, vo
bool ok = false;
TRI_ASSERT(datafile->_fid == compactor->_fid);
if (datafile->isPhysical(datafile)) {
if (datafile->isPhysical()) {
// construct a suitable tempname
std::string jname("temp-" + std::to_string(datafile->_fid) + ".db");
std::string tempFilename = arangodb::basics::FileUtils::buildFilename(collection->path(), jname);
@ -155,12 +156,12 @@ void MMFilesCompactorThread::RenameDatafileCallback(TRI_datafile_t* datafile, vo
int res = TRI_RenameDatafile(datafile, tempFilename.c_str());
if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename datafile '" << datafile->getName(datafile) << "' to '" << tempFilename << "': " << TRI_errno_string(res);
LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename datafile '" << datafile->getName() << "' to '" << tempFilename << "': " << TRI_errno_string(res);
} else {
res = TRI_RenameDatafile(compactor, realName.c_str());
if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename compaction file '" << compactor->getName(compactor) << "' to '" << realName << "': " << TRI_errno_string(res);
LOG_TOPIC(ERR, Logger::COMPACTOR) << "unable to rename compaction file '" << compactor->getName() << "' to '" << realName << "': " << TRI_errno_string(res);
}
}
@ -187,7 +188,7 @@ void MMFilesCompactorThread::RenameDatafileCallback(TRI_datafile_t* datafile, vo
/// @brief remove an empty compactor file
int MMFilesCompactorThread::removeCompactor(LogicalCollection* collection,
TRI_datafile_t* compactor) {
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty compaction file '" << compactor->getName(compactor) << "'";
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty compaction file '" << compactor->getName() << "'";
// remove the compactor from the list of compactors
bool ok = static_cast<MMFilesCollection*>(collection->getPhysical())->removeCompactor(compactor);
@ -199,15 +200,12 @@ int MMFilesCompactorThread::removeCompactor(LogicalCollection* collection,
}
// close the file & remove it
if (compactor->isPhysical(compactor)) {
std::string filename = compactor->getName(compactor);
TRI_CloseDatafile(compactor);
TRI_FreeDatafile(compactor);
if (compactor->isPhysical()) {
std::string filename = compactor->getName();
delete compactor;
TRI_UnlinkFile(filename.c_str());
} else {
TRI_CloseDatafile(compactor);
TRI_FreeDatafile(compactor);
delete compactor;
}
return TRI_ERROR_NO_ERROR;
@ -216,7 +214,7 @@ int MMFilesCompactorThread::removeCompactor(LogicalCollection* collection,
/// @brief remove an empty datafile
int MMFilesCompactorThread::removeDatafile(LogicalCollection* collection,
TRI_datafile_t* df) {
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty datafile '" << df->getName(df) << "'";
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "removing empty datafile '" << df->getName() << "'";
bool ok = static_cast<MMFilesCollection*>(collection->getPhysical())->removeDatafile(df);
@ -251,7 +249,7 @@ MMFilesCompactorThread::compaction_initial_context_t MMFilesCompactorThread::get
TRI_datafile_t* df = compaction._datafile;
// We will sequentially scan the logfile for collection:
if (df->isPhysical(df)) {
if (df->isPhysical()) {
TRI_MMFileAdvise(df->_data, df->_maximalSize, TRI_MADVISE_SEQUENTIAL);
TRI_MMFileAdvise(df->_data, df->_maximalSize, TRI_MADVISE_WILLNEED);
}
@ -317,7 +315,7 @@ MMFilesCompactorThread::compaction_initial_context_t MMFilesCompactorThread::get
}
}
if (df->isPhysical(df)) {
if (df->isPhysical()) {
TRI_MMFileAdvise(df->_data, df->_maximalSize, TRI_MADVISE_RANDOM);
}
@ -425,6 +423,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
trx.addHint(TRI_TRANSACTION_HINT_NO_BEGIN_MARKER, true);
trx.addHint(TRI_TRANSACTION_HINT_NO_ABORT_MARKER, true);
trx.addHint(TRI_TRANSACTION_HINT_NO_COMPACTION_LOCK, true);
trx.addHint(TRI_TRANSACTION_HINT_NO_THROTTLING, true);
compaction_initial_context_t initial = getCompactionContext(&trx, collection, toCompact);
@ -446,7 +445,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
return;
}
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "created new compactor file '" << compactor->getName(compactor) << "'";
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "created new compactor file '" << compactor->getName() << "'";
// these attributes remain the same for all datafiles we collect
context._collection = collection;
@ -465,7 +464,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
auto compaction = toCompact[i];
TRI_datafile_t* df = compaction._datafile;
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "compacting datafile '" << df->getName(df) << "' into '" << compactor->getName(compactor) << "', number: " << i << ", keep deletions: " << compaction._keepDeletions;
LOG_TOPIC(DEBUG, Logger::COMPACTOR) << "compacting datafile '" << df->getName() << "' into '" << compactor->getName() << "', number: " << i << ", keep deletions: " << compaction._keepDeletions;
// if this is the first datafile in the list of datafiles, we can also
// collect
@ -476,7 +475,7 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
bool ok = TRI_IterateDatafile(df, compactifier);
if (!ok) {
LOG_TOPIC(WARN, Logger::COMPACTOR) << "failed to compact datafile '" << df->getName(df) << "'";
LOG_TOPIC(WARN, Logger::COMPACTOR) << "failed to compact datafile '" << df->getName() << "'";
// compactor file does not need to be removed now. will be removed on next
// startup
// TODO: Remove file
@ -510,8 +509,8 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
auto compaction = toCompact[i];
TRI_datafile_t* datafile = compaction._datafile;
if (datafile->isPhysical(datafile)) {
std::string filename(datafile->getName(datafile));
if (datafile->isPhysical()) {
std::string filename(datafile->getName());
filename.append(".dead");
TRI_WriteFile(filename.c_str(), "", 0);
@ -544,8 +543,8 @@ void MMFilesCompactorThread::compactDatafiles(LogicalCollection* collection,
auto compaction = toCompact[i];
TRI_datafile_t* datafile = compaction._datafile;
if (datafile->isPhysical(datafile)) {
std::string filename(datafile->getName(datafile));
if (datafile->isPhysical()) {
std::string filename(datafile->getName());
filename.append(".dead");
TRI_WriteFile(filename.c_str(), "", 0);
@ -877,9 +876,9 @@ void MMFilesCompactorThread::run() {
// check whether someone else holds a read-lock on the compaction
// lock
TRY_WRITE_LOCKER(locker, document->_compactionLock);
TryCompactionLocker compactionLocker(collection);
if (!locker.isLocked()) {
if (!compactionLocker.isLocked()) {
// someone else is holding the compactor lock, we'll not compact
continue;
}
@ -958,6 +957,7 @@ uint64_t MMFilesCompactorThread::getNumberOfDocuments(LogicalCollection* collect
// only try to acquire the lock here
// if lock acquisition fails, we go on and report an (arbitrary) positive number
trx.addHint(TRI_TRANSACTION_HINT_TRY_LOCK, false);
trx.addHint(TRI_TRANSACTION_HINT_NO_THROTTLING, true);
int res = trx.begin();

15
arangod/StorageEngine/MMFilesEngine.cpp
View File

@ -90,7 +90,7 @@ static uint64_t getNumericFilenamePartFromDatabase(std::string const& filename)
}
static uint64_t getNumericFilenamePartFromDatafile(TRI_datafile_t const* datafile) {
return getNumericFilenamePartFromDatafile(datafile->getName(datafile));
return getNumericFilenamePartFromDatafile(datafile->getName());
}
@ -1306,12 +1306,11 @@ bool MMFilesEngine::iterateFiles(std::vector<std::string> const& files) {
for (auto const& filename : files) {
LOG(DEBUG) << "iterating over collection journal file '" << filename << "'";
TRI_datafile_t* datafile = TRI_OpenDatafile(filename.c_str(), true);
TRI_datafile_t* datafile = TRI_OpenDatafile(filename, true);
if (datafile != nullptr) {
TRI_IterateDatafile(datafile, cb);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
delete datafile;
}
}
@ -1858,7 +1857,7 @@ int MMFilesEngine::openCollection(TRI_vocbase_t* vocbase, LogicalCollection* col
}
TRI_datafile_t* datafile =
TRI_OpenDatafile(filename.c_str(), ignoreErrors);
TRI_OpenDatafile(filename, ignoreErrors);
if (datafile == nullptr) {
LOG_TOPIC(ERR, Logger::DATAFILES) << "cannot open datafile '"
@ -1963,10 +1962,8 @@ int MMFilesEngine::openCollection(TRI_vocbase_t* vocbase, LogicalCollection* col
// stop if necessary
if (stop) {
for (auto& datafile : all) {
LOG(TRACE) << "closing datafile '" << datafile->_filename << "'";
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
LOG(TRACE) << "closing datafile '" << datafile->getName() << "'";
delete datafile;
}
return TRI_ERROR_INTERNAL;

2
arangod/Utils/TransactionContext.cpp
View File

@ -38,7 +38,7 @@
using namespace arangodb;
// custom type value handler, used for deciphering the _id attribute
struct CustomTypeHandler : public VPackCustomTypeHandler {
struct CustomTypeHandler final : public VPackCustomTypeHandler {
CustomTypeHandler(TRI_vocbase_t* vocbase, CollectionNameResolver const* resolver)
: vocbase(vocbase), resolver(resolver) {}

2
arangod/V8Server/v8-collection.cpp
View File

@ -2332,7 +2332,7 @@ static void JS_TruncateDatafileVocbaseCol(
TRI_V8_THROW_EXCEPTION(TRI_ERROR_ARANGO_COLLECTION_NOT_UNLOADED);
}
int res = TRI_TruncateDatafile(path.c_str(), (TRI_voc_size_t)size);
int res = TRI_TruncateDatafile(path, (TRI_voc_size_t)size);
if (res != TRI_ERROR_NO_ERROR) {
TRI_V8_THROW_EXCEPTION_MESSAGE(res, "cannot truncate datafile");

102
arangod/VocBase/CompactionLocker.h Normal file
View File

@ -0,0 +1,102 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_VOCBASE_COMPACTION_LOCKER_H
#define ARANGOD_VOCBASE_COMPACTION_LOCKER_H 1
#include "Basics/Common.h"
#include "VocBase/LogicalCollection.h"
namespace arangodb {
class CompactionPreventer {
public:
explicit CompactionPreventer(LogicalCollection* collection)
: _collection(collection) {
_collection->preventCompaction();
}
~CompactionPreventer() { _collection->allowCompaction(); }
private:
LogicalCollection* _collection;
};
class TryCompactionPreventer {
public:
explicit TryCompactionPreventer(LogicalCollection* collection)
: _collection(collection), _isLocked(false) {
_isLocked = _collection->tryPreventCompaction();
}
~TryCompactionPreventer() {
if (_isLocked) {
_collection->allowCompaction();
}
}
bool isLocked() const { return _isLocked; }
private:
LogicalCollection* _collection;
bool _isLocked;
};
class CompactionLocker {
public:
explicit CompactionLocker(LogicalCollection* collection)
: _collection(collection) {
_collection->lockForCompaction();
}
~CompactionLocker() {
_collection->finishCompaction();
}
private:
LogicalCollection* _collection;
};
class TryCompactionLocker {
public:
explicit TryCompactionLocker(LogicalCollection* collection)
: _collection(collection), _isLocked(false) {
_isLocked = _collection->tryLockForCompaction();
}
~TryCompactionLocker() {
if (_isLocked) {
_collection->finishCompaction();
}
}
bool isLocked() const { return _isLocked; }
private:
LogicalCollection* _collection;
bool _isLocked;
};
} // namespace arangodb
#endif

44
arangod/VocBase/DatafileDescription.h
View File

@ -1,44 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Michael Hackstein
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_VOCBASE_DATAFILE_DESCRIPTION_H
#define ARANGOD_VOCBASE_DATAFILE_DESCRIPTION_H 1
#include "Basics/Common.h"
#include "VocBase/voc-types.h"
struct TRI_datafile_t;
namespace arangodb {
struct DatafileDescription {
TRI_datafile_t const* _data;
TRI_voc_tick_t _dataMin;
TRI_voc_tick_t _dataMax;
TRI_voc_tick_t _tickMax;
bool _isJournal;
};
} // namespace arangodb
#endif

2
arangod/VocBase/DatafileStatistics.h
View File

@ -28,8 +28,6 @@
#include "Basics/ReadWriteLock.h"
#include "VocBase/voc-types.h"
struct TRI_datafile_t;
namespace arangodb {
/// @brief datafile statistics

13
arangod/VocBase/LogicalCollection.h
View File

@ -25,7 +25,6 @@
#define ARANGOD_VOCBASE_LOGICAL_COLLECTION_H 1
#include "Basics/Common.h"
#include "VocBase/DatafileDescription.h"
#include "VocBase/MasterPointers.h"
#include "VocBase/PhysicalCollection.h"
#include "VocBase/voc-types.h"
@ -199,6 +198,17 @@ class LogicalCollection {
return getPhysical()->applyForTickRange(dataMin, dataMax, callback);
}
/// @brief disallow starting the compaction of the collection
void preventCompaction() { getPhysical()->preventCompaction(); }
bool tryPreventCompaction() { return getPhysical()->tryPreventCompaction(); }
/// @brief re-allow starting the compaction of the collection
void allowCompaction() { getPhysical()->allowCompaction(); }
/// @brief compaction finished
void lockForCompaction() { getPhysical()->lockForCompaction(); }
bool tryLockForCompaction() { return getPhysical()->tryLockForCompaction(); }
void finishCompaction() { getPhysical()->finishCompaction(); }
PhysicalCollection* getPhysical() const {
TRI_ASSERT(_physical != nullptr);
@ -444,6 +454,7 @@ class LogicalCollection {
mutable arangodb::basics::ReadWriteLock
_idxLock; // lock protecting the indexes
};
} // namespace arangodb
#endif

22
arangod/VocBase/PhysicalCollection.h
View File

@ -25,7 +25,6 @@
#define ARANGOD_VOCBASE_PHYSICAL_COLLECTION_H 1
#include "Basics/Common.h"
#include "VocBase/DatafileDescription.h"
#include "VocBase/voc-types.h"
#include <velocypack/Builder.h>
@ -63,6 +62,27 @@ class PhysicalCollection {
/// @brief iterates over a collection
virtual bool iterateDatafiles(std::function<bool(TRI_df_marker_t const*, TRI_datafile_t*)> const& cb) = 0;
/// @brief disallow compaction of the collection
/// after this call it is guaranteed that no compaction will be started until allowCompaction() is called
virtual void preventCompaction() = 0;
/// @brief try disallowing compaction of the collection
/// returns true if compaction is disallowed, and false if not
virtual bool tryPreventCompaction() = 0;
/// @brief re-allow compaction of the collection
virtual void allowCompaction() = 0;
/// @brief exclusively lock the collection for compaction
virtual void lockForCompaction() = 0;
/// @brief try to exclusively lock the collection for compaction
/// after this call it is guaranteed that no compaction will be started until allowCompaction() is called
virtual bool tryLockForCompaction() = 0;
/// @brief signal that compaction is finished
virtual void finishCompaction() = 0;
protected:
LogicalCollection* _logicalCollection;
};

3
arangod/VocBase/collection.h
View File

@ -86,7 +86,7 @@ class VocbaseCollectionInfo {
int64_t _initialCount; // initial count, used when loading a collection
uint32_t _indexBuckets; // number of buckets used in hash tables for indexes
char _name[TRI_COL_PATH_LENGTH]; // name of the collection
char _name[512]; // name of the collection
std::shared_ptr<arangodb::velocypack::Buffer<uint8_t> const>
_keyOptions; // options for key creation
@ -292,7 +292,6 @@ struct TRI_collection_t {
std::unique_ptr<arangodb::KeyGenerator> _keyGenerator;
std::atomic<int64_t> _uncollectedLogfileEntries;
arangodb::basics::ReadWriteLock _compactionLock;
private:
mutable arangodb::Ditches _ditches;

542
arangod/VocBase/datafile.cpp
View File

@ -41,10 +41,9 @@
using namespace arangodb;
using namespace arangodb::basics;
////////////////////////////////////////////////////////////////////////////////
/// @brief check if a marker appears to be created by ArangoDB 28
////////////////////////////////////////////////////////////////////////////////
namespace {
/// @brief check if a marker appears to be created by ArangoDB 28
static TRI_voc_crc_t Crc28(TRI_voc_crc_t crc, void const* data, size_t length) {
static TRI_voc_crc_t const CrcPolynomial = 0xEDB88320;
unsigned char* current = (unsigned char*) data;
@ -91,82 +90,7 @@ static bool IsMarker28 (void const* marker) {
return crc == m->_crc;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return whether the datafile is a physical file (true) or an
/// anonymous mapped region (false)
////////////////////////////////////////////////////////////////////////////////
static bool IsPhysicalDatafile(TRI_datafile_t const* datafile) {
return datafile->_filename != nullptr;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return the name of a datafile
////////////////////////////////////////////////////////////////////////////////
static char const* GetNameDatafile(TRI_datafile_t const* datafile) {
if (datafile->_filename == nullptr) {
// anonymous regions do not have a filename
return "anonymous region";
}
// return name of the physical file
return datafile->_filename;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief close a datafile
////////////////////////////////////////////////////////////////////////////////
static void CloseDatafile(TRI_datafile_t* datafile) {
TRI_ASSERT(datafile->_state != TRI_DF_STATE_CLOSED);
if (datafile->isPhysical(datafile)) {
int res = TRI_CLOSE(datafile->_fd);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "unable to close datafile '" << datafile->getName(datafile) << "': " << res;
}
}
datafile->_state = TRI_DF_STATE_CLOSED;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a datafile
////////////////////////////////////////////////////////////////////////////////
static void DestroyDatafile(TRI_datafile_t* datafile) {
if (datafile->_filename != nullptr) {
TRI_FreeString(TRI_CORE_MEM_ZONE, datafile->_filename);
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief sync the data of a datafile
////////////////////////////////////////////////////////////////////////////////
static bool SyncDatafile(TRI_datafile_t* datafile, char const* begin,
char const* end) {
if (datafile->_filename == nullptr) {
// anonymous regions do not need to be synced
return true;
}
TRI_ASSERT(datafile->_fd >= 0);
if (begin == end) {
// no need to sync
return true;
}
return TRI_MSync(datafile->_fd, begin, end);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief calculates the actual CRC of a marker, without bounds checks
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_crc_t CalculateCrcValue(TRI_df_marker_t const* marker) {
TRI_voc_size_t zero = 0;
off_t o = marker->offsetOfCrc();
@ -185,10 +109,7 @@ static TRI_voc_crc_t CalculateCrcValue(TRI_df_marker_t const* marker) {
return crc;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks a CRC of a marker, with bounds checks
////////////////////////////////////////////////////////////////////////////////
static bool CheckCrcMarker(TRI_df_marker_t const* marker, char const* end) {
TRI_voc_size_t const size = marker->getSize();
@ -204,17 +125,19 @@ static bool CheckCrcMarker(TRI_df_marker_t const* marker, char const* end) {
return marker->getCrc() == expected;
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief creates a new datafile
///
/// returns the file descriptor or -1 if the file cannot be created
////////////////////////////////////////////////////////////////////////////////
static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) {
static int CreateDatafile(std::string const& filename, TRI_voc_size_t maximalSize) {
TRI_ERRORBUF;
// open the file
int fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
int fd = TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
TRI_IF_FAILURE("CreateDatafile1") {
@ -265,7 +188,7 @@ static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) {
}
TRI_CLOSE(fd);
TRI_UnlinkFile(filename);
TRI_UnlinkFile(filename.c_str());
return -1;
}
@ -282,7 +205,7 @@ static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) {
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename);
TRI_UnlinkFile(filename.c_str());
LOG(ERR) << "cannot seek in datafile '" << filename << "': '" << TRI_GET_ERRORBUF << "'";
return -1;
@ -291,59 +214,6 @@ static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) {
return fd;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initializes a datafile
////////////////////////////////////////////////////////////////////////////////
static void InitDatafile(TRI_datafile_t* datafile, char* filename, int fd,
void* mmHandle, TRI_voc_size_t maximalSize,
TRI_voc_size_t currentSize, TRI_voc_fid_t fid,
char* data) {
// filename is a string for physical datafiles, and NULL for anonymous regions
// fd is a positive value for physical datafiles, and -1 for anonymous regions
if (filename == nullptr) {
TRI_ASSERT(fd == -1);
} else {
TRI_ASSERT(fd >= 0);
}
datafile->_state = TRI_DF_STATE_READ;
datafile->_fid = fid;
datafile->_filename = filename;
datafile->_fd = fd;
datafile->_mmHandle = mmHandle;
datafile->_initSize = maximalSize;
datafile->_maximalSize = maximalSize;
datafile->_currentSize = currentSize;
datafile->_footerSize = sizeof(TRI_df_footer_marker_t);
datafile->_isSealed = false;
datafile->_lastError = TRI_ERROR_NO_ERROR;
datafile->_full = false;
datafile->_data = data;
datafile->_next = data + currentSize;
datafile->_synced = data;
datafile->_written = nullptr;
// reset tick aggregates
datafile->_tickMin = 0;
datafile->_tickMax = 0;
datafile->_dataMin = 0;
datafile->_dataMax = 0;
// initialize function pointers
datafile->isPhysical = &IsPhysicalDatafile;
datafile->getName = &GetNameDatafile;
datafile->close = &CloseDatafile;
datafile->destroy = &DestroyDatafile;
datafile->sync = &SyncDatafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief truncates a datafile
///
@ -357,7 +227,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile,
void* mmHandle;
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
TRI_ASSERT(datafile->isPhysical());
size_t pageSize = PageSizeFeature::getPageSize();
// use multiples of page-size
@ -368,7 +238,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile,
// sanity check
if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) >
maximalSize) {
LOG(ERR) << "cannot create datafile '" << datafile->getName(datafile) << "', maximal size " << (unsigned int)maximalSize << " is too small";
LOG(ERR) << "cannot create datafile '" << datafile->getName() << "', maximal size " << maximalSize << " is too small";
return TRI_set_errno(TRI_ERROR_ARANGO_MAXIMAL_SIZE_TOO_SMALL);
}
@ -477,13 +347,13 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile,
// rename files
std::string oldname = arangodb::basics::FileUtils::buildFilename(datafile->_filename, ".corrupted");
res = TRI_RenameFile(datafile->_filename, oldname.c_str());
res = TRI_RenameFile(datafile->_filename.c_str(), oldname.c_str());
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
res = TRI_RenameFile(filename.c_str(), datafile->_filename);
res = TRI_RenameFile(filename.c_str(), datafile->_filename.c_str());
if (res != TRI_ERROR_NO_ERROR) {
return res;
@ -493,7 +363,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile,
// call will return an error
datafile->_state = TRI_DF_STATE_WRITE;
return TRI_SealDatafile(datafile);
return datafile->seal();
}
////////////////////////////////////////////////////////////////////////////////
@ -502,7 +372,7 @@ static int TruncateAndSealDatafile(TRI_datafile_t* datafile,
static bool TryRepairDatafile(TRI_datafile_t* datafile) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
TRI_ASSERT(datafile->isPhysical());
char* ptr = datafile->_data;
char* end = datafile->_data + datafile->_currentSize;
@ -552,7 +422,7 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) {
if (isFollowedByNullBytes) {
// only last marker in datafile was corrupt. fix the datafile in
// place
LOG(INFO) << "truncating datafile '" << datafile->getName(datafile) << "' at position " << currentSize;
LOG(INFO) << "truncating datafile '" << datafile->getName() << "' at position " << currentSize;
int res = TruncateAndSealDatafile(datafile, currentSize);
return (res == TRI_ERROR_NO_ERROR);
}
@ -572,32 +442,26 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) {
// next marker looks good.
// create a temporary buffer
auto buffer =
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, size, false);
if (buffer == nullptr) {
return false;
}
auto buffer = std::unique_ptr<char[]>(new char[size]);
// create a new marker in the temporary buffer
auto temp = reinterpret_cast<TRI_df_marker_t*>(buffer);
auto temp = reinterpret_cast<TRI_df_marker_t*>(buffer.get());
DatafileHelper::InitMarker(
reinterpret_cast<TRI_df_marker_t*>(buffer), TRI_DF_MARKER_BLANK,
reinterpret_cast<TRI_df_marker_t*>(buffer.get()), TRI_DF_MARKER_BLANK,
static_cast<uint32_t>(size));
temp->setCrc(CalculateCrcValue(temp));
// all done. now copy back the marker into the file
memcpy(static_cast<void*>(ptr), buffer,
memcpy(static_cast<void*>(ptr), buffer.get(),
static_cast<size_t>(size));
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
bool ok = datafile->sync(datafile, ptr, (ptr + size));
buffer.reset(); // don't need the buffer anymore
bool ok = datafile->sync(ptr, (ptr + size));
if (ok) {
LOG(INFO) << "zeroed single invalid marker in datafile '" << datafile->getName(datafile) << "' at position " << currentSize;
LOG(INFO) << "zeroed single invalid marker in datafile '" << datafile->getName() << "' at position " << currentSize;
} else {
LOG(ERR) << "could not zero single invalid marker in datafile '" << datafile->getName(datafile) << "' at position " << currentSize;
LOG(ERR) << "could not zero single invalid marker in datafile '" << datafile->getName() << "' at position " << currentSize;
return false;
}
} else {
@ -628,9 +492,9 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) {
////////////////////////////////////////////////////////////////////////////////
static bool FixDatafile(TRI_datafile_t* datafile, TRI_voc_size_t currentSize) {
LOG(WARN) << "datafile '" << datafile->getName(datafile) << "' is corrupted at position " << currentSize;
LOG(WARN) << "datafile '" << datafile->getName() << "' is corrupted at position " << currentSize;
LOG(WARN) << "setting datafile '" << datafile->getName(datafile) << "' to read-only and ignoring all data from this file beyond this position";
LOG(WARN) << "setting datafile '" << datafile->getName() << "' to read-only and ignoring all data from this file beyond this position";
datafile->_currentSize = currentSize;
TRI_ASSERT(datafile->_initSize == datafile->_maximalSize);
@ -649,14 +513,14 @@ static bool FixDatafile(TRI_datafile_t* datafile, TRI_voc_size_t currentSize) {
static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
TRI_ASSERT(datafile->isPhysical());
char* ptr = datafile->_data;
char* end = datafile->_data + datafile->_currentSize;
TRI_voc_size_t currentSize = 0;
if (datafile->_currentSize == 0) {
LOG(WARN) << "current size is 0 in read-only datafile '" << datafile->getName(datafile) << "', trying to fix";
LOG(WARN) << "current size is 0 in read-only datafile '" << datafile->getName() << "', trying to fix";
end = datafile->_data + datafile->_maximalSize;
}
@ -677,7 +541,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
#endif
if (size == 0) {
LOG(DEBUG) << "reached end of datafile '" << datafile->getName(datafile) << "' data, current size " << currentSize;
LOG(DEBUG) << "reached end of datafile '" << datafile->getName() << "' data, current size " << currentSize;
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
@ -698,7 +562,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' too small, size " << size << ", should be at least " << sizeof(TRI_df_marker_t);
LOG(WARN) << "marker in datafile '" << datafile->getName() << "' too small, size " << size << ", should be at least " << sizeof(TRI_df_marker_t);
updateTick(maxTick);
@ -717,7 +581,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' points with size " << size << " beyond end of file";
LOG(WARN) << "marker in datafile '" << datafile->getName() << "' points with size " << size << " beyond end of file";
updateTick(maxTick);
@ -730,7 +594,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
if (!TRI_IsValidMarkerDatafile(marker)) {
if (type == 0 && size < 128) {
// ignore markers with type 0 and a small size
LOG(WARN) << "ignoring suspicious marker in datafile '" << datafile->getName(datafile) << "': type: " << type << ", size: " << size;
LOG(WARN) << "ignoring suspicious marker in datafile '" << datafile->getName() << "': type: " << type << ", size: " << size;
} else {
if (ignoreFailures) {
return FixDatafile(datafile, currentSize);
@ -742,7 +606,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' is corrupt: type: " << type << ", size: " << size;
LOG(WARN) << "marker in datafile '" << datafile->getName() << "' is corrupt: type: " << type << ", size: " << size;
updateTick(maxTick);
@ -775,7 +639,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
if (isFollowedByNullBytes) {
// only last marker in datafile was corrupt. fix the datafile in
// place
LOG(WARN) << "datafile '" << datafile->getName(datafile) << "' automatically truncated at last marker";
LOG(WARN) << "datafile '" << datafile->getName() << "' automatically truncated at last marker";
ignoreFailures = true;
} else {
// there is some other stuff following. now inspect it...
@ -812,7 +676,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName() << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
if (nextMarkerOk) {
LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it";
@ -834,7 +698,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
currentSize += static_cast<TRI_voc_size_t>(alignedSize);
if (marker->getType() == TRI_DF_MARKER_FOOTER) {
LOG(DEBUG) << "found footer, reached end of datafile '" << datafile->getName(datafile) << "', current size " << currentSize;
LOG(DEBUG) << "found footer, reached end of datafile '" << datafile->getName() << "', current size " << currentSize;
datafile->_isSealed = true;
datafile->_currentSize = currentSize;
@ -945,21 +809,7 @@ static TRI_datafile_t* CreateAnonymousDatafile(TRI_voc_fid_t fid,
return nullptr;
}
// create datafile structure
TRI_datafile_t* datafile = static_cast<TRI_datafile_t*>(
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false));
if (datafile == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
LOG(ERR) << "out of memory";
return nullptr;
}
InitDatafile(datafile, nullptr, fd, mmHandle, maximalSize, 0, fid,
static_cast<char*>(data));
return datafile;
return new TRI_datafile_t(StaticStrings::Empty, fd, mmHandle, maximalSize, 0, fid, static_cast<char*>(data));
}
#endif
@ -968,10 +818,10 @@ static TRI_datafile_t* CreateAnonymousDatafile(TRI_voc_fid_t fid,
/// @brief creates a new physical datafile
////////////////////////////////////////////////////////////////////////////////
static TRI_datafile_t* CreatePhysicalDatafile(char const* filename,
static TRI_datafile_t* CreatePhysicalDatafile(std::string const& filename,
TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize) {
TRI_ASSERT(filename != nullptr);
TRI_ASSERT(!filename.empty());
int fd = CreateDatafile(filename, maximalSize);
@ -996,7 +846,7 @@ static TRI_datafile_t* CreatePhysicalDatafile(char const* filename,
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename);
TRI_UnlinkFile(filename.c_str());
LOG(ERR) << "cannot memory map file '" << filename << "': '" << TRI_errno_string((int)res) << "'";
LOG(ERR) << "The database directory might reside on a shared folder "
@ -1006,47 +856,33 @@ static TRI_datafile_t* CreatePhysicalDatafile(char const* filename,
}
// create datafile structure
auto datafile = static_cast<TRI_datafile_t*>(
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false));
if (datafile == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
try {
return new TRI_datafile_t(filename, fd, mmHandle, maximalSize, 0, fid, static_cast<char*>(data));
} catch (...) {
TRI_CLOSE(fd);
LOG(ERR) << "out of memory";
return nullptr;
}
InitDatafile(datafile, TRI_DuplicateString(filename), fd, mmHandle,
maximalSize, 0, fid, static_cast<char*>(data));
// Advise OS that sequential access is going to happen:
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_SEQUENTIAL);
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief opens a datafile
////////////////////////////////////////////////////////////////////////////////
static TRI_datafile_t* OpenDatafile(char const* filename, bool ignoreErrors) {
static TRI_datafile_t* OpenDatafile(std::string const& filename, bool ignoreErrors) {
TRI_ERRORBUF;
void* data;
TRI_stat_t status;
void* mmHandle;
// this function must not be called for non-physical datafiles
TRI_ASSERT(filename != nullptr);
TRI_voc_fid_t fid = GetNumericFilenamePart(filename);
TRI_ASSERT(!filename.empty());
TRI_voc_fid_t fid = GetNumericFilenamePart(filename.c_str());
// ..........................................................................
// attempt to open a datafile file
// ..........................................................................
int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename.c_str(), O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) {
TRI_SYSTEM_ERROR();
@ -1163,27 +999,21 @@ static TRI_datafile_t* OpenDatafile(char const* filename, bool ignoreErrors) {
}
// create datafile structure
TRI_datafile_t* datafile = static_cast<TRI_datafile_t*>(
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false));
if (datafile == nullptr) {
try {
return new TRI_datafile_t(filename, fd, mmHandle, size, size, fid, static_cast<char*>(data));
} catch (...) {
TRI_UNMMFile(data, size, fd, &mmHandle);
TRI_CLOSE(fd);
return nullptr;
}
InitDatafile(datafile, TRI_DuplicateString(filename), fd, mmHandle, size,
size, fid, static_cast<char*>(data));
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief creates either an anonymous or a physical datafile
////////////////////////////////////////////////////////////////////////////////
TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid,
TRI_datafile_t* TRI_CreateDatafile(std::string const& filename, TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize,
bool withInitialMarkers) {
TRI_datafile_t* datafile;
@ -1205,7 +1035,7 @@ TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid,
}
// create either an anonymous or a physical datafile
if (filename == nullptr) {
if (filename.empty()) {
#ifdef TRI_HAVE_ANONYMOUS_MMAP
datafile = CreateAnonymousDatafile(fid, maximalSize);
#else
@ -1227,40 +1057,22 @@ TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid,
int res = WriteInitialHeaderMarker(datafile, fid, maximalSize);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "cannot write header to datafile '" << datafile->getName(datafile) << "'";
LOG(ERR) << "cannot write header to datafile '" << datafile->getName() << "'";
TRI_UNMMFile(datafile->_data, datafile->_maximalSize, datafile->_fd,
&datafile->_mmHandle);
datafile->close(datafile);
datafile->destroy(datafile);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, datafile);
datafile->close();
delete datafile;
return nullptr;
}
}
LOG(DEBUG) << "created datafile '" << datafile->getName(datafile) << "' of size " << maximalSize << " and page-size " << pageSize;
LOG(DEBUG) << "created datafile '" << datafile->getName() << "' of size " << maximalSize << " and page-size " << pageSize;
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief frees the memory allocated, but does not free the pointer
////////////////////////////////////////////////////////////////////////////////
void TRI_DestroyDatafile(TRI_datafile_t* datafile) {
datafile->destroy(datafile);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief frees the memory allocated and but frees the pointer
////////////////////////////////////////////////////////////////////////////////
void TRI_FreeDatafile(TRI_datafile_t* datafile) {
TRI_DestroyDatafile(datafile);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, datafile);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the name for a marker
////////////////////////////////////////////////////////////////////////////////
@ -1436,15 +1248,15 @@ int TRI_WriteElementDatafile(TRI_datafile_t* datafile, void* position,
// out of bounds check for writing into a datafile
if (position == nullptr || position < (void*)datafile->_data ||
position >= (void*)(datafile->_data + datafile->_maximalSize)) {
LOG(ERR) << "logic error. writing out of bounds of datafile '" << datafile->getName(datafile) << "'";
LOG(ERR) << "logic error. writing out of bounds of datafile '" << datafile->getName() << "'";
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
}
memcpy(position, marker, static_cast<size_t>(marker->getSize()));
if (forceSync) {
bool ok = datafile->sync(datafile, static_cast<char const*>(position),
((char*)position) + marker->getSize());
bool ok = datafile->sync(static_cast<char const*>(position),
reinterpret_cast<char const*>(position) + marker->getSize());
if (!ok) {
datafile->_state = TRI_DF_STATE_WRITE_ERROR;
@ -1506,7 +1318,7 @@ int TRI_WriteCrcElementDatafile(TRI_datafile_t* datafile, void* position,
TRI_df_marker_t* marker, bool forceSync) {
TRI_ASSERT(marker->getTick() != 0);
if (datafile->isPhysical(datafile)) {
if (datafile->isPhysical()) {
TRI_voc_crc_t crc = TRI_InitialCrc32();
crc = TRI_BlockCrc32(crc, (char const*)marker, marker->getSize());
@ -1528,7 +1340,7 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile,
void* data) {
TRI_ASSERT(iterator != nullptr);
LOG(TRACE) << "iterating over datafile '" << datafile->getName(datafile) << "', fid: " << datafile->_fid;
LOG(TRACE) << "iterating over datafile '" << datafile->getName() << "', fid: " << datafile->_fid;
char const* ptr = datafile->_data;
char const* end = datafile->_data + datafile->_currentSize;
@ -1563,7 +1375,7 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile,
/// also may set datafile's min/max tick values
bool TRI_IterateDatafile(TRI_datafile_t* datafile,
std::function<bool(TRI_df_marker_t const*, TRI_datafile_t*)> const& cb) {
LOG(TRACE) << "iterating over datafile '" << datafile->getName(datafile) << "', fid: " << datafile->_fid;
LOG(TRACE) << "iterating over datafile '" << datafile->getName() << "', fid: " << datafile->_fid;
char const* ptr = datafile->_data;
char const* end = datafile->_data + datafile->_currentSize;
@ -1600,9 +1412,9 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile,
/// The datafile will be opened read-only if a footer is found
////////////////////////////////////////////////////////////////////////////////
TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) {
TRI_datafile_t* TRI_OpenDatafile(std::string const& filename, bool ignoreFailures) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(filename != nullptr);
TRI_ASSERT(!filename.empty());
TRI_datafile_t* datafile = OpenDatafile(filename, false);
@ -1618,9 +1430,9 @@ TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) {
&datafile->_mmHandle);
TRI_CLOSE(datafile->_fd);
LOG(ERR) << "datafile '" << datafile->getName(datafile) << "' is corrupt";
LOG(ERR) << "datafile '" << datafile->getName() << "' is corrupt";
// must free datafile here
TRI_FreeDatafile(datafile);
delete datafile;
return nullptr;
}
@ -1642,47 +1454,13 @@ TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) {
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief closes a datafile and all memory regions
////////////////////////////////////////////////////////////////////////////////
int TRI_CloseDatafile(TRI_datafile_t* datafile) {
if (datafile->_state == TRI_DF_STATE_READ ||
datafile->_state == TRI_DF_STATE_WRITE) {
int res = TRI_UNMMFile(datafile->_data, datafile->_initSize, datafile->_fd,
&datafile->_mmHandle);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "munmap failed with: " << res;
datafile->_state = TRI_DF_STATE_WRITE_ERROR;
datafile->_lastError = res;
return res;
}
datafile->close(datafile);
datafile->_data = nullptr;
datafile->_next = nullptr;
datafile->_fd = -1;
return TRI_ERROR_NO_ERROR;
}
if (datafile->_state == TRI_DF_STATE_CLOSED) {
LOG(WARN) << "closing an already closed datafile '" << datafile->getName(datafile) << "'";
return TRI_ERROR_NO_ERROR;
}
TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
return TRI_ERROR_ARANGO_ILLEGAL_STATE;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief renames a datafile
////////////////////////////////////////////////////////////////////////////////
int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
TRI_ASSERT(datafile->isPhysical());
TRI_ASSERT(filename != nullptr);
if (TRI_ExistsFile(filename)) {
@ -1693,7 +1471,7 @@ int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) {
return TRI_ERROR_ARANGO_DATAFILE_ALREADY_EXISTS;
}
int res = TRI_RenameFile(datafile->_filename, filename);
int res = TRI_RenameFile(datafile->_filename.c_str(), filename);
if (res != TRI_ERROR_NO_ERROR) {
datafile->_state = TRI_DF_STATE_RENAME_ERROR;
@ -1702,47 +1480,43 @@ int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) {
return res;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, datafile->_filename);
datafile->_filename = TRI_DuplicateString(filename);
datafile->_filename = filename;
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief seals a datafile, writes a footer, sets it to read-only
////////////////////////////////////////////////////////////////////////////////
int TRI_SealDatafile(TRI_datafile_t* datafile) {
if (datafile->_state == TRI_DF_STATE_READ) {
return TRI_set_errno(TRI_ERROR_ARANGO_READ_ONLY);
int TRI_datafile_t::seal() {
if (_state == TRI_DF_STATE_READ) {
return TRI_ERROR_ARANGO_READ_ONLY;
}
if (datafile->_state != TRI_DF_STATE_WRITE) {
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
if (_state != TRI_DF_STATE_WRITE) {
return TRI_ERROR_ARANGO_ILLEGAL_STATE;
}
if (datafile->_isSealed) {
return TRI_set_errno(TRI_ERROR_ARANGO_DATAFILE_SEALED);
if (_isSealed) {
return TRI_ERROR_ARANGO_DATAFILE_SEALED;
}
// set a proper tick value
if (datafile->_tickMax == 0) {
datafile->_tickMax = TRI_NewTickServer();
if (_tickMax == 0) {
_tickMax = TRI_NewTickServer();
}
// create the footer
TRI_df_footer_marker_t footer = DatafileHelper::CreateFooterMarker(datafile->_tickMax);
TRI_df_footer_marker_t footer = DatafileHelper::CreateFooterMarker(_tickMax);
// reserve space and write footer to file
datafile->_footerSize = 0;
_footerSize = 0;
TRI_df_marker_t* position;
int res =
TRI_ReserveElementDatafile(datafile, footer.base.getSize(), &position, 0);
TRI_ReserveElementDatafile(this, footer.base.getSize(), &position, 0);
if (res == TRI_ERROR_NO_ERROR) {
TRI_ASSERT(position != nullptr);
res = TRI_WriteCrcElementDatafile(datafile, position, &footer.base, false);
res = TRI_WriteCrcElementDatafile(this, position, &footer.base, false);
}
if (res != TRI_ERROR_NO_ERROR) {
@ -1750,44 +1524,42 @@ int TRI_SealDatafile(TRI_datafile_t* datafile) {
}
// sync file
bool ok = datafile->sync(datafile, datafile->_synced,
((char*)datafile->_data) + datafile->_currentSize);
bool ok = sync(_synced, reinterpret_cast<char const*>(_data) + _currentSize);
if (!ok) {
datafile->_state = TRI_DF_STATE_WRITE_ERROR;
_state = TRI_DF_STATE_WRITE_ERROR;
if (errno == ENOSPC) {
datafile->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL);
_lastError = TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL);
} else {
datafile->_lastError = TRI_errno();
_lastError = TRI_errno();
}
LOG(ERR) << "msync failed with: " << TRI_last_error();
}
// everything is now synced
datafile->_synced = datafile->_written;
_synced = _written;
TRI_ProtectMMFile(datafile->_data, datafile->_maximalSize, PROT_READ,
datafile->_fd, &datafile->_mmHandle);
TRI_ProtectMMFile(_data, _maximalSize, PROT_READ,
_fd, &_mmHandle);
// seal datafile
if (ok) {
datafile->_isSealed = true;
datafile->_state = TRI_DF_STATE_READ;
_isSealed = true;
_state = TRI_DF_STATE_READ;
// note: _initSize must remain constant
TRI_ASSERT(datafile->_initSize == datafile->_maximalSize);
datafile->_maximalSize = datafile->_currentSize;
TRI_ASSERT(_initSize == _maximalSize);
_maximalSize = _currentSize;
}
if (!ok) {
return datafile->_lastError;
return _lastError;
}
if (datafile->isPhysical(datafile)) {
if (isPhysical()) {
// From now on we predict random access (until collection or compaction):
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_RANDOM);
TRI_MMFileAdvise(_data, _maximalSize, TRI_MADVISE_RANDOM);
}
return TRI_ERROR_NO_ERROR;
@ -1798,9 +1570,9 @@ int TRI_SealDatafile(TRI_datafile_t* datafile) {
/// this is called from the recovery procedure only
////////////////////////////////////////////////////////////////////////////////
int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position) {
int TRI_TruncateDatafile(std::string const& path, TRI_voc_size_t position) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(path != nullptr);
TRI_ASSERT(!path.empty());
TRI_datafile_t* datafile = OpenDatafile(path, true);
@ -1809,8 +1581,7 @@ int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position) {
}
int res = TruncateAndSealDatafile(datafile, position);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
delete datafile;
return res;
}
@ -1835,8 +1606,7 @@ bool TRI_TryRepairDatafile(char const* path) {
&datafile->_mmHandle);
bool result = TryRepairDatafile(datafile);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
delete datafile;
return result;
}
@ -1909,7 +1679,7 @@ static std::string DiagnoseMarker(TRI_df_marker_t const* marker,
static DatafileScan ScanDatafile(TRI_datafile_t const* datafile) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
TRI_ASSERT(datafile->isPhysical());
char* ptr = datafile->_data;
char* end = datafile->_data + datafile->_currentSize;
@ -2027,8 +1797,7 @@ DatafileScan TRI_ScanDatafile(char const* path) {
if (datafile != nullptr) {
scan = ScanDatafile(datafile);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
delete datafile;
} else {
scan.currentSize = 0;
scan.maximalSize = 0;
@ -2041,3 +1810,112 @@ DatafileScan TRI_ScanDatafile(char const* path) {
return scan;
}
TRI_datafile_t::TRI_datafile_t(std::string const& filename, int fd, void* mmHandle, TRI_voc_size_t maximalSize,
TRI_voc_size_t currentSize, TRI_voc_fid_t fid, char* data)
: _fid(fid),
_state(TRI_DF_STATE_READ),
_fd(fd),
_mmHandle(mmHandle),
_initSize(maximalSize),
_maximalSize(maximalSize),
_currentSize(currentSize),
_footerSize(sizeof(TRI_df_footer_marker_t)),
_data(data),
_next(data + currentSize),
_tickMin(0),
_tickMax(0),
_dataMin(0),
_dataMax(0),
_filename(filename),
_lastError(TRI_ERROR_NO_ERROR),
_full(false),
_isSealed(false),
_synced(data),
_written(nullptr) {
// filename is a string for physical datafiles, and NULL for anonymous regions
// fd is a positive value for physical datafiles, and -1 for anonymous regions
if (filename.empty()) {
TRI_ASSERT(fd == -1);
} else {
TRI_ASSERT(fd >= 0);
// Advise OS that sequential access is going to happen:
TRI_MMFileAdvise(_data, _maximalSize, TRI_MADVISE_SEQUENTIAL);
}
}
TRI_datafile_t::~TRI_datafile_t() {
try {
this->close();
} catch (...) {
// silently continue as this is the destructor
}
}
/// @brief return the name of a datafile
std::string TRI_datafile_t::getName() const {
if (_filename.empty()) {
// anonymous regions do not have a filename
return "anonymous region";
}
// return name of the physical file
return _filename;
}
/// @brief close a datafile
int TRI_datafile_t::close() {
if (_state == TRI_DF_STATE_READ ||
_state == TRI_DF_STATE_WRITE) {
int res = TRI_UNMMFile(_data, _initSize, _fd, &_mmHandle);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "munmap failed with: " << res;
_state = TRI_DF_STATE_WRITE_ERROR;
_lastError = res;
return res;
}
if (isPhysical()) {
int res = TRI_CLOSE(_fd);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "unable to close datafile '" << getName() << "': " << res;
}
}
_state = TRI_DF_STATE_CLOSED;
_data = nullptr;
_next = nullptr;
_fd = -1;
return TRI_ERROR_NO_ERROR;
}
if (_state == TRI_DF_STATE_CLOSED) {
LOG(WARN) << "closing an already closed datafile '" << getName() << "'";
return TRI_ERROR_NO_ERROR;
}
return TRI_ERROR_ARANGO_ILLEGAL_STATE;
}
/// @brief sync the data of a datafile
bool TRI_datafile_t::sync(char const* begin, char const* end) {
if (!isPhysical()) {
// anonymous regions do not need to be synced
return true;
}
TRI_ASSERT(_fd >= 0);
if (begin == end) {
// no need to sync
return true;
}
return TRI_MSync(_fd, begin, end);
}

70
arangod/VocBase/datafile.h
View File

@ -153,6 +153,30 @@ typedef uint32_t TRI_df_version_t;
////////////////////////////////////////////////////////////////////////////////
struct TRI_datafile_t {
TRI_datafile_t(std::string const& filename, int fd, void* mmHandle, TRI_voc_size_t maximalSize,
TRI_voc_size_t currentsize, TRI_voc_fid_t fid, char* data);
~TRI_datafile_t();
/// @brief return whether the datafile is a physical file (true) or an
/// anonymous mapped region (false)
inline bool isPhysical() const { return !_filename.empty(); }
/// @brief return the name of a datafile
std::string getName() const;
/// @brief close a datafile
int close();
/// @brief destroy a datafile
void destroy();
/// @brief sync the data of a datafile
bool sync(char const* begin, char const* end);
/// @brief seals a datafile, writes a footer, sets it to read-only
int seal();
TRI_voc_fid_t _fid; // datafile identifier
TRI_df_state_e _state; // state of the datafile (READ or WRITE)
@ -174,24 +198,12 @@ struct TRI_datafile_t {
TRI_voc_tick_t _dataMin; // minimum tick value of document/edge marker
TRI_voc_tick_t _dataMax; // maximum tick value of document/edge marker
char* _filename; // underlying filename
// function pointers
bool (*isPhysical)(struct TRI_datafile_t const*); // returns true if
// the datafile is a
// physical file
char const* (*getName)(
struct TRI_datafile_t const*); // returns the name of a datafile
void (*close)(struct TRI_datafile_t*); // close the datafile
void (*destroy)(struct TRI_datafile_t*); // destroys the datafile
bool (*sync)(struct TRI_datafile_t*, char const*,
char const*); // syncs the datafile
std::string _filename; // underlying filename
int _lastError; // last (critical) error
bool _full; // at least one request was rejected because there is not enough
// room
bool _isSealed; // true, if footer has been written
// .............................................................................
// access to the following attributes must be protected by a _lock
// .............................................................................
@ -385,20 +397,8 @@ struct TRI_col_header_marker_t {
/// ref TRI_CreatePhysicalDatafile, based on the first parameter
////////////////////////////////////////////////////////////////////////////////
TRI_datafile_t* TRI_CreateDatafile(char const*, TRI_voc_fid_t fid,
TRI_voc_size_t, bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief frees the memory allocated, but does not free the pointer
////////////////////////////////////////////////////////////////////////////////
void TRI_DestroyDatafile(TRI_datafile_t*);
////////////////////////////////////////////////////////////////////////////////
/// @brief frees the memory allocated and but frees the pointer
////////////////////////////////////////////////////////////////////////////////
void TRI_FreeDatafile(TRI_datafile_t*);
TRI_datafile_t* TRI_CreateDatafile(std::string const& filename, TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize, bool withInitialMarkers);
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the name for a marker
@ -460,19 +460,7 @@ bool TRI_IterateDatafile(TRI_datafile_t*,
/// @brief opens an existing datafile read-only
////////////////////////////////////////////////////////////////////////////////
TRI_datafile_t* TRI_OpenDatafile(char const*, bool);
////////////////////////////////////////////////////////////////////////////////
/// @brief closes a datafile and all memory regions
////////////////////////////////////////////////////////////////////////////////
int TRI_CloseDatafile(TRI_datafile_t* datafile);
////////////////////////////////////////////////////////////////////////////////
/// @brief seals a database, writes a footer, sets it to read-only
////////////////////////////////////////////////////////////////////////////////
int TRI_SealDatafile(TRI_datafile_t* datafile) TRI_WARN_UNUSED_RESULT;
TRI_datafile_t* TRI_OpenDatafile(std::string const& filename, bool ignoreFailures);
////////////////////////////////////////////////////////////////////////////////
/// @brief renames a datafile
@ -484,7 +472,7 @@ int TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename);
/// @brief truncates a datafile and seals it, only called by arango-dfdd
////////////////////////////////////////////////////////////////////////////////
int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position);
int TRI_TruncateDatafile(std::string const& path, TRI_voc_size_t position);
////////////////////////////////////////////////////////////////////////////////
/// @brief try to repair a datafile, only called by arango-dfdd

4
arangod/VocBase/replication-common.cpp
View File

@ -22,11 +22,7 @@
////////////////////////////////////////////////////////////////////////////////
#include "replication-common.h"
#include "Basics/files.h"
#include "Basics/tri-strings.h"
#include "VocBase/collection.h"
#include "VocBase/vocbase.h"
////////////////////////////////////////////////////////////////////////////////
/// @brief generate a timestamp string in a target buffer

17
arangod/VocBase/replication-dump.cpp
View File

@ -25,11 +25,12 @@
#include "Basics/ReadLocker.h"
#include "Basics/VPackStringBufferAdapter.h"
#include "Logger/Logger.h"
#include "VocBase/collection.h"
#include "VocBase/CompactionLocker.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/LogicalCollection.h"
#include "VocBase/collection.h"
#include "VocBase/datafile.h"
#include "VocBase/collection.h"
#include "VocBase/LogicalCollection.h"
#include "VocBase/vocbase.h"
#include "Wal/Logfile.h"
#include "Wal/LogfileManager.h"
@ -447,17 +448,17 @@ static int DumpCollection(TRI_replication_dump_t* dump,
////////////////////////////////////////////////////////////////////////////////
int TRI_DumpCollectionReplication(TRI_replication_dump_t* dump,
arangodb::LogicalCollection* col,
arangodb::LogicalCollection* collection,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax, bool withTicks) {
TRI_ASSERT(col != nullptr);
TRI_ASSERT(col->_collection != nullptr);
TRI_ASSERT(collection != nullptr);
TRI_ASSERT(collection->_collection != nullptr);
// get a custom type handler
auto customTypeHandler = dump->_transactionContext->orderCustomTypeHandler();
dump->_vpackOptions.customTypeHandler = customTypeHandler.get();
TRI_collection_t* document = col->_collection;
TRI_collection_t* document = collection->_collection;
TRI_ASSERT(document != nullptr);
// create a barrier so the underlying collection is not unloaded
@ -470,10 +471,10 @@ int TRI_DumpCollectionReplication(TRI_replication_dump_t* dump,
// block compaction
int res;
{
READ_LOCKER(locker, document->_compactionLock);
CompactionPreventer compactionPreventer(collection);
try {
res = DumpCollection(dump, col, document->_vocbase->id(), document->_info.id(), dataMin, dataMax, withTicks);
res = DumpCollection(dump, collection, collection->vocbase()->id(), collection->cid(), dataMin, dataMax, withTicks);
} catch (...) {
res = TRI_ERROR_INTERNAL;
}

4
arangod/VocBase/transaction.cpp
View File

@ -491,7 +491,7 @@ static int UseCollections(TRI_transaction_t* trx, int nestingLevel) {
// read-lock the compaction lock
if (!HasHint(trx, TRI_TRANSACTION_HINT_NO_COMPACTION_LOCK)) {
if (!trxCollection->_compactionLocked) {
trxCollection->_collection->_collection->_compactionLock.readLock();
trxCollection->_collection->preventCompaction();
trxCollection->_compactionLocked = true;
}
}
@ -548,7 +548,7 @@ static int UnuseCollections(TRI_transaction_t* trx, int nestingLevel) {
if (trxCollection->_accessType == TRI_TRANSACTION_WRITE &&
trxCollection->_compactionLocked) {
// read-unlock the compaction lock
trxCollection->_collection->_collection->_compactionLock.unlock();
trxCollection->_collection->allowCompaction();
trxCollection->_compactionLocked = false;
}
}

6
arangod/VocBase/vocbase.h
View File

@ -63,12 +63,6 @@ class LogicalCollection;
constexpr auto TRI_VOC_SYSTEM_DATABASE = "_system";
////////////////////////////////////////////////////////////////////////////////
/// @brief maximal path length
////////////////////////////////////////////////////////////////////////////////
constexpr size_t TRI_COL_PATH_LENGTH = 512;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximal name length
////////////////////////////////////////////////////////////////////////////////

8
arangod/Wal/CollectorThread.cpp
View File

@ -37,10 +37,11 @@
#include "Utils/DatabaseGuard.h"
#include "Utils/SingleCollectionTransaction.h"
#include "Utils/StandaloneTransactionContext.h"
#include "VocBase/CompactionLocker.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/DatafileStatistics.h"
#include "VocBase/collection.h"
#include "VocBase/LogicalCollection.h"
#include "VocBase/collection.h"
#include "Wal/Logfile.h"
#include "Wal/LogfileManager.h"
@ -626,10 +627,9 @@ int CollectorThread::processCollectionOperations(CollectorCache* cache) {
// first try to read-lock the compactor-lock, afterwards try to write-lock the
// collection
// if any locking attempt fails, release and try again next time
TRY_READ_LOCKER(locker, document->_compactionLock);
TryCompactionPreventer compactionPreventer(collection);
if (!locker.isLocked()) {
if (!compactionPreventer.isLocked()) {
return TRI_ERROR_LOCK_TIMEOUT;
}

2
arangod/Wal/CollectorThread.h
View File

@ -37,8 +37,6 @@
#include "Wal/Logfile.h"
struct TRI_collection_t;
struct TRI_datafile_t;
struct TRI_df_marker_t;
namespace arangodb {
class LogicalCollection;

10
arangod/Wal/Logfile.cpp
View File

@ -35,17 +35,13 @@ Logfile::Logfile(Logfile::IdType id, TRI_datafile_t* df, StatusType status)
/// @brief destroy the logfile
Logfile::~Logfile() {
if (_df != nullptr) {
TRI_CloseDatafile(_df);
TRI_FreeDatafile(_df);
}
delete _df;
}
/// @brief create a new logfile
Logfile* Logfile::createNew(std::string const& filename, Logfile::IdType id,
uint32_t size) {
TRI_datafile_t* df = TRI_CreateDatafile(
filename.c_str(), id, static_cast<TRI_voc_size_t>(size), false);
TRI_datafile_t* df = TRI_CreateDatafile(filename, id, static_cast<TRI_voc_size_t>(size), false);
if (df == nullptr) {
int res = TRI_errno();
@ -62,7 +58,7 @@ Logfile* Logfile::createNew(std::string const& filename, Logfile::IdType id,
/// @brief open an existing logfile
Logfile* Logfile::openExisting(std::string const& filename, Logfile::IdType id,
bool wasCollected, bool ignoreErrors) {
TRI_datafile_t* df = TRI_OpenDatafile(filename.c_str(), ignoreErrors);
TRI_datafile_t* df = TRI_OpenDatafile(filename, ignoreErrors);
if (df == nullptr) {
int res = TRI_errno();

2
arangod/Wal/Logfile.h
View File

@ -76,7 +76,7 @@ class Logfile {
if (_df == nullptr) {
return "";
}
return _df->getName(_df);
return _df->getName();
}
/// @brief return the datafile pointer

12
lib/Basics/StringBuffer.h
View File

@ -210,22 +210,34 @@ int TRI_AppendString2StringBuffer(TRI_string_buffer_t* self, char const* str,
////////////////////////////////////////////////////////////////////////////////
static inline void TRI_AppendCharUnsafeStringBuffer(TRI_string_buffer_t* self, char chr) {
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
TRI_ASSERT(self->_len - static_cast<size_t>(self->_current - self->_buffer) > 0);
#endif
*self->_current++ = chr;
}
static inline void TRI_AppendStringUnsafeStringBuffer(TRI_string_buffer_t* self, char const* str) {
size_t len = strlen(str);
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
TRI_ASSERT(self->_len - static_cast<size_t>(self->_current - self->_buffer) >= len);
#endif
memcpy(self->_current, str, len);
self->_current += len;
}
static inline void TRI_AppendStringUnsafeStringBuffer(TRI_string_buffer_t* self, char const* str,
size_t len) {
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
TRI_ASSERT(self->_len - static_cast<size_t>(self->_current - self->_buffer) >= len);
#endif
memcpy(self->_current, str, len);
self->_current += len;
}
static inline void TRI_AppendStringUnsafeStringBuffer(TRI_string_buffer_t* self, std::string const& str) {
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
TRI_ASSERT(self->_len - static_cast<size_t>(self->_current - self->_buffer) >= str.size());
#endif
memcpy(self->_current, str.c_str(), str.size());
self->_current += str.size();
}

25
lib/Basics/VelocyPackHelper.cpp
View File

@ -47,13 +47,28 @@ unsigned long long XXH64(const void* input, size_t length,
unsigned long long seed);
}
using namespace arangodb;
using VelocyPackHelper = arangodb::basics::VelocyPackHelper;
static std::unique_ptr<VPackAttributeTranslator> Translator;
static std::unique_ptr<VPackAttributeExcludeHandler> ExcludeHandler;
static std::unique_ptr<VPackCustomTypeHandler> CustomTypeHandler;
// a default custom type handler that prevents throwing exceptions when
// custom types are encountered during Slice.toJson() and family
struct DefaultCustomTypeHandler final : public VPackCustomTypeHandler {
void dump(VPackSlice const&, VPackDumper* dumper, VPackSlice const&) override {
LOG(WARN) << "DefaultCustomTypeHandler called";
dumper->appendString("hello from CustomTypeHandler");
}
std::string toString(VPackSlice const&, VPackOptions const*, VPackSlice const&) override {
LOG(WARN) << "DefaultCustomTypeHandler called";
return "hello from CustomTypeHandler";
}
};
// attribute exclude handler for skipping over system attributes
struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler {
struct SystemAttributeExcludeHandler final : public VPackAttributeExcludeHandler {
bool shouldExclude(VPackSlice const& key, int nesting) override final {
VPackValueLength keyLength;
char const* p = key.getString(keyLength);
@ -81,10 +96,7 @@ struct SystemAttributeExcludeHandler : public VPackAttributeExcludeHandler {
}
};
////////////////////////////////////////////////////////////////////////////////
/// @brief static initializer for all VPack values
////////////////////////////////////////////////////////////////////////////////
void VelocyPackHelper::initialize() {
LOG(TRACE) << "initializing vpack";
@ -104,6 +116,11 @@ void VelocyPackHelper::initialize() {
VPackOptions::Defaults.attributeTranslator = Translator.get();
VPackOptions::Defaults.unsupportedTypeBehavior =
VPackOptions::ConvertUnsupportedType;
CustomTypeHandler.reset(new DefaultCustomTypeHandler);
VPackOptions::Defaults.customTypeHandler = CustomTypeHandler.get();
VPackOptions::Defaults.escapeUnicode = false; // false here, but will be set
// when converting to JSON for
// HTTP xfer