mirror of https://gitee.com/bigwinds/arangodb
fixed compactor locks
This commit is contained in:
parent
b54f0a15d8
commit
9af8a30a36
|
@ -179,7 +179,6 @@ void BootstrapFeature::unprepare() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
std::vector<std::string> names = databaseFeature->getDatabaseNames();
|
|
||||||
for (auto& name : databaseFeature->getDatabaseNames()) {
|
for (auto& name : databaseFeature->getDatabaseNames()) {
|
||||||
TRI_vocbase_t* vocbase = databaseFeature->useDatabase(name);
|
TRI_vocbase_t* vocbase = databaseFeature->useDatabase(name);
|
||||||
|
|
||||||
|
|
|
@ -1073,7 +1073,6 @@ int DatabaseFeature::createApplicationDirectory(std::string const& name, std::st
|
||||||
int DatabaseFeature::iterateDatabases(VPackSlice const& databases) {
|
int DatabaseFeature::iterateDatabases(VPackSlice const& databases) {
|
||||||
V8DealerFeature* dealer = ApplicationServer::getFeature<V8DealerFeature>("V8Dealer");
|
V8DealerFeature* dealer = ApplicationServer::getFeature<V8DealerFeature>("V8Dealer");
|
||||||
std::string const appPath = dealer->appPath();
|
std::string const appPath = dealer->appPath();
|
||||||
std::string const databasePath = ApplicationServer::getFeature<DatabasePathFeature>("DatabasePath")->subdirectoryName("databases");
|
|
||||||
|
|
||||||
StorageEngine* engine = ApplicationServer::getFeature<EngineSelectorFeature>("EngineSelector")->ENGINE;
|
StorageEngine* engine = ApplicationServer::getFeature<EngineSelectorFeature>("EngineSelector")->ENGINE;
|
||||||
|
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
#include "CollectionExport.h"
|
#include "CollectionExport.h"
|
||||||
|
#include "Basics/WriteLocker.h"
|
||||||
#include "Indexes/PrimaryIndex.h"
|
#include "Indexes/PrimaryIndex.h"
|
||||||
#include "Utils/CollectionGuard.h"
|
#include "Utils/CollectionGuard.h"
|
||||||
#include "Utils/SingleCollectionTransaction.h"
|
#include "Utils/SingleCollectionTransaction.h"
|
||||||
|
@ -63,18 +64,14 @@ CollectionExport::~CollectionExport() {
|
||||||
}
|
}
|
||||||
|
|
||||||
void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
|
void CollectionExport::run(uint64_t maxWaitTime, size_t limit) {
|
||||||
// try to acquire the exclusive lock on the compaction
|
{
|
||||||
while (!TRI_CheckAndLockCompactorVocBase(_document->_vocbase)) {
|
// try to acquire the exclusive lock on the compaction
|
||||||
// didn't get it. try again...
|
WRITE_LOCKER_EVENTUAL(locker, _document->_vocbase->_compactionBlockers._lock, 5000);
|
||||||
usleep(5000);
|
|
||||||
|
// create a ditch under the compaction lock
|
||||||
|
_ditch = _document->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
|
||||||
}
|
}
|
||||||
|
|
||||||
// create a ditch under the compaction lock
|
|
||||||
_ditch = _document->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
|
|
||||||
|
|
||||||
// release the lock
|
|
||||||
TRI_UnlockCompactorVocBase(_document->_vocbase);
|
|
||||||
|
|
||||||
// now we either have a ditch or not
|
// now we either have a ditch or not
|
||||||
if (_ditch == nullptr) {
|
if (_ditch == nullptr) {
|
||||||
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
|
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
|
||||||
|
|
|
@ -23,6 +23,7 @@
|
||||||
|
|
||||||
#include "CollectionKeys.h"
|
#include "CollectionKeys.h"
|
||||||
#include "Basics/StaticStrings.h"
|
#include "Basics/StaticStrings.h"
|
||||||
|
#include "Basics/WriteLocker.h"
|
||||||
#include "Utils/CollectionGuard.h"
|
#include "Utils/CollectionGuard.h"
|
||||||
#include "Utils/SingleCollectionTransaction.h"
|
#include "Utils/SingleCollectionTransaction.h"
|
||||||
#include "Utils/StandaloneTransactionContext.h"
|
#include "Utils/StandaloneTransactionContext.h"
|
||||||
|
@ -88,18 +89,14 @@ void CollectionKeys::create(TRI_voc_tick_t maxTick) {
|
||||||
arangodb::wal::LogfileManager::instance()->waitForCollectorQueue(
|
arangodb::wal::LogfileManager::instance()->waitForCollectorQueue(
|
||||||
_document->_info.id(), 30.0);
|
_document->_info.id(), 30.0);
|
||||||
|
|
||||||
// try to acquire the exclusive lock on the compaction
|
{
|
||||||
while (!TRI_CheckAndLockCompactorVocBase(_document->_vocbase)) {
|
// try to acquire the exclusive lock on the compaction
|
||||||
// didn't get it. try again...
|
WRITE_LOCKER_EVENTUAL(locker, _document->_vocbase->_compactionBlockers._lock, 5000);
|
||||||
usleep(5000);
|
|
||||||
|
// create a ditch under the compaction lock
|
||||||
|
_ditch = _document->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
|
||||||
}
|
}
|
||||||
|
|
||||||
// create a ditch under the compaction lock
|
|
||||||
_ditch = _document->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
|
|
||||||
|
|
||||||
// release the lock
|
|
||||||
TRI_UnlockCompactorVocBase(_document->_vocbase);
|
|
||||||
|
|
||||||
// now we either have a ditch or not
|
// now we either have a ditch or not
|
||||||
if (_ditch == nullptr) {
|
if (_ditch == nullptr) {
|
||||||
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
|
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
|
||||||
|
|
|
@ -25,8 +25,9 @@
|
||||||
|
|
||||||
#include "ApplicationFeatures/ApplicationServer.h"
|
#include "ApplicationFeatures/ApplicationServer.h"
|
||||||
#include "Basics/ReadLocker.h"
|
#include "Basics/ReadLocker.h"
|
||||||
|
#include "Basics/WriteLocker.h"
|
||||||
#include "Basics/files.h"
|
#include "Basics/files.h"
|
||||||
#include "Basics/tri-strings.h"
|
//#include "Basics/tri-strings.h"
|
||||||
#include "Logger/Logger.h"
|
#include "Logger/Logger.h"
|
||||||
#include "Utils/CursorRepository.h"
|
#include "Utils/CursorRepository.h"
|
||||||
#include "VocBase/Ditch.h"
|
#include "VocBase/Ditch.h"
|
||||||
|
@ -257,46 +258,49 @@ void TRI_CleanupVocBase(void* data) {
|
||||||
// and collections cannot be closed properly
|
// and collections cannot be closed properly
|
||||||
CleanupCursors(vocbase, true);
|
CleanupCursors(vocbase, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if we can get the compactor lock exclusively
|
// check if we can get the compactor lock exclusively
|
||||||
if (TRI_CheckAndLockCompactorVocBase(vocbase)) {
|
// check if compaction is currently disallowed
|
||||||
try {
|
{
|
||||||
READ_LOCKER(readLocker, vocbase->_collectionsLock);
|
TRY_WRITE_LOCKER(locker, vocbase->_compactionBlockers._lock);
|
||||||
// copy all collections
|
|
||||||
collections = vocbase->_collections;
|
if (locker.isLocked()) {
|
||||||
} catch (...) {
|
try {
|
||||||
collections.clear();
|
READ_LOCKER(readLocker, vocbase->_collectionsLock);
|
||||||
|
// copy all collections
|
||||||
|
collections = vocbase->_collections;
|
||||||
|
} catch (...) {
|
||||||
|
collections.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (auto& collection : collections) {
|
||||||
|
TRI_ASSERT(collection != nullptr);
|
||||||
|
TRI_document_collection_t* document;
|
||||||
|
|
||||||
|
{
|
||||||
|
READ_LOCKER(readLocker, collection->_lock);
|
||||||
|
document = collection->_collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (document == nullptr) {
|
||||||
|
// collection currently not loaded
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
TRI_ASSERT(document != nullptr);
|
||||||
|
|
||||||
|
// we're the only ones that can unload the collection, so using
|
||||||
|
// the collection pointer outside the lock is ok
|
||||||
|
|
||||||
|
// maybe cleanup indexes, unload the collection or some datafiles
|
||||||
|
// clean indexes?
|
||||||
|
if (iterations % (uint64_t)CLEANUP_INDEX_ITERATIONS == 0) {
|
||||||
|
document->cleanupIndexes(document);
|
||||||
|
}
|
||||||
|
|
||||||
|
CleanupDocumentCollection(collection, document);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& collection : collections) {
|
|
||||||
TRI_ASSERT(collection != nullptr);
|
|
||||||
TRI_document_collection_t* document;
|
|
||||||
|
|
||||||
{
|
|
||||||
READ_LOCKER(readLocker, collection->_lock);
|
|
||||||
document = collection->_collection;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (document == nullptr) {
|
|
||||||
// collection currently not loaded
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
TRI_ASSERT(document != nullptr);
|
|
||||||
|
|
||||||
// we're the only ones that can unload the collection, so using
|
|
||||||
// the collection pointer outside the lock is ok
|
|
||||||
|
|
||||||
// maybe cleanup indexes, unload the collection or some datafiles
|
|
||||||
// clean indexes?
|
|
||||||
if (iterations % (uint64_t)CLEANUP_INDEX_ITERATIONS == 0) {
|
|
||||||
document->cleanupIndexes(document);
|
|
||||||
}
|
|
||||||
|
|
||||||
CleanupDocumentCollection(collection, document);
|
|
||||||
}
|
|
||||||
|
|
||||||
TRI_UnlockCompactorVocBase(vocbase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (vocbase->_state >= 1) {
|
if (vocbase->_state >= 1) {
|
||||||
|
|
|
@ -165,10 +165,7 @@ struct compaction_info_t {
|
||||||
bool _keepDeletions;
|
bool _keepDeletions;
|
||||||
};
|
};
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief determine the number of documents in the collection
|
/// @brief determine the number of documents in the collection
|
||||||
///////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
static uint64_t GetNumberOfDocuments(TRI_document_collection_t* document) {
|
static uint64_t GetNumberOfDocuments(TRI_document_collection_t* document) {
|
||||||
TRI_vocbase_t* vocbase = document->_vocbase;
|
TRI_vocbase_t* vocbase = document->_vocbase;
|
||||||
|
|
||||||
|
@ -962,88 +959,15 @@ static bool CompactifyDocumentCollection(TRI_document_collection_t* document) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief try to write-lock the compaction
|
|
||||||
/// returns true if lock acquisition was successful. the caller is responsible
|
|
||||||
/// to free the write lock eventually
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
static bool TryLockCompaction(TRI_vocbase_t* vocbase) {
|
|
||||||
return TRI_TryWriteLockReadWriteLock(&vocbase->_compactionBlockers._lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief write-lock the compaction
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
static void LockCompaction(TRI_vocbase_t* vocbase) {
|
|
||||||
while (!TryLockCompaction(vocbase)) {
|
|
||||||
// cycle until we have acquired the write-lock
|
|
||||||
usleep(1000);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief write-unlock the compaction
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
static void UnlockCompaction(TRI_vocbase_t* vocbase) {
|
|
||||||
TRI_WriteUnlockReadWriteLock(&vocbase->_compactionBlockers._lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief atomic check and lock for running the compaction
|
|
||||||
/// if this function returns true, it has acquired a write-lock on the
|
|
||||||
/// compactionBlockers structure, which the caller must free eventually
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
static bool CheckAndLockCompaction(TRI_vocbase_t* vocbase) {
|
|
||||||
// check if we can acquire the write lock instantly
|
|
||||||
if (!TryLockCompaction(vocbase)) {
|
|
||||||
// couldn't acquire the write lock
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// we are now holding the write lock
|
|
||||||
double now = TRI_microtime();
|
|
||||||
|
|
||||||
// check if we have a still-valid compaction blocker
|
|
||||||
for (auto const& blocker : vocbase->_compactionBlockers._data) {
|
|
||||||
if (blocker._expires > now) {
|
|
||||||
// found a compaction blocker. unlock and return
|
|
||||||
UnlockCompaction(vocbase);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief initialize the compaction blockers structure
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
int TRI_InitCompactorVocBase(TRI_vocbase_t* vocbase) {
|
|
||||||
TRI_InitReadWriteLock(&vocbase->_compactionBlockers._lock);
|
|
||||||
|
|
||||||
return TRI_ERROR_NO_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief destroy the compaction blockers structure
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
void TRI_DestroyCompactorVocBase(TRI_vocbase_t* vocbase) {
|
|
||||||
TRI_DestroyReadWriteLock(&vocbase->_compactionBlockers._lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
/// @brief remove data of expired compaction blockers
|
/// @brief remove data of expired compaction blockers
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
bool TRI_CleanupCompactorVocBase(TRI_vocbase_t* vocbase) {
|
bool TRI_CleanupCompactorVocBase(TRI_vocbase_t* vocbase) {
|
||||||
// check if we can instantly acquire the lock
|
// check if we can instantly acquire the lock
|
||||||
if (!TryLockCompaction(vocbase)) {
|
TRY_WRITE_LOCKER(locker, vocbase->_compactionBlockers._lock);
|
||||||
|
|
||||||
|
if (!locker.isLocked()) {
|
||||||
// couldn't acquire lock
|
// couldn't acquire lock
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -1064,8 +988,6 @@ bool TRI_CleanupCompactorVocBase(TRI_vocbase_t* vocbase) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
UnlockCompaction(vocbase);
|
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1084,16 +1006,17 @@ int TRI_InsertBlockerCompactorVocBase(TRI_vocbase_t* vocbase, double lifetime,
|
||||||
blocker._expires = TRI_microtime() + lifetime;
|
blocker._expires = TRI_microtime() + lifetime;
|
||||||
|
|
||||||
int res = TRI_ERROR_NO_ERROR;
|
int res = TRI_ERROR_NO_ERROR;
|
||||||
LockCompaction(vocbase);
|
|
||||||
|
|
||||||
try {
|
{
|
||||||
vocbase->_compactionBlockers._data.push_back(blocker);
|
WRITE_LOCKER_EVENTUAL(locker, vocbase->_compactionBlockers._lock, 1000);
|
||||||
} catch (...) {
|
|
||||||
res = TRI_ERROR_OUT_OF_MEMORY;
|
try {
|
||||||
|
vocbase->_compactionBlockers._data.push_back(blocker);
|
||||||
|
} catch (...) {
|
||||||
|
res = TRI_ERROR_OUT_OF_MEMORY;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
UnlockCompaction(vocbase);
|
|
||||||
|
|
||||||
if (res != TRI_ERROR_NO_ERROR) {
|
if (res != TRI_ERROR_NO_ERROR) {
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
@ -1109,47 +1032,20 @@ int TRI_InsertBlockerCompactorVocBase(TRI_vocbase_t* vocbase, double lifetime,
|
||||||
|
|
||||||
int TRI_TouchBlockerCompactorVocBase(TRI_vocbase_t* vocbase, TRI_voc_tick_t id,
|
int TRI_TouchBlockerCompactorVocBase(TRI_vocbase_t* vocbase, TRI_voc_tick_t id,
|
||||||
double lifetime) {
|
double lifetime) {
|
||||||
bool found = false;
|
|
||||||
|
|
||||||
if (lifetime <= 0.0) {
|
if (lifetime <= 0.0) {
|
||||||
return TRI_ERROR_BAD_PARAMETER;
|
return TRI_ERROR_BAD_PARAMETER;
|
||||||
}
|
}
|
||||||
|
|
||||||
LockCompaction(vocbase);
|
WRITE_LOCKER_EVENTUAL(locker, vocbase->_compactionBlockers._lock, 1000);
|
||||||
|
|
||||||
for (auto& blocker : vocbase->_compactionBlockers._data) {
|
for (auto& blocker : vocbase->_compactionBlockers._data) {
|
||||||
if (blocker._id == id) {
|
if (blocker._id == id) {
|
||||||
blocker._expires = TRI_microtime() + lifetime;
|
blocker._expires = TRI_microtime() + lifetime;
|
||||||
found = true;
|
return TRI_ERROR_NO_ERROR;
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
UnlockCompaction(vocbase);
|
return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND;
|
||||||
|
|
||||||
if (!found) {
|
|
||||||
return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND;
|
|
||||||
}
|
|
||||||
|
|
||||||
return TRI_ERROR_NO_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief atomically check-and-lock the compactor
|
|
||||||
/// if the function returns true, then a write-lock on the compactor was
|
|
||||||
/// acquired, which must eventually be freed by the caller
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
bool TRI_CheckAndLockCompactorVocBase(TRI_vocbase_t* vocbase) {
|
|
||||||
return TryLockCompaction(vocbase);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief unlock the compactor
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
void TRI_UnlockCompactorVocBase(TRI_vocbase_t* vocbase) {
|
|
||||||
UnlockCompaction(vocbase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
@ -1158,9 +1054,7 @@ void TRI_UnlockCompactorVocBase(TRI_vocbase_t* vocbase) {
|
||||||
|
|
||||||
int TRI_RemoveBlockerCompactorVocBase(TRI_vocbase_t* vocbase,
|
int TRI_RemoveBlockerCompactorVocBase(TRI_vocbase_t* vocbase,
|
||||||
TRI_voc_tick_t id) {
|
TRI_voc_tick_t id) {
|
||||||
bool found = false;
|
WRITE_LOCKER_EVENTUAL(locker, vocbase->_compactionBlockers._lock, 1000);
|
||||||
|
|
||||||
LockCompaction(vocbase);
|
|
||||||
|
|
||||||
size_t const n = vocbase->_compactionBlockers._data.size();
|
size_t const n = vocbase->_compactionBlockers._data.size();
|
||||||
|
|
||||||
|
@ -1168,18 +1062,26 @@ int TRI_RemoveBlockerCompactorVocBase(TRI_vocbase_t* vocbase,
|
||||||
auto& blocker = vocbase->_compactionBlockers._data[i];
|
auto& blocker = vocbase->_compactionBlockers._data[i];
|
||||||
if (blocker._id == id) {
|
if (blocker._id == id) {
|
||||||
vocbase->_compactionBlockers._data.erase(vocbase->_compactionBlockers._data.begin() + i);
|
vocbase->_compactionBlockers._data.erase(vocbase->_compactionBlockers._data.begin() + i);
|
||||||
found = true;
|
return TRI_ERROR_NO_ERROR;
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
UnlockCompaction(vocbase);
|
return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND;
|
||||||
|
}
|
||||||
|
|
||||||
if (!found) {
|
/// @brief check whether there is an active compaction blocker
|
||||||
return TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND;
|
/// note that this must be called while holding the compactionBlockers lock
|
||||||
|
static bool HasActiveBlockers(TRI_vocbase_t* vocbase) {
|
||||||
|
double const now = TRI_microtime();
|
||||||
|
|
||||||
|
// check if we have a still-valid compaction blocker
|
||||||
|
for (auto const& blocker : vocbase->_compactionBlockers._data) {
|
||||||
|
if (blocker._expires > now) {
|
||||||
|
// found a compaction blocker
|
||||||
|
return true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
return TRI_ERROR_NO_ERROR;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
@ -1199,101 +1101,103 @@ void TRI_CompactorVocBase(void* data) {
|
||||||
// compaction loop
|
// compaction loop
|
||||||
int state = vocbase->_state;
|
int state = vocbase->_state;
|
||||||
|
|
||||||
// check if compaction is currently disallowed
|
{
|
||||||
if (CheckAndLockCompaction(vocbase)) {
|
// check if compaction is currently disallowed
|
||||||
// compaction is currently allowed
|
TRY_WRITE_LOCKER(compactionLocker, vocbase->_compactionBlockers._lock);
|
||||||
double now = TRI_microtime();
|
|
||||||
numCompacted = 0;
|
|
||||||
|
|
||||||
try {
|
if (compactionLocker.isLocked() && !HasActiveBlockers(vocbase)) {
|
||||||
READ_LOCKER(readLocker, vocbase->_collectionsLock);
|
// compaction is currently allowed
|
||||||
// copy all collections
|
double now = TRI_microtime();
|
||||||
collections = vocbase->_collections;
|
numCompacted = 0;
|
||||||
} catch (...) {
|
|
||||||
collections.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool worked;
|
try {
|
||||||
|
READ_LOCKER(readLocker, vocbase->_collectionsLock);
|
||||||
|
// copy all collections
|
||||||
|
collections = vocbase->_collections;
|
||||||
|
} catch (...) {
|
||||||
|
collections.clear();
|
||||||
|
}
|
||||||
|
|
||||||
for (auto& collection : collections) {
|
bool worked;
|
||||||
{
|
|
||||||
TRY_READ_LOCKER(readLocker, collection->_lock);
|
|
||||||
|
|
||||||
if (!readLocker.isLocked()) {
|
for (auto& collection : collections) {
|
||||||
// if we can't acquire the read lock instantly, we continue directly
|
{
|
||||||
// we don't want to stall here for too long
|
TRY_READ_LOCKER(readLocker, collection->_lock);
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
TRI_document_collection_t* document = collection->_collection;
|
if (!readLocker.isLocked()) {
|
||||||
|
// if we can't acquire the read lock instantly, we continue directly
|
||||||
if (document == nullptr) {
|
// we don't want to stall here for too long
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
worked = false;
|
|
||||||
bool doCompact = document->_info.doCompact();
|
|
||||||
|
|
||||||
// for document collection, compactify datafiles
|
|
||||||
if (collection->_status == TRI_VOC_COL_STATUS_LOADED && doCompact) {
|
|
||||||
// check whether someone else holds a read-lock on the compaction
|
|
||||||
// lock
|
|
||||||
|
|
||||||
TRY_WRITE_LOCKER(locker, document->_compactionLock);
|
|
||||||
|
|
||||||
if (!locker.isLocked()) {
|
|
||||||
// someone else is holding the compactor lock, we'll not compact
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
TRI_document_collection_t* document = collection->_collection;
|
||||||
if (document->_lastCompaction + COMPACTOR_COLLECTION_INTERVAL <=
|
|
||||||
now) {
|
|
||||||
auto ce = document->ditches()->createCompactionDitch(__FILE__,
|
|
||||||
__LINE__);
|
|
||||||
|
|
||||||
if (ce == nullptr) {
|
if (document == nullptr) {
|
||||||
// out of memory
|
continue;
|
||||||
LOG_TOPIC(WARN, Logger::COMPACTOR) << "out of memory when trying to create compaction ditch";
|
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
worked = CompactifyDocumentCollection(document);
|
|
||||||
|
|
||||||
if (!worked) {
|
|
||||||
// set compaction stamp
|
|
||||||
document->_lastCompaction = now;
|
|
||||||
}
|
|
||||||
// if we worked, then we don't set the compaction stamp to
|
|
||||||
// force
|
|
||||||
// another round of compaction
|
|
||||||
} catch (...) {
|
|
||||||
LOG_TOPIC(ERR, Logger::COMPACTOR) << "an unknown exception occurred during compaction";
|
|
||||||
// in case an error occurs, we must still free this ditch
|
|
||||||
}
|
|
||||||
|
|
||||||
document->ditches()->freeDitch(ce);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (...) {
|
|
||||||
// in case an error occurs, we must still relase the lock
|
|
||||||
LOG_TOPIC(ERR, Logger::COMPACTOR) << "an unknown exception occurred during compaction";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
worked = false;
|
||||||
|
bool doCompact = document->_info.doCompact();
|
||||||
|
|
||||||
|
// for document collection, compactify datafiles
|
||||||
|
if (collection->_status == TRI_VOC_COL_STATUS_LOADED && doCompact) {
|
||||||
|
// check whether someone else holds a read-lock on the compaction
|
||||||
|
// lock
|
||||||
|
|
||||||
|
TRY_WRITE_LOCKER(locker, document->_compactionLock);
|
||||||
|
|
||||||
|
if (!locker.isLocked()) {
|
||||||
|
// someone else is holding the compactor lock, we'll not compact
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (document->_lastCompaction + COMPACTOR_COLLECTION_INTERVAL <=
|
||||||
|
now) {
|
||||||
|
auto ce = document->ditches()->createCompactionDitch(__FILE__,
|
||||||
|
__LINE__);
|
||||||
|
|
||||||
|
if (ce == nullptr) {
|
||||||
|
// out of memory
|
||||||
|
LOG_TOPIC(WARN, Logger::COMPACTOR) << "out of memory when trying to create compaction ditch";
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
worked = CompactifyDocumentCollection(document);
|
||||||
|
|
||||||
|
if (!worked) {
|
||||||
|
// set compaction stamp
|
||||||
|
document->_lastCompaction = now;
|
||||||
|
}
|
||||||
|
// if we worked, then we don't set the compaction stamp to
|
||||||
|
// force
|
||||||
|
// another round of compaction
|
||||||
|
} catch (...) {
|
||||||
|
LOG_TOPIC(ERR, Logger::COMPACTOR) << "an unknown exception occurred during compaction";
|
||||||
|
// in case an error occurs, we must still free this ditch
|
||||||
|
}
|
||||||
|
|
||||||
|
document->ditches()->freeDitch(ce);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (...) {
|
||||||
|
// in case an error occurs, we must still relase the lock
|
||||||
|
LOG_TOPIC(ERR, Logger::COMPACTOR) << "an unknown exception occurred during compaction";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // end of lock
|
||||||
|
|
||||||
|
if (worked) {
|
||||||
|
++numCompacted;
|
||||||
|
|
||||||
|
// signal the cleanup thread that we worked and that it can now wake
|
||||||
|
// up
|
||||||
|
TRI_LockCondition(&vocbase->_cleanupCondition);
|
||||||
|
TRI_SignalCondition(&vocbase->_cleanupCondition);
|
||||||
|
TRI_UnlockCondition(&vocbase->_cleanupCondition);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // end of lock
|
|
||||||
|
|
||||||
if (worked) {
|
|
||||||
++numCompacted;
|
|
||||||
|
|
||||||
// signal the cleanup thread that we worked and that it can now wake
|
|
||||||
// up
|
|
||||||
TRI_LockCondition(&vocbase->_cleanupCondition);
|
|
||||||
TRI_SignalCondition(&vocbase->_cleanupCondition);
|
|
||||||
TRI_UnlockCondition(&vocbase->_cleanupCondition);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
UnlockCompaction(vocbase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (numCompacted > 0) {
|
if (numCompacted > 0) {
|
||||||
|
|
|
@ -25,65 +25,23 @@
|
||||||
#define ARANGOD_VOC_BASE_COMPACTOR_H 1
|
#define ARANGOD_VOC_BASE_COMPACTOR_H 1
|
||||||
|
|
||||||
#include "Basics/Common.h"
|
#include "Basics/Common.h"
|
||||||
|
|
||||||
#include "VocBase/voc-types.h"
|
#include "VocBase/voc-types.h"
|
||||||
|
|
||||||
struct TRI_vocbase_t;
|
struct TRI_vocbase_t;
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief initialize the compaction blockers structure
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
int TRI_InitCompactorVocBase(TRI_vocbase_t*);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief destroy the compaction blockers structure
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
void TRI_DestroyCompactorVocBase(TRI_vocbase_t*);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief remove data of expired compaction blockers
|
/// @brief remove data of expired compaction blockers
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
bool TRI_CleanupCompactorVocBase(TRI_vocbase_t*);
|
bool TRI_CleanupCompactorVocBase(TRI_vocbase_t*);
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief insert a compaction blocker
|
/// @brief insert a compaction blocker
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
int TRI_InsertBlockerCompactorVocBase(TRI_vocbase_t*, double, TRI_voc_tick_t*);
|
int TRI_InsertBlockerCompactorVocBase(TRI_vocbase_t*, double, TRI_voc_tick_t*);
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief touch an existing compaction blocker
|
/// @brief touch an existing compaction blocker
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
int TRI_TouchBlockerCompactorVocBase(TRI_vocbase_t*, TRI_voc_tick_t, double);
|
int TRI_TouchBlockerCompactorVocBase(TRI_vocbase_t*, TRI_voc_tick_t, double);
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief remove an existing compaction blocker
|
/// @brief remove an existing compaction blocker
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
int TRI_RemoveBlockerCompactorVocBase(TRI_vocbase_t*, TRI_voc_tick_t);
|
int TRI_RemoveBlockerCompactorVocBase(TRI_vocbase_t*, TRI_voc_tick_t);
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief atomically check-and-lock the compactor
|
|
||||||
/// if the function returns true, then a write-lock on the compactor was
|
|
||||||
/// acquired, which must eventually be freed by the caller
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
bool TRI_CheckAndLockCompactorVocBase(TRI_vocbase_t*);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief unlock the compactor
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
void TRI_UnlockCompactorVocBase(TRI_vocbase_t*);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
/// @brief compactor event loop
|
/// @brief compactor event loop
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
void TRI_CompactorVocBase(void*);
|
void TRI_CompactorVocBase(void*);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -1264,8 +1264,6 @@ TRI_vocbase_t* TRI_OpenVocBase(char const* path,
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
TRI_InitCompactorVocBase(vocbase);
|
|
||||||
|
|
||||||
// .............................................................................
|
// .............................................................................
|
||||||
// scan directory for collections
|
// scan directory for collections
|
||||||
// .............................................................................
|
// .............................................................................
|
||||||
|
@ -1277,7 +1275,6 @@ TRI_vocbase_t* TRI_OpenVocBase(char const* path,
|
||||||
int res = ScanPath(vocbase, vocbase->_path, isUpgrade, iterateMarkers);
|
int res = ScanPath(vocbase, vocbase->_path, isUpgrade, iterateMarkers);
|
||||||
|
|
||||||
if (res != TRI_ERROR_NO_ERROR) {
|
if (res != TRI_ERROR_NO_ERROR) {
|
||||||
TRI_DestroyCompactorVocBase(vocbase);
|
|
||||||
delete vocbase;
|
delete vocbase;
|
||||||
TRI_set_errno(res);
|
TRI_set_errno(res);
|
||||||
|
|
||||||
|
@ -1384,8 +1381,6 @@ void TRI_DestroyVocBase(TRI_vocbase_t* vocbase) {
|
||||||
for (auto& collection : vocbase->_collections) {
|
for (auto& collection : vocbase->_collections) {
|
||||||
delete collection;
|
delete collection;
|
||||||
}
|
}
|
||||||
|
|
||||||
TRI_DestroyCompactorVocBase(vocbase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
|
@ -64,12 +64,10 @@ struct compaction_blocker_t {
|
||||||
};
|
};
|
||||||
|
|
||||||
struct compaction_blockers_t {
|
struct compaction_blockers_t {
|
||||||
TRI_read_write_lock_t _lock;
|
arangodb::basics::ReadWriteLock _lock;
|
||||||
std::vector<compaction_blocker_t> _data;
|
std::vector<compaction_blocker_t> _data;
|
||||||
};
|
};
|
||||||
|
|
||||||
extern bool IGNORE_DATAFILE_ERRORS;
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
/// @brief tries to read lock the vocbase collection status
|
/// @brief tries to read lock the vocbase collection status
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
Loading…
Reference in New Issue