diff --git a/CHANGELOG b/CHANGELOG index d751f9a060..b9440b100c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -217,7 +217,16 @@ v2.0.0-rc1 (2014-02-28) * added collection status "loading" -v1.4.12 (XXXX-XX-XX) +v1.4.13 (XXXX-XX-XX) +-------------------- + +* fixed too eager compaction + + The compaction will now wait for several seconds before trying to re-compact the same + collection. Additionally, some other limits have been introduced for the compaction. + + +v1.4.12 (2014-03-05) -------------------- * fixed display bug in web interface which caused the following problems: diff --git a/arangod/VocBase/compactor.c b/arangod/VocBase/compactor.c index 6641162275..01149cd79e 100644 --- a/arangod/VocBase/compactor.c +++ b/arangod/VocBase/compactor.c @@ -77,12 +77,32 @@ #define COMPACTOR_MAX_FILES 4 +//////////////////////////////////////////////////////////////////////////////// +/// @brief maximum multiple of journal filesize of a compacted file +/// a value of 3 means that the maximum filesize of the compacted file is +/// 3 x (collection->journalSize) +//////////////////////////////////////////////////////////////////////////////// + +#define COMPACTOR_MAX_SIZE_FACTOR (3) + +//////////////////////////////////////////////////////////////////////////////// +/// @brief maximum filesize of resulting compacted file +//////////////////////////////////////////////////////////////////////////////// + +#define COMPACTOR_MAX_RESULT_FILESIZE (128 * 1024 * 1024) + //////////////////////////////////////////////////////////////////////////////// /// @brief datafiles smaller than the following value will be merged with others //////////////////////////////////////////////////////////////////////////////// #define COMPACTOR_MIN_SIZE (128 * 1024) +//////////////////////////////////////////////////////////////////////////////// +/// @brief re-try compaction of a specific collection in this interval (in s) +//////////////////////////////////////////////////////////////////////////////// + +#define COMPACTOR_COLLECTION_INTERVAL (15.0) + //////////////////////////////////////////////////////////////////////////////// /// @brief compactify interval in microseconds //////////////////////////////////////////////////////////////////////////////// @@ -266,6 +286,7 @@ static void DropDatafileCallback (TRI_datafile_t* datafile, void* data) { TRI_FreeString(TRI_CORE_MEM_ZONE, name); if (datafile->isPhysical(datafile)) { + // copy the current filename copy = TRI_DuplicateStringZ(TRI_CORE_MEM_ZONE, datafile->_filename); ok = TRI_RenameDatafile(datafile, filename); @@ -1031,6 +1052,7 @@ static void CompactifyDatafiles (TRI_document_collection_t* document, static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { TRI_primary_collection_t* primary; TRI_vector_t vector; + uint64_t maxSize; int64_t numAlive; size_t i, n; bool compactNext; @@ -1057,6 +1079,15 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { return false; } + // get maximum size of result file + maxSize = (uint64_t) COMPACTOR_MAX_SIZE_FACTOR * (uint64_t) primary->base._info._maximalSize; + if (maxSize < 8 * 1024 * 1024) { + maxSize = 8 * 1024 * 1024; + } + if (maxSize >= COMPACTOR_MAX_RESULT_FILESIZE) { + maxSize = COMPACTOR_MAX_RESULT_FILESIZE; + } + // copy datafile information TRI_InitVector(&vector, TRI_UNKNOWN_MEM_ZONE, sizeof(compaction_info_t)); numAlive = 0; @@ -1065,6 +1096,7 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { TRI_datafile_t* df; TRI_doc_datafile_info_t* dfi; compaction_info_t compaction; + uint64_t totalSize = 0; bool shouldCompact; df = primary->base._datafiles._buffer[i]; @@ -1076,7 +1108,7 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { if (dfi == NULL) { continue; } - + shouldCompact = false; if (! compactNext && @@ -1086,8 +1118,8 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { shouldCompact = true; compactNext = true; } - else if (numAlive == 0 && dfi->_numberDeletion > 0) { - // compact first datafile already if it has got some deletions + else if (numAlive == 0 && dfi->_numberAlive == 0 && dfi->_numberDeletion > 0) { + // compact first datafile(s) already if they have some deletions shouldCompact = true; compactNext = true; } @@ -1136,6 +1168,8 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { (unsigned long long) dfi->_sizeTransaction, (unsigned long long) dfi->_sizeShapes, (unsigned long long) dfi->_sizeAttributes); + + totalSize += (uint64_t) df->_maximalSize; compaction._datafile = df; compaction._keepDeletions = (numAlive > 0 && i > 0); @@ -1149,7 +1183,8 @@ static bool CompactifyDocumentCollection (TRI_document_collection_t* document) { // delete the collection in the middle of compaction, but the compactor // will not pick this up as it is read-locking the collection status) - if (TRI_LengthVector(&vector) >= COMPACTOR_MAX_FILES) { + if (TRI_LengthVector(&vector) >= COMPACTOR_MAX_FILES || + totalSize >= maxSize) { // found enough to compact break; } @@ -1454,10 +1489,11 @@ void TRI_CompactorVocBase (void* data) { while (true) { // keep initial _state value as vocbase->_state might change during compaction loop int state = vocbase->_state; - + // check if compaction is currently disallowed if (CheckAndLockCompaction(vocbase)) { // compaction is currently allowed + double now = TRI_microtime(); size_t i, n; numCompacted = 0; @@ -1503,16 +1539,24 @@ void TRI_CompactorVocBase (void* data) { continue; } - ce = TRI_CreateBarrierCompaction(&primary->_barrierList); + if (primary->_lastCompaction + COMPACTOR_COLLECTION_INTERVAL <= now) { + ce = TRI_CreateBarrierCompaction(&primary->_barrierList); - if (ce == NULL) { - // out of memory - LOG_WARNING("out of memory when trying to create a barrier element"); - } - else { - worked = CompactifyDocumentCollection((TRI_document_collection_t*) primary); + if (ce == NULL) { + // out of memory + LOG_WARNING("out of memory when trying to create a barrier element"); + } + else { + worked = CompactifyDocumentCollection((TRI_document_collection_t*) primary); + + if (! worked) { + // set compaction stamp + primary->_lastCompaction = now; + } + // if we worked, then we don't set the compaction stamp to force another round of compaction - TRI_FreeBarrier(ce); + TRI_FreeBarrier(ce); + } } // read-unlock the compaction lock diff --git a/arangod/VocBase/primary-collection.c b/arangod/VocBase/primary-collection.c index 3e4af328ce..0dc0125ef3 100644 --- a/arangod/VocBase/primary-collection.c +++ b/arangod/VocBase/primary-collection.c @@ -582,6 +582,7 @@ int TRI_InitPrimaryCollection (TRI_primary_collection_t* primary, primary->_capConstraint = NULL; primary->_keyGenerator = NULL; primary->_numberDocuments = 0; + primary->_lastCompaction = 0.0; primary->figures = Figures; primary->size = Count; diff --git a/arangod/VocBase/primary-collection.h b/arangod/VocBase/primary-collection.h index 0cd8c04438..140b2d2d98 100644 --- a/arangod/VocBase/primary-collection.h +++ b/arangod/VocBase/primary-collection.h @@ -311,6 +311,7 @@ typedef struct TRI_primary_collection_s { int64_t _numberDocuments; TRI_read_write_lock_t _compactionLock; + double _lastCompaction; int (*beginRead) (struct TRI_primary_collection_s*); int (*endRead) (struct TRI_primary_collection_s*);