diff --git a/arangod/VocBase/vocbase.c b/arangod/VocBase/vocbase.c index f934ef0f1d..46b6d49674 100644 --- a/arangod/VocBase/vocbase.c +++ b/arangod/VocBase/vocbase.c @@ -53,6 +53,8 @@ #include "VocBase/synchroniser.h" #include "VocBase/general-cursor.h" +#include "VocBase/index-garbage-collector.h" + #include "Ahuacatl/ahuacatl-functions.h" #include "Ahuacatl/ahuacatl-statementlist.h" @@ -1327,6 +1329,9 @@ TRI_vocbase_t* TRI_OpenVocBase (char const* path) { TRI_InitThread(&vocbase->_cleanup); TRI_StartThread(&vocbase->_cleanup, "[cleanup]", TRI_CleanupVocBase, vocbase); + TRI_InitThread(&(vocbase->_indexGC)); + TRI_StartThread(&(vocbase->_indexGC), "[indeX_garbage_collector]", TRI_IndexGCVocBase, vocbase); + // we are done return vocbase; } @@ -1349,6 +1354,9 @@ void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) { // this will signal the synchroniser and the compactor threads to do one last iteration vocbase->_state = 2; + // wait for the index garbage collector to finish what ever it is doing + TRI_JoinThread(&vocbase->_indexGC); + // wait until synchroniser and compactor are finished TRI_JoinThread(&vocbase->_synchroniser); TRI_JoinThread(&vocbase->_compactor); diff --git a/arangod/VocBase/vocbase.h b/arangod/VocBase/vocbase.h index 523894f3ba..a038b65b69 100644 --- a/arangod/VocBase/vocbase.h +++ b/arangod/VocBase/vocbase.h @@ -399,7 +399,11 @@ typedef struct TRI_vocbase_s { TRI_thread_t _synchroniser; TRI_thread_t _compactor; TRI_thread_t _cleanup; - + + // the index garbage collection + TRI_thread_t _indexGC; + + struct TRI_shadow_store_s* _cursors; TRI_associative_pointer_t* _functions; diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index 44398e05ac..37103f814c 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -152,6 +152,7 @@ "WARNING_ARANGO_INDEX_HASH_REMOVE_ITEM_MISSING" : { "code" : 3211, "message" : "hash index remove failure - item missing in index" }, "WARNING_ARANGO_INDEX_SKIPLIST_DOCUMENT_ATTRIBUTE_MISSING" : { "code" : 3300, "message" : "skiplist index insertion warning - attribute missing in document" }, "WARNING_ARANGO_INDEX_SKIPLIST_UPDATE_ATTRIBUTE_MISSING" : { "code" : 3302, "message" : "skiplist index update warning - attribute missing in revised document" }, + "WARNING_ARANGO_INDEX_SKIPLIST_INSERT_CAS_FAILURE" : { "code" : 3304, "message" : "skiplist index insertion warning - CAS failure while attempting to insert document" }, "WARNING_ARANGO_INDEX_SKIPLIST_REMOVE_ITEM_MISSING" : { "code" : 3311, "message" : "skiplist index remove failure - item missing in index" }, "WARNING_ARANGO_INDEX_BITARRAY_DOCUMENT_ATTRIBUTE_MISSING" : { "code" : 3400, "message" : "bitarray index insertion warning - attribute missing in document" }, "WARNING_ARANGO_INDEX_BITARRAY_UPDATE_ATTRIBUTE_MISSING" : { "code" : 3402, "message" : "bitarray index update warning - attribute missing in revised document" }, diff --git a/lib/BasicsC/errors.dat b/lib/BasicsC/errors.dat index c941761565..20654ef9d3 100755 --- a/lib/BasicsC/errors.dat +++ b/lib/BasicsC/errors.dat @@ -228,6 +228,7 @@ WARNING_ARANGO_INDEX_HASH_REMOVE_ITEM_MISSING,3211,"hash index remove failure - WARNING_ARANGO_INDEX_SKIPLIST_DOCUMENT_ATTRIBUTE_MISSING,3300,"skiplist index insertion warning - attribute missing in document","Will be raised when an attempt to insert a document into a skiplist index is caused by in the document not having one or more attributes which are required by the skiplist index." WARNING_ARANGO_INDEX_SKIPLIST_UPDATE_ATTRIBUTE_MISSING,3302,"skiplist index update warning - attribute missing in revised document","Will be raised when an attempt to update a document results in the revised document not having one or more attributes which are required by the skiplist index." +WARNING_ARANGO_INDEX_SKIPLIST_INSERT_CAS_FAILURE,3304,"skiplist index insertion warning - CAS failure while attempting to insert document","Will be raised when an attempt to insert a document into a skiplist index fails due to repeated CAS failures/clashes." WARNING_ARANGO_INDEX_SKIPLIST_REMOVE_ITEM_MISSING,3311,"skiplist index remove failure - item missing in index","Will be raised when an attempt to remove a document from a skiplist index fails when document can not be located within that index." WARNING_ARANGO_INDEX_BITARRAY_DOCUMENT_ATTRIBUTE_MISSING,3400,"bitarray index insertion warning - attribute missing in document","Will be raised when an attempt to insert a document into a bitarray index is caused by in the document not having one or more attributes which are required by the bitarray index." diff --git a/lib/BasicsC/locks-posix.c b/lib/BasicsC/locks-posix.c index 6ee8aa9641..d43543cca0 100644 --- a/lib/BasicsC/locks-posix.c +++ b/lib/BasicsC/locks-posix.c @@ -557,16 +557,22 @@ void TRI_UnlockCondition (TRI_condition_t* cond) { // ----------------------------------------------------------------------------- // COMPARE & SWAP operations below for MAC and GNUC +// Note that for the MAC OS we use the 'barrier' functions which ensure that +// read/write operations on the scalars are executed in order. According to the +// available documentation, the GCC variants of these COMPARE & SWAP operations +// are implemented with a memory barrier. The MS Windows variants of these +// operations (according to the documentation on MS site) also provide a full +// memory barrier. // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// -/// @brief atomically compares and swaps 32bit integers +/// @brief atomically compares and swaps 32bit integers with full memory barrier //////////////////////////////////////////////////////////////////////////////// bool TRI_CompareAndSwapIntegerInt32 (volatile int32_t* theValue, int32_t oldValue, int32_t newValue) { #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 - return OSAtomicCompareAndSwap32(oldValue, newValue, theValue); + return OSAtomicCompareAndSwap32Barrier(oldValue, newValue, theValue); #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 return __sync_val_compare_and_swap(theValue, oldValue, newValue); #else @@ -574,9 +580,19 @@ bool TRI_CompareAndSwapIntegerInt32 (volatile int32_t* theValue, int32_t oldValu #endif } +bool TRI_CompareIntegerInt32 (volatile int32_t* theValue, int32_t oldValue) { + #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwap32Barrier(oldValue, oldValue, theValue); + #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + return __sync_val_compare_and_swap(theValue, oldValue, oldValue); + #else + #error No TRI_CompareAndSwapIntegerInt32 implementation defined + #endif +} + bool TRI_CompareAndSwapIntegerUInt32 (volatile uint32_t* theValue, uint32_t oldValue, uint32_t newValue) { #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 - return OSAtomicCompareAndSwap32((volatile int32_t*)(oldValue), (int32_t)(newValue), (int32_t)(theValue)); + return OSAtomicCompareAndSwap32Barrier((int32_t)(oldValue), (int32_t)(newValue), (volatile int32_t*)(theValue)); #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 return __sync_val_compare_and_swap(theValue, oldValue, newValue); #else @@ -584,13 +600,23 @@ bool TRI_CompareAndSwapIntegerUInt32 (volatile uint32_t* theValue, uint32_t oldV #endif } +bool TRI_CompareIntegerUInt32 (volatile uint32_t* theValue, uint32_t oldValue) { + #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwap32Barrier((int32_t)(oldValue), (int32_t)(oldValue), (volatile int32_t*)(theValue)); + #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + return __sync_val_compare_and_swap(theValue, oldValue, oldValue); + #else + #error No TRI_CompareAndSwapIntegerUInt32 implementation defined + #endif +} + //////////////////////////////////////////////////////////////////////////////// -/// @brief atomically compares and swaps 64bit integers +/// @brief atomically compares and swaps 64bit integers with full memory barrier //////////////////////////////////////////////////////////////////////////////// bool TRI_CompareAndSwapIntegerInt64 (volatile int64_t* theValue, int64_t oldValue, int64_t newValue) { #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 - return OSAtomicCompareAndSwap64(oldValue, newValue, theValue); + return OSAtomicCompareAndSwap64Barrier(oldValue, newValue, theValue); #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 return __sync_val_compare_and_swap(theValue, oldValue, newValue); #else @@ -598,9 +624,19 @@ bool TRI_CompareAndSwapIntegerInt64 (volatile int64_t* theValue, int64_t oldValu #endif } +bool TRI_CompareIntegerInt64 (volatile int64_t* theValue, int64_t oldValue) { + #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwap64Barrier(oldValue, oldValue, theValue); + #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + return __sync_val_compare_and_swap(theValue, oldValue, oldValue); + #else + #error No TRI_CompareAndSwapIntegerInt64 implementation defined + #endif +} + bool TRI_CompareAndSwapIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldValue, uint64_t newValue) { #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 - return OSAtomicCompareAndSwap64((volatile int64_t*)(oldValue), (int64_t)(newValue), (int64_t)(theValue)); + return OSAtomicCompareAndSwap64Barrier((int64_t)(oldValue), (int64_t)(newValue), (volatile int64_t*)(theValue)); #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 return __sync_val_compare_and_swap(theValue, oldValue, newValue); #else @@ -608,18 +644,38 @@ bool TRI_CompareAndSwapIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldV #endif } +bool TRI_CompareIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldValue) { + #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwap64Barrier((int64_t)(oldValue), (int64_t)(oldValue), (volatile int64_t*)(theValue)); + #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + return __sync_val_compare_and_swap(theValue, oldValue, oldValue); + #else + #error No TRI_CompareAndSwapIntegerUInt64 implementation defined + #endif +} + //////////////////////////////////////////////////////////////////////////////// -/// @brief atomically compares and swaps pointers +/// @brief atomically compares and swaps pointers with full memory barrier //////////////////////////////////////////////////////////////////////////////// bool TRI_CompareAndSwapPointer(void* volatile* theValue, void* oldValue, void* newValue) { -#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 - return OSAtomicCompareAndSwapPtr(oldValue, newValue, theValue); -#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwapPtrBarrier(oldValue, newValue, theValue); + #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 return __sync_val_compare_and_swap(theValue, oldValue, newValue); -#else + #else #error No TRI_CompareAndSwapPointer implementation defined -#endif + #endif +} + +bool TRI_ComparePointer(void* volatile* theValue, void* oldValue) { + #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 + return OSAtomicCompareAndSwapPtrBarrier(oldValue, oldValue, theValue); + #elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + return __sync_val_compare_and_swap(theValue, oldValue, oldValue); + #else + #error No TRI_CompareAndSwapPointer implementation defined + #endif } //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/BasicsC/locks.h b/lib/BasicsC/locks.h index 786138be55..25e59209f7 100644 --- a/lib/BasicsC/locks.h +++ b/lib/BasicsC/locks.h @@ -407,9 +407,11 @@ void TRI_UnlockCondition (TRI_condition_t* cond); // returned), otherwise no operation is performed (false is returned). // ............................................................................. -bool TRI_CompareAndSwapIntegerInt32 (volatile int32_t* theValue, int32_t oldValue, int32_t newValue); +bool TRI_CompareAndSwapIntegerInt32 (volatile int32_t* theValue, int32_t oldValue, int32_t newValue); +bool TRI_CompareIntegerInt32 (volatile int32_t* theValue, int32_t oldValue); bool TRI_CompareAndSwapIntegerUInt32 (volatile uint32_t* theValue, uint32_t oldValue, uint32_t newValue); +bool TRI_CompareIntegerUInt32 (volatile uint32_t* theValue, uint32_t oldValue); //////////////////////////////////////////////////////////////////////////////// @@ -424,9 +426,11 @@ bool TRI_CompareAndSwapIntegerUInt32 (volatile uint32_t* theValue, uint32_t oldV // simply the 64bit equivalent of the function above. // ............................................................................. -bool TRI_CompareAndSwapIntegerInt64 (volatile int64_t* theValue, int64_t oldValue, int64_t newValue); +bool TRI_CompareAndSwapIntegerInt64 (volatile int64_t* theValue, int64_t oldValue, int64_t newValue); +bool TRI_CompareIntegerInt64 (volatile int64_t* theValue, int64_t oldValue); bool TRI_CompareAndSwapIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldValue, uint64_t newValue); +bool TRI_CompareIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldValue); //////////////////////////////////////////////////////////////////////////////// @@ -442,6 +446,7 @@ bool TRI_CompareAndSwapIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldV //////////////////////////////////////////////////////////////////////////////// bool TRI_CompareAndSwapPointer(void* volatile* theValue, void* oldValue, void* newValue); +bool TRI_ComparePointer(void* volatile* theValue, void* oldValue); //////////////////////////////////////////////////////////////////////////////// /// @} diff --git a/lib/BasicsC/voc-errors.c b/lib/BasicsC/voc-errors.c index b635666d5e..4843392363 100644 --- a/lib/BasicsC/voc-errors.c +++ b/lib/BasicsC/voc-errors.c @@ -2,7 +2,7 @@ /// @brief auto-generated file generated from errors.dat //////////////////////////////////////////////////////////////////////////////// -#include "BasicsC/common.h" +#include #include "./lib/BasicsC/voc-errors.h" //////////////////////////////////////////////////////////////////////////////// @@ -148,6 +148,7 @@ void TRI_InitialiseErrorMessages (void) { REG_ERROR(WARNING_ARANGO_INDEX_HASH_REMOVE_ITEM_MISSING, "hash index remove failure - item missing in index"); REG_ERROR(WARNING_ARANGO_INDEX_SKIPLIST_DOCUMENT_ATTRIBUTE_MISSING, "skiplist index insertion warning - attribute missing in document"); REG_ERROR(WARNING_ARANGO_INDEX_SKIPLIST_UPDATE_ATTRIBUTE_MISSING, "skiplist index update warning - attribute missing in revised document"); + REG_ERROR(WARNING_ARANGO_INDEX_SKIPLIST_INSERT_CAS_FAILURE, "skiplist index insertion warning - CAS failure while attempting to insert document"); REG_ERROR(WARNING_ARANGO_INDEX_SKIPLIST_REMOVE_ITEM_MISSING, "skiplist index remove failure - item missing in index"); REG_ERROR(WARNING_ARANGO_INDEX_BITARRAY_DOCUMENT_ATTRIBUTE_MISSING, "bitarray index insertion warning - attribute missing in document"); REG_ERROR(WARNING_ARANGO_INDEX_BITARRAY_UPDATE_ATTRIBUTE_MISSING, "bitarray index update warning - attribute missing in revised document"); diff --git a/lib/BasicsC/voc-errors.h b/lib/BasicsC/voc-errors.h index 75534c6059..665f8a8611 100644 --- a/lib/BasicsC/voc-errors.h +++ b/lib/BasicsC/voc-errors.h @@ -336,6 +336,9 @@ extern "C" { /// Will be raised when an attempt to update a document results in the /// revised document not having one or more attributes which are required by /// the skiplist index. +/// - 3304: @LIT{skiplist index insertion warning - CAS failure while attempting to insert document} +/// Will be raised when an attempt to insert a document into a skiplist index +/// fails due to repeated CAS failures/clashes. /// - 3311: @LIT{skiplist index remove failure - item missing in index} /// Will be raised when an attempt to remove a document from a skiplist index /// fails when document can not be located within that index. @@ -1801,6 +1804,18 @@ void TRI_InitialiseErrorMessages (void); #define TRI_WARNING_ARANGO_INDEX_SKIPLIST_UPDATE_ATTRIBUTE_MISSING (3302) +//////////////////////////////////////////////////////////////////////////////// +/// @brief 3304: WARNING_ARANGO_INDEX_SKIPLIST_INSERT_CAS_FAILURE +/// +/// skiplist index insertion warning - CAS failure while attempting to insert +/// document +/// +/// Will be raised when an attempt to insert a document into a skiplist index +/// fails due to repeated CAS failures/clashes. +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_WARNING_ARANGO_INDEX_SKIPLIST_INSERT_CAS_FAILURE (3304) + //////////////////////////////////////////////////////////////////////////////// /// @brief 3311: WARNING_ARANGO_INDEX_SKIPLIST_REMOVE_ITEM_MISSING /// diff --git a/lib/Makefile.files b/lib/Makefile.files index 9de06e98d1..a0ca0ebb89 100644 --- a/lib/Makefile.files +++ b/lib/Makefile.files @@ -51,6 +51,7 @@ lib_libarango_a_SOURCES = \ lib/BasicsC/mersenne.c \ lib/BasicsC/process-utils.c \ lib/BasicsC/random.c \ + lib/BasicsC/skip-list.c \ lib/BasicsC/socket-utils.c \ lib/BasicsC/string-buffer.c \ lib/BasicsC/strings.c \