diff --git a/arangod/VocBase/datafile.h b/arangod/VocBase/datafile.h index 666b9c68d7..ada5182604 100644 --- a/arangod/VocBase/datafile.h +++ b/arangod/VocBase/datafile.h @@ -29,8 +29,8 @@ #define TRIAGENS_VOC_BASE_DATAFILE_H 1 #include "Basics/Common.h" - #include "BasicsC/locks.h" +#include "ShapedJson/shaped-json.h" #include "VocBase/vocbase.h" @@ -405,6 +405,83 @@ typedef struct TRI_df_footer_marker_s { } TRI_df_footer_marker_t; +//////////////////////////////////////////////////////////////////////////////// +/// @brief datafile attribute marker +//////////////////////////////////////////////////////////////////////////////// + +typedef struct TRI_df_attribute_marker_s { + TRI_df_marker_t base; + + TRI_shape_aid_t _aid; + TRI_shape_size_t _size; + + // char name[] +} +TRI_df_attribute_marker_t; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief datafile shape marker +//////////////////////////////////////////////////////////////////////////////// + +typedef struct TRI_df_shape_marker_s { + TRI_df_marker_t base; +} +TRI_df_shape_marker_t; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief document datafile marker with key +//////////////////////////////////////////////////////////////////////////////// + +typedef struct TRI_doc_document_key_marker_s { + TRI_df_marker_t base; + + TRI_voc_rid_t _rid; // this is the tick for a create and update + TRI_voc_tid_t _tid; + + TRI_shape_sid_t _shape; + + uint16_t _offsetKey; + uint16_t _offsetJson; + +#ifdef TRI_PADDING_32 + char _padding_df_marker[4]; +#endif +} +TRI_doc_document_key_marker_t; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief edge datafile marker with key +//////////////////////////////////////////////////////////////////////////////// + +typedef struct TRI_doc_edge_key_marker_s { + TRI_doc_document_key_marker_t base; + + TRI_voc_cid_t _toCid; + TRI_voc_cid_t _fromCid; + + uint16_t _offsetToKey; + uint16_t _offsetFromKey; + +#ifdef TRI_PADDING_32 + char _padding_df_marker[4]; +#endif +} +TRI_doc_edge_key_marker_t; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief document datafile deletion marker +//////////////////////////////////////////////////////////////////////////////// + +typedef struct TRI_doc_deletion_key_marker_s { + TRI_df_marker_t base; + + TRI_voc_rid_t _rid; // this is the tick for the deletion + TRI_voc_tid_t _tid; + + uint16_t _offsetKey; +} +TRI_doc_deletion_key_marker_t; + //////////////////////////////////////////////////////////////////////////////// /// @brief begin transaction marker //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/VocBase/document-collection.h b/arangod/VocBase/document-collection.h index 25a25d2b58..4c843ee337 100644 --- a/arangod/VocBase/document-collection.h +++ b/arangod/VocBase/document-collection.h @@ -1,4 +1,4 @@ -//////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// /// @brief document collection with global read-write lock, derived from /// TRI_document_collection_t /// @@ -360,61 +360,6 @@ typedef struct TRI_document_collection_s { } TRI_document_collection_t; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief document datafile marker with key -//////////////////////////////////////////////////////////////////////////////// - -typedef struct TRI_doc_document_key_marker_s { - TRI_df_marker_t base; - - TRI_voc_rid_t _rid; // this is the tick for a create and update - TRI_voc_tid_t _tid; - - TRI_shape_sid_t _shape; - - uint16_t _offsetKey; - uint16_t _offsetJson; - -#ifdef TRI_PADDING_32 - char _padding_df_marker[4]; -#endif -} -TRI_doc_document_key_marker_t; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief edge datafile marker with key -//////////////////////////////////////////////////////////////////////////////// - -typedef struct TRI_doc_edge_key_marker_s { - TRI_doc_document_key_marker_t base; - - TRI_voc_cid_t _toCid; - TRI_voc_cid_t _fromCid; - - uint16_t _offsetToKey; - uint16_t _offsetFromKey; - -#ifdef TRI_PADDING_32 - char _padding_df_marker[4]; -#endif -} -TRI_doc_edge_key_marker_t; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief document datafile deletion marker -//////////////////////////////////////////////////////////////////////////////// - -typedef struct TRI_doc_deletion_key_marker_s { - TRI_df_marker_t base; - - TRI_voc_rid_t _rid; // this is the tick for the deletion - TRI_voc_tid_t _tid; - - uint16_t _offsetKey; -} -TRI_doc_deletion_key_marker_t; - // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- diff --git a/arangod/VocBase/voc-shaper.h b/arangod/VocBase/voc-shaper.h index 5bbca77509..92cc7c6a73 100644 --- a/arangod/VocBase/voc-shaper.h +++ b/arangod/VocBase/voc-shaper.h @@ -40,33 +40,6 @@ struct TRI_document_collection_s; -// ----------------------------------------------------------------------------- -// --SECTION-- public types -// ----------------------------------------------------------------------------- - -//////////////////////////////////////////////////////////////////////////////// -/// @brief datafile attribute marker -//////////////////////////////////////////////////////////////////////////////// - -typedef struct TRI_df_attribute_marker_s { - TRI_df_marker_t base; - - TRI_shape_aid_t _aid; - TRI_shape_size_t _size; - - // char name[] -} -TRI_df_attribute_marker_t; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief datafile shape marker -//////////////////////////////////////////////////////////////////////////////// - -typedef struct TRI_df_shape_marker_s { - TRI_df_marker_t base; -} -TRI_df_shape_marker_t; - // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- diff --git a/arangod/Wal/CollectorThread.cpp b/arangod/Wal/CollectorThread.cpp index 292442ada3..4288be2a48 100644 --- a/arangod/Wal/CollectorThread.cpp +++ b/arangod/Wal/CollectorThread.cpp @@ -26,6 +26,7 @@ //////////////////////////////////////////////////////////////////////////////// #include "CollectorThread.h" +#include "BasicsC/hashes.h" #include "BasicsC/logging.h" #include "Basics/ConditionLocker.h" #include "Utils/DatabaseGuard.h" @@ -290,13 +291,6 @@ bool CollectorThread::ScanMarker (TRI_df_marker_t const* marker, break; } - /* TODO: check whether these need to be handled - case TRI_WAL_MARKER_CREATE_COLLECTION: - case TRI_WAL_MARKER_DROP_COLLECTION: - case TRI_WAL_MARKER_RENAME_COLLECTION: - case TRI_WAL_MARKER_CHANGE_COLLECTION: - */ - default: { // do nothing intentionally } @@ -402,12 +396,160 @@ int CollectorThread::transferMarkers (TRI_voc_cid_t collectionId, TRI_voc_tick_t minTransferTick = 0; // TODO: find the actual max tick of a collection for (auto it2 = operations.begin(); it2 != operations.end(); ++it2) { - TRI_df_marker_t const* m = (*it2); + TRI_df_marker_t const* source = (*it2); - if (m->_tick <= minTransferTick) { + if (source->_tick <= minTransferTick) { // we have already transferred this marker in a previous run, nothing to do continue; } + + char const* base = reinterpret_cast(source); + + switch (source->_type) { + case TRI_WAL_MARKER_ATTRIBUTE: { + char const* name = base + sizeof(attribute_marker_t); + size_t n = strlen(name) + 1; // add NULL byte + TRI_voc_size_t const totalSize = sizeof(TRI_df_attribute_marker_t) + n; + + char* mem = nextFreeMarkerPosition(TRI_DF_MARKER_ATTRIBUTE, totalSize); + + if (mem == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + // set attribute id + TRI_df_attribute_marker_t* m = reinterpret_cast(mem); + m->_aid = reinterpret_cast(source)->_attributeId; + + // copy attribute name into marker + memcpy(mem + sizeof(TRI_df_attribute_marker_t), name, n); + + finishMarker(mem, source->_tick); + break; + } + + case TRI_WAL_MARKER_SHAPE: { + TRI_shape_t const* shape = reinterpret_cast(base + sizeof(shape_marker_t)); + TRI_voc_size_t const totalSize = sizeof(TRI_df_shape_marker_t) + shape->_size; + + char* mem = nextFreeMarkerPosition(TRI_DF_MARKER_SHAPE, totalSize); + + if (mem == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + // copy shape into marker + memcpy(mem + sizeof(TRI_df_shape_marker_t), shape, shape->_size); + + finishMarker(mem, source->_tick); + break; + } + + case TRI_WAL_MARKER_DOCUMENT: { + document_marker_t const* orig = reinterpret_cast(source); + + TRI_shape_t const* shape = reinterpret_cast(base + orig->_offsetJson); + char const* key = base + orig->_offsetKey; + size_t n = strlen(key) + 1; // add NULL byte + TRI_voc_size_t const totalSize = sizeof(TRI_doc_document_key_marker_t) + + TRI_DF_ALIGN_BLOCK(n) + + shape->_size; + + char* mem = nextFreeMarkerPosition(TRI_DOC_MARKER_KEY_DOCUMENT, totalSize); + + if (mem == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + TRI_doc_document_key_marker_t* m = reinterpret_cast(mem); + m->_rid = orig->_revisionId; + m->_tid = orig->_transactionId; + m->_shape = orig->_shape; + m->_offsetKey = sizeof(TRI_doc_document_key_marker_t); + m->_offsetJson = m->_offsetKey + TRI_DF_ALIGN_BLOCK(n); + + // copy key into marker + memcpy(mem + m->_offsetKey, key, n); + + // copy shape into marker + memcpy(mem + m->_offsetJson, shape, shape->_size); + + finishMarker(mem, source->_tick); + break; + } + + case TRI_WAL_MARKER_EDGE: { + edge_marker_t const* orig = reinterpret_cast(source); + + TRI_shape_t const* shape = reinterpret_cast(base + orig->_offsetJson); + char const* key = base + orig->_offsetKey; + size_t n = strlen(key) + 1; // add NULL byte + char const* toKey = base + orig->_offsetToKey; + size_t to = strlen(toKey) + 1; // add NULL byte + char const* fromKey = base + orig->_offsetFromKey; + size_t from = strlen(fromKey) + 1; // add NULL byte + TRI_voc_size_t const totalSize = sizeof(TRI_doc_edge_key_marker_t) + + TRI_DF_ALIGN_BLOCK(n) + + TRI_DF_ALIGN_BLOCK(to) + + TRI_DF_ALIGN_BLOCK(from) + + shape->_size; + + char* mem = nextFreeMarkerPosition(TRI_DOC_MARKER_KEY_EDGE, totalSize); + + if (mem == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + size_t offsetKey = sizeof(TRI_doc_edge_key_marker_t); + TRI_doc_edge_key_marker_t* m = reinterpret_cast(mem); + m->base._rid = orig->_revisionId; + m->base._tid = orig->_transactionId; + m->base._shape = orig->_shape; + m->base._offsetKey = offsetKey; + m->base._offsetJson = offsetKey + TRI_DF_ALIGN_BLOCK(n) + TRI_DF_ALIGN_BLOCK(to) + TRI_DF_ALIGN_BLOCK(from); + m->_toCid = orig->_toCid; + m->_fromCid = orig->_fromCid; + m->_offsetToKey = offsetKey + TRI_DF_ALIGN_BLOCK(n); + m->_offsetFromKey = offsetKey + TRI_DF_ALIGN_BLOCK(n) + TRI_DF_ALIGN_BLOCK(to); + + // copy key into marker + memcpy(mem + offsetKey, key, n); + memcpy(mem + m->_offsetToKey, toKey, to); + memcpy(mem + m->_offsetFromKey, fromKey, from); + + // copy shape into marker + memcpy(mem + m->base._offsetJson, shape, shape->_size); + + finishMarker(mem, source->_tick); + break; + } + + case TRI_WAL_MARKER_REMOVE: { + remove_marker_t const* orig = reinterpret_cast(source); + + char const* key = base + sizeof(remove_marker_t); + size_t n = strlen(key) + 1; // add NULL byte + TRI_voc_size_t const totalSize = sizeof(TRI_doc_deletion_key_marker_t) + n; + + char* mem = nextFreeMarkerPosition(TRI_DOC_MARKER_KEY_DELETION, totalSize); + + if (mem == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + + TRI_doc_deletion_key_marker_t* m = reinterpret_cast(mem); + m->_rid = orig->_revisionId; + m->_tid = orig->_transactionId; + m->_offsetKey = sizeof(TRI_doc_deletion_key_marker_t); + + // copy key into marker + memcpy(mem + m->_offsetKey, key, n); + + finishMarker(mem, source->_tick); + break; + } + } } TRI_ReleaseCollectionVocBase(vocbase, collection); @@ -415,6 +557,52 @@ int CollectorThread::transferMarkers (TRI_voc_cid_t collectionId, return TRI_ERROR_NO_ERROR; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the next position for a marker of the specified size +//////////////////////////////////////////////////////////////////////////////// + +char* CollectorThread::nextFreeMarkerPosition (TRI_df_marker_type_e type, + TRI_voc_size_t size) { + char* mem = nullptr; + + if (mem != nullptr) { + initMarker(reinterpret_cast(mem), type, size); + } + + return mem; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initialise a marker +//////////////////////////////////////////////////////////////////////////////// + +void CollectorThread::initMarker (TRI_df_marker_t* marker, + TRI_df_marker_type_e type, + TRI_voc_size_t size) { + assert(marker != nullptr); + + marker->_size = size; + marker->_type = (TRI_df_marker_type_t) type; + marker->_crc = 0; + marker->_tick = 0; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief set the tick of a marker and calculate its CRC value +//////////////////////////////////////////////////////////////////////////////// + +void CollectorThread::finishMarker (char* mem, + TRI_voc_tick_t tick) { + TRI_df_marker_t* marker = reinterpret_cast(mem); + + marker->_tick = tick; + + // calculate the CRC + TRI_voc_crc_t crc = TRI_InitialCrc32(); + crc = TRI_BlockCrc32(crc, (char const*) marker, marker->_size); + marker->_crc = TRI_FinalCrc32(crc); +} + // Local Variables: // mode: outline-minor // outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|/// @page\\|// --SECTION--\\|/// @\\}" diff --git a/arangod/Wal/CollectorThread.h b/arangod/Wal/CollectorThread.h index e2038d7e08..093713c568 100644 --- a/arangod/Wal/CollectorThread.h +++ b/arangod/Wal/CollectorThread.h @@ -31,6 +31,7 @@ #include "Basics/Common.h" #include "Basics/ConditionVariable.h" #include "Basics/Thread.h" +#include "VocBase/datafile.h" #include "VocBase/voc-types.h" struct TRI_datafile_s; @@ -165,6 +166,28 @@ namespace triagens { TRI_voc_tick_t, OperationsType const&); +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the next free position for a new marker of the specified size +//////////////////////////////////////////////////////////////////////////////// + + char* nextFreeMarkerPosition (TRI_df_marker_type_e, + TRI_voc_size_t); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initialise a marker +//////////////////////////////////////////////////////////////////////////////// + + void initMarker (struct TRI_df_marker_s*, + TRI_df_marker_type_e, + TRI_voc_size_t); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief set the tick of a marker and calculate its CRC value +//////////////////////////////////////////////////////////////////////////////// + + void finishMarker (char*, + TRI_voc_tick_t); + // ----------------------------------------------------------------------------- // --SECTION-- private variables // -----------------------------------------------------------------------------