diff --git a/CHANGELOG b/CHANGELOG index d50fba49d8..a619920924 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,17 @@ v2.2.1 (XXXX-XX-XX) ------------------- +* added startup option `--wal.suppress-shape-information` + + Setting this option to `true` will reduce memory and disk space usage and require + less CPU time when modifying documents or edges. It should therefore be turned on + for standalone ArangoDB servers. However, for servers that are used as replication + masters, setting this option to `true` will effectively disable the usage of the + write-ahead log for replication, so it should be set to `false` for any replication + master servers. + + The default value for this option is `false`. + * added optional `ttl` attribute to specify result cursor expiration for HTTP API method `POST /_api/cursor` diff --git a/Documentation/Books/Users/AdministratingArango/README.mdpp b/Documentation/Books/Users/AdministratingArango/README.mdpp index 580fd91679..874f2f3dbe 100644 --- a/Documentation/Books/Users/AdministratingArango/README.mdpp +++ b/Documentation/Books/Users/AdministratingArango/README.mdpp @@ -82,6 +82,10 @@ logfiles: @startDocuBlock WalLogfileAllowOversizeEntries + +@startDocuBlock WalLogfileSuppressShapeInformation + + When data gets copied from the write-ahead logfiles into the journals or datafiles of collections, files will be created on the collection level. How big these files are is determined by the following global configuration value: diff --git a/Documentation/Books/Users/ConfigureArango/Wal.mdpp b/Documentation/Books/Users/ConfigureArango/Wal.mdpp index d50a30f705..b011e4eec7 100644 --- a/Documentation/Books/Users/ConfigureArango/Wal.mdpp +++ b/Documentation/Books/Users/ConfigureArango/Wal.mdpp @@ -21,6 +21,10 @@ a replication backlog. @startDocuBlock WalLogfileAllowOversizeEntries +!SUBSECTION Suppress shape information + +@startDocuBlock WalLogfileSuppressShapeInformation + !SUBSECTION Number of reserve logfiles @startDocuBlock WalLogfileReserveLogfiles diff --git a/arangod/VocBase/document-collection.cpp b/arangod/VocBase/document-collection.cpp index 6bdd2b9c21..141ba46d19 100644 --- a/arangod/VocBase/document-collection.cpp +++ b/arangod/VocBase/document-collection.cpp @@ -5297,6 +5297,130 @@ int TRI_RemoveShapedJsonDocumentCollection (TRI_transaction_collection_t* trxCol return res; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief create a document or edge marker, without using a legend +//////////////////////////////////////////////////////////////////////////////// + +static int CreateMarkerNoLegend (triagens::wal::Marker*& marker, + TRI_document_collection_t* document, + TRI_voc_rid_t rid, + TRI_transaction_collection_t* trxCollection, + std::string const& keyString, + TRI_shaped_json_t const* shaped, + TRI_document_edge_t const* edge) { + + TRI_ASSERT(marker == nullptr); + + TRI_IF_FAILURE("InsertDocumentNoMarker") { + // test what happens when no marker can be created + return TRI_ERROR_DEBUG; + } + + TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") { + // test what happens if no marker can be created + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + + if (edge == nullptr) { + // document + marker = new triagens::wal::DocumentMarker(document->_vocbase->_id, + document->_info._cid, + rid, + TRI_MarkerIdTransaction(trxCollection->_transaction), + keyString, + 8, + shaped); + } + else { + // edge + marker = new triagens::wal::EdgeMarker(document->_vocbase->_id, + document->_info._cid, + rid, + TRI_MarkerIdTransaction(trxCollection->_transaction), + keyString, + edge, + 8, + shaped); + } + + TRI_ASSERT(marker != nullptr); + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief create a document or edge marker, with a legend +//////////////////////////////////////////////////////////////////////////////// + +static int CreateMarkerWithLegend (triagens::wal::Marker*& marker, + TRI_document_collection_t* document, + TRI_voc_rid_t rid, + TRI_transaction_collection_t* trxCollection, + std::string const& keyString, + TRI_shaped_json_t const* shaped, + TRI_document_edge_t const* edge) { + // construct a legend for the shaped json + triagens::basics::JsonLegend legend(document->getShaper()); // PROTECTED by trx in trxCollection + + TRI_IF_FAILURE("InsertDocumentNoLegend") { + // test what happens when no legend can be created + return TRI_ERROR_DEBUG; + } + + TRI_IF_FAILURE("InsertDocumentNoLegendExcept") { + // test what happens if no legend can be created + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + int res = legend.addShape(shaped->_sid, &shaped->_data); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + TRI_IF_FAILURE("InsertDocumentNoMarker") { + // test what happens when no marker can be created + return TRI_ERROR_DEBUG; + } + + TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") { + // test what happens if no marker can be created + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + if (edge == nullptr) { + // document + auto m = new triagens::wal::DocumentMarker(document->_vocbase->_id, + document->_info._cid, + rid, + TRI_MarkerIdTransaction(trxCollection->_transaction), + keyString, + legend.getSize(), + shaped); + + m->storeLegend(legend); + marker = m; // reinterpret_cast(m); + } + else { + // edge + auto m = new triagens::wal::EdgeMarker(document->_vocbase->_id, + document->_info._cid, + rid, + TRI_MarkerIdTransaction(trxCollection->_transaction), + keyString, + edge, + legend.getSize(), + shaped); + + m->storeLegend(legend); + + marker = m; //reinterpret_cast(m); + } + + return TRI_ERROR_NO_ERROR; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief insert a shaped-json document (or edge) /// note: key might be NULL. in this case, a key is auto-generated @@ -5345,70 +5469,25 @@ int TRI_InsertShapedJsonDocumentCollection (TRI_transaction_collection_t* trxCol keyString = key; } - uint64_t hash = TRI_HashKeyPrimaryIndex(keyString.c_str()); - - // construct a legend for the shaped json - triagens::basics::JsonLegend legend(document->getShaper()); // PROTECTED by trx in trxCollection - - TRI_IF_FAILURE("InsertDocumentNoLegend") { - // test what happens when no legend can be created - return TRI_ERROR_DEBUG; - } - - - if (marker == nullptr) { - TRI_IF_FAILURE("InsertDocumentNoLegendExcept") { - // test what happens if no legend can be created - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - int res = legend.addShape(shaped->_sid, &shaped->_data); - - if (res != TRI_ERROR_NO_ERROR) { - return res; - } - - TRI_IF_FAILURE("InsertDocumentNoMarker") { - // test what happens when no marker can be created - return TRI_ERROR_DEBUG; - } - - TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") { - // test what happens if no marker can be created - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - - if (edge == nullptr) { - // document - TRI_ASSERT(edge == nullptr); - - marker = new triagens::wal::DocumentMarker(document->_vocbase->_id, - document->_info._cid, - rid, - TRI_MarkerIdTransaction(trxCollection->_transaction), - keyString, - legend, - shaped); - } - else { - // edge - marker = new triagens::wal::EdgeMarker(document->_vocbase->_id, - document->_info._cid, - rid, - TRI_MarkerIdTransaction(trxCollection->_transaction), - keyString, - edge, - legend, - shaped); - } - } - - - TRI_ASSERT(marker != nullptr); - int res = TRI_ERROR_NO_ERROR; + if (marker == nullptr) { + if (triagens::wal::LogfileManager::instance()->suppressShapeInformation()) { + res = CreateMarkerNoLegend(marker, document, rid, trxCollection, keyString, shaped, edge); + } + else { + res = CreateMarkerWithLegend(marker, document, rid, trxCollection, keyString, shaped, edge); + } + } + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + TRI_ASSERT(marker != nullptr); + + uint64_t hash = TRI_HashKeyPrimaryIndex(keyString.c_str()); + // now insert into indexes { TRI_IF_FAILURE("InsertDocumentNoLock") { diff --git a/arangod/Wal/LogfileManager.cpp b/arangod/Wal/LogfileManager.cpp index c025ef2f93..324aeb5f58 100644 --- a/arangod/Wal/LogfileManager.cpp +++ b/arangod/Wal/LogfileManager.cpp @@ -138,6 +138,7 @@ LogfileManager::LogfileManager (TRI_server_t* server, _allowOversizeEntries(true), _ignoreLogfileErrors(false), _ignoreRecoveryErrors(false), + _suppressShapeInformation(false), _allowWrites(false), // start in read-only mode _hasFoundLastTick(false), _inRecovery(true), @@ -234,6 +235,7 @@ void LogfileManager::setupOptions (std::map_data.length) { + sizeof(document_marker_t) + alignedSize(key.size() + 1) + legendSize + shapedJson->_data.length) { document_marker_t* m = reinterpret_cast(begin()); m->_databaseId = databaseId; m->_collectionId = collectionId; @@ -1105,14 +1105,16 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId, m->_offsetKey = sizeof(document_marker_t); // start position of key m->_offsetLegend = static_cast(m->_offsetKey + alignedSize(key.size() + 1)); - m->_offsetJson = static_cast(m->_offsetLegend + alignedSize(legend.getSize())); + m->_offsetJson = static_cast(m->_offsetLegend + alignedSize(legendSize)); storeSizedString(m->_offsetKey, key); // store legend { + TRI_ASSERT(legendSize >= 8); + char* p = static_cast(begin()) + m->_offsetLegend; - legend.dump(p); + memset(p, 0, 8); // initialise initial bytes of legend with 0s } // store shapedJson @@ -1133,6 +1135,14 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId, DocumentMarker::~DocumentMarker () { } +//////////////////////////////////////////////////////////////////////////////// +/// @brief store legend in marker +//////////////////////////////////////////////////////////////////////////////// + +void DocumentMarker::storeLegend (triagens::basics::JsonLegend& legend) { + legend.dump((void*) this->legend()); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief dump marker //////////////////////////////////////////////////////////////////////////////// @@ -1153,8 +1163,6 @@ void DocumentMarker::dump () const { << "\n"; #ifdef DEBUG_WAL_DETAIL - std::cout << "LEGEND: '" << stringifyPart(legend(), legendLength()) << "'\n"; - std::cout << "LEGEND HEX: '" << hexifyPart(legend(), legendLength()) << "'\n"; std::cout << "JSON: '" << stringifyPart(json(), jsonLength()) << "'\n"; std::cout << "JSON HEX: '" << hexifyPart(json(), jsonLength()) << "'\n"; @@ -1179,13 +1187,15 @@ DocumentMarker* DocumentMarker::clone (TRI_df_marker_t const* other, if (other->_type == TRI_DOC_MARKER_KEY_DOCUMENT) { TRI_doc_document_key_marker_t const* original = reinterpret_cast(other); - return new DocumentMarker(databaseId, - collectionId, - revisionId, - transactionId, - std::string(base + original->_offsetKey), - legend, - shapedJson); + auto marker = new DocumentMarker(databaseId, + collectionId, + revisionId, + transactionId, + std::string(base + original->_offsetKey), + legend.getSize(), + shapedJson); + marker->storeLegend(legend); + return marker; } else { TRI_ASSERT(other->_type == TRI_WAL_MARKER_DOCUMENT); @@ -1195,13 +1205,15 @@ DocumentMarker* DocumentMarker::clone (TRI_df_marker_t const* other, TRI_ASSERT(original->_databaseId == databaseId); TRI_ASSERT(original->_collectionId == collectionId); - return new DocumentMarker(original->_databaseId, - original->_collectionId, - revisionId, - transactionId, - std::string(base + original->_offsetKey), - legend, - shapedJson); + auto marker = new DocumentMarker(original->_databaseId, + original->_collectionId, + revisionId, + transactionId, + std::string(base + original->_offsetKey), + legend.getSize(), + shapedJson); + marker->storeLegend(legend); + return marker; } } @@ -1223,10 +1235,10 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId, TRI_voc_tid_t transactionId, std::string const& key, TRI_document_edge_t const* edge, - triagens::basics::JsonLegend& legend, + size_t legendSize, TRI_shaped_json_t const* shapedJson) : Marker(TRI_WAL_MARKER_EDGE, - sizeof(edge_marker_t) + alignedSize(key.size() + 1) + alignedSize(strlen(edge->_fromKey) + 1) + alignedSize(strlen(edge->_toKey) + 1) + legend.getSize() + shapedJson->_data.length) { + sizeof(edge_marker_t) + alignedSize(key.size() + 1) + alignedSize(strlen(edge->_fromKey) + 1) + alignedSize(strlen(edge->_toKey) + 1) + legendSize + shapedJson->_data.length) { edge_marker_t* m = reinterpret_cast(begin()); @@ -1241,7 +1253,7 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId, m->_offsetToKey = static_cast(m->_offsetKey + alignedSize(key.size() + 1)); m->_offsetFromKey = static_cast(m->_offsetToKey + alignedSize(strlen(edge->_toKey) + 1)); m->_offsetLegend = static_cast(m->_offsetFromKey + alignedSize(strlen(edge->_fromKey) + 1)); - m->_offsetJson = static_cast(m->_offsetLegend + alignedSize(legend.getSize())); + m->_offsetJson = static_cast(m->_offsetLegend + alignedSize(legendSize)); // store keys storeSizedString(m->_offsetKey, key.c_str()); @@ -1250,8 +1262,9 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId, // store legend { + TRI_ASSERT(legendSize >= 8); char* p = static_cast(begin()) + m->_offsetLegend; - legend.dump(p); + memset(p, 0, 8); // initialise initial bytes of legend with 0s } // store shapedJson @@ -1272,6 +1285,14 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId, EdgeMarker::~EdgeMarker () { } +//////////////////////////////////////////////////////////////////////////////// +/// @brief store legend in marker +//////////////////////////////////////////////////////////////////////////////// + +void EdgeMarker::storeLegend (triagens::basics::JsonLegend& legend) { + legend.dump((void*) this->legend()); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief dump marker //////////////////////////////////////////////////////////////////////////////// @@ -1298,8 +1319,6 @@ void EdgeMarker::dump () const { << "\n"; #ifdef DEBUG_WAL_DETAIL - std::cout << "LEGEND: '" << stringifyPart(legend(), legendLength()) << "'\n"; - std::cout << "LEGEND HEX: '" << hexifyPart(legend(), legendLength()) << "'\n"; std::cout << "JSON: '" << stringifyPart(json(), jsonLength()) << "'\n"; std::cout << "JSON HEX: '" << hexifyPart(json(), jsonLength()) << "'\n"; @@ -1330,14 +1349,17 @@ EdgeMarker* EdgeMarker::clone (TRI_df_marker_t const* other, edge._toKey = (TRI_voc_key_t) base + original->_offsetToKey; edge._fromKey = (TRI_voc_key_t) base + original->_offsetFromKey; - return new EdgeMarker(databaseId, - collectionId, - revisionId, - transactionId, - std::string(base + original->base._offsetKey), - &edge, - legend, - shapedJson); + auto marker = new EdgeMarker(databaseId, + collectionId, + revisionId, + transactionId, + std::string(base + original->base._offsetKey), + &edge, + legend.getSize(), + shapedJson); + + marker->storeLegend(legend); + return marker; } else { TRI_ASSERT(other->_type == TRI_WAL_MARKER_EDGE); @@ -1353,14 +1375,17 @@ EdgeMarker* EdgeMarker::clone (TRI_df_marker_t const* other, edge._toKey = (TRI_voc_key_t) base + original->_offsetToKey; edge._fromKey = (TRI_voc_key_t) base + original->_offsetFromKey; - return new EdgeMarker(original->_databaseId, - original->_collectionId, - revisionId, - transactionId, - std::string(base + original->_offsetKey), - &edge, - legend, - shapedJson); + auto marker = new EdgeMarker(original->_databaseId, + original->_collectionId, + revisionId, + transactionId, + std::string(base + original->_offsetKey), + &edge, + legend.getSize(), + shapedJson); + + marker->storeLegend(legend); + return marker; } } diff --git a/arangod/Wal/Marker.h b/arangod/Wal/Marker.h index 8f9122a349..ddf11f8578 100644 --- a/arangod/Wal/Marker.h +++ b/arangod/Wal/Marker.h @@ -770,9 +770,9 @@ namespace triagens { TRI_voc_rid_t, TRI_voc_tid_t, std::string const&, - triagens::basics::JsonLegend&, + size_t, TRI_shaped_json_t const*); - + ~DocumentMarker (); public: @@ -814,6 +814,8 @@ namespace triagens { return static_cast(size() - m->_offsetJson); } + void storeLegend (triagens::basics::JsonLegend&); + void dump () const; static DocumentMarker* clone (TRI_df_marker_t const*, @@ -839,9 +841,9 @@ namespace triagens { TRI_voc_tid_t, std::string const&, TRI_document_edge_t const*, - triagens::basics::JsonLegend&, + size_t, TRI_shaped_json_t const*); - + ~EdgeMarker (); inline TRI_voc_rid_t revisionId () const { @@ -892,6 +894,8 @@ namespace triagens { edge_marker_t const* m = reinterpret_cast(begin()); return static_cast(size() - m->_offsetJson); } + + void storeLegend (triagens::basics::JsonLegend&); void dump () const;