From 666eae6b41c17f3685c2bc86c0ce570fd484e7ce Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Thu, 28 Mar 2013 14:41:40 +0100 Subject: [PATCH] allow writing too big markers into collections if maximalJournalSize is bigger than current datafile's size --- arangod/VocBase/compactor.c | 2 +- arangod/VocBase/datafile.c | 31 +++++++++++++++++++++++---- arangod/VocBase/datafile.h | 3 ++- arangod/VocBase/document-collection.c | 4 +++- arangod/VocBase/primary-collection.c | 2 +- arangod/VocBase/shape-collection.c | 6 +++--- 6 files changed, 37 insertions(+), 11 deletions(-) diff --git a/arangod/VocBase/compactor.c b/arangod/VocBase/compactor.c index 37b4c66629..e0e953aa25 100644 --- a/arangod/VocBase/compactor.c +++ b/arangod/VocBase/compactor.c @@ -96,7 +96,7 @@ static TRI_datafile_t* SelectCompactor (TRI_document_collection_t* document, datafile = document->base.base._compactors._buffer[i]; // try to reserve space - res = TRI_ReserveElementDatafile(datafile, size, result); + res = TRI_ReserveElementDatafile(datafile, size, result, document->base.base._info._maximalSize); // in case of full datafile, try next if (res == TRI_ERROR_NO_ERROR) { diff --git a/arangod/VocBase/datafile.c b/arangod/VocBase/datafile.c index 8baf26a5d3..ee47f47e1f 100644 --- a/arangod/VocBase/datafile.c +++ b/arangod/VocBase/datafile.c @@ -804,7 +804,7 @@ TRI_datafile_t* TRI_CreateDatafile (char const* filename, header._fid = fid; // reserve space and write header to file - result = TRI_ReserveElementDatafile(datafile, header.base._size, &position); + result = TRI_ReserveElementDatafile(datafile, header.base._size, &position, 0); if (result == TRI_ERROR_NO_ERROR) { result = TRI_WriteCrcElementDatafile(datafile, position, &header.base, header.base._size, true); @@ -1119,11 +1119,16 @@ void TRI_FillCrcKeyMarkerDatafile (TRI_datafile_t* datafile, //////////////////////////////////////////////////////////////////////////////// /// @brief reserves room for an element, advances the pointer +/// +/// note: maximalJournalSize is the collection's maximalJournalSize property, +/// which may be different from the size of the current datafile +/// some callers do not set the value of maximalJournalSize //////////////////////////////////////////////////////////////////////////////// int TRI_ReserveElementDatafile (TRI_datafile_t* datafile, TRI_voc_size_t size, - TRI_df_marker_t** position) { + TRI_df_marker_t** position, + TRI_voc_size_t maximalJournalSize) { *position = 0; size = TRI_DF_ALIGN_BLOCK(size); @@ -1139,7 +1144,25 @@ int TRI_ReserveElementDatafile (TRI_datafile_t* datafile, // check the maximal size if (size + TRI_JOURNAL_OVERHEAD > datafile->_maximalSize) { - return TRI_set_errno(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE); + // marker is bigger than journal size. + // adding the marker to this datafile will not work + + if (maximalJournalSize <= datafile->_maximalSize) { + // the collection property 'maximalJournalSize' is equal to + // or smaller than the size of this datafile + // creating a new file and writing the marker into it will not work either + return TRI_set_errno(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE); + } + + // if we get here, the collection's 'maximalJournalSize' property is + // higher than the size of this datafile. + // maybe the marker will fit into a new datafile with the bigger size? + if (size + TRI_JOURNAL_OVERHEAD > maximalJournalSize) { + // marker still won't fit + return TRI_set_errno(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE); + } + + // fall-through intentional } // add the marker, leave enough room for the footer @@ -1457,7 +1480,7 @@ int TRI_SealDatafile (TRI_datafile_t* datafile) { // reserve space and write footer to file datafile->_footerSize = 0; - res = TRI_ReserveElementDatafile(datafile, footer.base._size, &position); + res = TRI_ReserveElementDatafile(datafile, footer.base._size, &position, 0); if (res == TRI_ERROR_NO_ERROR) { res = TRI_WriteCrcElementDatafile(datafile, position, &footer.base, footer.base._size, true); diff --git a/arangod/VocBase/datafile.h b/arangod/VocBase/datafile.h index 9956ab5e30..f48d2e0c5d 100644 --- a/arangod/VocBase/datafile.h +++ b/arangod/VocBase/datafile.h @@ -532,7 +532,8 @@ void TRI_FillCrcKeyMarkerDatafile (TRI_datafile_t* datafile, int TRI_ReserveElementDatafile (TRI_datafile_t* datafile, TRI_voc_size_t size, - TRI_df_marker_t** position); + TRI_df_marker_t** position, + TRI_voc_size_t maximalJournalSize); //////////////////////////////////////////////////////////////////////////////// /// @brief writes a marker to the datafile diff --git a/arangod/VocBase/document-collection.c b/arangod/VocBase/document-collection.c index 6fc5c7b6dc..2a6beffb1c 100644 --- a/arangod/VocBase/document-collection.c +++ b/arangod/VocBase/document-collection.c @@ -601,7 +601,7 @@ static TRI_datafile_t* SelectJournal (TRI_document_collection_t* document, datafile = base->_journals._buffer[i]; // try to reserve space - res = TRI_ReserveElementDatafile(datafile, size, result); + res = TRI_ReserveElementDatafile(datafile, size, result, document->base.base._info._maximalSize); // in case of full datafile, try next if (res == TRI_ERROR_NO_ERROR) { @@ -612,6 +612,8 @@ static TRI_datafile_t* SelectJournal (TRI_document_collection_t* document, else if (res != TRI_ERROR_ARANGO_DATAFILE_FULL) { // some other error TRI_UNLOCK_JOURNAL_ENTRIES_DOC_COLLECTION(document); + + LOG_ERROR("cannot select datafile: '%s'", TRI_last_error()); return NULL; } diff --git a/arangod/VocBase/primary-collection.c b/arangod/VocBase/primary-collection.c index faddebecde..ed24ca3dad 100644 --- a/arangod/VocBase/primary-collection.c +++ b/arangod/VocBase/primary-collection.c @@ -171,7 +171,7 @@ static TRI_datafile_t* CreateJournal (TRI_primary_collection_t* primary, // create a collection header, still in the temporary file - res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position); + res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position, primary->base._info._maximalSize); if (res != TRI_ERROR_NO_ERROR) { collection->_lastError = journal->_lastError; diff --git a/arangod/VocBase/shape-collection.c b/arangod/VocBase/shape-collection.c index cef7723d9e..57b8dd7b8d 100644 --- a/arangod/VocBase/shape-collection.c +++ b/arangod/VocBase/shape-collection.c @@ -132,7 +132,7 @@ static bool CreateJournal (TRI_shape_collection_t* collection) { // create a collection header - res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position); + res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position, 0); if (res != TRI_ERROR_NO_ERROR) { LOG_ERROR("cannot create document header in journal '%s': %s", @@ -262,7 +262,7 @@ static TRI_datafile_t* SelectJournal (TRI_shape_collection_t* collection, datafile = collection->base._journals._buffer[0]; // try to reserve space - res = TRI_ReserveElementDatafile(datafile, size, result); + res = TRI_ReserveElementDatafile(datafile, size, result, 0); while (res == TRI_ERROR_ARANGO_DATAFILE_FULL) { ok = CloseJournal(collection, datafile); @@ -280,7 +280,7 @@ static TRI_datafile_t* SelectJournal (TRI_shape_collection_t* collection, datafile = collection->base._journals._buffer[0]; - res = TRI_ReserveElementDatafile(datafile, size, result); + res = TRI_ReserveElementDatafile(datafile, size, result, 0); } if (res != TRI_ERROR_NO_ERROR) {