From d7f35bcc7bc3bd174708df7bf6bae16d776796ee Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Fri, 13 Jun 2014 14:46:21 +0200 Subject: [PATCH] fixed datafile ticks issues --- .../HttpInterface/api-collection-spec.rb | 9 ++ arangod/RestServer/ArangoServer.cpp | 4 - arangod/V8Server/v8-vocbase.cpp | 2 + arangod/VocBase/compactor.cpp | 14 +- arangod/VocBase/datafile.cpp | 128 ++++++++---------- arangod/VocBase/datafile.h | 4 - arangod/VocBase/document-collection.cpp | 65 ++++++--- arangod/VocBase/document-collection.h | 1 + arangod/VocBase/replication-dump.cpp | 36 ----- arangod/VocBase/server.cpp | 8 +- arangod/VocBase/transaction.cpp | 4 +- arangod/Wal/CollectorThread.cpp | 13 +- arangod/Wal/CollectorThread.h | 3 +- js/common/modules/org/arangodb/test-helper.js | 27 +++- .../tests/shell-collection-noncluster.js | 15 +- js/common/tests/shell-document.js | 2 - .../tests/shell-compaction-noncluster.js | 31 ++--- 17 files changed, 176 insertions(+), 190 deletions(-) diff --git a/UnitTests/HttpInterface/api-collection-spec.rb b/UnitTests/HttpInterface/api-collection-spec.rb index 15ea5a781f..78c3cb78e0 100644 --- a/UnitTests/HttpInterface/api-collection-spec.rb +++ b/UnitTests/HttpInterface/api-collection-spec.rb @@ -290,6 +290,9 @@ describe ArangoDB do # get figures it "extracting the figures for a collection" do + # flush wal + ArangoDB.put("/_admin/wal/flush?waitForSync=true&waitForCollector=true", { }) + cmd = api + "/" + @cn + "/figures" doc = ArangoDB.log_get("#{prefix}-get-collection-figures", cmd) @@ -325,6 +328,9 @@ describe ArangoDB do body = "{ \"test\" : " + i.to_s + " }" doc = ArangoDB.log_post("#{prefix}-get-collection-figures", "/_api/document/?collection=" + @cn, :body => body) } + + # flush wal + ArangoDB.put("/_admin/wal/flush?waitForSync=true&waitForCollector=true", { }) doc = ArangoDB.log_get("#{prefix}-get-collection-figures", cmd) doc.code.should eq(200) @@ -346,6 +352,9 @@ describe ArangoDB do doc = ArangoDB.log_post("#{prefix}-get-collection-figures", "/_api/document/?collection=" + @cn, :body => body) } + # flush wal + ArangoDB.put("/_admin/wal/flush?waitForSync=true&waitForCollector=true", { }) + doc = ArangoDB.log_get("#{prefix}-get-collection-figures", cmd) doc.code.should eq(200) doc.headers['content-type'].should eq("application/json; charset=utf-8") diff --git a/arangod/RestServer/ArangoServer.cpp b/arangod/RestServer/ArangoServer.cpp index 752303515e..d08ea5d779 100644 --- a/arangod/RestServer/ArangoServer.cpp +++ b/arangod/RestServer/ArangoServer.cpp @@ -764,10 +764,6 @@ int ArangoServer::startupServer () { // open all databases bool const iterateMarkersOnOpen = ! wal::LogfileManager::instance()->hasFoundLastTick(); - if (iterateMarkersOnOpen) { - LOG_WARNING("no shutdown info found. scanning datafiles for last tick..."); - } - openDatabases(checkVersion, performUpgrade, iterateMarkersOnOpen); if (! wal::LogfileManager::instance()->open()) { diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index 55d35cc88c..b7c76fd874 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -7034,6 +7034,8 @@ static v8::Handle JS_RotateVocbaseCol (v8::Arguments const& argv) { // renaming a collection in a cluster is unsupported TRI_V8_EXCEPTION(scope, TRI_ERROR_CLUSTER_UNSUPPORTED); } + + PREVENT_EMBEDDED_TRANSACTION(scope); v8::Handle err; TRI_vocbase_col_t const* collection = UseCollection(argv.Holder(), &err); diff --git a/arangod/VocBase/compactor.cpp b/arangod/VocBase/compactor.cpp index 7cc036caea..b4d5881209 100644 --- a/arangod/VocBase/compactor.cpp +++ b/arangod/VocBase/compactor.cpp @@ -213,7 +213,7 @@ static int CopyMarker (TRI_document_collection_t* document, return TRI_ERROR_ARANGO_NO_JOURNAL; } - return TRI_WriteElementDatafile(compactor, *result, marker, marker->_size, false); + return TRI_WriteElementDatafile(compactor, *result, marker, false); } //////////////////////////////////////////////////////////////////////////////// @@ -445,7 +445,7 @@ static bool Compactifier (TRI_df_marker_t const* marker, TRI_df_marker_t* result; int res; - compaction_context_t* context = (compaction_context_t*) data; + compaction_context_t* context = static_cast(data); TRI_document_collection_t* document = context->_document; // new or updated document @@ -831,11 +831,11 @@ static void CompactifyDatafiles (TRI_document_collection_t* document, compaction_info_t* compaction = static_cast(TRI_AtVector(compactions, i)); TRI_datafile_t* df = compaction->_datafile; - LOG_DEBUG("compacting datafile '%s' into '%s', number: %d, keep deletions: %d", - df->getName(df), - compactor->getName(compactor), - (int) i, - (int) compaction->_keepDeletions); + LOG_TRACE("compacting datafile '%s' into '%s', number: %d, keep deletions: %d", + df->getName(df), + compactor->getName(compactor), + (int) i, + (int) compaction->_keepDeletions); // if this is the first datafile in the list of datafiles, we can also collect // deletion markers diff --git a/arangod/VocBase/datafile.cpp b/arangod/VocBase/datafile.cpp index 0b1b50bf44..745a13c2c2 100644 --- a/arangod/VocBase/datafile.cpp +++ b/arangod/VocBase/datafile.cpp @@ -1170,7 +1170,7 @@ int TRI_WriteInitialHeaderMarkerDatafile (TRI_datafile_t* datafile, res = TRI_ReserveElementDatafile(datafile, header.base._size, &position, 0); if (res == TRI_ERROR_NO_ERROR) { - res = TRI_WriteCrcElementDatafile(datafile, position, &header.base, header.base._size, false); + res = TRI_WriteCrcElementDatafile(datafile, position, &header.base, false); } return res; @@ -1280,54 +1280,15 @@ int TRI_ReserveElementDatafile (TRI_datafile_t* datafile, int TRI_WriteElementDatafile (TRI_datafile_t* datafile, void* position, TRI_df_marker_t const* marker, - TRI_voc_size_t markerSize, bool forceSync) { TRI_voc_tick_t tick = marker->_tick; - TRI_df_marker_type_e type = (TRI_df_marker_type_e) (int) marker->_type; TRI_ASSERT(tick > 0); + TRI_ASSERT(marker->_size > 0); - if (type != TRI_DF_MARKER_HEADER && - type != TRI_DF_MARKER_FOOTER && - type != TRI_COL_MARKER_HEADER && - type != TRI_DF_MARKER_ATTRIBUTE && - type != TRI_DF_MARKER_SHAPE) { - -#ifdef TRI_ENABLE_MAINTAINER_MODE - // check _tick value of marker and set min/max tick values for datafile - if (tick <= datafile->_tickMin || tick <= (TRI_voc_tick_t) datafile->_fid) { - LOG_FATAL_AND_EXIT("logic error. invalid tick value %llu encountered when writing marker of type %d into datafile '%s'. " - "expected tick value > tickMin %llu", - (unsigned long long) tick, - (int) marker->_type, - datafile->getName(datafile), - (unsigned long long) datafile->_tickMin); - TRI_ASSERT(false); - } - - if (tick <= datafile->_tickMax) { - LOG_FATAL_AND_EXIT("logic error. invalid tick value %llu encountered when writing marker of type %d into datafile '%s'. " - "expected tick value > tickMax %llu", - (unsigned long long) tick, - (int) marker->_type, - datafile->getName(datafile), - (unsigned long long) datafile->_tickMax); - TRI_ASSERT(false); - } -#endif - - TRI_UpdateTicksDatafile(datafile, marker); - } + TRI_UpdateTicksDatafile(datafile, marker); - TRI_ASSERT(markerSize > 0); - - if (markerSize != marker->_size) { - LOG_ERROR("marker size is %lu, but size is %lu", - (unsigned long) marker->_size, - (unsigned long) markerSize); - } - if (datafile->_state != TRI_DF_STATE_WRITE) { if (datafile->_state == TRI_DF_STATE_READ) { LOG_ERROR("cannot write marker, datafile is read-only"); @@ -1346,13 +1307,11 @@ int TRI_WriteElementDatafile (TRI_datafile_t* datafile, return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE); } - memcpy(position, marker, markerSize); + memcpy(position, marker, static_cast(marker->_size)); if (forceSync) { - bool ok; - - ok = datafile->sync(datafile, static_cast(position), ((char*) position) + markerSize); + bool ok = datafile->sync(datafile, static_cast(position), ((char*) position) + marker->_size); if (! ok) { datafile->_state = TRI_DF_STATE_WRITE_ERROR; @@ -1369,7 +1328,7 @@ int TRI_WriteElementDatafile (TRI_datafile_t* datafile, return datafile->_lastError; } else { - LOG_TRACE("msync succeeded %p, size %lu", position, (unsigned long) markerSize); + LOG_TRACE("msync succeeded %p, size %lu", position, (unsigned long) marker->_size); } } @@ -1383,24 +1342,53 @@ int TRI_WriteElementDatafile (TRI_datafile_t* datafile, void TRI_UpdateTicksDatafile (TRI_datafile_t* datafile, TRI_df_marker_t const* marker) { TRI_df_marker_type_e type = (TRI_df_marker_type_e) marker->_type; + TRI_voc_tick_t tick = marker->_tick; - if (type == TRI_DF_MARKER_HEADER || - type == TRI_DF_MARKER_FOOTER || - type == TRI_COL_MARKER_HEADER || - type == TRI_DF_MARKER_ATTRIBUTE || - type == TRI_DF_MARKER_SHAPE) { - // ignore these markers - return; - } + if (type != TRI_DF_MARKER_HEADER && + type != TRI_DF_MARKER_FOOTER && + type != TRI_COL_MARKER_HEADER && + type != TRI_DF_MARKER_ATTRIBUTE && + type != TRI_DF_MARKER_SHAPE) { - if (type == TRI_DOC_MARKER_KEY_DOCUMENT || - type == TRI_DOC_MARKER_KEY_EDGE) { - if (datafile->_dataMin == 0) { - datafile->_dataMin = marker->_tick; +#ifdef TRI_ENABLE_MAINTAINER_MODE + // check _tick value of marker and set min/max tick values for datafile + if (marker->_tick < datafile->_tickMin) { + LOG_FATAL_AND_EXIT("logic error. invalid tick value %llu encountered when writing marker of type %d into datafile '%s'. " + "expected tick value >= tickMin %llu", + (unsigned long long) tick, + (int) marker->_type, + datafile->getName(datafile), + (unsigned long long) datafile->_tickMin); } - if (datafile->_dataMax < marker->_tick) { - datafile->_dataMax = marker->_tick; + if (tick < datafile->_tickMax) { + LOG_FATAL_AND_EXIT("logic error. invalid tick value %llu encountered when writing marker of type %d into datafile '%s'. " + "expected tick value >= tickMax %llu", + (unsigned long long) tick, + (int) marker->_type, + datafile->getName(datafile), + (unsigned long long) datafile->_tickMax); + } + + if (tick < static_cast(datafile->_fid)) { + LOG_FATAL_AND_EXIT("logic error. invalid tick value %llu encountered when writing marker of type %d into datafile '%s'. " + "expected tick value >= fid %llu", + (unsigned long long) tick, + (int) marker->_type, + datafile->getName(datafile), + (unsigned long long) datafile->_fid); + } +#endif + + if (type == TRI_DOC_MARKER_KEY_DOCUMENT || + type == TRI_DOC_MARKER_KEY_EDGE) { + if (datafile->_dataMin == 0) { + datafile->_dataMin = tick; + } + + if (datafile->_dataMax < tick) { + datafile->_dataMax = tick; + } } } @@ -1408,39 +1396,33 @@ void TRI_UpdateTicksDatafile (TRI_datafile_t* datafile, type != TRI_DF_MARKER_SHAPE) { if (datafile->_tickMin == 0) { - datafile->_tickMin = marker->_tick; + datafile->_tickMin = tick; } if (datafile->_tickMax < marker->_tick) { - datafile->_tickMax = marker->_tick; + datafile->_tickMax = tick; } } } //////////////////////////////////////////////////////////////////////////////// /// @brief checksums and writes a marker to the datafile -/// this function will also assign a new tick value for the marker (so that -/// the tick values are increasing) //////////////////////////////////////////////////////////////////////////////// int TRI_WriteCrcElementDatafile (TRI_datafile_t* datafile, void* position, TRI_df_marker_t* marker, - TRI_voc_size_t markerSize, bool forceSync) { - if (marker->_tick == 0) { - // set a tick value for the marker - marker->_tick = TRI_NewTickServer(); - } + TRI_ASSERT(marker->_tick != 0); if (datafile->isPhysical(datafile)) { TRI_voc_crc_t crc = TRI_InitialCrc32(); - crc = TRI_BlockCrc32(crc, (char const*) marker, markerSize); + crc = TRI_BlockCrc32(crc, (char const*) marker, marker->_size); marker->_crc = TRI_FinalCrc32(crc); } - return TRI_WriteElementDatafile(datafile, position, marker, markerSize, forceSync); + return TRI_WriteElementDatafile(datafile, position, marker, forceSync); } //////////////////////////////////////////////////////////////////////////////// @@ -1668,7 +1650,7 @@ int TRI_SealDatafile (TRI_datafile_t* datafile) { res = TRI_ReserveElementDatafile(datafile, footer.base._size, &position, 0); if (res == TRI_ERROR_NO_ERROR) { - res = TRI_WriteCrcElementDatafile(datafile, position, &footer.base, footer.base._size, true); + res = TRI_WriteCrcElementDatafile(datafile, position, &footer.base, true); } if (res != TRI_ERROR_NO_ERROR) { diff --git a/arangod/VocBase/datafile.h b/arangod/VocBase/datafile.h index 572708794a..b6fe66fe9e 100644 --- a/arangod/VocBase/datafile.h +++ b/arangod/VocBase/datafile.h @@ -669,19 +669,15 @@ int TRI_ReserveElementDatafile (TRI_datafile_t* datafile, int TRI_WriteElementDatafile (TRI_datafile_t* datafile, void* position, TRI_df_marker_t const* marker, - TRI_voc_size_t markerSize, bool sync) TRI_WARN_UNUSED_RESULT; //////////////////////////////////////////////////////////////////////////////// /// @brief checksums and writes a marker to the datafile -/// this function will also assign a new tick value for the marker (so that -/// the tick values are increasing) //////////////////////////////////////////////////////////////////////////////// int TRI_WriteCrcElementDatafile (TRI_datafile_t* datafile, void* position, TRI_df_marker_t* marker, - TRI_voc_size_t markerSize, bool sync) TRI_WARN_UNUSED_RESULT; //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/VocBase/document-collection.cpp b/arangod/VocBase/document-collection.cpp index 8904ae3c63..20573506db 100644 --- a/arangod/VocBase/document-collection.cpp +++ b/arangod/VocBase/document-collection.cpp @@ -276,7 +276,7 @@ static TRI_datafile_t* CreateCompactor (TRI_document_collection_t* document, cm._type = (TRI_col_type_t) collection->_info._type; cm._cid = collection->_info._cid; - res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, sizeof(cm), false); + res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, false); if (res != TRI_ERROR_NO_ERROR) { collection->_lastError = journal->_lastError; @@ -2322,19 +2322,16 @@ TRI_doc_datafile_info_t* TRI_FindDatafileInfoDocumentCollection (TRI_document_co //////////////////////////////////////////////////////////////////////////////// TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* document, + TRI_voc_fid_t fid, TRI_voc_size_t journalSize) { TRI_col_header_marker_t cm; - TRI_collection_t* collection; TRI_datafile_t* journal; TRI_df_marker_t* position; - TRI_voc_fid_t fid; int res; - collection = document; + TRI_ASSERT(fid > 0); - fid = (TRI_voc_fid_t) TRI_NewTickServer(); - - if (collection->_info._isVolatile) { + if (document->_info._isVolatile) { // in-memory collection journal = TRI_CreateDatafile(nullptr, fid, journalSize, true); } @@ -2346,7 +2343,7 @@ TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* // construct a suitable filename (which is temporary at the beginning) number = TRI_StringUInt64(fid); jname = TRI_Concatenate3String("temp-", number, ".db"); - filename = TRI_Concatenate2File(collection->_directory, jname); + filename = TRI_Concatenate2File(document->_directory, jname); TRI_FreeString(TRI_CORE_MEM_ZONE, number); TRI_FreeString(TRI_CORE_MEM_ZONE, jname); @@ -2357,12 +2354,12 @@ TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* if (journal == NULL) { if (TRI_errno() == TRI_ERROR_OUT_OF_MEMORY_MMAP) { - collection->_lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP); - collection->_state = TRI_COL_STATE_READ; + document->_lastError = TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY_MMAP); + document->_state = TRI_COL_STATE_READ; } else { - collection->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL); - collection->_state = TRI_COL_STATE_WRITE_ERROR; + document->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_NO_JOURNAL); + document->_state = TRI_COL_STATE_WRITE_ERROR; } return NULL; @@ -2375,7 +2372,7 @@ TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* res = TRI_ReserveElementDatafile(journal, sizeof(TRI_col_header_marker_t), &position, journalSize); if (res != TRI_ERROR_NO_ERROR) { - collection->_lastError = journal->_lastError; + document->_lastError = journal->_lastError; LOG_ERROR("cannot create document header in journal '%s': %s", journal->getName(journal), TRI_last_error()); TRI_FreeDatafile(journal); @@ -2386,13 +2383,13 @@ TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* TRI_InitMarkerDatafile((char*) &cm, TRI_COL_MARKER_HEADER, sizeof(TRI_col_header_marker_t)); cm.base._tick = (TRI_voc_tick_t) fid; - cm._type = (TRI_col_type_t) collection->_info._type; - cm._cid = collection->_info._cid; + cm._type = (TRI_col_type_t) document->_info._type; + cm._cid = document->_info._cid; - res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, sizeof(cm), true); + res = TRI_WriteCrcElementDatafile(journal, position, &cm.base, true); if (res != TRI_ERROR_NO_ERROR) { - collection->_lastError = journal->_lastError; + document->_lastError = journal->_lastError; LOG_ERROR("cannot create document header in journal '%s': %s", journal->getName(journal), TRI_last_error()); TRI_FreeDatafile(journal); @@ -2414,7 +2411,7 @@ TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* number = TRI_StringUInt64(journal->_fid); jname = TRI_Concatenate3String("journal-", number, ".db"); - filename = TRI_Concatenate2File(collection->_directory, jname); + filename = TRI_Concatenate2File(document->_directory, jname); TRI_FreeString(TRI_CORE_MEM_ZONE, number); TRI_FreeString(TRI_CORE_MEM_ZONE, jname); @@ -2435,7 +2432,7 @@ TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t* TRI_FreeString(TRI_CORE_MEM_ZONE, filename); } - TRI_PushBackVectorPointer(&collection->_journals, journal); + TRI_PushBackVectorPointer(&document->_journals, journal); return journal; } @@ -5230,7 +5227,14 @@ int TRI_DeleteDocumentDocumentCollection (TRI_transaction_collection_t* trxColle TRI_doc_update_policy_t const* policy, TRI_doc_mptr_t* doc) { // no extra locking here as the collection is already locked - return TRI_RemoveShapedJsonDocumentCollection(trxCollection, (const TRI_voc_key_t) TRI_EXTRACT_MARKER_KEY(doc), 0, policy, false, false); // PROTECTED by trx in trxCollection + TRI_ASSERT(TRI_IsLockedCollectionTransaction(trxCollection, TRI_TRANSACTION_WRITE, 0)); + + return TRI_RemoveShapedJsonDocumentCollection(trxCollection, + (const TRI_voc_key_t) TRI_EXTRACT_MARKER_KEY(doc), + 0, + policy, + false, + false); // PROTECTED by trx in trxCollection } //////////////////////////////////////////////////////////////////////////////// @@ -5239,8 +5243,25 @@ int TRI_DeleteDocumentDocumentCollection (TRI_transaction_collection_t* trxColle //////////////////////////////////////////////////////////////////////////////// int TRI_RotateJournalDocumentCollection (TRI_document_collection_t* document) { - // TODO: re-create this functionality - return TRI_ERROR_NOT_IMPLEMENTED; + int res = TRI_ERROR_ARANGO_NO_JOURNAL; + + TRI_LOCK_JOURNAL_ENTRIES_DOC_COLLECTION(document); + + if (document->_state == TRI_COL_STATE_WRITE) { + size_t const n = document->_journals._length; + + if (n > 0) { + TRI_datafile_t* datafile = static_cast(document->_journals._buffer[0]); + TRI_ASSERT(datafile != nullptr); + TRI_CloseJournalDocumentCollection(document, 0); + + res = TRI_ERROR_NO_ERROR; + } + } + + TRI_UNLOCK_JOURNAL_ENTRIES_DOC_COLLECTION(document); + + return res; } // ----------------------------------------------------------------------------- diff --git a/arangod/VocBase/document-collection.h b/arangod/VocBase/document-collection.h index 0dcbb6a5f0..ea47d0646b 100644 --- a/arangod/VocBase/document-collection.h +++ b/arangod/VocBase/document-collection.h @@ -707,6 +707,7 @@ int TRI_RollbackOperationDocumentCollection (TRI_document_collection_t*, //////////////////////////////////////////////////////////////////////////////// TRI_datafile_t* TRI_CreateJournalDocumentCollection (TRI_document_collection_t*, + TRI_voc_fid_t, TRI_voc_size_t); //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/VocBase/replication-dump.cpp b/arangod/VocBase/replication-dump.cpp index 164df63766..acf43bfa54 100644 --- a/arangod/VocBase/replication-dump.cpp +++ b/arangod/VocBase/replication-dump.cpp @@ -53,11 +53,6 @@ using namespace triagens; // --SECTION-- private defines // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief shortcut function //////////////////////////////////////////////////////////////////////////////// @@ -97,19 +92,10 @@ using namespace triagens; #define APPEND_JSON(buffer, json) FAIL_IFNOT(TRI_StringifyJson, buffer, json) -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- private types // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief a datafile descriptor //////////////////////////////////////////////////////////////////////////////// @@ -133,19 +119,10 @@ typedef struct resolved_name_s { } resolved_name_t; -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- private functions // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief hashes a collection id //////////////////////////////////////////////////////////////////////////////// @@ -1016,19 +993,10 @@ NEXT_DF: return res; } -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- public functions // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief dump data from a collection //////////////////////////////////////////////////////////////////////////////// @@ -1172,10 +1140,6 @@ void TRI_DestroyDumpReplication (TRI_replication_dump_t* dump) { TRI_FreeStringBuffer(TRI_CORE_MEM_ZONE, dump->_buffer); } -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // Local Variables: // mode: outline-minor // outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|/// @page\\|// --SECTION--\\|/// @\\}" diff --git a/arangod/VocBase/server.cpp b/arangod/VocBase/server.cpp index 796d3f0be8..56125fc6c8 100644 --- a/arangod/VocBase/server.cpp +++ b/arangod/VocBase/server.cpp @@ -506,6 +506,10 @@ static int OpenDatabases (TRI_server_t* server, TRI_vector_string_t files; size_t i, n; int res; + + if (server->_iterateMarkersOnOpen) { + LOG_WARNING("no shutdown info found. scanning datafiles for last tick..."); + } res = TRI_ERROR_NO_ERROR; files = TRI_FilesDirectory(server->_databasePath); @@ -515,7 +519,7 @@ static int OpenDatabases (TRI_server_t* server, if (n > 1) { qsort(files._buffer, n, sizeof(char**), &DatabaseIdComparator); } - + for (i = 0; i < n; ++i) { TRI_vocbase_t* vocbase; TRI_json_t* json; @@ -1731,6 +1735,8 @@ int TRI_StartServer (TRI_server_t* server, return TRI_ERROR_ARANGO_DATADIR_NOT_WRITABLE; } + + server->_iterateMarkersOnOpen = false; } if (! TRI_IsWritable(server->_databasePath)) { diff --git a/arangod/VocBase/transaction.cpp b/arangod/VocBase/transaction.cpp index abd6f9da2e..78aa183836 100644 --- a/arangod/VocBase/transaction.cpp +++ b/arangod/VocBase/transaction.cpp @@ -830,8 +830,8 @@ int TRI_UnlockCollectionTransaction (TRI_transaction_collection_t* trxCollection //////////////////////////////////////////////////////////////////////////////// bool TRI_IsLockedCollectionTransaction (TRI_transaction_collection_t* trxCollection, - const TRI_transaction_type_e accessType, - const int nestingLevel) { + TRI_transaction_type_e accessType, + int nestingLevel) { if (accessType == TRI_TRANSACTION_WRITE && trxCollection->_accessType != TRI_TRANSACTION_WRITE) { // wrong lock type diff --git a/arangod/Wal/CollectorThread.cpp b/arangod/Wal/CollectorThread.cpp index dc465c03f3..7c10ac22e0 100644 --- a/arangod/Wal/CollectorThread.cpp +++ b/arangod/Wal/CollectorThread.cpp @@ -805,7 +805,7 @@ int CollectorThread::executeTransferMarkers (TRI_document_collection_t* document size_t n = strlen(name) + 1; // add NULL byte TRI_voc_size_t const totalSize = sizeof(TRI_df_attribute_marker_t) + n; - char* dst = nextFreeMarkerPosition(document, TRI_DF_MARKER_ATTRIBUTE, totalSize, cache); + char* dst = nextFreeMarkerPosition(document, source->_tick, TRI_DF_MARKER_ATTRIBUTE, totalSize, cache); if (dst == nullptr) { return TRI_ERROR_OUT_OF_MEMORY; @@ -832,7 +832,7 @@ int CollectorThread::executeTransferMarkers (TRI_document_collection_t* document ptrdiff_t shapeLength = source->_size - (shape - base); TRI_voc_size_t const totalSize = sizeof(TRI_df_shape_marker_t) + shapeLength; - char* dst = nextFreeMarkerPosition(document, TRI_DF_MARKER_SHAPE, totalSize, cache); + char* dst = nextFreeMarkerPosition(document, source->_tick, TRI_DF_MARKER_SHAPE, totalSize, cache); if (dst == nullptr) { return TRI_ERROR_OUT_OF_MEMORY; @@ -861,7 +861,7 @@ int CollectorThread::executeTransferMarkers (TRI_document_collection_t* document TRI_DF_ALIGN_BLOCK(n) + shapeLength; - char* dst = nextFreeMarkerPosition(document, TRI_DOC_MARKER_KEY_DOCUMENT, totalSize, cache); + char* dst = nextFreeMarkerPosition(document, source->_tick, TRI_DOC_MARKER_KEY_DOCUMENT, totalSize, cache); if (dst == nullptr) { return TRI_ERROR_OUT_OF_MEMORY; @@ -906,7 +906,7 @@ int CollectorThread::executeTransferMarkers (TRI_document_collection_t* document TRI_DF_ALIGN_BLOCK(from) + shapeLength; - char* dst = nextFreeMarkerPosition(document, TRI_DOC_MARKER_KEY_EDGE, totalSize, cache); + char* dst = nextFreeMarkerPosition(document, source->_tick, TRI_DOC_MARKER_KEY_EDGE, totalSize, cache); if (dst == nullptr) { return TRI_ERROR_OUT_OF_MEMORY; @@ -948,7 +948,7 @@ int CollectorThread::executeTransferMarkers (TRI_document_collection_t* document size_t n = strlen(key) + 1; // add NULL byte TRI_voc_size_t const totalSize = sizeof(TRI_doc_deletion_key_marker_t) + n; - char* dst = nextFreeMarkerPosition(document, TRI_DOC_MARKER_KEY_DELETION, totalSize, cache); + char* dst = nextFreeMarkerPosition(document, source->_tick, TRI_DOC_MARKER_KEY_DELETION, totalSize, cache); if (dst == nullptr) { return TRI_ERROR_OUT_OF_MEMORY; @@ -1091,6 +1091,7 @@ int CollectorThread::syncDatafileCollection (TRI_document_collection_t* document //////////////////////////////////////////////////////////////////////////////// char* CollectorThread::nextFreeMarkerPosition (TRI_document_collection_t* document, + TRI_voc_tick_t tick, TRI_df_marker_type_e type, TRI_voc_size_t size, CollectorCache* cache) { @@ -1138,7 +1139,7 @@ char* CollectorThread::nextFreeMarkerPosition (TRI_document_collection_t* docume TRI_CloseJournalDocumentCollection(document, i); } - TRI_datafile_t* datafile = TRI_CreateJournalDocumentCollection(document, targetSize); + TRI_datafile_t* datafile = TRI_CreateJournalDocumentCollection(document, tick, targetSize); if (datafile == nullptr) { LOG_ERROR("unable to create journal file"); diff --git a/arangod/Wal/CollectorThread.h b/arangod/Wal/CollectorThread.h index d627d40626..a7e1b185a0 100644 --- a/arangod/Wal/CollectorThread.h +++ b/arangod/Wal/CollectorThread.h @@ -316,7 +316,8 @@ namespace triagens { /// @brief get the next free position for a new marker of the specified size //////////////////////////////////////////////////////////////////////////////// - char* nextFreeMarkerPosition (struct TRI_document_collection_t*, + char* nextFreeMarkerPosition (struct TRI_document_collection_t*, + TRI_voc_tick_t, TRI_df_marker_type_e, TRI_voc_size_t, CollectorCache*); diff --git a/js/common/modules/org/arangodb/test-helper.js b/js/common/modules/org/arangodb/test-helper.js index e65e00b533..9bec11b06e 100644 --- a/js/common/modules/org/arangodb/test-helper.js +++ b/js/common/modules/org/arangodb/test-helper.js @@ -40,10 +40,6 @@ var processCsvFile = internal.processCsvFile; // --SECTION-- public variables // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @brief array query -//////////////////////////////////////////////////////////////////////////////// - exports.Helper = { process: function (file, processor) { processCsvFile(file, function (raw_row, index) { @@ -62,6 +58,29 @@ exports.Helper = { while (collection.status() !== arangodb.ArangoCollection.STATUS_UNLOADED) { collection.unload(); + internal.wait(1); + } + }, + + rotate: function (collection) { + var internal = require("internal"); + + internal.flushWal(true, true); + + var fig = collection.figures(); + var files = fig.datafiles.count + fig.journals.count; + + // wait for at most 10 seconds + var end = internal.time() + 10; + collection.rotate(); + + while (internal.time() < end) { + // wait until the figures change + fig = collection.figures(); + if (fig.datafiles.count + fig.journals.count !== files) { + break; + } + internal.wait(1); } } diff --git a/js/common/tests/shell-collection-noncluster.js b/js/common/tests/shell-collection-noncluster.js index 1ec199f51d..786b9d92ca 100644 --- a/js/common/tests/shell-collection-noncluster.js +++ b/js/common/tests/shell-collection-noncluster.js @@ -26,10 +26,9 @@ //////////////////////////////////////////////////////////////////////////////// var jsunity = require("jsunity"); - var arangodb = require("org/arangodb"); - var ArangoCollection = arangodb.ArangoCollection; +var testHelper = require("org/arangodb/test-helper").Helper; var db = arangodb.db; var ERRORS = arangodb.errors; @@ -58,19 +57,13 @@ function CollectionSuite () { var f = c1.figures(); assertEqual(0, f.datafiles.count); - c1.rotate(); - - // must wait so the synchroniser can catch up - require("internal").wait(5); + testHelper.rotate(c1); f = c1.figures(); assertEqual(1, f.datafiles.count); c1.save({ _key: "test2" }); - c1.rotate(); - - // must wait so the synchroniser can catch up - require("internal").wait(5); + testHelper.rotate(c1); f = c1.figures(); // we may have one or two datafiles, depending on the compaction @@ -92,7 +85,7 @@ function CollectionSuite () { var c1 = db._create(cn); try { - c1.rotate(); + testHelper.rotate(c1); fail(); } catch (err) { diff --git a/js/common/tests/shell-document.js b/js/common/tests/shell-document.js index 0959ad5dd1..83139a3eb3 100644 --- a/js/common/tests/shell-document.js +++ b/js/common/tests/shell-document.js @@ -26,9 +26,7 @@ //////////////////////////////////////////////////////////////////////////////// var jsunity = require("jsunity"); - var arangodb = require("org/arangodb"); - var ERRORS = arangodb.errors; var db = arangodb.db; var wait = require("internal").wait; diff --git a/js/server/tests/shell-compaction-noncluster.js b/js/server/tests/shell-compaction-noncluster.js index 9cf311c4aa..5c0bc12e43 100644 --- a/js/server/tests/shell-compaction-noncluster.js +++ b/js/server/tests/shell-compaction-noncluster.js @@ -43,7 +43,7 @@ function CompactionSuite () { //////////////////////////////////////////////////////////////////////////////// /// @brief create movement of shapes //////////////////////////////////////////////////////////////////////////////// - +/* testShapesMovement : function () { var cn = "example"; internal.db._drop(cn); @@ -142,7 +142,7 @@ function CompactionSuite () { c1.toArray(); c1.truncate(); - c1.rotate(); + testHelper.rotate(c1); // create lots of different shapes for (i = 0; i < 100; ++i) { @@ -154,8 +154,7 @@ function CompactionSuite () { } // make sure compaction moves the shapes - c1.rotate(); - internal.wait(5); + testHelper.rotate(c1); c1.truncate(); internal.wait(5); @@ -196,7 +195,7 @@ function CompactionSuite () { c1.save({ _key: "test" + i }); } c1.truncate(); - c1.rotate(); + testHelper.rotate(c1); // create lots of different shapes for (i = 0; i < 100; ++i) { @@ -216,8 +215,7 @@ function CompactionSuite () { } // make sure compaction moves the shapes - c1.rotate(); - internal.wait(5); + testHelper.rotate(c1); var doc = c1.document("foo"); assertTrue(doc.hasOwnProperty("name")); @@ -265,8 +263,7 @@ function CompactionSuite () { } // make sure compaction moves the shapes - c1.rotate(); - internal.wait(5); + testHelper.rotate(c1); // unload the collection testHelper.waitUnload(c1); @@ -308,7 +305,7 @@ function CompactionSuite () { internal.db._drop(cn); }, - +*/ //////////////////////////////////////////////////////////////////////////////// /// @brief test journals //////////////////////////////////////////////////////////////////////////////// @@ -318,6 +315,8 @@ function CompactionSuite () { internal.db._drop(cn); var c1 = internal.db._create(cn, { "journalSize" : 1048576 }); + internal.flushWal(true, true); + // empty collection var fig = c1.figures(); assertEqual(0, c1.count()); @@ -330,6 +329,7 @@ function CompactionSuite () { assertEqual(0, fig["compactors"]["count"]); c1.save({ "foo": "bar" }); + internal.flushWal(true, true); fig = c1.figures(); assertEqual(1, c1.count()); @@ -342,8 +342,7 @@ function CompactionSuite () { assertEqual(0, fig["datafiles"]["count"]); assertEqual(0, fig["compactors"]["count"]); - c1.rotate(); - internal.wait(5); + testHelper.rotate(c1); fig = c1.figures(); assertEqual(1, c1.count()); @@ -358,6 +357,7 @@ function CompactionSuite () { c1.save({ "bar": "baz" }); + internal.flushWal(true, true); fig = c1.figures(); assertEqual(2, c1.count()); assertEqual(2, fig["alive"]["count"]); @@ -369,8 +369,7 @@ function CompactionSuite () { assertEqual(1, fig["datafiles"]["count"]); assertEqual(0, fig["compactors"]["count"]); - c1.rotate(); - internal.wait(5); + testHelper.rotate(c1); fig = c1.figures(); assertEqual(2, c1.count()); @@ -384,9 +383,7 @@ function CompactionSuite () { assertEqual(0, fig["compactors"]["count"]); c1.truncate(); - c1.rotate(); - - internal.wait(10); + testHelper.rotate(c1); fig = c1.figures(); assertEqual(0, c1.count());