diff --git a/arangod/Aql/EnumerateCollectionExecutor.cpp b/arangod/Aql/EnumerateCollectionExecutor.cpp index e6655478ab..7a160d496c 100644 --- a/arangod/Aql/EnumerateCollectionExecutor.cpp +++ b/arangod/Aql/EnumerateCollectionExecutor.cpp @@ -173,13 +173,14 @@ std::tuple EnumerateCollection std::tie(_state, _input) = _fetcher.fetchRow(); if (_state == ExecutionState::WAITING) { - return {_state, stats, 0}; + return std::make_tuple(_state, stats, 0); // tupple, cannot use initializer list due to build failure } if (!_input) { TRI_ASSERT(_state == ExecutionState::DONE); - return {_state, stats, 0}; + return std::make_tuple(_state, stats, 0); // tupple, cannot use initializer list due to build failure } + _cursor->reset(); _cursorHasMore = _cursor->hasMore(); } @@ -192,10 +193,10 @@ std::tuple EnumerateCollection stats.incrScanned(actuallySkipped); if (_state == ExecutionState::DONE && !_cursorHasMore) { - return {ExecutionState::DONE, stats, actuallySkipped}; + return std::make_tuple(ExecutionState::DONE, stats, actuallySkipped); // tupple, cannot use initializer list due to build failure } - return {ExecutionState::HASMORE, stats, actuallySkipped}; + return std::make_tuple(ExecutionState::HASMORE, stats, actuallySkipped); // tupple, cannot use initializer list due to build failure } void EnumerateCollectionExecutor::initializeCursor() { diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp index 656b87f047..8454fc9b82 100644 --- a/arangod/Aql/ExecutionBlockImpl.cpp +++ b/arangod/Aql/ExecutionBlockImpl.cpp @@ -229,7 +229,7 @@ struct ExecuteSkipVariant { static std::tuple executeSkip( Executor& executor, typename Executor::Fetcher& fetcher, size_t toSkip) { auto res = fetcher.skipRows(toSkip); - return {res.first, typename Executor::Stats{}, res.second}; + return std::make_tuple(res.first, typename Executor::Stats{}, res.second); // tupple, cannot use initializer list due to build failure } }; @@ -250,7 +250,7 @@ struct ExecuteSkipVariant { // this function should never be executed TRI_ASSERT(false); // Make MSVC happy: - return {ExecutionState::DONE, {}, 0}; + return std::make_tuple(ExecutionState::DONE, typename Executor::Stats{}, 0); // tupple, cannot use initializer list due to build failure } }; diff --git a/arangod/Aql/IResearchViewExecutor.cpp b/arangod/Aql/IResearchViewExecutor.cpp index 73b87c4cf2..c3ef45e387 100644 --- a/arangod/Aql/IResearchViewExecutor.cpp +++ b/arangod/Aql/IResearchViewExecutor.cpp @@ -294,17 +294,17 @@ IResearchViewExecutorBase::skipRows(size_t toSkip) { if (!_inputRow.isInitialized()) { if (_upstreamState == ExecutionState::DONE) { // There will be no more rows, stop fetching. - return {ExecutionState::DONE, stats, 0}; + return std::make_tuple(ExecutionState::DONE, stats, 0); // tupple, cannot use initializer list due to build failure } std::tie(_upstreamState, _inputRow) = _fetcher.fetchRow(); if (_upstreamState == ExecutionState::WAITING) { - return {_upstreamState, stats, 0}; + return std::make_tuple(_upstreamState, stats, 0); // tupple, cannot use initializer list due to build failure } if (!_inputRow.isInitialized()) { - return {ExecutionState::DONE, stats, 0}; + return std::make_tuple(ExecutionState::DONE, stats, 0); // tupple, cannot use initializer list due to build failure } // reset must be called exactly after we've got a new and valid input row. @@ -321,7 +321,7 @@ IResearchViewExecutorBase::skipRows(size_t toSkip) { _inputRow = InputAqlItemRow{CreateInvalidInputRowHint{}}; } - return {ExecutionState::HASMORE, stats, skipped}; + return std::make_tuple(ExecutionState::HASMORE, stats, skipped); // tupple, cannot use initializer list due to build failure } template diff --git a/arangod/Aql/IndexExecutor.cpp b/arangod/Aql/IndexExecutor.cpp index 033d4968f1..80067510df 100644 --- a/arangod/Aql/IndexExecutor.cpp +++ b/arangod/Aql/IndexExecutor.cpp @@ -537,21 +537,25 @@ std::tuple IndexExecutor::skipRows if (!_input) { if (_state == ExecutionState::DONE) { size_t skipped = _skipped; + _skipped = 0; - return {_state, stats, skipped}; + + return std::make_tuple(_state, stats, skipped); // tupple, cannot use initializer list due to build failure } std::tie(_state, _input) = _fetcher.fetchRow(); if (_state == ExecutionState::WAITING) { - return {_state, stats, 0}; + return std::make_tuple(_state, stats, 0); // tupple, cannot use initializer list due to build failure } if (!_input) { TRI_ASSERT(_state == ExecutionState::DONE); size_t skipped = _skipped; + _skipped = 0; - return {_state, stats, skipped}; + + return std::make_tuple(_state, stats, skipped); // tupple, cannot use initializer list due to build failure } initIndexes(_input); @@ -577,10 +581,12 @@ std::tuple IndexExecutor::skipRows } size_t skipped = _skipped; + _skipped = 0; + if (_state == ExecutionState::DONE && !_input) { - return {ExecutionState::DONE, stats, skipped}; - } else { - return {ExecutionState::HASMORE, stats, skipped}; + return std::make_tuple(ExecutionState::DONE, stats, skipped); // tupple, cannot use initializer list due to build failure } + + return std::make_tuple(ExecutionState::HASMORE, stats, skipped); // tupple, cannot use initializer list due to build failure } diff --git a/arangod/IResearch/IResearchLink.cpp b/arangod/IResearch/IResearchLink.cpp index 09691bb4ca..1576d0466c 100644 --- a/arangod/IResearch/IResearchLink.cpp +++ b/arangod/IResearch/IResearchLink.cpp @@ -50,6 +50,12 @@ namespace { +//////////////////////////////////////////////////////////////////////////////// +/// @brief the suffix appened to the index_meta filename to generate the +/// backup filename to be used for renaming +//////////////////////////////////////////////////////////////////////////////// +const irs::string_ref IRESEARCH_BACKUP_SUFFIX(".backup"); + //////////////////////////////////////////////////////////////////////////////// /// @brief the suffix appened to the index_meta filename to generate the /// corresponding checkpoint file @@ -1003,7 +1009,7 @@ arangodb::Result IResearchLink::initDataStore(InitCallback const& initCallback, try { recovery_reader = irs::directory_reader::open(*(_dataStore._directory)); } catch (irs::index_not_found const&) { - // ingore + // ignore } } @@ -1011,19 +1017,54 @@ arangodb::Result IResearchLink::initDataStore(InitCallback const& initCallback, // '.checkpoint' file for the last state of the data store // if it's missing them probably the WAL tail was lost if (recovery_reader) { - auto& checkpoint = recovery_reader.meta().filename; - auto checkpointFile = checkpoint + std::string(IRESEARCH_CHECKPOINT_SUFFIX); - auto ref = irs::directory_utils::reference( // create a reference - *(_dataStore._directory), checkpointFile, false // args - ); + irs::index_file_refs::ref_t ref; - if (!ref) { - return arangodb::Result( // result - TRI_ERROR_ARANGO_ILLEGAL_STATE, // code - std::string("failed to find checkpoint file matching the latest data store state for arangosearch link '") + std::to_string(id()) + "', expecting file '" + checkpointFile + "' in path: " + _dataStore._path.utf8() + // find the latest segment state with a checkpoint file + for(;;) { + auto& filename = recovery_reader.meta().filename; // segment state filename + auto checkpointFile = // checkpoint filename + filename + std::string(IRESEARCH_CHECKPOINT_SUFFIX); + + ref = irs::directory_utils::reference( // create a reference + *(_dataStore._directory), checkpointFile, false // args ); + + if (ref) { + break; // found checkpoint file for latest state + } + + auto src = _dataStore._path; + auto& srcFilename = filename; + auto dst = src; + auto dstFilename = filename + std::string(IRESEARCH_BACKUP_SUFFIX); + + src /= srcFilename; + dst /= dstFilename; + + // move segment state file without a matching checkpint out of the way + if (!src.rename(dst)) { + return arangodb::Result( // result + TRI_ERROR_ARANGO_ILLEGAL_STATE, // code + std::string("failed rename the latest data store state file for arangosearch link '") + std::to_string(id()) + "', source '" + srcFilename + "' destination '" + dstFilename + "' in path: " + _dataStore._path.utf8() + ); + } + + try { + recovery_reader.reset(); // unset to allow for checking for success below + recovery_reader = irs::directory_reader::open(*(_dataStore._directory)); // retry opening + } catch (irs::index_not_found const&) { + // ignore + } + + if (!recovery_reader) { + return arangodb::Result( // result + TRI_ERROR_ARANGO_ILLEGAL_STATE, // code + std::string("failed to find checkpoint file matching the latest data store state for arangosearch link '") + std::to_string(id()) + "', expecting file '" + checkpointFile + "' in path: " + _dataStore._path.utf8() + ); + } } + auto& checkpointFile = *ref; // ref non-null ensured by above loop auto in = _dataStore._directory->open( // open checkpoint file checkpointFile, irs::IOAdvice::NORMAL // args, use 'NORMAL' since the file could be empty ); @@ -1791,4 +1832,4 @@ arangodb::Result IResearchLink::unload() { // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/arangod/IResearch/IResearchLinkHelper.cpp b/arangod/IResearch/IResearchLinkHelper.cpp index 31af5e17c7..febd1c4298 100644 --- a/arangod/IResearch/IResearchLinkHelper.cpp +++ b/arangod/IResearch/IResearchLinkHelper.cpp @@ -103,20 +103,29 @@ arangodb::Result canUseAnalyzers( // validate return arangodb::Result(); } -bool createLink( // create link +arangodb::Result createLink( // create link arangodb::LogicalCollection& collection, // link collection arangodb::LogicalView const& view, // link view arangodb::velocypack::Slice definition // link definition ) { - bool isNew = false; - auto link = collection.createIndex(definition, isNew); - LOG_TOPIC_IF("2c861", DEBUG, arangodb::iresearch::TOPIC, link) - << "added link '" << link->id() << "'"; + try { + bool isNew = false; + auto link = collection.createIndex(definition, isNew); - return link && isNew; + if (!(link && isNew)) { + return arangodb::Result( // result + TRI_ERROR_INTERNAL, // code + std::string("failed to create link between arangosearch view '") + view.name() + "' and collection '" + collection.name() + "'" + ); + } + } catch (arangodb::basics::Exception const& e) { + return arangodb::Result(e.code(), e.what()); + } + + return arangodb::Result(); } -bool createLink( // create link +arangodb::Result createLink( // create link arangodb::LogicalCollection& collection, // link collection arangodb::iresearch::IResearchViewCoordinator const& view, // link view arangodb::velocypack::Slice definition // link definition @@ -141,7 +150,10 @@ bool createLink( // create link ); if (!arangodb::iresearch::mergeSliceSkipKeys(builder, definition, acceptor)) { - return false; + return arangodb::Result( // result + TRI_ERROR_INTERNAL, // code + std::string("failed to generate definition while creating link between arangosearch view '") + view.name() + "' and collection '" + collection.name() + "'" + ); } builder.close(); @@ -150,20 +162,27 @@ bool createLink( // create link return arangodb::methods::Indexes::ensureIndex( // ensure index &collection, builder.slice(), true, tmp // args - ).ok(); + ); } template -bool dropLink( // drop link +arangodb::Result dropLink( // drop link arangodb::LogicalCollection& collection, // link collection arangodb::iresearch::IResearchLink const& link // link to drop ) { // don't need to create an extra transaction inside arangodb::methods::Indexes::drop(...) - return collection.dropIndex(link.id()); + if (!collection.dropIndex(link.id())) { + return arangodb::Result( // result + TRI_ERROR_INTERNAL, // code + std::string("failed to drop link '") + std::to_string(link.id()) + "' from collection '" + collection.name() + "'" + ); + } + + return arangodb::Result(); } template<> -bool dropLink( // drop link +arangodb::Result dropLink( // drop link arangodb::LogicalCollection& collection, // link collection arangodb::iresearch::IResearchLink const& link // link to drop ) { @@ -176,7 +195,7 @@ bool dropLink( // drop link ); builder.close(); - return arangodb::methods::Indexes::drop(&collection, builder.slice()).ok(); + return arangodb::methods::Indexes::drop(&collection, builder.slice()); } template @@ -195,17 +214,15 @@ arangodb::Result modifyLinks( // modify links struct State { std::shared_ptr _collection; - size_t _collectionsToLockOffset; // std::numeric_limits::max() == - // removal only + size_t _collectionsToLockOffset; // std::numeric_limits::max() == removal only std::shared_ptr _link; size_t _linkDefinitionsOffset; - bool _stale = false; // request came from the stale list - bool _valid = true; + arangodb::Result _result; // operation result + bool _stale = false; // request came from the stale list explicit State(size_t collectionsToLockOffset) - : State(collectionsToLockOffset, std::numeric_limits::max()) {} + : State(collectionsToLockOffset, std::numeric_limits::max()) {} State(size_t collectionsToLockOffset, size_t linkDefinitionsOffset) - : _collectionsToLockOffset(collectionsToLockOffset), - _linkDefinitionsOffset(linkDefinitionsOffset) {} + : _collectionsToLockOffset(collectionsToLockOffset), _linkDefinitionsOffset(linkDefinitionsOffset) {} }; std::vector collectionsToLock; std::vector> linkDefinitions; @@ -215,12 +232,10 @@ arangodb::Result modifyLinks( // modify links auto collection = linksItr.key(); if (!collection.isString()) { - return arangodb::Result(TRI_ERROR_BAD_PARAMETER, - std::string( - "error parsing link parameters from json for " - "arangosearch view '") + - view.name() + "' offset '" + - arangodb::basics::StringUtils::itoa(linksItr.index()) + '"'); + return arangodb::Result( // result + TRI_ERROR_BAD_PARAMETER, // code + std::string("error parsing link parameters from json for arangosearch view '") + view.name() + "' offset '" + arangodb::basics::StringUtils::itoa(linksItr.index()) + '"' + ); } auto link = linksItr.value(); @@ -465,19 +480,20 @@ arangodb::Result modifyLinks( // modify links // execute removals for (auto& state : linkModifications) { if (state._link) { // link removal or recreate request - LOG_TOPIC("9da74", DEBUG, arangodb::iresearch::TOPIC) - << "removed link '" << state._link->id() << "'"; - state._valid = dropLink(*(state._collection), *(state._link)); + state._result = dropLink(*(state._collection), *(state._link)); modified.emplace(state._collection->id()); } } // execute additions - for (auto& state : linkModifications) { - if (state._valid && state._linkDefinitionsOffset < linkDefinitions.size()) { - state._valid = - createLink(*(state._collection), view, - linkDefinitions[state._linkDefinitionsOffset].first.slice()); + for (auto& state: linkModifications) { + if (state._result.ok() // valid state (unmodified or after removal) + && state._linkDefinitionsOffset < linkDefinitions.size()) { + state._result = createLink( // create link + *(state._collection), // collection + view, // view + linkDefinitions[state._linkDefinitionsOffset].first.slice() // definition + ); modified.emplace(state._collection->id()); } } @@ -485,9 +501,12 @@ arangodb::Result modifyLinks( // modify links std::string error; // validate success - for (auto& state : linkModifications) { - if (!state._valid) { - error.append(error.empty() ? "" : ", ").append(collectionsToLock[state._collectionsToLockOffset]); + for (auto& state: linkModifications) { + if (!state._result.ok()) { + error.append(error.empty() ? "" : ", ") // separator + .append(collectionsToLock[state._collectionsToLockOffset]) // collection name + .append(": ").append(std::to_string(state._result.errorNumber())) // error code + .append(" ").append(state._result.errorMessage()); // error message } } @@ -495,11 +514,10 @@ arangodb::Result modifyLinks( // modify links return arangodb::Result(); } - return arangodb::Result( - TRI_ERROR_ARANGO_ILLEGAL_STATE, - std::string("failed to update links while updating arangosearch view '") + - view.name() + - "', retry same request or examine errors for collections: " + error); + return arangodb::Result( // result + TRI_ERROR_ARANGO_ILLEGAL_STATE, // code + std::string("failed to update links while updating arangosearch view '") + view.name() + "', retry same request or examine errors for collections: " + error + ); } } // namespace @@ -835,4 +853,4 @@ namespace iresearch { // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/arangod/V8Server/v8-analyzers.cpp b/arangod/V8Server/v8-analyzers.cpp index 3220c1a4c3..f30279bff1 100644 --- a/arangod/V8Server/v8-analyzers.cpp +++ b/arangod/V8Server/v8-analyzers.cpp @@ -561,11 +561,11 @@ void JS_List(v8::FunctionCallbackInfo const& args) { for (size_t i = 0, count = result.size(); i < count; ++i) { auto analyzer = WrapAnalyzer(isolate, result[i]); - if (analyzer.IsEmpty()) { + if (analyzer.IsEmpty() || i > std::numeric_limits::max()) { TRI_V8_THROW_EXCEPTION_MEMORY(); } - v8Result->Set(i, analyzer); + v8Result->Set(static_cast(i), analyzer); // cast safe because of check above } TRI_V8_RETURN(v8Result); diff --git a/tests/IResearch/IResearchLink-test.cpp b/tests/IResearch/IResearchLink-test.cpp index c95caedca7..fecd29a05c 100644 --- a/tests/IResearch/IResearchLink-test.cpp +++ b/tests/IResearch/IResearchLink-test.cpp @@ -366,7 +366,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -382,7 +382,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -398,7 +398,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -414,7 +414,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -430,7 +430,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -446,7 +446,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -472,7 +472,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -488,7 +488,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -496,47 +496,221 @@ SECTION("test_flush_marker") { CHECK((0 == state.errorCount)); } - // open existing without checkpoint file + // will commit 'link' and set RecoveryState to DONE + { + StorageEngineMock::inRecoveryResult = false; + dbFeature->recoveryDone(); + } + + // commit failed write WAL + { + auto before = StorageEngineMock::flushSubscriptionResult; + auto restore = irs::make_finally([&before]()->void { StorageEngineMock::flushSubscriptionResult = before; }); + StorageEngineMock::flushSubscriptionResult = arangodb::Result(TRI_ERROR_INTERNAL); + + arangodb::transaction::Methods trx( + arangodb::transaction::StandaloneContext::Create(*vocbase), + EMPTY, + EMPTY, + EMPTY, + arangodb::transaction::Options() + ); + CHECK((trx.begin().ok())); + CHECK((link->insert(trx, arangodb::LocalDocumentId(1), doc0->slice(), arangodb::Index::OperationMode::normal).ok())); + CHECK((trx.commit().ok())); + CHECK((!link->commit().ok())); + } + + // commit failed write checkpoint + { + arangodb::transaction::Methods trx( + arangodb::transaction::StandaloneContext::Create(*vocbase), + EMPTY, + EMPTY, + EMPTY, + arangodb::transaction::Options() + ); + CHECK((trx.begin().ok())); + CHECK((link->insert(trx, arangodb::LocalDocumentId(2), doc1->slice(), arangodb::Index::OperationMode::normal).ok())); + CHECK((trx.commit().ok())); + irs::utf8_path path; + path /= s.testFilesystemPath; + path /= "databases"; + path /= std::string("database-") + std::to_string(vocbase->id()); + path /= std::string("arangosearch-") + std::to_string(logicalCollection->id()) + "_" + std::to_string(link->id()); + path /= "segments_3.checkpoint"; + CHECK((path.mkdir())); //create a directory by same name as the checkpoint file to force error + CHECK((!link->commit().ok())); + } + + // commit success + { + arangodb::transaction::Methods trx( + arangodb::transaction::StandaloneContext::Create(*vocbase), + EMPTY, + EMPTY, + EMPTY, + arangodb::transaction::Options() + ); + CHECK((trx.begin().ok())); + CHECK((link->insert(trx, arangodb::LocalDocumentId(3), doc2->slice(), arangodb::Index::OperationMode::normal).ok())); + CHECK((trx.commit().ok())); + CHECK((link->commit().ok())); + } +} + +SECTION("test_flush_marker_reopen") { + static std::vector const EMPTY; + auto doc0 = arangodb::velocypack::Parser::fromJson("{ \"abc\": \"def\" }"); + + auto before = StorageEngineMock::inRecoveryResult; + StorageEngineMock::inRecoveryResult = true; + auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; }); + + auto* dbFeature = arangodb::application_features::ApplicationServer::lookupFeature("Database"); + REQUIRE((dbFeature)); + TRI_vocbase_t* vocbase; + REQUIRE((TRI_ERROR_NO_ERROR == dbFeature->createDatabase(1, "testDatabase", vocbase))); + auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\", \"id\": 100 }"); + auto logicalCollection = vocbase->createCollection(collectionJson->slice()); + REQUIRE((false == !logicalCollection)); + bool created; + + // open existing without any checkpoint files + { + auto linkJson1 = arangodb::velocypack::Parser::fromJson("{ \"id\": 42, \"includeAllFields\": true, \"type\": \"arangosearch\", \"view\": \"testView\" }"); + + // initial population of link + { + auto before = StorageEngineMock::inRecoveryResult; + StorageEngineMock::inRecoveryResult = false; + auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; }); + std::shared_ptr index1; + CHECK((arangodb::iresearch::IResearchMMFilesLink::factory().instantiate(index1, *logicalCollection, linkJson1->slice(), 42, false).ok())); + CHECK((false == !index1)); + auto link1 = std::dynamic_pointer_cast(index1); + CHECK((false == !link1)); + + // remove initial 'checkpoint' file + { + irs::utf8_path path; + path /= s.testFilesystemPath; + path /= "databases"; + path /= std::string("database-") + std::to_string(vocbase->id()); + path /= std::string("arangosearch-") + std::to_string(logicalCollection->id()) + "_42"; + irs::fs_directory dir(path.utf8()); + auto reader = irs::directory_reader::open(dir); + path /= reader.meta().filename + ".checkpoint"; + bool exists; + CHECK((path.exists_file(exists) && exists)); + CHECK((path.remove())); + } + + // insert doc0 + { + arangodb::transaction::Methods trx( + arangodb::transaction::StandaloneContext::Create(*vocbase), + EMPTY, + EMPTY, + EMPTY, + arangodb::transaction::Options() + ); + CHECK((trx.begin().ok())); + CHECK((link1->insert(trx, arangodb::LocalDocumentId(1), doc0->slice(), arangodb::Index::OperationMode::normal).ok())); + CHECK((trx.commit().ok())); + CHECK((link1->commit().ok())); + } + + // remove initial 'checkpoint' file + { + irs::utf8_path path; + path /= s.testFilesystemPath; + path /= "databases"; + path /= std::string("database-") + std::to_string(vocbase->id()); + path /= std::string("arangosearch-") + std::to_string(logicalCollection->id()) + "_42"; + irs::fs_directory dir(path.utf8()); + auto reader = irs::directory_reader::open(dir); + path /= reader.meta().filename + ".checkpoint"; + bool exists; + CHECK((path.exists_file(exists) && exists)); + CHECK((path.remove())); // remove post-commit 'checkpoint' file + } + } + + auto index1 = logicalCollection->createIndex(linkJson1->slice(), created); + CHECK((true == !index1)); + } + + // open existing without last checkpoint file { auto linkJson1 = arangodb::velocypack::Parser::fromJson("{ \"id\": 43, \"includeAllFields\": true, \"type\": \"arangosearch\", \"view\": \"testView\" }"); // initial population of link { + auto before = StorageEngineMock::inRecoveryResult; + StorageEngineMock::inRecoveryResult = false; + auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; }); std::shared_ptr index1; CHECK((arangodb::iresearch::IResearchMMFilesLink::factory().instantiate(index1, *logicalCollection, linkJson1->slice(), 43, false).ok())); CHECK((false == !index1)); auto link1 = std::dynamic_pointer_cast(index1); CHECK((false == !link1)); - arangodb::transaction::Methods trx( - arangodb::transaction::StandaloneContext::Create(*vocbase), - EMPTY, - EMPTY, - EMPTY, - arangodb::transaction::Options() - ); - CHECK((trx.begin().ok())); - CHECK((link1->insert(trx, arangodb::LocalDocumentId(1), doc0->slice(), arangodb::Index::OperationMode::normal).ok())); - CHECK((trx.commit().ok())); - auto before = StorageEngineMock::inRecoveryResult; - StorageEngineMock::inRecoveryResult = false; - auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; }); - dbFeature->recoveryDone(); // will commit 'link1' (it will also commit 'link' and set RecoveryState to DONE) + // insert doc0 + { + arangodb::transaction::Methods trx( + arangodb::transaction::StandaloneContext::Create(*vocbase), + EMPTY, + EMPTY, + EMPTY, + arangodb::transaction::Options() + ); + CHECK((trx.begin().ok())); + CHECK((link1->insert(trx, arangodb::LocalDocumentId(1), doc0->slice(), arangodb::Index::OperationMode::normal).ok())); + CHECK((trx.commit().ok())); + CHECK((link1->commit().ok())); + } + + // remove initial 'checkpoint' file + { + irs::utf8_path path; + path /= s.testFilesystemPath; + path /= "databases"; + path /= std::string("database-") + std::to_string(vocbase->id()); + path /= std::string("arangosearch-") + std::to_string(logicalCollection->id()) + "_43"; + irs::fs_directory dir(path.utf8()); + auto reader = irs::directory_reader::open(dir); + path /= reader.meta().filename + ".checkpoint"; + bool exists; + CHECK((path.exists_file(exists) && exists)); + CHECK((path.remove())); // remove post-commit 'checkpoint' file + } } - irs::utf8_path path; - path /= s.testFilesystemPath; - path /= "databases"; - path /= std::string("database-") + std::to_string(vocbase->id()); - path /= std::string("arangosearch-") + std::to_string(logicalCollection->id()) + "_43"; - irs::fs_directory dir(path.utf8()); - auto reader = irs::directory_reader::open(dir); - path /= reader.meta().filename + ".checkpoint"; - bool exists; - CHECK((path.exists_file(exists) && exists)); - CHECK((path.remove())); auto index1 = logicalCollection->createIndex(linkJson1->slice(), created); - CHECK((true == !index1)); + CHECK((false == !index1)); // link creation success in recovery + + // first and only marker + { + auto json = arangodb::velocypack::Parser::fromJson("{ \"type\": \"arangosearch\", \"data\": { \"cid\": 100, \"iid\": 45, \"value\": \"segments_1\" } }"); + std::basic_string buf; + buf.resize(sizeof(::MMFilesMarker) + sizeof(TRI_voc_tick_t)); // reserve space for header + arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); + buf.append(json->slice().begin(), json->slice().byteSize()); + auto* marker = reinterpret_cast(&buf[0]); + marker->setSize(static_cast(buf.size())); + marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); + arangodb::MMFilesWalRecoverState state(false); + CHECK((0 == state.errorCount)); + CHECK((arangodb::MMFilesWalRecoverState::ReplayMarker(marker, &state, nullptr))); + CHECK((0 == state.errorCount)); + } + + auto before = StorageEngineMock::inRecoveryResult; + StorageEngineMock::inRecoveryResult = false; + auto restore = irs::make_finally([&before]()->void { StorageEngineMock::inRecoveryResult = before; }); + dbFeature->recoveryDone(); // will commit 'link1' (it will also commit 'link' and set RecoveryState to DONE) + logicalCollection->dropIndex(index1->id()); } // open existing with checkpoint file unmatched by marker (missing WAL tail) @@ -622,7 +796,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -632,6 +806,7 @@ SECTION("test_flush_marker") { StorageEngineMock::inRecoveryResult = false; dbFeature->recoveryDone(); // will commit 'link1' (it will also commit 'link' and set RecoveryState to DONE) + logicalCollection->dropIndex(index1->id()); } // open exisiting with checkpoint file matching second marker (i.e. DURING_CHECKPOINT then again DURING_CHECKPOINT) @@ -677,7 +852,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -693,7 +868,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -703,6 +878,7 @@ SECTION("test_flush_marker") { StorageEngineMock::inRecoveryResult = false; dbFeature->recoveryDone(); // will commit 'link1' (it will also commit 'link' and set RecoveryState to DONE) + logicalCollection->dropIndex(index1->id()); } // open exisiting with checkpoint file matching third marker (ensure initial recovery state changes, i.e. DURING_CHECKPOINT then BEFORE_CHECKPOINT then DURING_CHECKPOINT)) @@ -748,7 +924,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -764,7 +940,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -780,7 +956,7 @@ SECTION("test_flush_marker") { arangodb::encoding::storeNumber(&buf[sizeof(::MMFilesMarker)], TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -790,62 +966,7 @@ SECTION("test_flush_marker") { StorageEngineMock::inRecoveryResult = false; dbFeature->recoveryDone(); // will commit 'link1' (it will also commit 'link' and set RecoveryState to DONE) - } - - // commit failed write WAL - { - auto before = StorageEngineMock::flushSubscriptionResult; - auto restore = irs::make_finally([&before]()->void { StorageEngineMock::flushSubscriptionResult = before; }); - StorageEngineMock::flushSubscriptionResult = arangodb::Result(TRI_ERROR_INTERNAL); - - arangodb::transaction::Methods trx( - arangodb::transaction::StandaloneContext::Create(*vocbase), - EMPTY, - EMPTY, - EMPTY, - arangodb::transaction::Options() - ); - CHECK((trx.begin().ok())); - CHECK((link->insert(trx, arangodb::LocalDocumentId(1), doc0->slice(), arangodb::Index::OperationMode::normal).ok())); - CHECK((trx.commit().ok())); - CHECK((!link->commit().ok())); - } - - // commit failed write checkpoint - { - arangodb::transaction::Methods trx( - arangodb::transaction::StandaloneContext::Create(*vocbase), - EMPTY, - EMPTY, - EMPTY, - arangodb::transaction::Options() - ); - CHECK((trx.begin().ok())); - CHECK((link->insert(trx, arangodb::LocalDocumentId(2), doc1->slice(), arangodb::Index::OperationMode::normal).ok())); - CHECK((trx.commit().ok())); - irs::utf8_path path; - path /= s.testFilesystemPath; - path /= "databases"; - path /= std::string("database-") + std::to_string(vocbase->id()); - path /= std::string("arangosearch-") + std::to_string(logicalCollection->id()) + "_" + std::to_string(link->id()); - path /= "segments_3.checkpoint"; - CHECK((path.mkdir())); //create a directory by same name as the checkpoint file to force error - CHECK((!link->commit().ok())); - } - - // commit success - { - arangodb::transaction::Methods trx( - arangodb::transaction::StandaloneContext::Create(*vocbase), - EMPTY, - EMPTY, - EMPTY, - arangodb::transaction::Options() - ); - CHECK((trx.begin().ok())); - CHECK((link->insert(trx, arangodb::LocalDocumentId(3), doc2->slice(), arangodb::Index::OperationMode::normal).ok())); - CHECK((trx.commit().ok())); - CHECK((link->commit().ok())); + logicalCollection->dropIndex(index1->id()); } } @@ -1242,8 +1363,12 @@ SECTION("test_write") { CHECK_THROWS((reader.reopen())); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief generate tests +//////////////////////////////////////////////////////////////////////////////// + } // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- \ No newline at end of file diff --git a/tests/RestServer/FlushFeature-test.cpp b/tests/RestServer/FlushFeature-test.cpp index 88d7c09d04..5485bff3e9 100644 --- a/tests/RestServer/FlushFeature-test.cpp +++ b/tests/RestServer/FlushFeature-test.cpp @@ -161,7 +161,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -201,7 +201,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -241,7 +241,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -282,7 +282,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -324,7 +324,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(42), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -366,7 +366,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); @@ -408,7 +408,7 @@ TEST_CASE("FlushFeature", "[serverfeature][serverfeature-flush]") { TRI_voc_tick_t(1), sizeof(TRI_voc_tick_t)); buf.append(json->slice().begin(), json->slice().byteSize()); auto* marker = reinterpret_cast(&buf[0]); - marker->setSize(buf.size()); + marker->setSize(static_cast(buf.size())); marker->setType(::MMFilesMarkerType::TRI_DF_MARKER_VPACK_FLUSH_SYNC); arangodb::MMFilesWalRecoverState state(false); CHECK((0 == state.errorCount)); diff --git a/tests/js/server/recovery/view-arangosearch-link-populate-persistentids-mmfiles.js b/tests/js/server/recovery/view-arangosearch-link-populate-persistentids-mmfiles.js index 743d0afbf6..30d40f678a 100644 --- a/tests/js/server/recovery/view-arangosearch-link-populate-persistentids-mmfiles.js +++ b/tests/js/server/recovery/view-arangosearch-link-populate-persistentids-mmfiles.js @@ -77,7 +77,7 @@ function recoverySuite () { view.properties(meta); fail(); } catch (e) { - assertEqual(e.errorNum, errors.ERROR_ARANGO_INDEX_CREATION_FAILED.code); + assertEqual(e.errorNum, errors.ERROR_ARANGO_ILLEGAL_STATE.code); } var links = view.properties().links;