diff --git a/UnitTests/Makefile.unittests b/UnitTests/Makefile.unittests index 5a41faad3b..29adc9c2c9 100755 --- a/UnitTests/Makefile.unittests +++ b/UnitTests/Makefile.unittests @@ -208,6 +208,15 @@ unittests-recovery: @echo "================================================================================" @echo + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-indexes" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-indexes" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-collections" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-databases" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-single-collection" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-collections" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-databases" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="collections-reuse" + $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="collections-different-attributes" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes-hash" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes-skiplist" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes-geo" @@ -215,13 +224,6 @@ unittests-recovery: $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="edges" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="cap-constraint" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-indexes" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-collections" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-databases" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-single-collection" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-collections" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-databases" - $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-indexes" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="many-inserts" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="many-updates" $(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="wait-for-sync" diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index 35b6dc3757..1e486705ab 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -3203,6 +3203,7 @@ static v8::Handle CreateVocBase (v8::Arguments const& argv, // extract the parameters TRI_col_info_t parameter; + TRI_voc_cid_t cid = 0; if (2 <= argv.Length()) { if (! argv[1]->IsObject()) { @@ -3270,10 +3271,15 @@ static v8::Handle CreateVocBase (v8::Arguments const& argv, TRI_FreeCollectionInfoOptions(¶meter); TRI_V8_EXCEPTION_PARAMETER(scope, "volatile collections do not support the waitForSync option"); } + + if (p->Has(v8g->IdKey)) { + // specify collection id - used for testing only + cid = TRI_ObjectToUInt64(p->Get(v8g->IdKey), true); + } } else { - TRI_InitCollectionInfo(vocbase, ¶meter, name.c_str(), collectionType, effectiveSize, 0); + TRI_InitCollectionInfo(vocbase, ¶meter, name.c_str(), collectionType, effectiveSize, nullptr); } @@ -3286,7 +3292,7 @@ static v8::Handle CreateVocBase (v8::Arguments const& argv, TRI_vocbase_col_t const* collection = TRI_CreateCollectionVocBase(vocbase, ¶meter, - 0, + cid, true); TRI_FreeCollectionInfoOptions(¶meter); @@ -9088,7 +9094,8 @@ static v8::Handle JS_CreateDatabase (v8::Arguments const& argv) { } - TRI_v8_global_t* v8g = (TRI_v8_global_t*) v8::Isolate::GetCurrent()->GetData(); + TRI_v8_global_t* v8g = static_cast(v8::Isolate::GetCurrent()->GetData()); + TRI_voc_tick_t id = 0; // get database defaults from server TRI_vocbase_defaults_t defaults; @@ -9128,12 +9135,17 @@ static v8::Handle JS_CreateDatabase (v8::Arguments const& argv) { if (options->Has(keyAuthenticateSystemOnly)) { defaults.authenticateSystemOnly = options->Get(keyAuthenticateSystemOnly)->BooleanValue(); } + + if (options->Has(v8g->IdKey)) { + // only used for testing to create database with a specific id + id = TRI_ObjectToUInt64(options->Get(v8g->IdKey), true); + } } string const name = TRI_ObjectToString(argv[0]); TRI_vocbase_t* database; - int res = TRI_CreateDatabaseServer(static_cast(v8g->_server), name.c_str(), &defaults, &database, true); + int res = TRI_CreateDatabaseServer(static_cast(v8g->_server), id, name.c_str(), &defaults, &database, true); if (res != TRI_ERROR_NO_ERROR) { TRI_V8_EXCEPTION(scope, res); diff --git a/arangod/VocBase/cleanup.cpp b/arangod/VocBase/cleanup.cpp index ec4249ce1b..7b831cca97 100644 --- a/arangod/VocBase/cleanup.cpp +++ b/arangod/VocBase/cleanup.cpp @@ -167,18 +167,14 @@ static void CleanupDocumentCollection (TRI_vocbase_col_t* collection, // execute callback, sone of the callbacks might delete or free our collection if (element->_type == TRI_BARRIER_DATAFILE_DROP_CALLBACK) { - TRI_barrier_datafile_drop_cb_t* de; - - de = (TRI_barrier_datafile_drop_cb_t*) element; + TRI_barrier_datafile_drop_cb_t* de = (TRI_barrier_datafile_drop_cb_t*) element; de->callback(de->_datafile, de->_data); TRI_Free(TRI_UNKNOWN_MEM_ZONE, element); // next iteration } else if (element->_type == TRI_BARRIER_DATAFILE_RENAME_CALLBACK) { - TRI_barrier_datafile_rename_cb_t* de; - - de = (TRI_barrier_datafile_rename_cb_t*) element; + TRI_barrier_datafile_rename_cb_t* de = (TRI_barrier_datafile_rename_cb_t*) element; de->callback(de->_datafile, de->_data); TRI_Free(TRI_UNKNOWN_MEM_ZONE, element); @@ -187,6 +183,7 @@ static void CleanupDocumentCollection (TRI_vocbase_col_t* collection, else if (element->_type == TRI_BARRIER_COLLECTION_UNLOAD_CALLBACK) { // collection is unloaded TRI_barrier_collection_cb_t* ce = (TRI_barrier_collection_cb_t*) element; + bool hasUnloaded = ce->callback(ce->_collection, ce->_data); TRI_Free(TRI_UNKNOWN_MEM_ZONE, element); @@ -198,6 +195,7 @@ static void CleanupDocumentCollection (TRI_vocbase_col_t* collection, else if (element->_type == TRI_BARRIER_COLLECTION_DROP_CALLBACK) { // collection is dropped TRI_barrier_collection_cb_t* ce = (TRI_barrier_collection_cb_t*) element; + bool hasUnloaded = ce->callback(ce->_collection, ce->_data); TRI_Free(TRI_UNKNOWN_MEM_ZONE, element); diff --git a/arangod/VocBase/collection.cpp b/arangod/VocBase/collection.cpp index 00488f3910..61c014bf70 100644 --- a/arangod/VocBase/collection.cpp +++ b/arangod/VocBase/collection.cpp @@ -1477,6 +1477,27 @@ bool TRI_IterateCollection (TRI_collection_t* collection, return result; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief removes an index file from the indexFiles vector +//////////////////////////////////////////////////////////////////////////////// + +int TRI_RemoveFileIndexCollection (TRI_collection_t* collection, + TRI_idx_iid_t iid) { + size_t const n = collection->_indexFiles._length; + + for (size_t i = 0; i < n; ++i) { + char const* filename = collection->_indexFiles._buffer[i]; + + if (GetNumericFilenamePart(filename) == iid) { + // found + TRI_RemoveVectorString(&collection->_indexFiles, i); + return true; + } + } + + return false; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief iterates over all index files of a collection //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/VocBase/collection.h b/arangod/VocBase/collection.h index bfbfd976f9..fda8112a7d 100644 --- a/arangod/VocBase/collection.h +++ b/arangod/VocBase/collection.h @@ -352,6 +352,13 @@ int TRI_RenameCollection (TRI_collection_t*, char const*); // --SECTION-- protected functions // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief removes an index file from the indexFiles vector +//////////////////////////////////////////////////////////////////////////////// + +int TRI_RemoveFileIndexCollection (TRI_collection_t*, + TRI_idx_iid_t); + //////////////////////////////////////////////////////////////////////////////// /// @brief iterates over a collection //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/VocBase/server.cpp b/arangod/VocBase/server.cpp index 78c6dd219f..5fae9628b1 100644 --- a/arangod/VocBase/server.cpp +++ b/arangod/VocBase/server.cpp @@ -1074,8 +1074,7 @@ static int SaveDatabaseParameters (TRI_voc_tick_t id, TRI_FreeString(TRI_CORE_MEM_ZONE, tickString); if (! TRI_SaveJson(file, json, false)) { - LOG_ERROR("cannot save database information in file '%s'", - file); + LOG_ERROR("cannot save database information in file '%s'", file); TRI_FreeJson(TRI_CORE_MEM_ZONE, json); TRI_FreeString(TRI_CORE_MEM_ZONE, file); @@ -1415,7 +1414,7 @@ static void DatabaseManager (void* data) { } // found a database to delete - database = (TRI_vocbase_t*) TRI_RemoveVectorPointer(&server->_droppedDatabases, i); + database = static_cast(TRI_RemoveVectorPointer(&server->_droppedDatabases, i)); break; } } @@ -2062,6 +2061,7 @@ int TRI_CreateCoordinatorDatabaseServer (TRI_server_t* server, //////////////////////////////////////////////////////////////////////////////// int TRI_CreateDatabaseServer (TRI_server_t* server, + TRI_voc_tick_t databaseId, char const* name, TRI_vocbase_defaults_t const* defaults, TRI_vocbase_t** database, @@ -2096,11 +2096,14 @@ int TRI_CreateDatabaseServer (TRI_server_t* server, return TRI_ERROR_OUT_OF_MEMORY; } - // create the database directory char* file; - TRI_voc_tick_t tick = TRI_NewTickServer(); - int res = CreateDatabaseDirectory(server, tick, name, defaults, &file); + + if (databaseId == 0) { + databaseId = TRI_NewTickServer(); + } + + int res = CreateDatabaseDirectory(server, databaseId, name, defaults, &file); if (res != TRI_ERROR_NO_ERROR) { TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); @@ -2116,7 +2119,7 @@ int TRI_CreateDatabaseServer (TRI_server_t* server, name, path); - TRI_vocbase_t* vocbase = TRI_OpenVocBase(server, path, tick, name, defaults, false, false); + TRI_vocbase_t* vocbase = TRI_OpenVocBase(server, path, databaseId, name, defaults, false, false); TRI_FreeString(TRI_CORE_MEM_ZONE, path); if (vocbase == nullptr) { @@ -2139,34 +2142,41 @@ int TRI_CreateDatabaseServer (TRI_server_t* server, } TRI_ASSERT(vocbase != nullptr); + + char* tickString = TRI_StringUInt64(databaseId); + TRI_Insert3ArrayJson(TRI_UNKNOWN_MEM_ZONE, json, "id", TRI_CreateStringCopyJson(TRI_UNKNOWN_MEM_ZONE, tickString)); + TRI_FreeString(TRI_CORE_MEM_ZONE, tickString); + TRI_Insert3ArrayJson(TRI_UNKNOWN_MEM_ZONE, json, "name", TRI_CreateStringCopyJson(TRI_UNKNOWN_MEM_ZONE, name)); + // create application directories CreateApplicationDirectory(vocbase->_name, server->_appPath); CreateApplicationDirectory(vocbase->_name, server->_devAppPath); - TRI_ReloadAuthInfo(vocbase); - TRI_StartCompactorVocBase(vocbase); + if (! triagens::wal::LogfileManager::instance()->isInRecovery()) { + TRI_ReloadAuthInfo(vocbase); + TRI_StartCompactorVocBase(vocbase); - // start the replication applier - if (vocbase->_replicationApplier->_configuration._autoStart) { - if (server->_disableReplicationAppliers) { - LOG_INFO("replication applier explicitly deactivated for database '%s'", name); - } - else { - res = TRI_StartReplicationApplier(vocbase->_replicationApplier, 0, false); + // start the replication applier + if (vocbase->_replicationApplier->_configuration._autoStart) { + if (server->_disableReplicationAppliers) { + LOG_INFO("replication applier explicitly deactivated for database '%s'", name); + } + else { + res = TRI_StartReplicationApplier(vocbase->_replicationApplier, 0, false); - if (res != TRI_ERROR_NO_ERROR) { - LOG_WARNING("unable to start replication applier for database '%s': %s", - name, - TRI_errno_string(res)); + if (res != TRI_ERROR_NO_ERROR) { + LOG_WARNING("unable to start replication applier for database '%s': %s", + name, + TRI_errno_string(res)); + } } } + + // increase reference counter + TRI_UseVocBase(vocbase); } - - // increase reference counter - TRI_UseVocBase(vocbase); - { DatabaseWriteLocker locker(&server->_databasesLock); TRI_InsertKeyAssociativePointer(&server->_databases, vocbase->_name, vocbase, false); @@ -2459,6 +2469,46 @@ TRI_vocbase_t* TRI_UseDatabaseServer (TRI_server_t* server, return vocbase; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookup a database by its id +//////////////////////////////////////////////////////////////////////////////// + +TRI_vocbase_t* TRI_LookupDatabaseByIdServer (TRI_server_t* server, + TRI_voc_tick_t id) { + DatabaseReadLocker locker(&server->_databasesLock); + size_t const n = server->_databases._nrAlloc; + + for (size_t i = 0; i < n; ++i) { + TRI_vocbase_t* vocbase = static_cast(server->_databases._table[i]); + + if (vocbase != nullptr && vocbase->_id == id) { + return vocbase; + } + } + + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookup a database by its name +//////////////////////////////////////////////////////////////////////////////// + +TRI_vocbase_t* TRI_LookupDatabaseByNameServer (TRI_server_t* server, + char const* name) { + DatabaseReadLocker locker(&server->_databasesLock); + size_t const n = server->_databases._nrAlloc; + + for (size_t i = 0; i < n; ++i) { + TRI_vocbase_t* vocbase = static_cast(server->_databases._table[i]); + + if (vocbase != nullptr && TRI_EqualString(vocbase->_name, name)) { + return vocbase; + } + } + + return nullptr; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief get a database by its id /// this will increase the reference-counter for the database diff --git a/arangod/VocBase/server.h b/arangod/VocBase/server.h index b9d6fca0df..e1971ee47d 100644 --- a/arangod/VocBase/server.h +++ b/arangod/VocBase/server.h @@ -175,6 +175,7 @@ int TRI_CreateCoordinatorDatabaseServer (TRI_server_t*, //////////////////////////////////////////////////////////////////////////////// int TRI_CreateDatabaseServer (TRI_server_t*, + TRI_voc_tick_t, char const*, TRI_vocbase_defaults_t const*, struct TRI_vocbase_s**, @@ -242,6 +243,20 @@ struct TRI_vocbase_s* TRI_UseCoordinatorDatabaseServer (TRI_server_t*, struct TRI_vocbase_s* TRI_UseDatabaseServer (TRI_server_t*, char const*); +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookup a database by its id +//////////////////////////////////////////////////////////////////////////////// + +struct TRI_vocbase_s* TRI_LookupDatabaseByIdServer (TRI_server_t*, + TRI_voc_tick_t); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookup a database by its name +//////////////////////////////////////////////////////////////////////////////// + +struct TRI_vocbase_s* TRI_LookupDatabaseByNameServer (TRI_server_t*, + char const*); + //////////////////////////////////////////////////////////////////////////////// /// @brief use a database by its id /// this will increase the reference-counter for the database diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index f02397816e..6980edf9bb 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -591,8 +591,6 @@ static TRI_vocbase_col_t* CreateCollection (TRI_vocbase_t* vocbase, if (found != nullptr) { TRI_WRITE_UNLOCK_COLLECTIONS_VOCBASE(vocbase); - LOG_DEBUG("collection named '%s' already exists", name); - TRI_set_errno(TRI_ERROR_ARANGO_DUPLICATE_NAME); return nullptr; } @@ -1501,7 +1499,9 @@ TRI_vocbase_t* TRI_OpenVocBase (TRI_server_t* server, void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) { // stop replication - TRI_StopReplicationApplier(vocbase->_replicationApplier, false); + if (vocbase->_replicationApplier != nullptr) { + TRI_StopReplicationApplier(vocbase->_replicationApplier, false); + } TRI_vector_pointer_t collections; TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE); @@ -1532,11 +1532,12 @@ void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) { TRI_SignalCondition(&vocbase->_compactorCondition); TRI_UnlockCondition(&vocbase->_compactorCondition); - TRI_ASSERT(vocbase->_hasCompactor); - int res = TRI_JoinThread(&vocbase->_compactor); + if (vocbase->_hasCompactor) { + int res = TRI_JoinThread(&vocbase->_compactor); - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("unable to join compactor thread: %s", TRI_errno_string(res)); + if (res != TRI_ERROR_NO_ERROR) { + LOG_ERROR("unable to join compactor thread: %s", TRI_errno_string(res)); + } } // this will signal the cleanup thread to do one last iteration @@ -1546,7 +1547,7 @@ void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) { TRI_SignalCondition(&vocbase->_cleanupCondition); TRI_UnlockCondition(&vocbase->_cleanupCondition); - res = TRI_JoinThread(&vocbase->_cleanup); + int res = TRI_JoinThread(&vocbase->_cleanup); if (res != TRI_ERROR_NO_ERROR) { LOG_ERROR("unable to join cleanup thread: %s", TRI_errno_string(res)); @@ -1987,7 +1988,7 @@ int TRI_DropCollectionVocBase (TRI_vocbase_t* vocbase, TRI_ASSERT(collection != nullptr); - if (! collection->_canDrop) { + if (! collection->_canDrop && ! triagens::wal::LogfileManager::instance()->isInRecovery()) { return TRI_set_errno(TRI_ERROR_FORBIDDEN); } @@ -2062,7 +2063,7 @@ int TRI_DropCollectionVocBase (TRI_vocbase_t* vocbase, TRI_WRITE_UNLOCK_STATUS_VOCBASE_COL(collection); - DropCollectionCallback(0, collection); + DropCollectionCallback(nullptr, collection); TRI_ReadUnlockReadWriteLock(&vocbase->_inventoryLock); @@ -2121,17 +2122,22 @@ int TRI_DropCollectionVocBase (TRI_vocbase_t* vocbase, TRI_WRITE_UNLOCK_STATUS_VOCBASE_COL(collection); TRI_ReadUnlockReadWriteLock(&vocbase->_inventoryLock); + + if (triagens::wal::LogfileManager::instance()->isInRecovery()) { + DropCollectionCallback(nullptr, collection); + } + else { + // added callback for dropping + TRI_CreateBarrierDropCollection(&collection->_collection->_barrierList, + collection->_collection, + DropCollectionCallback, + collection); - // added callback for dropping - TRI_CreateBarrierDropCollection(&collection->_collection->_barrierList, - collection->_collection, - DropCollectionCallback, - collection); - - // wake up the cleanup thread - TRI_LockCondition(&vocbase->_cleanupCondition); - TRI_SignalCondition(&vocbase->_cleanupCondition); - TRI_UnlockCondition(&vocbase->_cleanupCondition); + // wake up the cleanup thread + TRI_LockCondition(&vocbase->_cleanupCondition); + TRI_SignalCondition(&vocbase->_cleanupCondition); + TRI_UnlockCondition(&vocbase->_cleanupCondition); + } return TRI_ERROR_NO_ERROR; } @@ -2347,6 +2353,7 @@ bool TRI_DropVocBase (TRI_vocbase_t* vocbase) { else { vocbase->_usage._isDeleted = true; result = true; + } TRI_UnlockSpin(&vocbase->_usage._lock); diff --git a/arangod/Wal/LogfileManager.cpp b/arangod/Wal/LogfileManager.cpp index 12c1e98f73..38805741c8 100644 --- a/arangod/Wal/LogfileManager.cpp +++ b/arangod/Wal/LogfileManager.cpp @@ -1457,8 +1457,6 @@ int LogfileManager::readShutdownInfo () { // read if of last sealed logfile (maybe 0) uint64_t lastSealedId = basics::JsonHelper::stringUInt64(json, "lastSealed"); - TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); - if (lastSealedId < lastCollectedId) { // should not happen normally lastSealedId = lastCollectedId; @@ -1471,6 +1469,8 @@ int LogfileManager::readShutdownInfo () { else { LOG_TRACE("previous shutdown was at '%s'", shutdownTime.c_str()); } + + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); { WRITE_LOCKER(_logfilesLock); diff --git a/arangod/Wal/LogfileManager.h b/arangod/Wal/LogfileManager.h index c7a66d5d59..f0890cd386 100644 --- a/arangod/Wal/LogfileManager.h +++ b/arangod/Wal/LogfileManager.h @@ -339,6 +339,7 @@ namespace triagens { inline void throttleWhenPending (uint64_t value) { _throttleWhenPending = value; + if (_throttleWhenPending == 0) { deactivateWriteThrottling(); } diff --git a/arangod/Wal/RecoverState.cpp b/arangod/Wal/RecoverState.cpp index c2a52b6a5e..6041916549 100644 --- a/arangod/Wal/RecoverState.cpp +++ b/arangod/Wal/RecoverState.cpp @@ -29,6 +29,8 @@ #include "RecoverState.h" #include "Basics/FileUtils.h" +#include "BasicsC/conversions.h" +#include "BasicsC/files.h" #include "VocBase/collection.h" #include "VocBase/replication-applier.h" #include "VocBase/voc-shaper.h" @@ -42,10 +44,81 @@ using namespace triagens::wal; // --SECTION-- helper functions // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief whether or not a collection is volatile +//////////////////////////////////////////////////////////////////////////////// + static inline bool IsVolatile (TRI_transaction_collection_t const* trxCollection) { return trxCollection->_collection->_collection->_info._isVolatile; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the directory for a database +//////////////////////////////////////////////////////////////////////////////// + +static std::string GetDatabaseDirectory (TRI_server_t* server, + TRI_voc_tick_t databaseId) { + char* idString = TRI_StringUInt64(databaseId); + char* dname = TRI_Concatenate2String("database-", idString); + TRI_FreeString(TRI_CORE_MEM_ZONE, idString); + char* filename = TRI_Concatenate2File(server->_databasePath, dname); + TRI_FreeString(TRI_CORE_MEM_ZONE, dname); + + std::string result(filename); + TRI_FreeString(TRI_CORE_MEM_ZONE, filename); + + return result; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the directory for a collection +//////////////////////////////////////////////////////////////////////////////// + +static std::string GetCollectionDirectory (TRI_vocbase_t* vocbase, + TRI_voc_cid_t collectionId) { + char* dirname = TRI_GetDirectoryCollection(vocbase->_path, + "empty", // does not matter + TRI_COL_TYPE_DOCUMENT, // does not matter + collectionId); + + std::string result(dirname); + TRI_FreeString(TRI_CORE_MEM_ZONE, dirname); + + return result; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief wait until a database directory disappears +//////////////////////////////////////////////////////////////////////////////// + +static int WaitForDeletion (TRI_server_t* server, + TRI_voc_tick_t databaseId) { + std::string result = GetDatabaseDirectory(server, databaseId); + + int iterations = 0; + while (TRI_IsDirectory(result.c_str()) && iterations++ < 60 * 100) { + usleep(10000); + } + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief wait until a collection directory disappears +//////////////////////////////////////////////////////////////////////////////// + +static int WaitForDeletion (TRI_vocbase_t* vocbase, + TRI_voc_cid_t collectionId) { + std::string result = GetCollectionDirectory(vocbase, collectionId); + + int iterations = 0; + while (TRI_IsDirectory(result.c_str()) && iterations++ < 60 * 100) { + usleep(10000); + } + + return TRI_ERROR_NO_ERROR; +} + // ----------------------------------------------------------------------------- // --SECTION-- constructors / destructors // ----------------------------------------------------------------------------- @@ -61,9 +134,6 @@ RecoverState::RecoverState (TRI_server_t* server, remoteTransactions(), remoteTransactionCollections(), remoteTransactionDatabases(), - droppedCollections(), - droppedDatabases(), - droppedIndexes(), lastTick(0), logfilesToProcess(), openedCollections(), @@ -137,33 +207,6 @@ void RecoverState::releaseResources () { openedDatabases.clear(); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief checks if a database is dropped already -//////////////////////////////////////////////////////////////////////////////// - -bool RecoverState::isDropped (TRI_voc_tick_t databaseId) const { - return (droppedDatabases.find(databaseId) != droppedDatabases.end()); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief checks if a database or collection is dropped already -//////////////////////////////////////////////////////////////////////////////// - -bool RecoverState::isDropped (TRI_voc_tick_t databaseId, - TRI_voc_cid_t collectionId) const { - if (isDropped(databaseId)) { - // database has been dropped - return true; - } - - if (droppedCollections.find(collectionId) != droppedCollections.end()) { - // collection has been dropped - return true; - } - - return false; -} - //////////////////////////////////////////////////////////////////////////////// /// @brief gets a database (and inserts it into the cache if not in it) //////////////////////////////////////////////////////////////////////////////// @@ -180,11 +223,47 @@ TRI_vocbase_t* RecoverState::useDatabase (TRI_voc_tick_t databaseId) { if (vocbase == nullptr) { return nullptr; } - + openedDatabases.insert(it, std::make_pair(databaseId, vocbase)); return vocbase; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief release a database (so it can be dropped) +//////////////////////////////////////////////////////////////////////////////// + +TRI_vocbase_t* RecoverState::releaseDatabase (TRI_voc_tick_t databaseId) { + auto it = openedDatabases.find(databaseId); + + if (it == openedDatabases.end()) { + return nullptr; + } + + TRI_vocbase_t* vocbase = (*it).second; + TRI_ReleaseDatabaseServer(server, vocbase); + openedDatabases.erase(databaseId); + + return vocbase; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief release a collection (so it can be dropped) +//////////////////////////////////////////////////////////////////////////////// + +TRI_vocbase_col_t* RecoverState::releaseCollection (TRI_voc_cid_t collectionId) { + auto it = openedCollections.find(collectionId); + + if (it == openedCollections.end()) { + return nullptr; + } + + TRI_vocbase_col_t* collection = (*it).second; + TRI_ReleaseCollectionVocBase(collection->_vocbase, collection); + openedCollections.erase(collectionId); + + return collection; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief gets a collection (and inserts it into the cache if not in it) //////////////////////////////////////////////////////////////////////////////// @@ -454,8 +533,7 @@ bool RecoverState::InitialScanMarker (TRI_df_marker_t const* marker, state->remoteTransactions.erase(m->_transactionId); break; } - - + // ----------------------------------------------------------------------------- // create markers // ----------------------------------------------------------------------------- @@ -466,46 +544,42 @@ bool RecoverState::InitialScanMarker (TRI_df_marker_t const* marker, state->droppedCollections.erase(m->_collectionId); break; } - + case TRI_WAL_MARKER_CREATE_DATABASE: { database_create_marker_t const* m = reinterpret_cast(marker); // undo a potential drop marker discovered before for the same database state->droppedDatabases.erase(m->_databaseId); break; } - + case TRI_WAL_MARKER_CREATE_INDEX: { - index_create_marker_t const* m = reinterpret_cast(marker); - // undo a potential drop marker discovered before for the same index - state->droppedIndexes.erase(m->_indexId); + // ignored break; } - - + // ----------------------------------------------------------------------------- // drop markers // ----------------------------------------------------------------------------- - + case TRI_WAL_MARKER_DROP_COLLECTION: { collection_drop_marker_t const* m = reinterpret_cast(marker); // note that the collection was dropped and doesn't need to be recovered state->droppedCollections.insert(m->_collectionId); break; } - + case TRI_WAL_MARKER_DROP_DATABASE: { database_drop_marker_t const* m = reinterpret_cast(marker); // note that the database was dropped and doesn't need to be recovered state->droppedDatabases.insert(m->_databaseId); break; } - + case TRI_WAL_MARKER_DROP_INDEX: { - index_drop_marker_t const* m = reinterpret_cast(marker); - // note that the index was dropped and doesn't need to be recovered - state->droppedIndexes.insert(m->_indexId); + // ignored break; } + } return true; @@ -551,7 +625,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, if (res != TRI_ERROR_NO_ERROR) { LOG_WARNING("could not apply attribute marker: %s", TRI_errno_string(res)); - return state->shouldAbort(); + return state->canContinue(); } break; } @@ -562,7 +636,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, shape_marker_t const* m = reinterpret_cast(marker); TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; - + if (state->isDropped(databaseId, collectionId)) { return true; } @@ -577,7 +651,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, if (res != TRI_ERROR_NO_ERROR) { LOG_WARNING("could not apply shape marker: %s", TRI_errno_string(res)); - return state->shouldAbort(); + return state->canContinue(); } break; } @@ -592,6 +666,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, document_marker_t const* m = reinterpret_cast(marker); TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; + if (state->isDropped(databaseId, collectionId)) { return true; } @@ -655,7 +730,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, (unsigned long long) collectionId, (unsigned long long) databaseId, TRI_errno_string(res)); - return state->shouldAbort(); + return state->canContinue(); } break; } @@ -666,6 +741,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, edge_marker_t const* m = reinterpret_cast(marker); TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; + if (state->isDropped(databaseId, collectionId)) { return true; } @@ -734,7 +810,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, (unsigned long long) collectionId, (unsigned long long) databaseId, TRI_errno_string(res)); - return state->shouldAbort(); + return state->canContinue(); } break; @@ -746,6 +822,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, remove_marker_t const* m = reinterpret_cast(marker); TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; + if (state->isDropped(databaseId, collectionId)) { return true; } @@ -797,7 +874,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, (unsigned long long) collectionId, (unsigned long long) databaseId, TRI_errno_string(res)); - return state->shouldAbort(); + return state->canContinue(); } break; } @@ -813,6 +890,10 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, TRI_voc_tid_t externalId = m->_externalId; // start a remote transaction + if (state->isDropped(databaseId)) { + return true; + } + TRI_vocbase_t* vocbase = state->useDatabase(databaseId); if (vocbase == nullptr) { LOG_WARNING("cannot start remote transaction in database %llu: %s", @@ -824,7 +905,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, if (trx == nullptr) { LOG_WARNING("unable to start transaction: %s", TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY)); - return state->shouldAbort(); + return state->canContinue(); } trx->addHint(TRI_TRANSACTION_HINT_NO_BEGIN_MARKER, true); @@ -834,7 +915,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, if (res != TRI_ERROR_NO_ERROR) { LOG_WARNING("unable to start transaction: %s", TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY)); delete trx; - return state->shouldAbort(); + return state->canContinue(); } state->runningRemoteTransactions.insert(std::make_pair(m->_externalId, trx)); @@ -851,9 +932,19 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; TRI_idx_iid_t indexId = m->_indexId; + if (state->isDropped(databaseId, collectionId)) { return true; } + + TRI_vocbase_t* vocbase = state->useDatabase(databaseId); + if (vocbase == nullptr) { + LOG_WARNING("cannot create index for collection %llu in database %llu: %s", + (unsigned long long) collectionId, + (unsigned long long) databaseId, + TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND)); + return state->canContinue(); + } TRI_document_collection_t* document = state->getCollection(databaseId, collectionId); if (document == nullptr) { @@ -862,7 +953,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, (unsigned long long) collectionId, (unsigned long long) databaseId, TRI_errno_string(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND)); - return state->shouldAbort(); + return state->canContinue(); } char const* properties = reinterpret_cast(m) + sizeof(index_create_marker_t); @@ -873,33 +964,35 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, (unsigned long long) indexId, (unsigned long long) collectionId, (unsigned long long) databaseId); - return state->shouldAbort(); + return state->canContinue(); } - return true; - // TODO: must handle indexes differently -/* // fake transaction to satisfy assertions triagens::arango::TransactionBase trx(true); - - TRI_index_t* idx = nullptr; - int res = TRI_FromJsonIndexDocumentCollection(document, json, &idx); + + std::string collectionDirectory = GetCollectionDirectory(vocbase, collectionId); + char* idString = TRI_StringUInt64(indexId); + char* indexName = TRI_Concatenate3String("index-", idString, ".json"); + TRI_FreeString(TRI_CORE_MEM_ZONE, idString); + char* filename = TRI_Concatenate2File(collectionDirectory.c_str(), indexName); + TRI_FreeString(TRI_CORE_MEM_ZONE, indexName); + + bool ok = TRI_SaveJson(filename, json, false); TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); - if (res == TRI_ERROR_NO_ERROR) { - res = TRI_SaveIndex(document, idx, false); - } - - if (res != TRI_ERROR_NO_ERROR) { - LOG_WARNING("cannot create index %llu, collection %llu in database %llu: %s", + if (! ok) { + TRI_FreeString(TRI_CORE_MEM_ZONE, filename); + LOG_WARNING("cannot create index %llu, collection %llu in database %llu", (unsigned long long) indexId, (unsigned long long) collectionId, - (unsigned long long) databaseId, - TRI_errno_string(res)); - return state->shouldAbort(); + (unsigned long long) databaseId); + return state->canContinue(); } + else { + TRI_PushBackVectorString(&document->_indexFiles, filename); + } + break; -*/ } @@ -907,20 +1000,31 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, collection_create_marker_t const* m = reinterpret_cast(marker); TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; - if (state->isDropped(databaseId, collectionId)) { - return true; - } - TRI_vocbase_t* vocbase = state->useDatabase(databaseId); - if (vocbase == nullptr) { - LOG_WARNING("cannot open database %llu", (unsigned long long) databaseId); - return state->shouldAbort(); + // remove the drop marker + state->droppedCollections.erase(collectionId); + + if (state->isDropped(databaseId)) { + return true; + } + + TRI_vocbase_t* vocbase = state->useDatabase(databaseId); + + if (vocbase == nullptr) { + LOG_WARNING("cannot open database %llu", (unsigned long long) databaseId); + return state->canContinue(); + } + + TRI_vocbase_col_t* collection = state->releaseCollection(collectionId); + + if (collection == nullptr) { + collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId); } - TRI_vocbase_col_t* collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId); if (collection != nullptr) { - // collection already exists - nothing to do - return true; + // drop an existing collection + TRI_DropCollectionVocBase(vocbase, collection, false); + WaitForDeletion(vocbase, collectionId); } char const* properties = reinterpret_cast(m) + sizeof(collection_create_marker_t); @@ -930,8 +1034,25 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, LOG_WARNING("cannot unpack collection properties for collection %llu in database %llu", (unsigned long long) collectionId, (unsigned long long) databaseId); - return state->shouldAbort(); + return state->canContinue(); } + + // check if there is another collection with the same name as the one that we attempt to create + TRI_json_t const* name = TRI_LookupArrayJson(json, "name"); + + if (TRI_IsStringJson(name)) { + collection = TRI_LookupCollectionByNameVocBase(vocbase, name->_value._string.data); + + if (collection != nullptr && ! TRI_IsSystemNameCollection(name->_value._string.data)) { + // if yes, delete it + TRI_voc_cid_t otherCid = collection->_cid; + + state->releaseCollection(otherCid); + TRI_DropCollectionVocBase(vocbase, collection, false); + WaitForDeletion(vocbase, otherCid); + } + } + TRI_col_info_t info; memset(&info, 0, sizeof(TRI_col_info_t)); @@ -942,6 +1063,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, // fake transaction to satisfy assertions triagens::arango::TransactionBase trx(true); + WaitForDeletion(vocbase, collectionId); collection = TRI_CreateCollectionVocBase(vocbase, &info, collectionId, false); TRI_FreeCollectionInfoOptions(&info); @@ -950,7 +1072,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, LOG_WARNING("cannot create collection %llu in database %llu", (unsigned long long) collectionId, (unsigned long long) databaseId); - return state->shouldAbort(); + return state->canContinue(); } break; } @@ -959,43 +1081,58 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, case TRI_WAL_MARKER_CREATE_DATABASE: { database_create_marker_t const* m = reinterpret_cast(marker); TRI_voc_tick_t databaseId = m->_databaseId; - if (state->isDropped(databaseId)) { - return true; + + // remove the drop marker + state->droppedDatabases.erase(databaseId); + TRI_vocbase_t* vocbase = state->releaseDatabase(databaseId); + + if (vocbase != nullptr) { + // remove already existing database + TRI_DropByIdDatabaseServer(state->server, databaseId, false); + WaitForDeletion(state->server, databaseId); } - - if (TRI_ExistsDatabaseByIdServer(state->server, databaseId)) { - // database already exists - nothing to do - return true; - } - + char const* properties = reinterpret_cast(m) + sizeof(database_create_marker_t); TRI_json_t* json = triagens::basics::JsonHelper::fromString(properties); if (json == nullptr) { LOG_WARNING("cannot unpack database properties for database %llu", (unsigned long long) databaseId); - return state->shouldAbort(); + return state->canContinue(); } TRI_json_t const* nameValue = TRI_LookupArrayJson(json, "name"); - TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); if (! TRI_IsStringJson(nameValue)) { + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); LOG_WARNING("cannot unpack database properties for database %llu", (unsigned long long) databaseId); - return state->shouldAbort(); + return state->canContinue(); } - + + std::string nameString(nameValue->_value._string.data); + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); + + // remove already existing database with same name + vocbase = TRI_LookupDatabaseByNameServer(state->server, nameString.c_str()); + + if (vocbase != nullptr) { + TRI_voc_tick_t otherId = vocbase->_id; + state->releaseDatabase(otherId); + TRI_DropDatabaseServer(state->server, nameString.c_str(), false); + WaitForDeletion(state->server, otherId); + } + TRI_vocbase_defaults_t defaults; TRI_GetDatabaseDefaultsServer(state->server, &defaults); - TRI_vocbase_t* vocbase = nullptr; // fake transaction to satisfy assertions triagens::arango::TransactionBase trx(true); - int res = TRI_CreateDatabaseServer(state->server, nameValue->_value._string.data, &defaults, &vocbase, false); + vocbase = nullptr; + int res = TRI_CreateDatabaseServer(state->server, databaseId, nameString.c_str(), &defaults, &vocbase, false); if (res != TRI_ERROR_NO_ERROR) { LOG_WARNING("cannot create database %llu: %s", (unsigned long long) databaseId, TRI_errno_string(res)); - return state->shouldAbort(); + return state->canContinue(); } break; } @@ -1010,23 +1147,40 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; TRI_idx_iid_t indexId = m->_indexId; - + if (state->isDropped(databaseId, collectionId)) { return true; } - - if (state->droppedIndexes.find(indexId) == state->droppedIndexes.end()) { - TRI_document_collection_t* document = state->getCollection(databaseId, collectionId); - if (document == nullptr) { - return state->shouldAbort(); - } - // fake transaction to satisfy assertions - triagens::arango::TransactionBase trx(true); + TRI_vocbase_t* vocbase = state->useDatabase(databaseId); - // ignore any potential error returned by this call - TRI_DropIndexDocumentCollection(document, indexId, false); + if (vocbase == nullptr) { + LOG_WARNING("cannot open database %llu", (unsigned long long) databaseId); + return state->canContinue(); } + + TRI_document_collection_t* document = state->getCollection(databaseId, collectionId); + if (document == nullptr) { + return state->canContinue(); + } + + // fake transaction to satisfy assertions + triagens::arango::TransactionBase trx(true); + + // ignore any potential error returned by this call + TRI_DropIndexDocumentCollection(document, indexId, false); + TRI_RemoveFileIndexCollection(document, indexId); + + // additionally remove the index file + std::string collectionDirectory = GetCollectionDirectory(vocbase, collectionId); + char* idString = TRI_StringUInt64(indexId); + char* indexName = TRI_Concatenate3String("index-", idString, ".json"); + TRI_FreeString(TRI_CORE_MEM_ZONE, idString); + char* filename = TRI_Concatenate2File(collectionDirectory.c_str(), indexName); + TRI_FreeString(TRI_CORE_MEM_ZONE, indexName); + + TRI_UnlinkFile(filename); + TRI_FreeString(TRI_CORE_MEM_ZONE, filename); break; } @@ -1035,26 +1189,29 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, collection_drop_marker_t const* m = reinterpret_cast(marker); TRI_voc_cid_t collectionId = m->_collectionId; TRI_voc_tick_t databaseId = m->_databaseId; - - if (state->isDropped(databaseId, collectionId)) { - TRI_vocbase_t* vocbase = state->useDatabase(databaseId); - if (vocbase == nullptr) { - // database already deleted - do nothing - return true; - } - - TRI_vocbase_col_t* collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId); - - if (collection == nullptr) { - // collection already deleted - do nothing - return true; - } + // insert the drop marker + state->droppedCollections.insert(collectionId); + + TRI_vocbase_t* vocbase = state->useDatabase(databaseId); + if (vocbase == nullptr) { + // database already deleted - do nothing + return true; + } + + // ignore any potential error returned by this call + TRI_vocbase_col_t* collection = state->releaseCollection(collectionId); + + if (collection == nullptr) { + collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId); + } + + if (collection != nullptr) { // fake transaction to satisfy assertions triagens::arango::TransactionBase trx(true); - // ignore any potential error returned by this call TRI_DropCollectionVocBase(vocbase, collection, false); + WaitForDeletion(vocbase, collectionId); } break; } @@ -1063,8 +1220,12 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, case TRI_WAL_MARKER_DROP_DATABASE: { database_drop_marker_t const* m = reinterpret_cast(marker); TRI_voc_tick_t databaseId = m->_databaseId; + + // insert the drop marker + state->droppedDatabases.insert(databaseId); + TRI_vocbase_t* vocbase = state->releaseDatabase(databaseId); - if (state->isDropped(databaseId)) { + if (vocbase != nullptr) { // fake transaction to satisfy assertions triagens::arango::TransactionBase trx(true); @@ -1086,6 +1247,9 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker, int RecoverState::replayLogfile (Logfile* logfile) { LOG_TRACE("replaying logfile '%s'", logfile->filename().c_str()); + droppedCollections.clear(); + droppedDatabases.clear(); + if (! TRI_IterateDatafile(logfile->df(), &RecoverState::ReplayMarker, static_cast(this))) { LOG_WARNING("WAL inspection failed when scanning logfile '%s'", logfile->filename().c_str()); return TRI_ERROR_ARANGO_RECOVERY; @@ -1138,11 +1302,6 @@ int RecoverState::abortOpenTransactions () { TRI_voc_tick_t databaseId = (*it).second.first; - // only write abort markers for databases that haven't been deleted yet - if (isDropped(databaseId)) { - continue; - } - AbortTransactionMarker marker(databaseId, transactionId); SlotInfoCopy slotInfo = triagens::wal::LogfileManager::instance()->allocateAndWrite(marker.mem(), marker.size(), false); @@ -1198,6 +1357,7 @@ int RecoverState::fillIndexes () { TRI_vocbase_col_t* collection = (*it).second; TRI_document_collection_t* document = collection->_collection; + TRI_ASSERT(document != nullptr); document->useSecondaryIndexes(true); diff --git a/arangod/Wal/RecoverState.h b/arangod/Wal/RecoverState.h index 09308698b5..0a6b8b81c6 100644 --- a/arangod/Wal/RecoverState.h +++ b/arangod/Wal/RecoverState.h @@ -93,12 +93,39 @@ namespace triagens { // --SECTION-- public functions // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief checks if a database is dropped already +//////////////////////////////////////////////////////////////////////////////// + + bool isDropped (TRI_voc_tick_t databaseId) const { + return (droppedDatabases.find(databaseId) != droppedDatabases.end()); + } + +//////////////////////////////////////////////////////////////////////////////// +/// @brief checks if a database or collection is dropped already +//////////////////////////////////////////////////////////////////////////////// + + bool isDropped (TRI_voc_tick_t databaseId, + TRI_voc_cid_t collectionId) const { + if (isDropped(databaseId)) { + // database has been dropped + return true; + } + + if (droppedCollections.find(collectionId) != droppedCollections.end()) { + // collection has been dropped + return true; + } + + return false; + } + //////////////////////////////////////////////////////////////////////////////// /// @brief whether or not to abort recovery on first error //////////////////////////////////////////////////////////////////////////////// - inline bool shouldAbort () const { - return ! ignoreRecoveryErrors; + inline bool canContinue () const { + return ignoreRecoveryErrors; } //////////////////////////////////////////////////////////////////////////////// @@ -158,25 +185,24 @@ namespace triagens { void releaseResources (); -//////////////////////////////////////////////////////////////////////////////// -/// @brief checks if a database is dropped already -//////////////////////////////////////////////////////////////////////////////// - - bool isDropped (TRI_voc_tick_t) const; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief checks if a database or collection is dropped already -//////////////////////////////////////////////////////////////////////////////// - - bool isDropped (TRI_voc_tick_t, - TRI_voc_cid_t) const; - //////////////////////////////////////////////////////////////////////////////// /// @brief gets a database (and inserts it into the cache if not in it) //////////////////////////////////////////////////////////////////////////////// TRI_vocbase_t* useDatabase (TRI_voc_tick_t); +//////////////////////////////////////////////////////////////////////////////// +/// @brief releases a database +//////////////////////////////////////////////////////////////////////////////// + + TRI_vocbase_t* releaseDatabase (TRI_voc_tick_t); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief release a collection (so it can be dropped) +//////////////////////////////////////////////////////////////////////////////// + + TRI_vocbase_col_t* releaseCollection (TRI_voc_cid_t); + //////////////////////////////////////////////////////////////////////////////// /// @brief gets a collection (and inserts it into the cache if not in it) //////////////////////////////////////////////////////////////////////////////// @@ -274,7 +300,7 @@ namespace triagens { std::unordered_set remoteTransactionDatabases; std::unordered_set droppedCollections; std::unordered_set droppedDatabases; - std::unordered_set droppedIndexes; + TRI_voc_tick_t lastTick; std::vector logfilesToProcess; std::unordered_map openedCollections; diff --git a/js/server/tests/recovery/collections-different-attributes.js b/js/server/tests/recovery/collections-different-attributes.js new file mode 100644 index 0000000000..f9135bb734 --- /dev/null +++ b/js/server/tests/recovery/collections-different-attributes.js @@ -0,0 +1,79 @@ + +var db = require("org/arangodb").db; +var internal = require("internal"); +var jsunity = require("jsunity"); + + +function runSetup () { + internal.debugClearFailAt(); + + var c, i; + db._drop("UnitTestsRecovery"); + c = db._create("UnitTestsRecovery", { "id" : 9999990 }); + for (i = 0; i < 10; ++i) { + c.save({ _key: "test" + i, value1: i, value2: "test", value3: [ "foo", i ] }); + } + + // drop the collection + c.drop(); + internal.wait(5); + + c = db._create("UnitTestsRecovery", { "id" : 9999990 }); + for (i = 0; i < 10; ++i) { + c.save({ _key: "test" + i, value3: i, value1: "test", value2: [ "foo", i ] }); + } + + db._drop("test"); + c = db._create("test"); + c.save({ _key: "crashme" }, true); + + internal.debugSegfault("crashing server"); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite +//////////////////////////////////////////////////////////////////////////////// + +function recoverySuite () { + jsunity.jsUnity.attachAssertions(); + + return { + setUp: function () { + }, + tearDown: function () { + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test whether we can restore the trx data +//////////////////////////////////////////////////////////////////////////////// + + testCollectionsDifferentAttributes : function () { + var c, i, doc; + c = db._collection("UnitTestsRecovery"); + + for (i = 0; i < 10; ++i) { + doc = c.document("test" + i); + assertEqual(i, doc.value3); + assertEqual("test", doc.value1); + assertEqual([ "foo", i ], doc.value2); + } + } + + }; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief executes the test suite +//////////////////////////////////////////////////////////////////////////////// + +function main (argv) { + if (argv[1] === "setup") { + runSetup(); + return 0; + } + else { + jsunity.run(recoverySuite); + return jsunity.done() ? 0 : 1; + } +} + diff --git a/js/server/tests/recovery/collections-reuse.js b/js/server/tests/recovery/collections-reuse.js new file mode 100644 index 0000000000..c627abf4ba --- /dev/null +++ b/js/server/tests/recovery/collections-reuse.js @@ -0,0 +1,104 @@ + +var db = require("org/arangodb").db; +var internal = require("internal"); +var jsunity = require("jsunity"); + + +function runSetup () { + internal.debugClearFailAt(); + + var c, i, j; + for (i = 0 ; i < 10; ++i) { + db._drop("UnitTestsRecovery" + i); + // use fake collection ids + c = db._create("UnitTestsRecovery" + i, { "id" : 9999990 + i }); + for (j = 0; j < 10; ++j) { + c.save({ _key: "test" + j, value: j }); + } + } + + // drop 'em all + for (i = 0; i < 10; ++i) { + db._drop("UnitTestsRecovery" + i); + } + internal.wait(5); + + for (i = 0; i < 10; ++i) { + c = db._create("UnitTestsRecoveryX" + i, { "id" : 9999990 + i }); + for (j = 0; j < 10; ++j) { + c.save({ _key: "test" + j, value: "X" + j }); + } + } + + // drop 'em all + for (i = 0; i < 10; ++i) { + db._drop("UnitTestsRecoveryX" + i); + } + internal.wait(5); + + for (i = 0; i < 10; ++i) { + c = db._create("UnitTestsRecoveryY" + i, { "id" : 9999990 + i }); + for (j = 0; j < 10; ++j) { + c.save({ _key: "test" + j, value: "peterY" + j }); + } + } + + db._drop("test"); + c = db._create("test"); + c.save({ _key: "crashme" }, true); + + internal.debugSegfault("crashing server"); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test suite +//////////////////////////////////////////////////////////////////////////////// + +function recoverySuite () { + jsunity.jsUnity.attachAssertions(); + + return { + setUp: function () { + }, + tearDown: function () { + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test whether we can restore the trx data +//////////////////////////////////////////////////////////////////////////////// + + testCollectionsReuse : function () { + var c, i, j, doc; + + for (i = 0; i < 10; ++i) { + assertNull(db._collection("UnitTestsRecovery" + i)); + assertNull(db._collection("UnitTestsRecoverX" + i)); + + c = db._collection("UnitTestsRecoveryY" + i); + assertEqual(10, c.count()); + + for (j = 0; j < 10; ++j) { + doc = c.document("test" + j); + assertEqual("peterY" + j, doc.value); + } + } + } + + }; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief executes the test suite +//////////////////////////////////////////////////////////////////////////////// + +function main (argv) { + if (argv[1] === "setup") { + runSetup(); + return 0; + } + else { + jsunity.run(recoverySuite); + return jsunity.done() ? 0 : 1; + } +} + diff --git a/js/server/tests/recovery/create-databases.js b/js/server/tests/recovery/create-databases.js index 8f10cafdda..6138e12c18 100644 --- a/js/server/tests/recovery/create-databases.js +++ b/js/server/tests/recovery/create-databases.js @@ -7,30 +7,18 @@ var jsunity = require("jsunity"); function runSetup () { internal.debugClearFailAt(); - var i, j, k, c; - for (i = 0; i < 5; ++i) { - db._useDatabase("_system"); + var i, j, c; + db._drop("footest"); + c = db._create("footest", { id: 99999999 }); + c.save({ foo: "bar" }); + c.save({ foo: "bart" }); + db._drop("footest"); - try { - db._dropDatabase("UnitTestsRecovery" + i); - } - catch (err) { - // ignore this error - } + internal.wait(2); - db._createDatabase("UnitTestsRecovery" + i); - db._useDatabase("UnitTestsRecovery" + i); + c = db._create("footest", { id: 99999999 }); + c.save({ foo: "baz" }); - for (j = 0; j < 10; ++j) { - c = db._create("test" + j); - - for (k = 0; k < 10; ++k) { - c.save({ _key: i + "-" + j + "-" + k, value1: i, value2: j, value3: k }); - } - } - } - - db._useDatabase("_system"); db._drop("test"); c = db._create("test"); c.save({ _key: "crashme" }, true); @@ -55,23 +43,10 @@ function recoverySuite () { /// @brief test whether we can restore the trx data //////////////////////////////////////////////////////////////////////////////// - testCreateDatabases : function () { - var i, j, k, c, doc; - for (i = 0; i < 5; ++i) { - db._useDatabase("UnitTestsRecovery" + i); - - for (j = 0; j < 10; ++j) { - c = db._collection("test" + j); - assertEqual(10, c.count()); - - for (k = 0; k < 10; ++k) { - doc = c.document(i + "-" + j + "-" + k); - assertEqual(i, doc.value1); - assertEqual(j, doc.value2); - assertEqual(k, doc.value3); - } - } - } + testCreateDatabase : function () { + var c = db._collection("footest"); + assertEqual(1, c.count()); + assertEqual("baz", c.toArray()[0].foo); } }; diff --git a/js/server/tests/shell-wal-noncluster.js b/js/server/tests/shell-wal-noncluster.js index e2b2ba7e40..9818a043b0 100644 --- a/js/server/tests/shell-wal-noncluster.js +++ b/js/server/tests/shell-wal-noncluster.js @@ -131,7 +131,8 @@ function walFailureSuite () { db._drop(cn); c = db._create(cn); - internal.wal.flush(true, true); + internal.wal.flush(true, false); + internal.wait(5); internal.debugSetFailAt("CollectorThreadProcessQueuedOperations"); internal.wal.properties({ throttleWait: 1000, throttleWhenPending: 1000 }); @@ -149,7 +150,7 @@ function walFailureSuite () { internal.debugClearFailAt(); }, - +/* //////////////////////////////////////////////////////////////////////////////// /// @brief test write throttling //////////////////////////////////////////////////////////////////////////////// @@ -160,7 +161,7 @@ function walFailureSuite () { db._drop(cn); c = db._create(cn); - internal.wal.flush(true, true); + internal.wal.flush(true, false); internal.debugSetFailAt("CollectorThreadProcessQueuedOperations"); internal.wal.properties({ throttleWait: 1000, throttleWhenPending: 1000 }); @@ -171,9 +172,7 @@ function walFailureSuite () { } internal.wal.flush(true, false); - - // let the collector build up its queue - internal.wait(7); + internal.wait(5); try { c.save({ _key: "foo" }); @@ -186,7 +185,7 @@ function walFailureSuite () { internal.debugClearFailAt(); assertEqual(1005, c.count()); } - +*/ }; }