Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

xxx

This commit is contained in:
Jan Steemann 2014-06-30 19:00:36 +02:00
parent 9d0f781289
commit d55626c038
16 changed files with 711 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
UnitTests/Makefile.unittests
View File

@ -208,6 +208,15 @@ unittests-recovery:
@echo "================================================================================"
@echo
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-indexes"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-indexes"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-collections"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-databases"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-single-collection"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-collections"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-databases"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="collections-reuse"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="collections-different-attributes"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes-hash"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes-skiplist"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes-geo"
@ -215,13 +224,6 @@ unittests-recovery:
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="edges"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="cap-constraint"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="indexes"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-indexes"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-collections"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="create-databases"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-single-collection"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-collections"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-databases"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="drop-indexes"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="many-inserts"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="many-updates"
$(MAKE) execute-recovery-test PID=$(PID) RECOVERY_SCRIPT="wait-for-sync"

20
arangod/V8Server/v8-vocbase.cpp
View File

@ -3203,6 +3203,7 @@ static v8::Handle<v8::Value> CreateVocBase (v8::Arguments const& argv,
// extract the parameters
TRI_col_info_t parameter;
TRI_voc_cid_t cid = 0;
if (2 <= argv.Length()) {
if (! argv[1]->IsObject()) {
@ -3270,10 +3271,15 @@ static v8::Handle<v8::Value> CreateVocBase (v8::Arguments const& argv,
TRI_FreeCollectionInfoOptions(&parameter);
TRI_V8_EXCEPTION_PARAMETER(scope, "volatile collections do not support the waitForSync option");
}
if (p->Has(v8g->IdKey)) {
// specify collection id - used for testing only
cid = TRI_ObjectToUInt64(p->Get(v8g->IdKey), true);
}
}
else {
TRI_InitCollectionInfo(vocbase, &parameter, name.c_str(), collectionType, effectiveSize, 0);
TRI_InitCollectionInfo(vocbase, &parameter, name.c_str(), collectionType, effectiveSize, nullptr);
}
@ -3286,7 +3292,7 @@ static v8::Handle<v8::Value> CreateVocBase (v8::Arguments const& argv,
TRI_vocbase_col_t const* collection = TRI_CreateCollectionVocBase(vocbase,
&parameter,
0,
cid,
true);
TRI_FreeCollectionInfoOptions(&parameter);
@ -9088,7 +9094,8 @@ static v8::Handle<v8::Value> JS_CreateDatabase (v8::Arguments const& argv) {
}
TRI_v8_global_t* v8g = (TRI_v8_global_t*) v8::Isolate::GetCurrent()->GetData();
TRI_v8_global_t* v8g = static_cast<TRI_v8_global_t*>(v8::Isolate::GetCurrent()->GetData());
TRI_voc_tick_t id = 0;
// get database defaults from server
TRI_vocbase_defaults_t defaults;
@ -9128,12 +9135,17 @@ static v8::Handle<v8::Value> JS_CreateDatabase (v8::Arguments const& argv) {
if (options->Has(keyAuthenticateSystemOnly)) {
defaults.authenticateSystemOnly = options->Get(keyAuthenticateSystemOnly)->BooleanValue();
}
if (options->Has(v8g->IdKey)) {
// only used for testing to create database with a specific id
id = TRI_ObjectToUInt64(options->Get(v8g->IdKey), true);
}
}
string const name = TRI_ObjectToString(argv[0]);
TRI_vocbase_t* database;
int res = TRI_CreateDatabaseServer(static_cast<TRI_server_t*>(v8g->_server), name.c_str(), &defaults, &database, true);
int res = TRI_CreateDatabaseServer(static_cast<TRI_server_t*>(v8g->_server), id, name.c_str(), &defaults, &database, true);
if (res != TRI_ERROR_NO_ERROR) {
TRI_V8_EXCEPTION(scope, res);

10
arangod/VocBase/cleanup.cpp
View File

@ -167,18 +167,14 @@ static void CleanupDocumentCollection (TRI_vocbase_col_t* collection,
// execute callback, sone of the callbacks might delete or free our collection
if (element->_type == TRI_BARRIER_DATAFILE_DROP_CALLBACK) {
TRI_barrier_datafile_drop_cb_t* de;
de = (TRI_barrier_datafile_drop_cb_t*) element;
TRI_barrier_datafile_drop_cb_t* de = (TRI_barrier_datafile_drop_cb_t*) element;
de->callback(de->_datafile, de->_data);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, element);
// next iteration
}
else if (element->_type == TRI_BARRIER_DATAFILE_RENAME_CALLBACK) {
TRI_barrier_datafile_rename_cb_t* de;
de = (TRI_barrier_datafile_rename_cb_t*) element;
TRI_barrier_datafile_rename_cb_t* de = (TRI_barrier_datafile_rename_cb_t*) element;
de->callback(de->_datafile, de->_data);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, element);
@ -187,6 +183,7 @@ static void CleanupDocumentCollection (TRI_vocbase_col_t* collection,
else if (element->_type == TRI_BARRIER_COLLECTION_UNLOAD_CALLBACK) {
// collection is unloaded
TRI_barrier_collection_cb_t* ce = (TRI_barrier_collection_cb_t*) element;
bool hasUnloaded = ce->callback(ce->_collection, ce->_data);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, element);
@ -198,6 +195,7 @@ static void CleanupDocumentCollection (TRI_vocbase_col_t* collection,
else if (element->_type == TRI_BARRIER_COLLECTION_DROP_CALLBACK) {
// collection is dropped
TRI_barrier_collection_cb_t* ce = (TRI_barrier_collection_cb_t*) element;
bool hasUnloaded = ce->callback(ce->_collection, ce->_data);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, element);

21
arangod/VocBase/collection.cpp
View File

@ -1477,6 +1477,27 @@ bool TRI_IterateCollection (TRI_collection_t* collection,
return result;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief removes an index file from the indexFiles vector
////////////////////////////////////////////////////////////////////////////////
int TRI_RemoveFileIndexCollection (TRI_collection_t* collection,
TRI_idx_iid_t iid) {
size_t const n = collection->_indexFiles._length;
for (size_t i = 0; i < n; ++i) {
char const* filename = collection->_indexFiles._buffer[i];
if (GetNumericFilenamePart(filename) == iid) {
// found
TRI_RemoveVectorString(&collection->_indexFiles, i);
return true;
}
}
return false;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief iterates over all index files of a collection
////////////////////////////////////////////////////////////////////////////////

7
arangod/VocBase/collection.h
View File

@ -352,6 +352,13 @@ int TRI_RenameCollection (TRI_collection_t*, char const*);
// --SECTION-- protected functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief removes an index file from the indexFiles vector
////////////////////////////////////////////////////////////////////////////////
int TRI_RemoveFileIndexCollection (TRI_collection_t*,
TRI_idx_iid_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief iterates over a collection
////////////////////////////////////////////////////////////////////////////////

98
arangod/VocBase/server.cpp
View File

@ -1074,8 +1074,7 @@ static int SaveDatabaseParameters (TRI_voc_tick_t id,
TRI_FreeString(TRI_CORE_MEM_ZONE, tickString);
if (! TRI_SaveJson(file, json, false)) {
LOG_ERROR("cannot save database information in file '%s'",
file);
LOG_ERROR("cannot save database information in file '%s'", file);
TRI_FreeJson(TRI_CORE_MEM_ZONE, json);
TRI_FreeString(TRI_CORE_MEM_ZONE, file);
@ -1415,7 +1414,7 @@ static void DatabaseManager (void* data) {
}
// found a database to delete
database = (TRI_vocbase_t*) TRI_RemoveVectorPointer(&server->_droppedDatabases, i);
database = static_cast<TRI_vocbase_t*>(TRI_RemoveVectorPointer(&server->_droppedDatabases, i));
break;
}
}
@ -2062,6 +2061,7 @@ int TRI_CreateCoordinatorDatabaseServer (TRI_server_t* server,
////////////////////////////////////////////////////////////////////////////////
int TRI_CreateDatabaseServer (TRI_server_t* server,
TRI_voc_tick_t databaseId,
char const* name,
TRI_vocbase_defaults_t const* defaults,
TRI_vocbase_t** database,
@ -2096,11 +2096,14 @@ int TRI_CreateDatabaseServer (TRI_server_t* server,
return TRI_ERROR_OUT_OF_MEMORY;
}
// create the database directory
char* file;
TRI_voc_tick_t tick = TRI_NewTickServer();
int res = CreateDatabaseDirectory(server, tick, name, defaults, &file);
if (databaseId == 0) {
databaseId = TRI_NewTickServer();
}
int res = CreateDatabaseDirectory(server, databaseId, name, defaults, &file);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
@ -2116,7 +2119,7 @@ int TRI_CreateDatabaseServer (TRI_server_t* server,
name,
path);
TRI_vocbase_t* vocbase = TRI_OpenVocBase(server, path, tick, name, defaults, false, false);
TRI_vocbase_t* vocbase = TRI_OpenVocBase(server, path, databaseId, name, defaults, false, false);
TRI_FreeString(TRI_CORE_MEM_ZONE, path);
if (vocbase == nullptr) {
@ -2139,34 +2142,41 @@ int TRI_CreateDatabaseServer (TRI_server_t* server,
}
TRI_ASSERT(vocbase != nullptr);
char* tickString = TRI_StringUInt64(databaseId);
TRI_Insert3ArrayJson(TRI_UNKNOWN_MEM_ZONE, json, "id", TRI_CreateStringCopyJson(TRI_UNKNOWN_MEM_ZONE, tickString));
TRI_FreeString(TRI_CORE_MEM_ZONE, tickString);
TRI_Insert3ArrayJson(TRI_UNKNOWN_MEM_ZONE, json, "name", TRI_CreateStringCopyJson(TRI_UNKNOWN_MEM_ZONE, name));
// create application directories
CreateApplicationDirectory(vocbase->_name, server->_appPath);
CreateApplicationDirectory(vocbase->_name, server->_devAppPath);
TRI_ReloadAuthInfo(vocbase);
TRI_StartCompactorVocBase(vocbase);
if (! triagens::wal::LogfileManager::instance()->isInRecovery()) {
TRI_ReloadAuthInfo(vocbase);
TRI_StartCompactorVocBase(vocbase);
// start the replication applier
if (vocbase->_replicationApplier->_configuration._autoStart) {
if (server->_disableReplicationAppliers) {
LOG_INFO("replication applier explicitly deactivated for database '%s'", name);
}
else {
res = TRI_StartReplicationApplier(vocbase->_replicationApplier, 0, false);
// start the replication applier
if (vocbase->_replicationApplier->_configuration._autoStart) {
if (server->_disableReplicationAppliers) {
LOG_INFO("replication applier explicitly deactivated for database '%s'", name);
}
else {
res = TRI_StartReplicationApplier(vocbase->_replicationApplier, 0, false);
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("unable to start replication applier for database '%s': %s",
name,
TRI_errno_string(res));
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("unable to start replication applier for database '%s': %s",
name,
TRI_errno_string(res));
}
}
}
// increase reference counter
TRI_UseVocBase(vocbase);
}
// increase reference counter
TRI_UseVocBase(vocbase);
{
DatabaseWriteLocker locker(&server->_databasesLock);
TRI_InsertKeyAssociativePointer(&server->_databases, vocbase->_name, vocbase, false);
@ -2459,6 +2469,46 @@ TRI_vocbase_t* TRI_UseDatabaseServer (TRI_server_t* server,
return vocbase;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup a database by its id
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_t* TRI_LookupDatabaseByIdServer (TRI_server_t* server,
TRI_voc_tick_t id) {
DatabaseReadLocker locker(&server->_databasesLock);
size_t const n = server->_databases._nrAlloc;
for (size_t i = 0; i < n; ++i) {
TRI_vocbase_t* vocbase = static_cast<TRI_vocbase_t*>(server->_databases._table[i]);
if (vocbase != nullptr && vocbase->_id == id) {
return vocbase;
}
}
return nullptr;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup a database by its name
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_t* TRI_LookupDatabaseByNameServer (TRI_server_t* server,
char const* name) {
DatabaseReadLocker locker(&server->_databasesLock);
size_t const n = server->_databases._nrAlloc;
for (size_t i = 0; i < n; ++i) {
TRI_vocbase_t* vocbase = static_cast<TRI_vocbase_t*>(server->_databases._table[i]);
if (vocbase != nullptr && TRI_EqualString(vocbase->_name, name)) {
return vocbase;
}
}
return nullptr;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get a database by its id
/// this will increase the reference-counter for the database

15
arangod/VocBase/server.h
View File

@ -175,6 +175,7 @@ int TRI_CreateCoordinatorDatabaseServer (TRI_server_t*,
////////////////////////////////////////////////////////////////////////////////
int TRI_CreateDatabaseServer (TRI_server_t*,
TRI_voc_tick_t,
char const*,
TRI_vocbase_defaults_t const*,
struct TRI_vocbase_s**,
@ -242,6 +243,20 @@ struct TRI_vocbase_s* TRI_UseCoordinatorDatabaseServer (TRI_server_t*,
struct TRI_vocbase_s* TRI_UseDatabaseServer (TRI_server_t*,
char const*);
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup a database by its id
////////////////////////////////////////////////////////////////////////////////
struct TRI_vocbase_s* TRI_LookupDatabaseByIdServer (TRI_server_t*,
TRI_voc_tick_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup a database by its name
////////////////////////////////////////////////////////////////////////////////
struct TRI_vocbase_s* TRI_LookupDatabaseByNameServer (TRI_server_t*,
char const*);
////////////////////////////////////////////////////////////////////////////////
/// @brief use a database by its id
/// this will increase the reference-counter for the database

47
arangod/VocBase/vocbase.cpp
View File

@ -591,8 +591,6 @@ static TRI_vocbase_col_t* CreateCollection (TRI_vocbase_t* vocbase,
if (found != nullptr) {
TRI_WRITE_UNLOCK_COLLECTIONS_VOCBASE(vocbase);
LOG_DEBUG("collection named '%s' already exists", name);
TRI_set_errno(TRI_ERROR_ARANGO_DUPLICATE_NAME);
return nullptr;
}
@ -1501,7 +1499,9 @@ TRI_vocbase_t* TRI_OpenVocBase (TRI_server_t* server,
void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) {
// stop replication
TRI_StopReplicationApplier(vocbase->_replicationApplier, false);
if (vocbase->_replicationApplier != nullptr) {
TRI_StopReplicationApplier(vocbase->_replicationApplier, false);
}
TRI_vector_pointer_t collections;
TRI_InitVectorPointer(&collections, TRI_UNKNOWN_MEM_ZONE);
@ -1532,11 +1532,12 @@ void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) {
TRI_SignalCondition(&vocbase->_compactorCondition);
TRI_UnlockCondition(&vocbase->_compactorCondition);
TRI_ASSERT(vocbase->_hasCompactor);
int res = TRI_JoinThread(&vocbase->_compactor);
if (vocbase->_hasCompactor) {
int res = TRI_JoinThread(&vocbase->_compactor);
if (res != TRI_ERROR_NO_ERROR) {
LOG_ERROR("unable to join compactor thread: %s", TRI_errno_string(res));
if (res != TRI_ERROR_NO_ERROR) {
LOG_ERROR("unable to join compactor thread: %s", TRI_errno_string(res));
}
}
// this will signal the cleanup thread to do one last iteration
@ -1546,7 +1547,7 @@ void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) {
TRI_SignalCondition(&vocbase->_cleanupCondition);
TRI_UnlockCondition(&vocbase->_cleanupCondition);
res = TRI_JoinThread(&vocbase->_cleanup);
int res = TRI_JoinThread(&vocbase->_cleanup);
if (res != TRI_ERROR_NO_ERROR) {
LOG_ERROR("unable to join cleanup thread: %s", TRI_errno_string(res));
@ -1987,7 +1988,7 @@ int TRI_DropCollectionVocBase (TRI_vocbase_t* vocbase,
TRI_ASSERT(collection != nullptr);
if (! collection->_canDrop) {
if (! collection->_canDrop && ! triagens::wal::LogfileManager::instance()->isInRecovery()) {
return TRI_set_errno(TRI_ERROR_FORBIDDEN);
}
@ -2062,7 +2063,7 @@ int TRI_DropCollectionVocBase (TRI_vocbase_t* vocbase,
TRI_WRITE_UNLOCK_STATUS_VOCBASE_COL(collection);
DropCollectionCallback(0, collection);
DropCollectionCallback(nullptr, collection);
TRI_ReadUnlockReadWriteLock(&vocbase->_inventoryLock);
@ -2121,17 +2122,22 @@ int TRI_DropCollectionVocBase (TRI_vocbase_t* vocbase,
TRI_WRITE_UNLOCK_STATUS_VOCBASE_COL(collection);
TRI_ReadUnlockReadWriteLock(&vocbase->_inventoryLock);
if (triagens::wal::LogfileManager::instance()->isInRecovery()) {
DropCollectionCallback(nullptr, collection);
}
else {
// added callback for dropping
TRI_CreateBarrierDropCollection(&collection->_collection->_barrierList,
collection->_collection,
DropCollectionCallback,
collection);
// added callback for dropping
TRI_CreateBarrierDropCollection(&collection->_collection->_barrierList,
collection->_collection,
DropCollectionCallback,
collection);
// wake up the cleanup thread
TRI_LockCondition(&vocbase->_cleanupCondition);
TRI_SignalCondition(&vocbase->_cleanupCondition);
TRI_UnlockCondition(&vocbase->_cleanupCondition);
// wake up the cleanup thread
TRI_LockCondition(&vocbase->_cleanupCondition);
TRI_SignalCondition(&vocbase->_cleanupCondition);
TRI_UnlockCondition(&vocbase->_cleanupCondition);
}
return TRI_ERROR_NO_ERROR;
}
@ -2347,6 +2353,7 @@ bool TRI_DropVocBase (TRI_vocbase_t* vocbase) {
else {
vocbase->_usage._isDeleted = true;
result = true;
}
TRI_UnlockSpin(&vocbase->_usage._lock);

4
arangod/Wal/LogfileManager.cpp
View File

@ -1457,8 +1457,6 @@ int LogfileManager::readShutdownInfo () {
// read if of last sealed logfile (maybe 0)
uint64_t lastSealedId = basics::JsonHelper::stringUInt64(json, "lastSealed");
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
if (lastSealedId < lastCollectedId) {
// should not happen normally
lastSealedId = lastCollectedId;
@ -1471,6 +1469,8 @@ int LogfileManager::readShutdownInfo () {
else {
LOG_TRACE("previous shutdown was at '%s'", shutdownTime.c_str());
}
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
{
WRITE_LOCKER(_logfilesLock);

1
arangod/Wal/LogfileManager.h
View File

@ -339,6 +339,7 @@ namespace triagens {
inline void throttleWhenPending (uint64_t value) {
_throttleWhenPending = value;
if (_throttleWhenPending == 0) {
deactivateWriteThrottling();
}

422
arangod/Wal/RecoverState.cpp
View File

@ -29,6 +29,8 @@
#include "RecoverState.h"
#include "Basics/FileUtils.h"
#include "BasicsC/conversions.h"
#include "BasicsC/files.h"
#include "VocBase/collection.h"
#include "VocBase/replication-applier.h"
#include "VocBase/voc-shaper.h"
@ -42,10 +44,81 @@ using namespace triagens::wal;
// --SECTION-- helper functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a collection is volatile
////////////////////////////////////////////////////////////////////////////////
static inline bool IsVolatile (TRI_transaction_collection_t const* trxCollection) {
return trxCollection->_collection->_collection->_info._isVolatile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the directory for a database
////////////////////////////////////////////////////////////////////////////////
static std::string GetDatabaseDirectory (TRI_server_t* server,
TRI_voc_tick_t databaseId) {
char* idString = TRI_StringUInt64(databaseId);
char* dname = TRI_Concatenate2String("database-", idString);
TRI_FreeString(TRI_CORE_MEM_ZONE, idString);
char* filename = TRI_Concatenate2File(server->_databasePath, dname);
TRI_FreeString(TRI_CORE_MEM_ZONE, dname);
std::string result(filename);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
return result;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the directory for a collection
////////////////////////////////////////////////////////////////////////////////
static std::string GetCollectionDirectory (TRI_vocbase_t* vocbase,
TRI_voc_cid_t collectionId) {
char* dirname = TRI_GetDirectoryCollection(vocbase->_path,
"empty", // does not matter
TRI_COL_TYPE_DOCUMENT, // does not matter
collectionId);
std::string result(dirname);
TRI_FreeString(TRI_CORE_MEM_ZONE, dirname);
return result;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief wait until a database directory disappears
////////////////////////////////////////////////////////////////////////////////
static int WaitForDeletion (TRI_server_t* server,
TRI_voc_tick_t databaseId) {
std::string result = GetDatabaseDirectory(server, databaseId);
int iterations = 0;
while (TRI_IsDirectory(result.c_str()) && iterations++ < 60 * 100) {
usleep(10000);
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief wait until a collection directory disappears
////////////////////////////////////////////////////////////////////////////////
static int WaitForDeletion (TRI_vocbase_t* vocbase,
TRI_voc_cid_t collectionId) {
std::string result = GetCollectionDirectory(vocbase, collectionId);
int iterations = 0;
while (TRI_IsDirectory(result.c_str()) && iterations++ < 60 * 100) {
usleep(10000);
}
return TRI_ERROR_NO_ERROR;
}
// -----------------------------------------------------------------------------
// --SECTION-- constructors / destructors
// -----------------------------------------------------------------------------
@ -61,9 +134,6 @@ RecoverState::RecoverState (TRI_server_t* server,
remoteTransactions(),
remoteTransactionCollections(),
remoteTransactionDatabases(),
droppedCollections(),
droppedDatabases(),
droppedIndexes(),
lastTick(0),
logfilesToProcess(),
openedCollections(),
@ -137,33 +207,6 @@ void RecoverState::releaseResources () {
openedDatabases.clear();
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks if a database is dropped already
////////////////////////////////////////////////////////////////////////////////
bool RecoverState::isDropped (TRI_voc_tick_t databaseId) const {
return (droppedDatabases.find(databaseId) != droppedDatabases.end());
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks if a database or collection is dropped already
////////////////////////////////////////////////////////////////////////////////
bool RecoverState::isDropped (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId) const {
if (isDropped(databaseId)) {
// database has been dropped
return true;
}
if (droppedCollections.find(collectionId) != droppedCollections.end()) {
// collection has been dropped
return true;
}
return false;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief gets a database (and inserts it into the cache if not in it)
////////////////////////////////////////////////////////////////////////////////
@ -180,11 +223,47 @@ TRI_vocbase_t* RecoverState::useDatabase (TRI_voc_tick_t databaseId) {
if (vocbase == nullptr) {
return nullptr;
}
openedDatabases.insert(it, std::make_pair(databaseId, vocbase));
return vocbase;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief release a database (so it can be dropped)
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_t* RecoverState::releaseDatabase (TRI_voc_tick_t databaseId) {
auto it = openedDatabases.find(databaseId);
if (it == openedDatabases.end()) {
return nullptr;
}
TRI_vocbase_t* vocbase = (*it).second;
TRI_ReleaseDatabaseServer(server, vocbase);
openedDatabases.erase(databaseId);
return vocbase;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief release a collection (so it can be dropped)
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_col_t* RecoverState::releaseCollection (TRI_voc_cid_t collectionId) {
auto it = openedCollections.find(collectionId);
if (it == openedCollections.end()) {
return nullptr;
}
TRI_vocbase_col_t* collection = (*it).second;
TRI_ReleaseCollectionVocBase(collection->_vocbase, collection);
openedCollections.erase(collectionId);
return collection;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief gets a collection (and inserts it into the cache if not in it)
////////////////////////////////////////////////////////////////////////////////
@ -454,8 +533,7 @@ bool RecoverState::InitialScanMarker (TRI_df_marker_t const* marker,
state->remoteTransactions.erase(m->_transactionId);
break;
}
// -----------------------------------------------------------------------------
// create markers
// -----------------------------------------------------------------------------
@ -466,46 +544,42 @@ bool RecoverState::InitialScanMarker (TRI_df_marker_t const* marker,
state->droppedCollections.erase(m->_collectionId);
break;
}
case TRI_WAL_MARKER_CREATE_DATABASE: {
database_create_marker_t const* m = reinterpret_cast<database_create_marker_t const*>(marker);
// undo a potential drop marker discovered before for the same database
state->droppedDatabases.erase(m->_databaseId);
break;
}
case TRI_WAL_MARKER_CREATE_INDEX: {
index_create_marker_t const* m = reinterpret_cast<index_create_marker_t const*>(marker);
// undo a potential drop marker discovered before for the same index
state->droppedIndexes.erase(m->_indexId);
// ignored
break;
}
// -----------------------------------------------------------------------------
// drop markers
// -----------------------------------------------------------------------------
case TRI_WAL_MARKER_DROP_COLLECTION: {
collection_drop_marker_t const* m = reinterpret_cast<collection_drop_marker_t const*>(marker);
// note that the collection was dropped and doesn't need to be recovered
state->droppedCollections.insert(m->_collectionId);
break;
}
case TRI_WAL_MARKER_DROP_DATABASE: {
database_drop_marker_t const* m = reinterpret_cast<database_drop_marker_t const*>(marker);
// note that the database was dropped and doesn't need to be recovered
state->droppedDatabases.insert(m->_databaseId);
break;
}
case TRI_WAL_MARKER_DROP_INDEX: {
index_drop_marker_t const* m = reinterpret_cast<index_drop_marker_t const*>(marker);
// note that the index was dropped and doesn't need to be recovered
state->droppedIndexes.insert(m->_indexId);
// ignored
break;
}
}
return true;
@ -551,7 +625,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("could not apply attribute marker: %s", TRI_errno_string(res));
return state->shouldAbort();
return state->canContinue();
}
break;
}
@ -562,7 +636,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
shape_marker_t const* m = reinterpret_cast<shape_marker_t const*>(marker);
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
@ -577,7 +651,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("could not apply shape marker: %s", TRI_errno_string(res));
return state->shouldAbort();
return state->canContinue();
}
break;
}
@ -592,6 +666,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
document_marker_t const* m = reinterpret_cast<document_marker_t const*>(marker);
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
@ -655,7 +730,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
(unsigned long long) collectionId,
(unsigned long long) databaseId,
TRI_errno_string(res));
return state->shouldAbort();
return state->canContinue();
}
break;
}
@ -666,6 +741,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
edge_marker_t const* m = reinterpret_cast<edge_marker_t const*>(marker);
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
@ -734,7 +810,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
(unsigned long long) collectionId,
(unsigned long long) databaseId,
TRI_errno_string(res));
return state->shouldAbort();
return state->canContinue();
}
break;
@ -746,6 +822,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
remove_marker_t const* m = reinterpret_cast<remove_marker_t const*>(marker);
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
@ -797,7 +874,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
(unsigned long long) collectionId,
(unsigned long long) databaseId,
TRI_errno_string(res));
return state->shouldAbort();
return state->canContinue();
}
break;
}
@ -813,6 +890,10 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
TRI_voc_tid_t externalId = m->_externalId;
// start a remote transaction
if (state->isDropped(databaseId)) {
return true;
}
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
if (vocbase == nullptr) {
LOG_WARNING("cannot start remote transaction in database %llu: %s",
@ -824,7 +905,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
if (trx == nullptr) {
LOG_WARNING("unable to start transaction: %s", TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY));
return state->shouldAbort();
return state->canContinue();
}
trx->addHint(TRI_TRANSACTION_HINT_NO_BEGIN_MARKER, true);
@ -834,7 +915,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("unable to start transaction: %s", TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY));
delete trx;
return state->shouldAbort();
return state->canContinue();
}
state->runningRemoteTransactions.insert(std::make_pair(m->_externalId, trx));
@ -851,9 +932,19 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
TRI_idx_iid_t indexId = m->_indexId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
if (vocbase == nullptr) {
LOG_WARNING("cannot create index for collection %llu in database %llu: %s",
(unsigned long long) collectionId,
(unsigned long long) databaseId,
TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND));
return state->canContinue();
}
TRI_document_collection_t* document = state->getCollection(databaseId, collectionId);
if (document == nullptr) {
@ -862,7 +953,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
(unsigned long long) collectionId,
(unsigned long long) databaseId,
TRI_errno_string(TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND));
return state->shouldAbort();
return state->canContinue();
}
char const* properties = reinterpret_cast<char const*>(m) + sizeof(index_create_marker_t);
@ -873,33 +964,35 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
(unsigned long long) indexId,
(unsigned long long) collectionId,
(unsigned long long) databaseId);
return state->shouldAbort();
return state->canContinue();
}
return true;
// TODO: must handle indexes differently
/*
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
TRI_index_t* idx = nullptr;
int res = TRI_FromJsonIndexDocumentCollection(document, json, &idx);
std::string collectionDirectory = GetCollectionDirectory(vocbase, collectionId);
char* idString = TRI_StringUInt64(indexId);
char* indexName = TRI_Concatenate3String("index-", idString, ".json");
TRI_FreeString(TRI_CORE_MEM_ZONE, idString);
char* filename = TRI_Concatenate2File(collectionDirectory.c_str(), indexName);
TRI_FreeString(TRI_CORE_MEM_ZONE, indexName);
bool ok = TRI_SaveJson(filename, json, false);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
if (res == TRI_ERROR_NO_ERROR) {
res = TRI_SaveIndex(document, idx, false);
}
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("cannot create index %llu, collection %llu in database %llu: %s",
if (! ok) {
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
LOG_WARNING("cannot create index %llu, collection %llu in database %llu",
(unsigned long long) indexId,
(unsigned long long) collectionId,
(unsigned long long) databaseId,
TRI_errno_string(res));
return state->shouldAbort();
(unsigned long long) databaseId);
return state->canContinue();
}
else {
TRI_PushBackVectorString(&document->_indexFiles, filename);
}
break;
*/
}
@ -907,20 +1000,31 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
collection_create_marker_t const* m = reinterpret_cast<collection_create_marker_t const*>(marker);
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
if (vocbase == nullptr) {
LOG_WARNING("cannot open database %llu", (unsigned long long) databaseId);
return state->shouldAbort();
// remove the drop marker
state->droppedCollections.erase(collectionId);
if (state->isDropped(databaseId)) {
return true;
}
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
if (vocbase == nullptr) {
LOG_WARNING("cannot open database %llu", (unsigned long long) databaseId);
return state->canContinue();
}
TRI_vocbase_col_t* collection = state->releaseCollection(collectionId);
if (collection == nullptr) {
collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId);
}
TRI_vocbase_col_t* collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId);
if (collection != nullptr) {
// collection already exists - nothing to do
return true;
// drop an existing collection
TRI_DropCollectionVocBase(vocbase, collection, false);
WaitForDeletion(vocbase, collectionId);
}
char const* properties = reinterpret_cast<char const*>(m) + sizeof(collection_create_marker_t);
@ -930,8 +1034,25 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
LOG_WARNING("cannot unpack collection properties for collection %llu in database %llu",
(unsigned long long) collectionId,
(unsigned long long) databaseId);
return state->shouldAbort();
return state->canContinue();
}
// check if there is another collection with the same name as the one that we attempt to create
TRI_json_t const* name = TRI_LookupArrayJson(json, "name");
if (TRI_IsStringJson(name)) {
collection = TRI_LookupCollectionByNameVocBase(vocbase, name->_value._string.data);
if (collection != nullptr && ! TRI_IsSystemNameCollection(name->_value._string.data)) {
// if yes, delete it
TRI_voc_cid_t otherCid = collection->_cid;
state->releaseCollection(otherCid);
TRI_DropCollectionVocBase(vocbase, collection, false);
WaitForDeletion(vocbase, otherCid);
}
}
TRI_col_info_t info;
memset(&info, 0, sizeof(TRI_col_info_t));
@ -942,6 +1063,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
WaitForDeletion(vocbase, collectionId);
collection = TRI_CreateCollectionVocBase(vocbase, &info, collectionId, false);
TRI_FreeCollectionInfoOptions(&info);
@ -950,7 +1072,7 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
LOG_WARNING("cannot create collection %llu in database %llu",
(unsigned long long) collectionId,
(unsigned long long) databaseId);
return state->shouldAbort();
return state->canContinue();
}
break;
}
@ -959,43 +1081,58 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
case TRI_WAL_MARKER_CREATE_DATABASE: {
database_create_marker_t const* m = reinterpret_cast<database_create_marker_t const*>(marker);
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId)) {
return true;
// remove the drop marker
state->droppedDatabases.erase(databaseId);
TRI_vocbase_t* vocbase = state->releaseDatabase(databaseId);
if (vocbase != nullptr) {
// remove already existing database
TRI_DropByIdDatabaseServer(state->server, databaseId, false);
WaitForDeletion(state->server, databaseId);
}
if (TRI_ExistsDatabaseByIdServer(state->server, databaseId)) {
// database already exists - nothing to do
return true;
}
char const* properties = reinterpret_cast<char const*>(m) + sizeof(database_create_marker_t);
TRI_json_t* json = triagens::basics::JsonHelper::fromString(properties);
if (json == nullptr) {
LOG_WARNING("cannot unpack database properties for database %llu", (unsigned long long) databaseId);
return state->shouldAbort();
return state->canContinue();
}
TRI_json_t const* nameValue = TRI_LookupArrayJson(json, "name");
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
if (! TRI_IsStringJson(nameValue)) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
LOG_WARNING("cannot unpack database properties for database %llu", (unsigned long long) databaseId);
return state->shouldAbort();
return state->canContinue();
}
std::string nameString(nameValue->_value._string.data);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
// remove already existing database with same name
vocbase = TRI_LookupDatabaseByNameServer(state->server, nameString.c_str());
if (vocbase != nullptr) {
TRI_voc_tick_t otherId = vocbase->_id;
state->releaseDatabase(otherId);
TRI_DropDatabaseServer(state->server, nameString.c_str(), false);
WaitForDeletion(state->server, otherId);
}
TRI_vocbase_defaults_t defaults;
TRI_GetDatabaseDefaultsServer(state->server, &defaults);
TRI_vocbase_t* vocbase = nullptr;
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
int res = TRI_CreateDatabaseServer(state->server, nameValue->_value._string.data, &defaults, &vocbase, false);
vocbase = nullptr;
int res = TRI_CreateDatabaseServer(state->server, databaseId, nameString.c_str(), &defaults, &vocbase, false);
if (res != TRI_ERROR_NO_ERROR) {
LOG_WARNING("cannot create database %llu: %s", (unsigned long long) databaseId, TRI_errno_string(res));
return state->shouldAbort();
return state->canContinue();
}
break;
}
@ -1010,23 +1147,40 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
TRI_idx_iid_t indexId = m->_indexId;
if (state->isDropped(databaseId, collectionId)) {
return true;
}
if (state->droppedIndexes.find(indexId) == state->droppedIndexes.end()) {
TRI_document_collection_t* document = state->getCollection(databaseId, collectionId);
if (document == nullptr) {
return state->shouldAbort();
}
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
// ignore any potential error returned by this call
TRI_DropIndexDocumentCollection(document, indexId, false);
if (vocbase == nullptr) {
LOG_WARNING("cannot open database %llu", (unsigned long long) databaseId);
return state->canContinue();
}
TRI_document_collection_t* document = state->getCollection(databaseId, collectionId);
if (document == nullptr) {
return state->canContinue();
}
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
// ignore any potential error returned by this call
TRI_DropIndexDocumentCollection(document, indexId, false);
TRI_RemoveFileIndexCollection(document, indexId);
// additionally remove the index file
std::string collectionDirectory = GetCollectionDirectory(vocbase, collectionId);
char* idString = TRI_StringUInt64(indexId);
char* indexName = TRI_Concatenate3String("index-", idString, ".json");
TRI_FreeString(TRI_CORE_MEM_ZONE, idString);
char* filename = TRI_Concatenate2File(collectionDirectory.c_str(), indexName);
TRI_FreeString(TRI_CORE_MEM_ZONE, indexName);
TRI_UnlinkFile(filename);
TRI_FreeString(TRI_CORE_MEM_ZONE, filename);
break;
}
@ -1035,26 +1189,29 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
collection_drop_marker_t const* m = reinterpret_cast<collection_drop_marker_t const*>(marker);
TRI_voc_cid_t collectionId = m->_collectionId;
TRI_voc_tick_t databaseId = m->_databaseId;
if (state->isDropped(databaseId, collectionId)) {
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
if (vocbase == nullptr) {
// database already deleted - do nothing
return true;
}
TRI_vocbase_col_t* collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId);
if (collection == nullptr) {
// collection already deleted - do nothing
return true;
}
// insert the drop marker
state->droppedCollections.insert(collectionId);
TRI_vocbase_t* vocbase = state->useDatabase(databaseId);
if (vocbase == nullptr) {
// database already deleted - do nothing
return true;
}
// ignore any potential error returned by this call
TRI_vocbase_col_t* collection = state->releaseCollection(collectionId);
if (collection == nullptr) {
collection = TRI_LookupCollectionByIdVocBase(vocbase, collectionId);
}
if (collection != nullptr) {
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
// ignore any potential error returned by this call
TRI_DropCollectionVocBase(vocbase, collection, false);
WaitForDeletion(vocbase, collectionId);
}
break;
}
@ -1063,8 +1220,12 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
case TRI_WAL_MARKER_DROP_DATABASE: {
database_drop_marker_t const* m = reinterpret_cast<database_drop_marker_t const*>(marker);
TRI_voc_tick_t databaseId = m->_databaseId;
// insert the drop marker
state->droppedDatabases.insert(databaseId);
TRI_vocbase_t* vocbase = state->releaseDatabase(databaseId);
if (state->isDropped(databaseId)) {
if (vocbase != nullptr) {
// fake transaction to satisfy assertions
triagens::arango::TransactionBase trx(true);
@ -1086,6 +1247,9 @@ bool RecoverState::ReplayMarker (TRI_df_marker_t const* marker,
int RecoverState::replayLogfile (Logfile* logfile) {
LOG_TRACE("replaying logfile '%s'", logfile->filename().c_str());
droppedCollections.clear();
droppedDatabases.clear();
if (! TRI_IterateDatafile(logfile->df(), &RecoverState::ReplayMarker, static_cast<void*>(this))) {
LOG_WARNING("WAL inspection failed when scanning logfile '%s'", logfile->filename().c_str());
return TRI_ERROR_ARANGO_RECOVERY;
@ -1138,11 +1302,6 @@ int RecoverState::abortOpenTransactions () {
TRI_voc_tick_t databaseId = (*it).second.first;
// only write abort markers for databases that haven't been deleted yet
if (isDropped(databaseId)) {
continue;
}
AbortTransactionMarker marker(databaseId, transactionId);
SlotInfoCopy slotInfo = triagens::wal::LogfileManager::instance()->allocateAndWrite(marker.mem(), marker.size(), false);
@ -1198,6 +1357,7 @@ int RecoverState::fillIndexes () {
TRI_vocbase_col_t* collection = (*it).second;
TRI_document_collection_t* document = collection->_collection;
TRI_ASSERT(document != nullptr);
document->useSecondaryIndexes(true);

58
arangod/Wal/RecoverState.h
View File

@ -93,12 +93,39 @@ namespace triagens {
// --SECTION-- public functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief checks if a database is dropped already
////////////////////////////////////////////////////////////////////////////////
bool isDropped (TRI_voc_tick_t databaseId) const {
return (droppedDatabases.find(databaseId) != droppedDatabases.end());
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks if a database or collection is dropped already
////////////////////////////////////////////////////////////////////////////////
bool isDropped (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId) const {
if (isDropped(databaseId)) {
// database has been dropped
return true;
}
if (droppedCollections.find(collectionId) != droppedCollections.end()) {
// collection has been dropped
return true;
}
return false;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not to abort recovery on first error
////////////////////////////////////////////////////////////////////////////////
inline bool shouldAbort () const {
return ! ignoreRecoveryErrors;
inline bool canContinue () const {
return ignoreRecoveryErrors;
}
////////////////////////////////////////////////////////////////////////////////
@ -158,25 +185,24 @@ namespace triagens {
void releaseResources ();
////////////////////////////////////////////////////////////////////////////////
/// @brief checks if a database is dropped already
////////////////////////////////////////////////////////////////////////////////
bool isDropped (TRI_voc_tick_t) const;
////////////////////////////////////////////////////////////////////////////////
/// @brief checks if a database or collection is dropped already
////////////////////////////////////////////////////////////////////////////////
bool isDropped (TRI_voc_tick_t,
TRI_voc_cid_t) const;
////////////////////////////////////////////////////////////////////////////////
/// @brief gets a database (and inserts it into the cache if not in it)
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_t* useDatabase (TRI_voc_tick_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief releases a database
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_t* releaseDatabase (TRI_voc_tick_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief release a collection (so it can be dropped)
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_col_t* releaseCollection (TRI_voc_cid_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief gets a collection (and inserts it into the cache if not in it)
////////////////////////////////////////////////////////////////////////////////
@ -274,7 +300,7 @@ namespace triagens {
std::unordered_set<TRI_voc_tick_t> remoteTransactionDatabases;
std::unordered_set<TRI_voc_cid_t> droppedCollections;
std::unordered_set<TRI_voc_tick_t> droppedDatabases;
std::unordered_set<TRI_idx_iid_t> droppedIndexes;
TRI_voc_tick_t lastTick;
std::vector<Logfile*> logfilesToProcess;
std::unordered_map<TRI_voc_cid_t, TRI_vocbase_col_t*> openedCollections;

79
js/server/tests/recovery/collections-different-attributes.js Normal file
View File

@ -0,0 +1,79 @@
var db = require("org/arangodb").db;
var internal = require("internal");
var jsunity = require("jsunity");
function runSetup () {
internal.debugClearFailAt();
var c, i;
db._drop("UnitTestsRecovery");
c = db._create("UnitTestsRecovery", { "id" : 9999990 });
for (i = 0; i < 10; ++i) {
c.save({ _key: "test" + i, value1: i, value2: "test", value3: [ "foo", i ] });
}
// drop the collection
c.drop();
internal.wait(5);
c = db._create("UnitTestsRecovery", { "id" : 9999990 });
for (i = 0; i < 10; ++i) {
c.save({ _key: "test" + i, value3: i, value1: "test", value2: [ "foo", i ] });
}
db._drop("test");
c = db._create("test");
c.save({ _key: "crashme" }, true);
internal.debugSegfault("crashing server");
}
////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {
},
tearDown: function () {
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test whether we can restore the trx data
////////////////////////////////////////////////////////////////////////////////
testCollectionsDifferentAttributes : function () {
var c, i, doc;
c = db._collection("UnitTestsRecovery");
for (i = 0; i < 10; ++i) {
doc = c.document("test" + i);
assertEqual(i, doc.value3);
assertEqual("test", doc.value1);
assertEqual([ "foo", i ], doc.value2);
}
}
};
}
////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////
function main (argv) {
if (argv[1] === "setup") {
runSetup();
return 0;
}
else {
jsunity.run(recoverySuite);
return jsunity.done() ? 0 : 1;
}
}

104
js/server/tests/recovery/collections-reuse.js Normal file
View File

@ -0,0 +1,104 @@
var db = require("org/arangodb").db;
var internal = require("internal");
var jsunity = require("jsunity");
function runSetup () {
internal.debugClearFailAt();
var c, i, j;
for (i = 0 ; i < 10; ++i) {
db._drop("UnitTestsRecovery" + i);
// use fake collection ids
c = db._create("UnitTestsRecovery" + i, { "id" : 9999990 + i });
for (j = 0; j < 10; ++j) {
c.save({ _key: "test" + j, value: j });
}
}
// drop 'em all
for (i = 0; i < 10; ++i) {
db._drop("UnitTestsRecovery" + i);
}
internal.wait(5);
for (i = 0; i < 10; ++i) {
c = db._create("UnitTestsRecoveryX" + i, { "id" : 9999990 + i });
for (j = 0; j < 10; ++j) {
c.save({ _key: "test" + j, value: "X" + j });
}
}
// drop 'em all
for (i = 0; i < 10; ++i) {
db._drop("UnitTestsRecoveryX" + i);
}
internal.wait(5);
for (i = 0; i < 10; ++i) {
c = db._create("UnitTestsRecoveryY" + i, { "id" : 9999990 + i });
for (j = 0; j < 10; ++j) {
c.save({ _key: "test" + j, value: "peterY" + j });
}
}
db._drop("test");
c = db._create("test");
c.save({ _key: "crashme" }, true);
internal.debugSegfault("crashing server");
}
////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {
},
tearDown: function () {
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test whether we can restore the trx data
////////////////////////////////////////////////////////////////////////////////
testCollectionsReuse : function () {
var c, i, j, doc;
for (i = 0; i < 10; ++i) {
assertNull(db._collection("UnitTestsRecovery" + i));
assertNull(db._collection("UnitTestsRecoverX" + i));
c = db._collection("UnitTestsRecoveryY" + i);
assertEqual(10, c.count());
for (j = 0; j < 10; ++j) {
doc = c.document("test" + j);
assertEqual("peterY" + j, doc.value);
}
}
}
};
}
////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////
function main (argv) {
if (argv[1] === "setup") {
runSetup();
return 0;
}
else {
jsunity.run(recoverySuite);
return jsunity.done() ? 0 : 1;
}
}

51
js/server/tests/recovery/create-databases.js
View File

@ -7,30 +7,18 @@ var jsunity = require("jsunity");
function runSetup () {
internal.debugClearFailAt();
var i, j, k, c;
for (i = 0; i < 5; ++i) {
db._useDatabase("_system");
var i, j, c;
db._drop("footest");
c = db._create("footest", { id: 99999999 });
c.save({ foo: "bar" });
c.save({ foo: "bart" });
db._drop("footest");
try {
db._dropDatabase("UnitTestsRecovery" + i);
}
catch (err) {
// ignore this error
}
internal.wait(2);
db._createDatabase("UnitTestsRecovery" + i);
db._useDatabase("UnitTestsRecovery" + i);
c = db._create("footest", { id: 99999999 });
c.save({ foo: "baz" });
for (j = 0; j < 10; ++j) {
c = db._create("test" + j);
for (k = 0; k < 10; ++k) {
c.save({ _key: i + "-" + j + "-" + k, value1: i, value2: j, value3: k });
}
}
}
db._useDatabase("_system");
db._drop("test");
c = db._create("test");
c.save({ _key: "crashme" }, true);
@ -55,23 +43,10 @@ function recoverySuite () {
/// @brief test whether we can restore the trx data
////////////////////////////////////////////////////////////////////////////////
testCreateDatabases : function () {
var i, j, k, c, doc;
for (i = 0; i < 5; ++i) {
db._useDatabase("UnitTestsRecovery" + i);
for (j = 0; j < 10; ++j) {
c = db._collection("test" + j);
assertEqual(10, c.count());
for (k = 0; k < 10; ++k) {
doc = c.document(i + "-" + j + "-" + k);
assertEqual(i, doc.value1);
assertEqual(j, doc.value2);
assertEqual(k, doc.value3);
}
}
}
testCreateDatabase : function () {
var c = db._collection("footest");
assertEqual(1, c.count());
assertEqual("baz", c.toArray()[0].foo);
}
};

13
js/server/tests/shell-wal-noncluster.js
View File

@ -131,7 +131,8 @@ function walFailureSuite () {
db._drop(cn);
c = db._create(cn);
internal.wal.flush(true, true);
internal.wal.flush(true, false);
internal.wait(5);
internal.debugSetFailAt("CollectorThreadProcessQueuedOperations");
internal.wal.properties({ throttleWait: 1000, throttleWhenPending: 1000 });
@ -149,7 +150,7 @@ function walFailureSuite () {
internal.debugClearFailAt();
},
/*
////////////////////////////////////////////////////////////////////////////////
/// @brief test write throttling
////////////////////////////////////////////////////////////////////////////////
@ -160,7 +161,7 @@ function walFailureSuite () {
db._drop(cn);
c = db._create(cn);
internal.wal.flush(true, true);
internal.wal.flush(true, false);
internal.debugSetFailAt("CollectorThreadProcessQueuedOperations");
internal.wal.properties({ throttleWait: 1000, throttleWhenPending: 1000 });
@ -171,9 +172,7 @@ function walFailureSuite () {
}
internal.wal.flush(true, false);
// let the collector build up its queue
internal.wait(7);
internal.wait(5);
try {
c.save({ _key: "foo" });
@ -186,7 +185,7 @@ function walFailureSuite () {
internal.debugClearFailAt();
assertEqual(1005, c.count());
}
*/
};
}