Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
arangodb/arangod/VocBase/replication-dump.c

1215 lines
36 KiB
C
Raw Blame History

////////////////////////////////////////////////////////////////////////////////
/// @brief replication dump functions
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2004-2013 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "replication-dump.h"
#include "BasicsC/conversions.h"
#include "BasicsC/files.h"
#include "BasicsC/json.h"
#include "BasicsC/logging.h"
#include "BasicsC/string-buffer.h"
#include "BasicsC/tri-strings.h"
#include "VocBase/collection.h"
#include "VocBase/datafile.h"
#include "VocBase/document-collection.h"
#include "VocBase/transaction.h"
#include "VocBase/vocbase.h"
// -----------------------------------------------------------------------------
// --SECTION-- REPLICATION
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- private defines
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @addtogroup VocBase
/// @{
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief shortcut function
////////////////////////////////////////////////////////////////////////////////
#define FAIL_IFNOT(func, buffer, val) \
if (func(buffer, val) != TRI_ERROR_NO_ERROR) { \
return false; \
}
////////////////////////////////////////////////////////////////////////////////
/// @brief create a string-buffer function name
////////////////////////////////////////////////////////////////////////////////
#define APPEND_FUNC(name) TRI_ ## name ## StringBuffer
////////////////////////////////////////////////////////////////////////////////
/// @brief append a character to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_CHAR(buffer, c) FAIL_IFNOT(APPEND_FUNC(AppendChar), buffer, c)
////////////////////////////////////////////////////////////////////////////////
/// @brief append a string to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_STRING(buffer, str) FAIL_IFNOT(APPEND_FUNC(AppendString), buffer, str)
////////////////////////////////////////////////////////////////////////////////
/// @brief append uint64 to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_UINT64(buffer, val) FAIL_IFNOT(APPEND_FUNC(AppendUInt64), buffer, val)
////////////////////////////////////////////////////////////////////////////////
/// @brief append json to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_JSON(buffer, json) FAIL_IFNOT(TRI_StringifyJson, buffer, json)
////////////////////////////////////////////////////////////////////////////////
/// @}
////////////////////////////////////////////////////////////////////////////////
// -----------------------------------------------------------------------------
// --SECTION-- private types
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @addtogroup VocBase
/// @{
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief a datafile descriptor
////////////////////////////////////////////////////////////////////////////////
typedef struct df_entry_s {
TRI_datafile_t* _data;
TRI_voc_tick_t _dataMin;
TRI_voc_tick_t _dataMax;
TRI_voc_tick_t _tickMax;
bool _isJournal;
}
df_entry_t;
////////////////////////////////////////////////////////////////////////////////
/// @brief container for a resolved collection name (cid => name)
////////////////////////////////////////////////////////////////////////////////
typedef struct resolved_name_s {
TRI_voc_cid_t _cid;
char* _name;
}
resolved_name_t;
////////////////////////////////////////////////////////////////////////////////
/// @}
////////////////////////////////////////////////////////////////////////////////
// -----------------------------------------------------------------------------
// --SECTION-- private functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @addtogroup VocBase
/// @{
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief hashes a collection id
////////////////////////////////////////////////////////////////////////////////
static uint64_t HashKeyCid (TRI_associative_pointer_t* array,
void const* key) {
TRI_voc_cid_t const* k = key;
return *k;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief hashes a collection name
////////////////////////////////////////////////////////////////////////////////
static uint64_t HashElementCid (TRI_associative_pointer_t* array,
void const* element) {
resolved_name_t const* e = element;
return e->_cid;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief compares a collection
////////////////////////////////////////////////////////////////////////////////
static bool IsEqualKeyElementCid (TRI_associative_pointer_t* array,
void const* key,
void const* element) {
TRI_voc_cid_t const* k = key;
resolved_name_t const* e = element;
return *k == e->_cid;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup a collection name
////////////////////////////////////////////////////////////////////////////////
static bool LookupCollectionName (TRI_replication_dump_t* dump,
TRI_voc_cid_t cid,
char** result) {
resolved_name_t* found;
assert(cid > 0);
found = (resolved_name_t*) TRI_LookupByKeyAssociativePointer(&dump->_collectionNames, &cid);
if (found == NULL) {
found = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(resolved_name_t), false);
if (found == NULL) {
// out of memory;
return false;
}
found->_cid = cid;
// name can be NULL if collection is not found.
// but we will still cache a NULL result!
found->_name = TRI_GetCollectionNameByIdVocBase(dump->_vocbase, cid);
TRI_InsertKeyAssociativePointer(&dump->_collectionNames, &found->_cid, found, false);
}
*result = found->_name;
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief append a collection name or id to a string buffer
////////////////////////////////////////////////////////////////////////////////
static bool AppendCollection (TRI_replication_dump_t* dump,
TRI_voc_cid_t cid,
bool translateCollectionIds) {
if (translateCollectionIds) {
if (cid > 0) {
char* name;
if (! LookupCollectionName(dump, cid, &name)) {
return false;
}
if (name != NULL) {
APPEND_STRING(dump->_buffer, name);
return true;
}
}
APPEND_STRING(dump->_buffer, "_unknown");
}
else {
APPEND_UINT64(dump->_buffer, (uint64_t) cid);
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief iterate over a vector of datafiles and pick those with a specific
/// data range
////////////////////////////////////////////////////////////////////////////////
static int IterateDatafiles (TRI_vector_pointer_t const* datafiles,
TRI_vector_t* result,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
bool isJournal) {
size_t i, n;
int res;
res = TRI_ERROR_NO_ERROR;
n = datafiles->_length;
for (i = 0; i < n; ++i) {
TRI_datafile_t* df = TRI_AtVectorPointer(datafiles, i);
df_entry_t entry = {
df,
df->_dataMin,
df->_dataMax,
df->_tickMax,
isJournal
};
LOG_TRACE("checking datafile %llu with data range %llu - %llu, tick max: %llu",
(unsigned long long) df->_fid,
(unsigned long long) df->_dataMin,
(unsigned long long) df->_dataMax,
(unsigned long long) df->_tickMax);
if (df->_dataMin == 0 || df->_dataMax == 0) {
// datafile doesn't have any data
continue;
}
assert(df->_tickMin <= df->_tickMax);
assert(df->_dataMin <= df->_dataMax);
if (dataMax < df->_dataMin) {
// datafile is newer than requested range
continue;
}
if (dataMin > df->_dataMax) {
// datafile is older than requested range
continue;
}
res = TRI_PushBackVector(result, &entry);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the datafiles of a collection for a specific tick range
////////////////////////////////////////////////////////////////////////////////
static TRI_vector_t GetRangeDatafiles (TRI_primary_collection_t* primary,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax) {
TRI_vector_t datafiles;
LOG_TRACE("getting datafiles in data range %llu - %llu",
(unsigned long long) dataMin,
(unsigned long long) dataMax);
// determine the datafiles of the collection
TRI_InitVector(&datafiles, TRI_CORE_MEM_ZONE, sizeof(df_entry_t));
TRI_READ_LOCK_DATAFILES_DOC_COLLECTION(primary);
IterateDatafiles(&primary->base._datafiles, &datafiles, dataMin, dataMax, false);
IterateDatafiles(&primary->base._journals, &datafiles, dataMin, dataMax, true);
TRI_READ_UNLOCK_DATAFILES_DOC_COLLECTION(primary);
return datafiles;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a raw marker from a datafile for a collection dump
////////////////////////////////////////////////////////////////////////////////
static bool StringifyMarkerDump (TRI_replication_dump_t* dump,
TRI_document_collection_t* document,
TRI_df_marker_t const* marker,
bool withTicks,
bool translateCollectionIds) {
TRI_string_buffer_t* buffer;
TRI_replication_operation_e type;
TRI_voc_key_t key;
TRI_voc_rid_t rid;
buffer = dump->_buffer;
if (buffer == NULL) {
return false;
}
if (marker->_type == TRI_DOC_MARKER_KEY_DELETION) {
TRI_doc_deletion_key_marker_t const* m = (TRI_doc_deletion_key_marker_t const*) marker;
key = ((char*) m) + m->_offsetKey;
type = MARKER_REMOVE;
rid = m->_rid;
}
else if (marker->_type == TRI_DOC_MARKER_KEY_DOCUMENT) {
TRI_doc_document_key_marker_t const* m = (TRI_doc_document_key_marker_t const*) marker;
key = ((char*) m) + m->_offsetKey;
type = MARKER_DOCUMENT;
rid = m->_rid;
}
else if (marker->_type == TRI_DOC_MARKER_KEY_EDGE) {
TRI_doc_document_key_marker_t const* m = (TRI_doc_document_key_marker_t const*) marker;
key = ((char*) m) + m->_offsetKey;
type = MARKER_EDGE;
rid = m->_rid;
}
else {
return false;
}
if (withTicks) {
APPEND_STRING(buffer, "{\"tick\":\"");
APPEND_UINT64(buffer, (uint64_t) marker->_tick);
APPEND_STRING(buffer, "\",\"type\":");
}
else {
APPEND_STRING(buffer, "{\"type\":");
}
APPEND_UINT64(buffer, (uint64_t) type);
APPEND_STRING(buffer, ",\"key\":\"");
// key is user-defined, but does not need escaping
APPEND_STRING(buffer, key);
APPEND_STRING(buffer, "\",\"rev\":\"");
APPEND_UINT64(buffer, (uint64_t) rid);
// document
if (marker->_type == TRI_DOC_MARKER_KEY_DOCUMENT ||
marker->_type == TRI_DOC_MARKER_KEY_EDGE) {
TRI_doc_document_key_marker_t const* m = (TRI_doc_document_key_marker_t const*) marker;
TRI_shaped_json_t shaped;
APPEND_STRING(buffer, "\",\"data\":{");
// common document meta-data
APPEND_STRING(buffer, "\"" TRI_VOC_ATTRIBUTE_KEY "\":\"");
APPEND_STRING(buffer, key);
APPEND_STRING(buffer, "\",\"" TRI_VOC_ATTRIBUTE_REV "\":\"");
APPEND_UINT64(buffer, (uint64_t) rid);
APPEND_CHAR(buffer, '"');
if (marker->_type == TRI_DOC_MARKER_KEY_EDGE) {
TRI_doc_edge_key_marker_t const* e = (TRI_doc_edge_key_marker_t const*) marker;
TRI_voc_key_t fromKey = ((char*) e) + e->_offsetFromKey;
TRI_voc_key_t toKey = ((char*) e) + e->_offsetToKey;
APPEND_STRING(buffer, ",\"" TRI_VOC_ATTRIBUTE_FROM "\":\"");
if (! AppendCollection(dump, e->_fromCid, translateCollectionIds)) {
return false;
}
APPEND_STRING(buffer, "\\/");
APPEND_STRING(buffer, fromKey);
APPEND_STRING(buffer, "\",\"" TRI_VOC_ATTRIBUTE_TO "\":\"");
if (! AppendCollection(dump, e->_toCid, translateCollectionIds)) {
return false;
}
APPEND_STRING(buffer, "\\/");
APPEND_STRING(buffer, toKey);
APPEND_CHAR(buffer, '"');
}
// the actual document data
TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, m);
TRI_StringifyArrayShapedJson(document->base._shaper, buffer, &shaped, true);
APPEND_STRING(buffer, "}}\n");
}
else {
APPEND_STRING(buffer, "\"}\n");
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief iterate over the attributes of a replication log marker (shaped json)
////////////////////////////////////////////////////////////////////////////////
static bool IterateShape (TRI_shaper_t* shaper,
TRI_shape_t const* shape,
char const* name,
char const* data,
uint64_t size,
void* ptr) {
bool append = false;
bool withName = false;
if (TRI_EqualString(name, "data")) {
append = true;
withName = false;
}
else if (TRI_EqualString(name, "type") ||
TRI_EqualString(name, "tid")) {
append = true;
withName = true;
}
if (append) {
TRI_replication_dump_t* dump;
TRI_string_buffer_t* buffer;
int res;
dump = (TRI_replication_dump_t*) ptr;
buffer = dump->_buffer;
// append ,
res = TRI_AppendCharStringBuffer(buffer, ',');
if (res != TRI_ERROR_NO_ERROR) {
dump->_failed = true;
return false;
}
if (withName) {
// append attribute name and value
res = TRI_AppendCharStringBuffer(buffer, '"');
if (res != TRI_ERROR_NO_ERROR) {
dump->_failed = true;
return false;
}
res = TRI_AppendStringStringBuffer(buffer, name);
if (res != TRI_ERROR_NO_ERROR) {
dump->_failed = true;
return false;
}
res = TRI_AppendStringStringBuffer(buffer, "\":");
if (shape->_type == TRI_SHAPE_NUMBER) {
if (! TRI_StringifyJsonShapeData(shaper, buffer, shape, data, size)) {
res = TRI_ERROR_OUT_OF_MEMORY;
}
}
else if (shape->_type == TRI_SHAPE_SHORT_STRING ||
shape->_type == TRI_SHAPE_LONG_STRING) {
char* value;
size_t length;
res = TRI_AppendCharStringBuffer(buffer, '"');
if (res != TRI_ERROR_NO_ERROR) {
dump->_failed = true;
return false;
}
TRI_StringValueShapedJson(shape, data, &value, &length);
if (value != NULL && length > 0) {
res = TRI_AppendString2StringBuffer(dump->_buffer, value, length);
if (res != TRI_ERROR_NO_ERROR) {
dump->_failed = true;
return false;
}
}
res = TRI_AppendCharStringBuffer(buffer, '"');
}
}
else {
// append raw value
char* value;
size_t length;
TRI_StringValueShapedJson(shape, data, &value, &length);
if (value != NULL && length > 2) {
res = TRI_AppendString2StringBuffer(dump->_buffer, value + 1, length - 2);
}
}
if (res != TRI_ERROR_NO_ERROR) {
dump->_failed = true;
return false;
}
}
// continue iterating
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a raw marker from a datafile for a log dump
////////////////////////////////////////////////////////////////////////////////
static bool StringifyMarkerLog (TRI_replication_dump_t* dump,
TRI_document_collection_t* document,
TRI_df_marker_t const* marker) {
TRI_doc_document_key_marker_t const* m = (TRI_doc_document_key_marker_t const*) marker;
TRI_shaper_t* shaper;
TRI_shaped_json_t shaped;
assert(marker->_type == TRI_DOC_MARKER_KEY_DOCUMENT);
shaper = document->base._shaper;
TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, m);
if (shaped._sid != 0) {
TRI_shape_t const* shape;
if (shaped._sid != dump->_lastSid || dump->_lastShape == NULL) {
shape = shaper->lookupShapeId(shaper, shaped._sid);
dump->_lastSid = shaped._sid;
dump->_lastShape = shape;
}
else {
shape = dump->_lastShape;
}
APPEND_STRING(dump->_buffer, "{\"tick\":\"");
APPEND_UINT64(dump->_buffer, (uint64_t) marker->_tick);
APPEND_CHAR(dump->_buffer, '"');
TRI_IterateShapeDataArray(shaper, shape, shaped._data.data, &IterateShape, dump);
APPEND_STRING(dump->_buffer, "}\n");
}
else {
return false;
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief check if a transaction id is contained in the list of failed
/// transactions
////////////////////////////////////////////////////////////////////////////////
static bool InFailedList (TRI_vector_t const* list, TRI_voc_tid_t search) {
size_t n;
assert(list != NULL);
n = list->_length;
// decide how to search based on size of list
if (n == 0) {
// simple case: list is empty
return false;
}
else if (n < 16) {
// list is small: use a linear search
size_t i;
for (i = 0; i < n; ++i) {
TRI_voc_tid_t* tid = TRI_AtVector(list, i);
if (*tid == search) {
return true;
}
}
return false;
}
else {
// list is somewhat bigger, use a binary search
size_t l = 0;
size_t r = (size_t) (n - 1);
while (true) {
// determine midpoint
TRI_voc_tid_t* tid;
size_t m;
m = l + ((r - l) / 2);
tid = TRI_AtVector(list, m);
if (*tid == search) {
return true;
}
if (*tid > search) {
if (m == 0) {
// we must abort because the following subtraction would
// make the size_t underflow
return false;
}
r = m - 1;
}
else {
l = m + 1;
}
if (r < l) {
return false;
}
}
}
// we should never get here
assert(false);
return false;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from a collection
////////////////////////////////////////////////////////////////////////////////
static int DumpCollection (TRI_replication_dump_t* dump,
TRI_primary_collection_t* primary,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
uint64_t chunkSize,
bool withTicks,
bool translateCollectionIds) {
TRI_vector_t datafiles;
TRI_document_collection_t* document;
TRI_string_buffer_t* buffer;
TRI_voc_tick_t lastFoundTick;
TRI_voc_tid_t lastTid;
size_t i, n;
int res;
bool hasMore;
bool bufferFull;
bool ignoreMarkers;
LOG_TRACE("dumping collection %llu, tick range %llu - %llu, chunk size %llu",
(unsigned long long) primary->base._info._cid,
(unsigned long long) dataMin,
(unsigned long long) dataMax,
(unsigned long long) chunkSize);
buffer = dump->_buffer;
datafiles = GetRangeDatafiles(primary, dataMin, dataMax);
document = (TRI_document_collection_t*) primary;
// setup some iteration state
lastFoundTick = 0;
lastTid = 0;
res = TRI_ERROR_NO_ERROR;
hasMore = true;
bufferFull = false;
ignoreMarkers = false;
n = datafiles._length;
for (i = 0; i < n; ++i) {
df_entry_t* e = (df_entry_t*) TRI_AtVector(&datafiles, i);
TRI_datafile_t* datafile = e->_data;
TRI_vector_t* failedList;
char const* ptr;
char const* end;
failedList = NULL;
// we are reading from a journal that might be modified in parallel
// so we must read-lock it
if (e->_isJournal) {
TRI_READ_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
if (document->_failedTransactions._length > 0) {
// there are failed transactions. just reference them
failedList = &document->_failedTransactions;
}
}
else {
assert(datafile->_isSealed);
TRI_READ_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
if (document->_failedTransactions._length > 0) {
// there are failed transactions. copy the list of ids
failedList = TRI_CopyVector(TRI_UNKNOWN_MEM_ZONE, &document->_failedTransactions);
if (failedList == NULL) {
res = TRI_ERROR_OUT_OF_MEMORY;
}
}
TRI_READ_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
}
ptr = datafile->_data;
if (res == TRI_ERROR_NO_ERROR) {
// no error so far. start iterating
end = ptr + datafile->_currentSize;
}
else {
// some error occurred. don't iterate
end = ptr;
}
while (ptr < end) {
TRI_df_marker_t* marker = (TRI_df_marker_t*) ptr;
TRI_voc_tick_t foundTick;
TRI_voc_tid_t tid;
if (marker->_size == 0 || marker->_type <= TRI_MARKER_MIN) {
// end of datafile
break;
}
ptr += TRI_DF_ALIGN_BLOCK(marker->_size);
if (marker->_type == TRI_DF_MARKER_ATTRIBUTE ||
marker->_type == TRI_DF_MARKER_SHAPE) {
// fully ignore these marker types. they don't need to be replicated,
// but we also cannot stop iteration if we find one of these
continue;
}
// get the marker's tick and check whether we should include it
foundTick = marker->_tick;
if (foundTick <= dataMin) {
// marker too old
continue;
}
if (foundTick > dataMax) {
// marker too new
hasMore = false;
goto NEXT_DF;
}
if (marker->_type != TRI_DOC_MARKER_KEY_DOCUMENT &&
marker->_type != TRI_DOC_MARKER_KEY_EDGE &&
marker->_type != TRI_DOC_MARKER_KEY_DELETION) {
// found a non-data marker...
// check if we can abort searching
if (foundTick >= dataMax ||
(foundTick >= e->_tickMax && i == (n - 1))) {
// fetched the last available marker
hasMore = false;
goto NEXT_DF;
}
continue;
}
// note the last tick we processed
lastFoundTick = foundTick;
// handle aborted/unfinished transactions
if (failedList == NULL) {
// there are no failed transactions
ignoreMarkers = false;
}
else {
// get transaction id of marker
if (marker->_type == TRI_DOC_MARKER_KEY_DELETION) {
tid = ((TRI_doc_deletion_key_marker_t const*) marker)->_tid;
}
else {
tid = ((TRI_doc_document_key_marker_t const*) marker)->_tid;
}
// check if marker is from an aborted transaction
if (tid > 0) {
if (tid != lastTid) {
ignoreMarkers = InFailedList(failedList, tid);
}
lastTid = tid;
}
if (ignoreMarkers) {
continue;
}
}
if (! StringifyMarkerDump(dump, document, marker, withTicks, translateCollectionIds)) {
res = TRI_ERROR_INTERNAL;
goto NEXT_DF;
}
if (foundTick >= dataMax ||
(foundTick >= e->_tickMax && i == (n - 1))) {
// fetched the last available marker
hasMore = false;
goto NEXT_DF;
}
if ((uint64_t) TRI_LengthStringBuffer(buffer) > chunkSize) {
// abort the iteration
bufferFull = true;
goto NEXT_DF;
}
}
NEXT_DF:
if (e->_isJournal) {
// read-unlock the journal
TRI_READ_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
}
else {
// free our copy of the failed list
if (failedList != NULL) {
TRI_FreeVector(TRI_UNKNOWN_MEM_ZONE, failedList);
}
}
if (res != TRI_ERROR_NO_ERROR || ! hasMore || bufferFull) {
break;
}
}
TRI_DestroyVector(&datafiles);
if (res == TRI_ERROR_NO_ERROR) {
if (lastFoundTick > 0) {
// data available for requested range
dump->_lastFoundTick = lastFoundTick;
dump->_hasMore = hasMore;
dump->_bufferFull = bufferFull;
}
else {
// no data available for requested range
dump->_lastFoundTick = 0;
dump->_hasMore = false;
dump->_bufferFull = false;
}
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from the replication log
////////////////////////////////////////////////////////////////////////////////
static int DumpLog (TRI_replication_dump_t* dump,
TRI_primary_collection_t* primary,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
uint64_t chunkSize) {
TRI_vector_t datafiles;
TRI_document_collection_t* document;
TRI_string_buffer_t* buffer;
TRI_voc_tick_t lastFoundTick;
size_t i, n;
int res;
bool hasMore;
bool bufferFull;
LOG_TRACE("dumping collection %llu, tick range %llu - %llu, chunk size %llu",
(unsigned long long) primary->base._info._cid,
(unsigned long long) dataMin,
(unsigned long long) dataMax,
(unsigned long long) chunkSize);
buffer = dump->_buffer;
datafiles = GetRangeDatafiles(primary, dataMin, dataMax);
document = (TRI_document_collection_t*) primary;
// setup some iteration state
lastFoundTick = 0;
res = TRI_ERROR_NO_ERROR;
hasMore = true;
bufferFull = false;
n = datafiles._length;
for (i = 0; i < n; ++i) {
df_entry_t* e = (df_entry_t*) TRI_AtVector(&datafiles, i);
TRI_datafile_t* datafile = e->_data;
char const* ptr;
char const* end;
// we are reading from a journal that might be modified in parallel
// so we must read-lock it
if (e->_isJournal) {
TRI_READ_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
}
else {
assert(datafile->_isSealed);
}
ptr = datafile->_data;
end = ptr + datafile->_currentSize;
while (ptr < end) {
TRI_df_marker_t* marker = (TRI_df_marker_t*) ptr;
TRI_voc_tick_t foundTick;
if (marker->_size == 0 || marker->_type <= TRI_MARKER_MIN) {
// end of datafile
break;
}
ptr += TRI_DF_ALIGN_BLOCK(marker->_size);
// get the marker's tick and check whether we should include it
foundTick = marker->_tick;
if (foundTick <= dataMin) {
// marker too old
continue;
}
if (foundTick > dataMax) {
// marker too new
hasMore = false;
goto NEXT_DF;
}
if (marker->_type != TRI_DOC_MARKER_KEY_DOCUMENT) {
// we're only interested in document markers here
// the replication collection does not contain any edge markers
// and deletion markers in the replication collection
// will not be replicated
// check if we can abort searching
if (foundTick >= dataMax ||
(foundTick >= e->_tickMax && i == (n - 1))) {
// fetched the last available marker
hasMore = false;
goto NEXT_DF;
}
continue;
}
// note the last tick we processed
lastFoundTick = foundTick;
if (! StringifyMarkerLog(dump, document, marker)) {
res = TRI_ERROR_INTERNAL;
goto NEXT_DF;
}
if (foundTick >= dataMax ||
(foundTick >= e->_dataMax && i == (n - 1))) {
// fetched the last available marker
hasMore = false;
goto NEXT_DF;
}
if ((uint64_t) TRI_LengthStringBuffer(buffer) > chunkSize) {
// abort the iteration
bufferFull = true;
goto NEXT_DF;
}
}
NEXT_DF:
if (e->_isJournal) {
// read-unlock the journal
TRI_READ_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
}
if (res != TRI_ERROR_NO_ERROR || ! hasMore || bufferFull) {
break;
}
}
TRI_DestroyVector(&datafiles);
if (res == TRI_ERROR_NO_ERROR) {
if (lastFoundTick > 0) {
// data available for requested range
dump->_lastFoundTick = lastFoundTick;
dump->_hasMore = hasMore;
dump->_bufferFull = bufferFull;
}
else {
// no data available for requested range
dump->_lastFoundTick = 0;
dump->_hasMore = false;
dump->_bufferFull = false;
}
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @}
////////////////////////////////////////////////////////////////////////////////
// -----------------------------------------------------------------------------
// --SECTION-- public functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @addtogroup VocBase
/// @{
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from a collection
////////////////////////////////////////////////////////////////////////////////
int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump,
TRI_vocbase_col_t* col,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
uint64_t chunkSize,
bool withTicks,
bool translateCollectionIds) {
TRI_primary_collection_t* primary;
TRI_barrier_t* b;
int res;
assert(col != NULL);
assert(col->_collection != NULL);
primary = (TRI_primary_collection_t*) col->_collection;
// create a barrier so the underlying collection is not unloaded
b = TRI_CreateBarrierReplication(&primary->_barrierList);
if (b == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
// block compaction
TRI_ReadLockReadWriteLock(&primary->_compactionLock);
res = DumpCollection(dump, primary, dataMin, dataMax, chunkSize, withTicks, translateCollectionIds);
TRI_ReadUnlockReadWriteLock(&primary->_compactionLock);
TRI_FreeBarrier(b);
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from the replication log
////////////////////////////////////////////////////////////////////////////////
int TRI_DumpLogReplication (TRI_vocbase_t* vocbase,
TRI_replication_dump_t* dump,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
uint64_t chunkSize) {
TRI_vocbase_col_t* col;
TRI_primary_collection_t* primary;
TRI_barrier_t* b;
int res;
col = TRI_UseCollectionByNameVocBase(vocbase, TRI_COL_NAME_REPLICATION);
if (col == NULL || col->_collection == NULL) {
return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
}
primary = (TRI_primary_collection_t*) col->_collection;
// create a barrier so the underlying collection is not unloaded
b = TRI_CreateBarrierReplication(&primary->_barrierList);
if (b == NULL) {
TRI_ReleaseCollectionVocBase(vocbase, col);
return TRI_ERROR_OUT_OF_MEMORY;
}
// block compaction
TRI_ReadLockReadWriteLock(&primary->_compactionLock);
res = DumpLog(dump, primary, dataMin, dataMax, chunkSize);
TRI_ReadUnlockReadWriteLock(&primary->_compactionLock);
TRI_FreeBarrier(b);
TRI_ReleaseCollectionVocBase(vocbase, col);
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initialise a replication dump container
////////////////////////////////////////////////////////////////////////////////
int TRI_InitDumpReplication (TRI_replication_dump_t* dump,
TRI_vocbase_t* vocbase,
size_t bufferSize) {
int res;
assert(vocbase != NULL);
dump->_vocbase = vocbase;
dump->_lastFoundTick = 0;
dump->_lastSid = 0;
dump->_lastShape = NULL;
dump->_failed = false;
dump->_bufferFull = false;
dump->_hasMore = false;
dump->_buffer = TRI_CreateSizedStringBuffer(TRI_CORE_MEM_ZONE, bufferSize);
if (dump->_buffer == NULL) {
return TRI_ERROR_OUT_OF_MEMORY;
}
res = TRI_InitAssociativePointer(&dump->_collectionNames,
TRI_UNKNOWN_MEM_ZONE,
HashKeyCid,
HashElementCid,
IsEqualKeyElementCid,
NULL);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeStringBuffer(TRI_CORE_MEM_ZONE, dump->_buffer);
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a replication dump container
////////////////////////////////////////////////////////////////////////////////
void TRI_DestroyDumpReplication (TRI_replication_dump_t* dump) {
size_t i;
for (i = 0; i < dump->_collectionNames._nrAlloc; ++i) {
resolved_name_t* found = dump->_collectionNames._table[i];
if (found != NULL) {
if (found->_name != NULL) {
// name can be NULL
TRI_Free(TRI_UNKNOWN_MEM_ZONE, found->_name);
}
TRI_Free(TRI_UNKNOWN_MEM_ZONE, found);
}
}
TRI_DestroyAssociativePointer(&dump->_collectionNames);
TRI_FreeStringBuffer(TRI_CORE_MEM_ZONE, dump->_buffer);
}
////////////////////////////////////////////////////////////////////////////////
/// @}
////////////////////////////////////////////////////////////////////////////////
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:
Reference in New Issue View Git Blame Copy Permalink