Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
arangodb/arangod/VocBase/replication-dump.cpp

1485 lines
51 KiB
C++
Raw Blame History

////////////////////////////////////////////////////////////////////////////////
/// @brief replication dump functions
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "replication-dump.h"
#include "Basics/conversions.h"
#include "Basics/files.h"
#include "Basics/json.h"
#include "Basics/logging.h"
#include "Basics/tri-strings.h"
#include "Utils/CollectionNameResolver.h"
#include "VocBase/collection.h"
#include "VocBase/datafile.h"
#include "VocBase/document-collection.h"
#include "VocBase/server.h"
#include "VocBase/transaction.h"
#include "VocBase/vocbase.h"
#include "VocBase/voc-shaper.h"
#include "Wal/Logfile.h"
#include "Wal/LogfileManager.h"
#include "Wal/Marker.h"
#include "Cluster/ServerState.h"
using namespace triagens;
// -----------------------------------------------------------------------------
// --SECTION-- REPLICATION
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- private defines
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief shortcut function
////////////////////////////////////////////////////////////////////////////////
#define FAIL_IFNOT(func, buffer, val) \
if (func(buffer, val) != TRI_ERROR_NO_ERROR) { \
return TRI_ERROR_OUT_OF_MEMORY; \
}
////////////////////////////////////////////////////////////////////////////////
/// @brief create a string-buffer function name
////////////////////////////////////////////////////////////////////////////////
#define APPEND_FUNC(name) TRI_ ## name ## StringBuffer
////////////////////////////////////////////////////////////////////////////////
/// @brief append a character to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_CHAR(buffer, c) FAIL_IFNOT(APPEND_FUNC(AppendChar), buffer, c)
////////////////////////////////////////////////////////////////////////////////
/// @brief append a string to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_STRING(buffer, str) FAIL_IFNOT(APPEND_FUNC(AppendString), buffer, str)
////////////////////////////////////////////////////////////////////////////////
/// @brief append uint64 to a string-buffer or fail
////////////////////////////////////////////////////////////////////////////////
#define APPEND_UINT64(buffer, val) FAIL_IFNOT(APPEND_FUNC(AppendUInt64), buffer, val)
// -----------------------------------------------------------------------------
// --SECTION-- private types
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief a datafile descriptor
////////////////////////////////////////////////////////////////////////////////
typedef struct df_entry_s {
TRI_datafile_t* _data;
TRI_voc_tick_t _dataMin;
TRI_voc_tick_t _dataMax;
TRI_voc_tick_t _tickMax;
bool _isJournal;
}
df_entry_t;
// -----------------------------------------------------------------------------
// --SECTION-- private functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief translate a (local) collection id into a collection name
////////////////////////////////////////////////////////////////////////////////
char const* NameFromCid (TRI_replication_dump_t* dump,
TRI_voc_cid_t cid) {
auto it = dump->_collectionNames.find(cid);
if (it != dump->_collectionNames.end()) {
// collection name is in cache already
return (*it).second.c_str();
}
// collection name not in cache yet
char* name = TRI_GetCollectionNameByIdVocBase(dump->_vocbase, cid);
if (name != nullptr) {
// insert into cache
try {
dump->_collectionNames.emplace(std::make_pair(cid, std::string(name)));
}
catch (...) {
TRI_FreeString(TRI_UNKNOWN_MEM_ZONE, name);
return nullptr;
}
TRI_FreeString(TRI_UNKNOWN_MEM_ZONE, name);
// and look it up again
return NameFromCid(dump, cid);
}
return nullptr;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief append a collection name or id to a string buffer
////////////////////////////////////////////////////////////////////////////////
static int AppendCollection (TRI_replication_dump_t* dump,
TRI_voc_cid_t cid,
bool translateCollectionIds,
triagens::arango::CollectionNameResolver* resolver) {
if (translateCollectionIds) {
if (cid > 0) {
std::string name;
if (triagens::arango::ServerState::instance()->isDBserver()) {
name = resolver->getCollectionNameCluster(cid);
}
else {
name = resolver->getCollectionName(cid);
}
APPEND_STRING(dump->_buffer, name.c_str());
return TRI_ERROR_NO_ERROR;
}
APPEND_STRING(dump->_buffer, "_unknown");
}
else {
APPEND_UINT64(dump->_buffer, (uint64_t) cid);
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief iterate over a vector of datafiles and pick those with a specific
/// data range
////////////////////////////////////////////////////////////////////////////////
static int IterateDatafiles (TRI_vector_pointer_t const* datafiles,
TRI_vector_t* result,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
bool isJournal) {
int res = TRI_ERROR_NO_ERROR;
size_t const n = datafiles->_length;
for (size_t i = 0; i < n; ++i) {
TRI_datafile_t* df = static_cast<TRI_datafile_t*>(TRI_AtVectorPointer(datafiles, i));
df_entry_t entry = {
df,
df->_dataMin,
df->_dataMax,
df->_tickMax,
isJournal
};
LOG_TRACE("checking datafile %llu with data range %llu - %llu, tick max: %llu",
(unsigned long long) df->_fid,
(unsigned long long) df->_dataMin,
(unsigned long long) df->_dataMax,
(unsigned long long) df->_tickMax);
if (df->_dataMin == 0 || df->_dataMax == 0) {
// datafile doesn't have any data
continue;
}
TRI_ASSERT(df->_tickMin <= df->_tickMax);
TRI_ASSERT(df->_dataMin <= df->_dataMax);
if (dataMax < df->_dataMin) {
// datafile is newer than requested range
continue;
}
if (dataMin > df->_dataMax) {
// datafile is older than requested range
continue;
}
res = TRI_PushBackVector(result, &entry);
if (res != TRI_ERROR_NO_ERROR) {
break;
}
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the datafiles of a collection for a specific tick range
////////////////////////////////////////////////////////////////////////////////
static TRI_vector_t GetRangeDatafiles (TRI_document_collection_t* document,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax) {
TRI_vector_t datafiles;
LOG_TRACE("getting datafiles in data range %llu - %llu",
(unsigned long long) dataMin,
(unsigned long long) dataMax);
// determine the datafiles of the collection
TRI_InitVector(&datafiles, TRI_CORE_MEM_ZONE, sizeof(df_entry_t));
TRI_READ_LOCK_DATAFILES_DOC_COLLECTION(document);
IterateDatafiles(&document->_datafiles, &datafiles, dataMin, dataMax, false);
IterateDatafiles(&document->_journals, &datafiles, dataMin, dataMax, true);
TRI_READ_UNLOCK_DATAFILES_DOC_COLLECTION(document);
return datafiles;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief append database id plus collection id
////////////////////////////////////////////////////////////////////////////////
static int AppendContext (TRI_replication_dump_t* dump,
TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId) {
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, collectionId);
// also include collection name
char const* cname = NameFromCid(dump, collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, "\",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
}
APPEND_STRING(dump->_buffer, "\",");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a raw marker from a datafile for a collection dump
////////////////////////////////////////////////////////////////////////////////
static int StringifyMarkerDump (TRI_replication_dump_t* dump,
TRI_document_collection_t* document,
TRI_df_marker_t const* marker,
bool withTicks,
bool translateCollectionIds,
triagens::arango::CollectionNameResolver* resolver) {
// This covers two cases:
// 1. document is not nullptr and marker points into a data file
// 2. document is a nullptr and marker points into a WAL file
// no other combinations are allowed.
TRI_string_buffer_t* buffer;
TRI_replication_operation_e type;
TRI_voc_key_t key;
TRI_voc_rid_t rid;
bool haveData = true;
bool isWal = false;
buffer = dump->_buffer;
if (buffer == nullptr) {
return TRI_ERROR_INTERNAL;
}
switch (marker->_type) {
case TRI_DOC_MARKER_KEY_DELETION: {
TRI_ASSERT(nullptr != document);
auto m = reinterpret_cast<TRI_doc_deletion_key_marker_t const*>(marker);
key = ((char*) m) + m->_offsetKey;
type = REPLICATION_MARKER_REMOVE;
rid = m->_rid;
haveData = false;
break;
}
case TRI_DOC_MARKER_KEY_DOCUMENT: {
TRI_ASSERT(nullptr != document);
auto m = reinterpret_cast<TRI_doc_document_key_marker_t const*>(marker);
key = ((char*) m) + m->_offsetKey;
type = REPLICATION_MARKER_DOCUMENT;
rid = m->_rid;
break;
}
case TRI_DOC_MARKER_KEY_EDGE: {
TRI_ASSERT(nullptr != document);
auto m = reinterpret_cast<TRI_doc_document_key_marker_t const*>(marker);
key = ((char*) m) + m->_offsetKey;
type = REPLICATION_MARKER_EDGE;
rid = m->_rid;
break;
}
case TRI_WAL_MARKER_REMOVE: {
TRI_ASSERT(nullptr == document);
auto m = static_cast<wal::remove_marker_t const*>(marker);
key = ((char*) m) + sizeof(wal::remove_marker_t);
type = REPLICATION_MARKER_REMOVE;
rid = m->_revisionId;
haveData = false;
isWal = true;
break;
}
case TRI_WAL_MARKER_DOCUMENT: {
TRI_ASSERT(nullptr == document);
auto m = static_cast<wal::document_marker_t const*>(marker);
key = ((char*) m) + m->_offsetKey;
type = REPLICATION_MARKER_DOCUMENT;
rid = m->_revisionId;
isWal = true;
break;
}
case TRI_WAL_MARKER_EDGE: {
TRI_ASSERT(nullptr == document);
auto m = static_cast<wal::edge_marker_t const*>(marker);
key = ((char*) m) + m->_offsetKey;
type = REPLICATION_MARKER_EDGE;
rid = m->_revisionId;
isWal = true;
break;
}
default: {
return TRI_ERROR_INTERNAL;
}
}
if (withTicks) {
APPEND_STRING(buffer, "{\"tick\":\"");
APPEND_UINT64(buffer, (uint64_t) marker->_tick);
APPEND_STRING(buffer, "\",\"type\":");
}
else {
APPEND_STRING(buffer, "{\"type\":");
}
APPEND_UINT64(buffer, (uint64_t) type);
APPEND_STRING(buffer, ",\"key\":\"");
// key is user-defined, but does not need escaping
APPEND_STRING(buffer, key);
APPEND_STRING(buffer, "\",\"rev\":\"");
APPEND_UINT64(buffer, (uint64_t) rid);
// document
if (haveData) {
APPEND_STRING(buffer, "\",\"data\":{");
// common document meta-data
APPEND_STRING(buffer, "\"" TRI_VOC_ATTRIBUTE_KEY "\":\"");
APPEND_STRING(buffer, key);
APPEND_STRING(buffer, "\",\"" TRI_VOC_ATTRIBUTE_REV "\":\"");
APPEND_UINT64(buffer, (uint64_t) rid);
APPEND_CHAR(buffer, '"');
// Is it an edge marker?
if (marker->_type == TRI_DOC_MARKER_KEY_EDGE ||
marker->_type == TRI_WAL_MARKER_EDGE) {
TRI_voc_key_t fromKey;
TRI_voc_key_t toKey;
TRI_voc_cid_t fromCid;
TRI_voc_cid_t toCid;
if (marker->_type == TRI_DOC_MARKER_KEY_EDGE) {
auto e = reinterpret_cast<TRI_doc_edge_key_marker_t const*>(marker);
fromKey = ((char*) e) + e->_offsetFromKey;
toKey = ((char*) e) + e->_offsetToKey;
fromCid = e->_fromCid;
toCid = e->_toCid;
}
else { // TRI_WAL_MARKER_EDGE
auto e = reinterpret_cast<wal::edge_marker_t const*>(marker);
fromKey = ((char*) e) + e->_offsetFromKey;
toKey = ((char*) e) + e->_offsetToKey;
fromCid = e->_fromCid;
toCid = e->_toCid;
}
int res;
APPEND_STRING(buffer, ",\"" TRI_VOC_ATTRIBUTE_FROM "\":\"");
res = AppendCollection(dump, fromCid, translateCollectionIds, resolver);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(buffer, "\\/");
APPEND_STRING(buffer, fromKey);
APPEND_STRING(buffer, "\",\"" TRI_VOC_ATTRIBUTE_TO "\":\"");
res = AppendCollection(dump, toCid, translateCollectionIds, resolver);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(buffer, "\\/");
APPEND_STRING(buffer, toKey);
APPEND_CHAR(buffer, '"');
}
// the actual document data
TRI_shaped_json_t shaped;
if (! isWal) {
auto m = reinterpret_cast<TRI_doc_document_key_marker_t const*>(marker);
TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, m);
TRI_StringifyArrayShapedJson(document->getShaper(), buffer, &shaped, true); // ONLY IN DUMP, PROTECTED by fake trx above
}
else { // isWal == true
auto m = static_cast<wal::document_marker_t const*>(marker);
TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, m);
char const* legend = reinterpret_cast<char const*>(m) + m->_offsetLegend;
if (m->_offsetJson - m->_offsetLegend == 8) {
auto p = reinterpret_cast<int64_t const*>(legend);
legend += *p;
}
basics::LegendReader legendReader(legend);
TRI_StringifyArrayShapedJson(&legendReader, buffer, &shaped, true);
}
APPEND_STRING(buffer, "}}\n");
}
else { // deletion marker, so no data
APPEND_STRING(buffer, "\"}\n");
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief append the document attributes of a marker
////////////////////////////////////////////////////////////////////////////////
static int AppendDocument (triagens::wal::document_marker_t const* marker,
TRI_replication_dump_t* dump) {
TRI_shaped_json_t shaped;
shaped._sid = marker->_shape;
shaped._data.length = marker->_size - marker->_offsetJson;
shaped._data.data = (char*) marker + marker->_offsetJson;
// check if the marker contains a legend
char const* legend = reinterpret_cast<char const*>(marker) + marker->_offsetLegend;
if (*((uint64_t*) legend) == 0ULL) {
// marker has no legend
// create a fake transaction so no assertion fails
triagens::arango::TransactionBase trx(true);
// try to open the collection and use the shaper
TRI_vocbase_t* vocbase = TRI_UseDatabaseByIdServer(dump->_vocbase->_server, marker->_databaseId);
if (vocbase == nullptr) {
// we'll not return an error in this case but an empty document
// this is intentional so the replication can still continue with documents of collections
// that survived
return TRI_ERROR_NO_ERROR;
}
TRI_vocbase_col_status_e status;
TRI_vocbase_col_t* collection = TRI_UseCollectionByIdVocBase(vocbase, marker->_collectionId, status);
int res = TRI_ERROR_NO_ERROR;
if (collection == nullptr) {
// we'll not return an error in this case but an empty document
// this is intentional so the replication can still continue with documents of collections
// that survived
res = TRI_ERROR_NO_ERROR;
}
else {
TRI_document_collection_t* document = collection->_collection;
if (! TRI_StringifyArrayShapedJson(document->getShaper(), dump->_buffer, &shaped, true)) {
res = TRI_ERROR_OUT_OF_MEMORY;
}
TRI_ReleaseCollectionVocBase(vocbase, collection);
}
TRI_ReleaseDatabaseServer(dump->_vocbase->_server, vocbase);
return res;
}
else {
// marker has a legend, so use it
if (marker->_offsetJson - marker->_offsetLegend == 8) {
auto p = reinterpret_cast<int64_t const*>(legend);
legend += *p;
}
triagens::basics::LegendReader lr(legend);
if (! TRI_StringifyArrayShapedJson(&lr, dump->_buffer, &shaped, true)) {
return TRI_ERROR_OUT_OF_MEMORY;
}
return TRI_ERROR_NO_ERROR;
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a document marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerDocument (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::document_marker_t const*>(marker);
int res = AppendContext(dump, m->_databaseId, m->_collectionId);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetKey);
APPEND_STRING(dump->_buffer, "\",\"rev\":\"");
APPEND_UINT64(dump->_buffer, m->_revisionId);
APPEND_STRING(dump->_buffer, "\",\"data\":{");
// common document meta-data
APPEND_STRING(dump->_buffer, "\"" TRI_VOC_ATTRIBUTE_KEY "\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetKey);
APPEND_STRING(dump->_buffer, "\",\"" TRI_VOC_ATTRIBUTE_REV "\":\"");
APPEND_UINT64(dump->_buffer, (uint64_t) m->_revisionId);
APPEND_STRING(dump->_buffer, "\"");
res = AppendDocument(m, dump);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(dump->_buffer, "}");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify an edge marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerEdge (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::edge_marker_t const*>(marker);
int res = AppendContext(dump, m->_databaseId, m->_collectionId);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetKey);
APPEND_STRING(dump->_buffer, "\",\"rev\":\"");
APPEND_UINT64(dump->_buffer, m->_revisionId);
APPEND_STRING(dump->_buffer, "\",\"data\":{");
// common document meta-data
APPEND_STRING(dump->_buffer, "\"" TRI_VOC_ATTRIBUTE_KEY "\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetKey);
APPEND_STRING(dump->_buffer, "\",\"" TRI_VOC_ATTRIBUTE_REV "\":\"");
APPEND_UINT64(dump->_buffer, (uint64_t) m->_revisionId);
// from
APPEND_STRING(dump->_buffer, "\",\"" TRI_VOC_ATTRIBUTE_FROM "\":\"");
APPEND_UINT64(dump->_buffer, (uint64_t) m->_fromCid);
APPEND_STRING(dump->_buffer, "\\/");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetFromKey);
// to
APPEND_STRING(dump->_buffer, "\",\"" TRI_VOC_ATTRIBUTE_TO "\":\"");
APPEND_UINT64(dump->_buffer, (uint64_t) m->_toCid);
APPEND_STRING(dump->_buffer, "\\/");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetToKey);
APPEND_STRING(dump->_buffer, "\"");
res = AppendDocument(reinterpret_cast<triagens::wal::document_marker_t const*>(m), dump);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(dump->_buffer, "}");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a remove marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerRemove (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::remove_marker_t const*>(marker);
int res = AppendContext(dump, m->_databaseId, m->_collectionId);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + sizeof(triagens::wal::remove_marker_t));
APPEND_STRING(dump->_buffer, "\",\"rev\":\"");
APPEND_UINT64(dump->_buffer, m->_revisionId);
APPEND_STRING(dump->_buffer, "\"");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a transaction marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerTransaction (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
// note: the data layout of begin / commit / abort markers is identical, so
// we cast to a begin transaction marker in all cases
auto m = reinterpret_cast<triagens::wal::transaction_begin_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\"");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a create collection marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerCreateCollection (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::collection_create_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
APPEND_STRING(dump->_buffer, "\",\"collection\":");
APPEND_STRING(dump->_buffer, (char const*) m + sizeof(triagens::wal::collection_create_marker_t));
char const* cname = NameFromCid(dump, m->_collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, ",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
APPEND_STRING(dump->_buffer, "\"");
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a drop collection marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerDropCollection (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::collection_drop_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
char const* cname = NameFromCid(dump, m->_collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, "\",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
}
APPEND_STRING(dump->_buffer, "\"");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a rename collection marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerRenameCollection (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::collection_rename_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
char const* cname = NameFromCid(dump, m->_collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, "\",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
}
APPEND_STRING(dump->_buffer, "\",\"collection\":{\"name\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + sizeof(triagens::wal::collection_rename_marker_t));
APPEND_STRING(dump->_buffer, "\"}");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a change collection marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerChangeCollection (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::collection_change_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
char const* cname = NameFromCid(dump, m->_collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, "\",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
}
APPEND_STRING(dump->_buffer, "\",\"collection\":");
APPEND_STRING(dump->_buffer, (char const*) m + sizeof(triagens::wal::collection_change_marker_t));
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a create index marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerCreateIndex (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::index_create_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
char const* cname = NameFromCid(dump, m->_collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, "\",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
}
APPEND_STRING(dump->_buffer, "\",\"id\":\"");
APPEND_UINT64(dump->_buffer, m->_indexId);
APPEND_STRING(dump->_buffer, "\",\"index\":");
APPEND_STRING(dump->_buffer, (char const*) m + sizeof(triagens::wal::index_create_marker_t));
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a drop index marker
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarkerDropIndex (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::index_drop_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
char const* cname = NameFromCid(dump, m->_collectionId);
if (cname != nullptr) {
APPEND_STRING(dump->_buffer, "\",\"cname\":\"");
APPEND_STRING(dump->_buffer, cname);
}
APPEND_STRING(dump->_buffer, "\",\"id\":\"");
APPEND_UINT64(dump->_buffer, m->_indexId);
APPEND_STRING(dump->_buffer, "\"");
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a marker should be replicated
////////////////////////////////////////////////////////////////////////////////
static inline bool MustReplicateWalMarkerType (TRI_df_marker_t const* marker) {
return (marker->_type == TRI_WAL_MARKER_DOCUMENT ||
marker->_type == TRI_WAL_MARKER_EDGE ||
marker->_type == TRI_WAL_MARKER_REMOVE ||
marker->_type == TRI_WAL_MARKER_BEGIN_TRANSACTION ||
marker->_type == TRI_WAL_MARKER_COMMIT_TRANSACTION ||
marker->_type == TRI_WAL_MARKER_ABORT_TRANSACTION ||
marker->_type == TRI_WAL_MARKER_CREATE_COLLECTION ||
marker->_type == TRI_WAL_MARKER_DROP_COLLECTION ||
marker->_type == TRI_WAL_MARKER_RENAME_COLLECTION ||
marker->_type == TRI_WAL_MARKER_CHANGE_COLLECTION ||
marker->_type == TRI_WAL_MARKER_CREATE_INDEX ||
marker->_type == TRI_WAL_MARKER_DROP_INDEX);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief translate a marker type to a replication type
////////////////////////////////////////////////////////////////////////////////
static TRI_replication_operation_e TranslateType (TRI_df_marker_t const* marker) {
switch (marker->_type) {
case TRI_WAL_MARKER_DOCUMENT:
return REPLICATION_MARKER_DOCUMENT;
case TRI_WAL_MARKER_EDGE:
return REPLICATION_MARKER_EDGE;
case TRI_WAL_MARKER_REMOVE:
return REPLICATION_MARKER_REMOVE;
case TRI_WAL_MARKER_BEGIN_TRANSACTION:
return REPLICATION_TRANSACTION_START;
case TRI_WAL_MARKER_COMMIT_TRANSACTION:
return REPLICATION_TRANSACTION_COMMIT;
case TRI_WAL_MARKER_ABORT_TRANSACTION:
return REPLICATION_TRANSACTION_ABORT;
case TRI_WAL_MARKER_CREATE_COLLECTION:
return REPLICATION_COLLECTION_CREATE;
case TRI_WAL_MARKER_DROP_COLLECTION:
return REPLICATION_COLLECTION_DROP;
case TRI_WAL_MARKER_RENAME_COLLECTION:
return REPLICATION_COLLECTION_RENAME;
case TRI_WAL_MARKER_CHANGE_COLLECTION:
return REPLICATION_COLLECTION_CHANGE;
case TRI_WAL_MARKER_CREATE_INDEX:
return REPLICATION_INDEX_CREATE;
case TRI_WAL_MARKER_DROP_INDEX:
return REPLICATION_INDEX_DROP;
default:
return REPLICATION_INVALID;
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a raw marker from a WAL logfile for a log dump
////////////////////////////////////////////////////////////////////////////////
static int StringifyWalMarker (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
TRI_ASSERT_EXPENSIVE(MustReplicateWalMarkerType(marker));
APPEND_STRING(dump->_buffer, "{\"tick\":\"");
APPEND_UINT64(dump->_buffer, (uint64_t) marker->_tick);
APPEND_STRING(dump->_buffer, "\",\"type\":");
APPEND_UINT64(dump->_buffer, (uint64_t) TranslateType(marker));
APPEND_STRING(dump->_buffer, ",");
int res = TRI_ERROR_INTERNAL;
switch (marker->_type) {
case TRI_WAL_MARKER_DOCUMENT: {
res = StringifyWalMarkerDocument(dump, marker);
break;
}
case TRI_WAL_MARKER_EDGE: {
res = StringifyWalMarkerEdge(dump, marker);
break;
}
case TRI_WAL_MARKER_REMOVE: {
res = StringifyWalMarkerRemove(dump, marker);
break;
}
case TRI_WAL_MARKER_BEGIN_TRANSACTION:
case TRI_WAL_MARKER_COMMIT_TRANSACTION:
case TRI_WAL_MARKER_ABORT_TRANSACTION: {
res = StringifyWalMarkerTransaction(dump, marker);
break;
}
case TRI_WAL_MARKER_CREATE_COLLECTION: {
res = StringifyWalMarkerCreateCollection(dump, marker);
break;
}
case TRI_WAL_MARKER_DROP_COLLECTION: {
res = StringifyWalMarkerDropCollection(dump, marker);
break;
}
case TRI_WAL_MARKER_RENAME_COLLECTION: {
res = StringifyWalMarkerRenameCollection(dump, marker);
break;
}
case TRI_WAL_MARKER_CHANGE_COLLECTION: {
res = StringifyWalMarkerChangeCollection(dump, marker);
break;
}
case TRI_WAL_MARKER_CREATE_INDEX: {
res = StringifyWalMarkerCreateIndex(dump, marker);
break;
}
case TRI_WAL_MARKER_DROP_INDEX: {
res = StringifyWalMarkerDropIndex(dump, marker);
break;
}
case TRI_WAL_MARKER_BEGIN_REMOTE_TRANSACTION:
case TRI_WAL_MARKER_COMMIT_REMOTE_TRANSACTION:
case TRI_WAL_MARKER_ABORT_REMOTE_TRANSACTION:
case TRI_WAL_MARKER_ATTRIBUTE:
case TRI_WAL_MARKER_SHAPE: {
TRI_ASSERT(false);
return TRI_ERROR_INTERNAL;
}
}
APPEND_STRING(dump->_buffer, "}\n");
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief helper function to extract a database id from a marker
////////////////////////////////////////////////////////////////////////////////
template<typename T>
static TRI_voc_tick_t GetDatabaseId (TRI_df_marker_t const* marker) {
T const* m = reinterpret_cast<T const*>(marker);
return m->_databaseId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the database id from a marker
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_tick_t GetDatabaseFromWalMarker (TRI_df_marker_t const* marker) {
TRI_ASSERT_EXPENSIVE(MustReplicateWalMarkerType(marker));
switch (marker->_type) {
case TRI_WAL_MARKER_ATTRIBUTE:
return GetDatabaseId<triagens::wal::attribute_marker_t>(marker);
case TRI_WAL_MARKER_SHAPE:
return GetDatabaseId<triagens::wal::shape_marker_t>(marker);
case TRI_WAL_MARKER_DOCUMENT:
return GetDatabaseId<triagens::wal::document_marker_t>(marker);
case TRI_WAL_MARKER_EDGE:
return GetDatabaseId<triagens::wal::edge_marker_t>(marker);
case TRI_WAL_MARKER_REMOVE:
return GetDatabaseId<triagens::wal::remove_marker_t>(marker);
case TRI_WAL_MARKER_BEGIN_TRANSACTION:
return GetDatabaseId<triagens::wal::transaction_begin_marker_t>(marker);
case TRI_WAL_MARKER_COMMIT_TRANSACTION:
return GetDatabaseId<triagens::wal::transaction_commit_marker_t>(marker);
case TRI_WAL_MARKER_ABORT_TRANSACTION:
return GetDatabaseId<triagens::wal::transaction_abort_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_COLLECTION:
return GetDatabaseId<triagens::wal::collection_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_COLLECTION:
return GetDatabaseId<triagens::wal::collection_drop_marker_t>(marker);
case TRI_WAL_MARKER_RENAME_COLLECTION:
return GetDatabaseId<triagens::wal::collection_rename_marker_t>(marker);
case TRI_WAL_MARKER_CHANGE_COLLECTION:
return GetDatabaseId<triagens::wal::collection_change_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_INDEX:
return GetDatabaseId<triagens::wal::index_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_INDEX:
return GetDatabaseId<triagens::wal::index_drop_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_DATABASE:
return GetDatabaseId<triagens::wal::database_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_DATABASE:
return GetDatabaseId<triagens::wal::database_drop_marker_t>(marker);
default: {
return 0;
}
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief helper function to extract a collection id from a marker
////////////////////////////////////////////////////////////////////////////////
template<typename T>
static TRI_voc_tick_t GetCollectionId (TRI_df_marker_t const* marker) {
T const* m = reinterpret_cast<T const*>(marker);
return m->_collectionId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the collection id from a marker
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_tick_t GetCollectionFromWalMarker (TRI_df_marker_t const* marker) {
switch (marker->_type) {
case TRI_WAL_MARKER_ATTRIBUTE:
return GetCollectionId<triagens::wal::attribute_marker_t>(marker);
case TRI_WAL_MARKER_SHAPE:
return GetCollectionId<triagens::wal::shape_marker_t>(marker);
case TRI_WAL_MARKER_DOCUMENT:
return GetCollectionId<triagens::wal::document_marker_t>(marker);
case TRI_WAL_MARKER_EDGE:
return GetCollectionId<triagens::wal::edge_marker_t>(marker);
case TRI_WAL_MARKER_REMOVE:
return GetCollectionId<triagens::wal::remove_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_COLLECTION:
return GetCollectionId<triagens::wal::collection_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_COLLECTION:
return GetCollectionId<triagens::wal::collection_drop_marker_t>(marker);
case TRI_WAL_MARKER_RENAME_COLLECTION:
return GetCollectionId<triagens::wal::collection_rename_marker_t>(marker);
case TRI_WAL_MARKER_CHANGE_COLLECTION:
return GetCollectionId<triagens::wal::collection_change_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_INDEX:
return GetCollectionId<triagens::wal::index_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_INDEX:
return GetCollectionId<triagens::wal::index_drop_marker_t>(marker);
default: {
return 0;
}
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a marker is replicated
////////////////////////////////////////////////////////////////////////////////
static bool MustReplicateWalMarker (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
// first check the marker type
if (! MustReplicateWalMarkerType(marker)) {
return false;
}
// then check if the marker belongs to the "correct" database
if (dump->_vocbase->_id != GetDatabaseFromWalMarker(marker)) {
return false;
}
// finally check if the marker is for a collection that we want to ignore
TRI_voc_cid_t cid = GetCollectionFromWalMarker(marker);
if (cid != 0) {
char const* name = NameFromCid(dump, cid);
if (name != nullptr && TRI_ExcludeCollectionReplication(name, dump->_includeSystem)) {
return false;
}
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from a collection
////////////////////////////////////////////////////////////////////////////////
static int DumpCollection (TRI_replication_dump_t* dump,
TRI_document_collection_t* document,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
bool withTicks,
bool translateCollectionIds,
triagens::arango::CollectionNameResolver* resolver) {
TRI_vector_t datafiles;
TRI_string_buffer_t* buffer;
TRI_voc_tick_t lastFoundTick;
TRI_voc_tid_t lastTid;
size_t i, n;
int res;
bool hasMore;
bool bufferFull;
bool ignoreMarkers;
// The following fake transaction allows us to access data pointers
// and shapers, essentially disabling the runtime checks. This is OK,
// since the dump only considers data files (and not WAL files), so
// the collector has no trouble. Also, the data files of the collection
// are protected from the compactor by a barrier and the dump only goes
// until a certain tick.
triagens::arango::TransactionBase trx(true);
LOG_TRACE("dumping collection %llu, tick range %llu - %llu",
(unsigned long long) document->_info._cid,
(unsigned long long) dataMin,
(unsigned long long) dataMax);
buffer = dump->_buffer;
datafiles = GetRangeDatafiles(document, dataMin, dataMax);
// setup some iteration state
lastFoundTick = 0;
lastTid = 0;
res = TRI_ERROR_NO_ERROR;
hasMore = true;
bufferFull = false;
ignoreMarkers = false;
n = datafiles._length;
for (i = 0; i < n; ++i) {
df_entry_t* e = (df_entry_t*) TRI_AtVector(&datafiles, i);
TRI_datafile_t* datafile = e->_data;
char const* ptr;
char const* end;
// we are reading from a journal that might be modified in parallel
// so we must read-lock it
if (e->_isJournal) {
TRI_READ_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(document);
}
else {
TRI_ASSERT(datafile->_isSealed);
}
ptr = datafile->_data;
if (res == TRI_ERROR_NO_ERROR) {
// no error so far. start iterating
end = ptr + datafile->_currentSize;
}
else {
// some error occurred. don't iterate
end = ptr;
}
while (ptr < end) {
TRI_df_marker_t* marker = (TRI_df_marker_t*) ptr;
TRI_voc_tick_t foundTick;
TRI_voc_tid_t tid;
if (marker->_size == 0 || marker->_type <= TRI_MARKER_MIN) {
// end of datafile
break;
}
ptr += TRI_DF_ALIGN_BLOCK(marker->_size);
if (marker->_type == TRI_DF_MARKER_ATTRIBUTE ||
marker->_type == TRI_DF_MARKER_SHAPE) {
// fully ignore these marker types. they don't need to be replicated,
// but we also cannot stop iteration if we find one of these
continue;
}
// get the marker's tick and check whether we should include it
foundTick = marker->_tick;
if (foundTick <= dataMin) {
// marker too old
continue;
}
if (foundTick > dataMax) {
// marker too new
hasMore = false;
goto NEXT_DF;
}
if (marker->_type != TRI_DOC_MARKER_KEY_DOCUMENT &&
marker->_type != TRI_DOC_MARKER_KEY_EDGE &&
marker->_type != TRI_DOC_MARKER_KEY_DELETION) {
// found a non-data marker...
// check if we can abort searching
if (foundTick >= dataMax ||
(foundTick >= e->_tickMax && i == (n - 1))) {
// fetched the last available marker
hasMore = false;
goto NEXT_DF;
}
continue;
}
// note the last tick we processed
lastFoundTick = foundTick;
// handle aborted/unfinished transactions
if (document->_failedTransactions == nullptr) {
// there are no failed transactions
ignoreMarkers = false;
}
else {
// get transaction id of marker
if (marker->_type == TRI_DOC_MARKER_KEY_DELETION) {
tid = ((TRI_doc_deletion_key_marker_t const*) marker)->_tid;
}
else {
tid = ((TRI_doc_document_key_marker_t const*) marker)->_tid;
}
// check if marker is from an aborted transaction
if (tid > 0) {
if (tid != lastTid) {
ignoreMarkers = (document->_failedTransactions->find(tid) != document->_failedTransactions->end());
}
lastTid = tid;
}
if (ignoreMarkers) {
continue;
}
}
res = StringifyMarkerDump(dump, document, marker, withTicks, translateCollectionIds, resolver);
if (res != TRI_ERROR_NO_ERROR) {
break; // will go to NEXT_DF
}
if (foundTick >= dataMax ||
(foundTick >= e->_tickMax && i == (n - 1))) {
// fetched the last available marker
hasMore = false;
goto NEXT_DF;
}
if ((uint64_t) TRI_LengthStringBuffer(buffer) > dump->_chunkSize) {
// abort the iteration
bufferFull = true;
goto NEXT_DF;
}
}
NEXT_DF:
if (e->_isJournal) {
// read-unlock the journal
TRI_READ_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(document);
}
if (res != TRI_ERROR_NO_ERROR || ! hasMore || bufferFull) {
break;
}
}
TRI_DestroyVector(&datafiles);
if (res == TRI_ERROR_NO_ERROR) {
if (lastFoundTick > 0) {
// data available for requested range
dump->_lastFoundTick = lastFoundTick;
dump->_hasMore = hasMore;
dump->_bufferFull = bufferFull;
}
else {
// no data available for requested range
dump->_lastFoundTick = 0;
dump->_hasMore = false;
dump->_bufferFull = false;
}
}
return res;
}
// -----------------------------------------------------------------------------
// --SECTION-- public functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from a collection
////////////////////////////////////////////////////////////////////////////////
int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump,
TRI_vocbase_col_t* col,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
bool withTicks,
bool translateCollectionIds) {
TRI_ASSERT(col != nullptr);
TRI_ASSERT(col->_collection != nullptr);
triagens::arango::CollectionNameResolver resolver(col->_vocbase);
TRI_document_collection_t* document = col->_collection;
// create a barrier so the underlying collection is not unloaded
TRI_barrier_t* b = TRI_CreateBarrierReplication(&document->_barrierList);
if (b == nullptr) {
return TRI_ERROR_OUT_OF_MEMORY;
}
// block compaction
TRI_ReadLockReadWriteLock(&document->_compactionLock);
int res = DumpCollection(dump, document, dataMin, dataMax, withTicks,
translateCollectionIds, &resolver);
TRI_ReadUnlockReadWriteLock(&document->_compactionLock);
TRI_FreeBarrier(b);
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from the replication log
////////////////////////////////////////////////////////////////////////////////
int TRI_DumpLogReplication (TRI_replication_dump_t* dump,
TRI_voc_tick_t tickMin,
TRI_voc_tick_t tickMax,
bool outputAsArray) {
LOG_TRACE("dumping log, tick range %llu - %llu",
(unsigned long long) tickMin,
(unsigned long long) tickMax);
// ask the logfile manager which datafiles qualify
std::vector<triagens::wal::Logfile*> logfiles = triagens::wal::LogfileManager::instance()->getLogfilesForTickRange(tickMin, tickMax);
size_t const n = logfiles.size();
// setup some iteration state
int res = TRI_ERROR_NO_ERROR;
TRI_voc_tick_t lastFoundTick = 0;
bool hasMore = true;
bool bufferFull = false;
if (outputAsArray) {
TRI_AppendStringStringBuffer(dump->_buffer, "[\n");
}
try {
bool first = true;
// iterate over the datafiles found
for (size_t i = 0; i < n; ++i) {
triagens::wal::Logfile* logfile = logfiles[i];
char const* ptr;
char const* end;
triagens::wal::LogfileManager::instance()->getActiveLogfileRegion(logfile, ptr, end);
while (ptr < end) {
TRI_df_marker_t const* marker = reinterpret_cast<TRI_df_marker_t const*>(ptr);
if (marker->_size == 0 || marker->_type <= TRI_MARKER_MIN) {
// end of datafile
break;
}
ptr += TRI_DF_ALIGN_BLOCK(marker->_size);
// get the marker's tick and check whether we should include it
TRI_voc_tick_t foundTick = marker->_tick;
if (foundTick <= tickMin) {
// marker too old
continue;
}
if (foundTick >= tickMax) {
hasMore = false;
}
if (foundTick > tickMax) {
// marker too new
break;
}
if (! MustReplicateWalMarker(dump, marker)) {
// check if we can abort searching
continue;
}
// note the last tick we processed
lastFoundTick = foundTick;
if (outputAsArray) {
if (! first) {
TRI_AppendStringStringBuffer(dump->_buffer, ", ");
}
else {
first = false;
}
}
res = StringifyWalMarker(dump, marker);
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
if ((uint64_t) TRI_LengthStringBuffer(dump->_buffer) >= dump->_chunkSize) {
// abort the iteration
bufferFull = true;
break;
}
}
if (! hasMore || bufferFull) {
break;
}
}
}
catch (triagens::basics::Exception const& ex) {
res = ex.code();
}
catch (...) {
res = TRI_ERROR_INTERNAL;
}
// always return the logfiles we have used
triagens::wal::LogfileManager::instance()->returnLogfiles(logfiles);
if (outputAsArray) {
TRI_AppendStringStringBuffer(dump->_buffer, "\n]");
}
if (res == TRI_ERROR_NO_ERROR) {
if (lastFoundTick > 0) {
// data available for requested range
dump->_lastFoundTick = lastFoundTick;
dump->_hasMore = hasMore;
dump->_bufferFull = bufferFull;
}
else {
// no data available for requested range
dump->_lastFoundTick = 0;
dump->_hasMore = false;
dump->_bufferFull = false;
}
}
return res;
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:
Reference in New Issue View Git Blame Copy Permalink