1
0
Fork 0

slight cleanup of replication-related code (#4286)

This commit is contained in:
Jan 2018-01-17 16:56:40 +01:00 committed by GitHub
parent b9a489eb9e
commit 1e116a9f6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 120 additions and 148 deletions

View File

@ -349,6 +349,7 @@ SET(ARANGOD_SOURCES
Replication/ReplicationFeature.cpp Replication/ReplicationFeature.cpp
Replication/Syncer.cpp Replication/Syncer.cpp
Replication/TailingSyncer.cpp Replication/TailingSyncer.cpp
Replication/common-defines.cpp
RestHandler/RestAdminLogHandler.cpp RestHandler/RestAdminLogHandler.cpp
RestHandler/RestAdminRoutingHandler.cpp RestHandler/RestAdminRoutingHandler.cpp
RestHandler/RestAdminServerHandler.cpp RestHandler/RestAdminServerHandler.cpp
@ -481,7 +482,6 @@ SET(ARANGOD_SOURCES
VocBase/LogicalCollection.cpp VocBase/LogicalCollection.cpp
VocBase/LogicalView.cpp VocBase/LogicalView.cpp
VocBase/ManagedDocumentResult.cpp VocBase/ManagedDocumentResult.cpp
VocBase/replication-common.cpp
VocBase/ticks.cpp VocBase/ticks.cpp
VocBase/vocbase.cpp VocBase/vocbase.cpp
${ADDITIONAL_BIN_ARANGOD_SOURCES} ${ADDITIONAL_BIN_ARANGOD_SOURCES}

View File

@ -22,6 +22,7 @@
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#include "MMFilesRestReplicationHandler.h" #include "MMFilesRestReplicationHandler.h"
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h" #include "Basics/VelocyPackHelper.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "MMFiles/MMFilesCollectionKeys.h" #include "MMFiles/MMFilesCollectionKeys.h"
@ -418,14 +419,14 @@ void MMFilesRestReplicationHandler::handleCommandLoggerFollow() {
_response->setContentType(rest::ContentType::DUMP); _response->setContentType(rest::ContentType::DUMP);
// set headers // set headers
_response->setHeaderNC(TRI_REPLICATION_HEADER_CHECKMORE, _response->setHeaderNC(StaticStrings::ReplicationHeaderCheckMore,
checkMore ? "true" : "false"); checkMore ? "true" : "false");
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTINCLUDED, _response->setHeaderNC(StaticStrings::ReplicationHeaderLastIncluded,
StringUtils::itoa(dump._lastFoundTick)); StringUtils::itoa(dump._lastFoundTick));
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTTICK, _response->setHeaderNC(StaticStrings::ReplicationHeaderLastTick,
StringUtils::itoa(state.lastCommittedTick)); StringUtils::itoa(state.lastCommittedTick));
_response->setHeaderNC(TRI_REPLICATION_HEADER_ACTIVE, "true"); _response->setHeaderNC(StaticStrings::ReplicationHeaderActive, "true");
_response->setHeaderNC(TRI_REPLICATION_HEADER_FROMPRESENT, _response->setHeaderNC(StaticStrings::ReplicationHeaderFromPresent,
dump._fromTickIncluded ? "true" : "false"); dump._fromTickIncluded ? "true" : "false");
if (length > 0) { if (length > 0) {
@ -521,10 +522,10 @@ void MMFilesRestReplicationHandler::handleCommandDetermineOpenTransactions() {
_response->setContentType(rest::ContentType::DUMP); _response->setContentType(rest::ContentType::DUMP);
_response->setHeaderNC(TRI_REPLICATION_HEADER_FROMPRESENT, _response->setHeaderNC(StaticStrings::ReplicationHeaderFromPresent,
dump._fromTickIncluded ? "true" : "false"); dump._fromTickIncluded ? "true" : "false");
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTTICK, _response->setHeaderNC(StaticStrings::ReplicationHeaderLastTick,
StringUtils::itoa(dump._lastFoundTick)); StringUtils::itoa(dump._lastFoundTick));
if (length > 0) { if (length > 0) {
@ -1031,10 +1032,10 @@ void MMFilesRestReplicationHandler::handleCommandDump() {
response->setContentType(rest::ContentType::DUMP); response->setContentType(rest::ContentType::DUMP);
// set headers // set headers
_response->setHeaderNC(TRI_REPLICATION_HEADER_CHECKMORE, _response->setHeaderNC(StaticStrings::ReplicationHeaderCheckMore,
(dump._hasMore ? "true" : "false")); (dump._hasMore ? "true" : "false"));
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTINCLUDED, _response->setHeaderNC(StaticStrings::ReplicationHeaderLastIncluded,
StringUtils::itoa(dump._lastFoundTick)); StringUtils::itoa(dump._lastFoundTick));
// transfer ownership of the buffer contents // transfer ownership of the buffer contents

View File

@ -25,7 +25,7 @@
#define ARANGOD_MMFILES_MMFILES_REPLICATION_COMMON_H 1 #define ARANGOD_MMFILES_MMFILES_REPLICATION_COMMON_H 1
#include "MMFiles/MMFilesDatafile.h" #include "MMFiles/MMFilesDatafile.h"
#include "VocBase/replication-common.h" #include "Replication/common-defines.h"
#include "VocBase/voc-types.h" #include "VocBase/voc-types.h"
namespace arangodb { namespace arangodb {

View File

@ -59,14 +59,22 @@ static void Append(MMFilesReplicationDumpContext* dump, char const* value) {
} }
} }
static void Append(MMFilesReplicationDumpContext* dump, std::string const& value) {
int res = TRI_AppendString2StringBuffer(dump->_buffer, value.c_str(), value.size());
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
}
/// @brief translate a (local) collection id into a collection name /// @brief translate a (local) collection id into a collection name
static char const* NameFromCid(MMFilesReplicationDumpContext* dump, static std::string const& nameFromCid(MMFilesReplicationDumpContext* dump,
TRI_voc_cid_t cid) { TRI_voc_cid_t cid) {
auto it = dump->_collectionNames.find(cid); auto it = dump->_collectionNames.find(cid);
if (it != dump->_collectionNames.end()) { if (it != dump->_collectionNames.end()) {
// collection name is in cache already // collection name is in cache already
return (*it).second.c_str(); return (*it).second;
} }
// collection name not in cache yet // collection name not in cache yet
@ -76,15 +84,15 @@ static char const* NameFromCid(MMFilesReplicationDumpContext* dump,
// insert into cache // insert into cache
try { try {
dump->_collectionNames.emplace(cid, std::move(name)); dump->_collectionNames.emplace(cid, std::move(name));
// and look it up again
return nameFromCid(dump, cid);
} catch (...) { } catch (...) {
return nullptr; // fall through to returning empty string
} }
// and look it up again
return NameFromCid(dump, cid);
} }
return nullptr; return StaticStrings::Empty;
} }
/// @brief stringify a raw marker from a logfile for a log dump or logger /// @brief stringify a raw marker from a logfile for a log dump or logger
@ -128,9 +136,9 @@ static int StringifyMarker(MMFilesReplicationDumpContext* dump,
Append(dump, collectionId); Append(dump, collectionId);
Append(dump, "\""); Append(dump, "\"");
// also include collection name // also include collection name
char const* cname = NameFromCid(dump, collectionId); std::string const& cname = nameFromCid(dump, collectionId);
if (cname != nullptr) { if (!cname.empty()) {
Append(dump, ",\"cname\":\""); Append(dump, ",\"cname\":\"");
Append(dump, cname); Append(dump, cname);
Append(dump, "\""); Append(dump, "\"");
@ -229,8 +237,8 @@ static int SliceifyMarker(MMFilesReplicationDumpContext* dump,
if (collectionId > 0) { if (collectionId > 0) {
builder.add("cid", VPackValue(collectionId)); builder.add("cid", VPackValue(collectionId));
// also include collection name // also include collection name
char const* cname = NameFromCid(dump, collectionId); std::string const& cname = nameFromCid(dump, collectionId);
if (cname != nullptr) { if (!cname.empty()) {
builder.add("cname", VPackValue(cname)); builder.add("cname", VPackValue(cname));
} }
} }
@ -321,9 +329,9 @@ static bool MustReplicateWalMarker(
TRI_voc_cid_t cid = collectionId; TRI_voc_cid_t cid = collectionId;
if (cid != 0) { if (cid != 0) {
char const* name = NameFromCid(dump, cid); std::string const& name = nameFromCid(dump, cid);
if (name != nullptr && if (!name.empty() &&
TRI_ExcludeCollectionReplication(name, dump->_includeSystem)) { TRI_ExcludeCollectionReplication(name, dump->_includeSystem)) {
return false; return false;
} }

View File

@ -27,8 +27,8 @@
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Basics/Exceptions.h" #include "Basics/Exceptions.h"
#include "Basics/StringBuffer.h" #include "Basics/StringBuffer.h"
#include "Replication/common-defines.h"
#include "Transaction/Context.h" #include "Transaction/Context.h"
#include "VocBase/replication-common.h"
#include "VocBase/voc-types.h" #include "VocBase/voc-types.h"
#include "VocBase/vocbase.h" #include "VocBase/vocbase.h"
@ -38,8 +38,6 @@
#include <velocypack/Options.h> #include <velocypack/Options.h>
#include <velocypack/velocypack-aliases.h> #include <velocypack/velocypack-aliases.h>
#include <vector>
/// @brief replication dump container /// @brief replication dump container
struct MMFilesReplicationDumpContext { struct MMFilesReplicationDumpContext {
MMFilesReplicationDumpContext(std::shared_ptr<arangodb::transaction::Context> const& MMFilesReplicationDumpContext(std::shared_ptr<arangodb::transaction::Context> const&

View File

@ -456,10 +456,10 @@ Result DatabaseInitialSyncer::handleCollectionDump(arangodb::LogicalCollection*
} }
bool found; bool found;
std::string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found); std::string header = response->getHeaderField(StaticStrings::ReplicationHeaderCheckMore, found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + url + ": required header " + TRI_REPLICATION_HEADER_CHECKMORE + _masterInfo._endpoint + url + ": required header " + StaticStrings::ReplicationHeaderCheckMore +
" is missing in dump response"); " is missing in dump response");
} }
@ -467,11 +467,11 @@ Result DatabaseInitialSyncer::handleCollectionDump(arangodb::LogicalCollection*
bool checkMore = StringUtils::boolean(header); bool checkMore = StringUtils::boolean(header);
if (checkMore) { if (checkMore) {
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, header = response->getHeaderField(StaticStrings::ReplicationHeaderLastIncluded,
found); found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + url + ": required header " + TRI_REPLICATION_HEADER_LASTINCLUDED _masterInfo._endpoint + url + ": required header " + StaticStrings::ReplicationHeaderLastIncluded
+ " is missing in dump response"); + " is missing in dump response");
} }

View File

@ -139,23 +139,23 @@ Result DatabaseTailingSyncer::syncCollectionFinalize(std::string const& collecti
} }
bool found; bool found;
std::string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found); std::string header = response->getHeaderField(StaticStrings::ReplicationHeaderCheckMore, found);
bool checkMore = false; bool checkMore = false;
if (found) { if (found) {
checkMore = StringUtils::boolean(header); checkMore = StringUtils::boolean(header);
} }
header = header =
response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found); response->getHeaderField(StaticStrings::ReplicationHeaderLastIncluded, found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + ": required header " + TRI_REPLICATION_HEADER_LASTINCLUDED + " is missing"); _masterInfo._endpoint + ": required header " + StaticStrings::ReplicationHeaderLastIncluded + " is missing");
} }
TRI_voc_tick_t lastIncludedTick = StringUtils::uint64(header); TRI_voc_tick_t lastIncludedTick = StringUtils::uint64(header);
// was the specified from value included the result? // was the specified from value included the result?
bool fromIncluded = false; bool fromIncluded = false;
header = response->getHeaderField(TRI_REPLICATION_HEADER_FROMPRESENT, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderFromPresent, found);
if (found) { if (found) {
fromIncluded = StringUtils::boolean(header); fromIncluded = StringUtils::boolean(header);
} }

View File

@ -33,9 +33,9 @@
#include "Cluster/ServerState.h" #include "Cluster/ServerState.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "Replication/TailingSyncer.h" #include "Replication/TailingSyncer.h"
#include "Replication/common-defines.h"
#include "Rest/Version.h" #include "Rest/Version.h"
#include "RestServer/ServerIdFeature.h" #include "RestServer/ServerIdFeature.h"
#include "VocBase/replication-common.h"
using namespace arangodb; using namespace arangodb;

View File

@ -22,7 +22,7 @@
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#include "ReplicationApplierState.h" #include "ReplicationApplierState.h"
#include "VocBase/replication-common.h" #include "Replication/common-defines.h"
#include <velocypack/Builder.h> #include <velocypack/Builder.h>
#include <velocypack/velocypack-aliases.h> #include <velocypack/velocypack-aliases.h>

View File

@ -25,7 +25,7 @@
#define ARANGOD_REPLICATION_REPLICATION_APPLIER_STATE_H 1 #define ARANGOD_REPLICATION_REPLICATION_APPLIER_STATE_H 1
#include "Basics/Common.h" #include "Basics/Common.h"
#include "VocBase/replication-common.h" #include "Replication/common-defines.h"
#include "VocBase/voc-types.h" #include "VocBase/voc-types.h"
#include <velocypack/Builder.h> #include <velocypack/Builder.h>

View File

@ -26,8 +26,8 @@
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Replication/ReplicationApplierConfiguration.h" #include "Replication/ReplicationApplierConfiguration.h"
#include "Replication/common-defines.h"
#include "Utils/DatabaseGuard.h" #include "Utils/DatabaseGuard.h"
#include "VocBase/replication-common.h"
#include "VocBase/ticks.h" #include "VocBase/ticks.h"
struct TRI_vocbase_t; struct TRI_vocbase_t;

View File

@ -1269,20 +1269,20 @@ Result TailingSyncer::fetchOpenTransactions(TRI_voc_tick_t fromTick,
bool found; bool found;
std::string header = std::string header =
response->getHeaderField(TRI_REPLICATION_HEADER_FROMPRESENT, found); response->getHeaderField(StaticStrings::ReplicationHeaderFromPresent, found);
if (found) { if (found) {
fromIncluded = StringUtils::boolean(header); fromIncluded = StringUtils::boolean(header);
} }
// fetch the tick from where we need to start scanning later // fetch the tick from where we need to start scanning later
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderLastIncluded, found);
if (!found) { if (!found) {
// we changed the API in 3.3 to use last included // we changed the API in 3.3 to use last included
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTTICK, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderLastTick, found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + ": required header " + TRI_REPLICATION_HEADER_LASTTICK + _masterInfo._endpoint + ": required header " + StaticStrings::ReplicationHeaderLastTick +
" is missing in determine-open-transactions response"); " is missing in determine-open-transactions response");
} }
} }
@ -1389,32 +1389,32 @@ Result TailingSyncer::followMasterLog(TRI_voc_tick_t& fetchTick,
} }
bool found; bool found;
std::string header = response->getHeaderField(TRI_REPLICATION_HEADER_CHECKMORE, found); std::string header = response->getHeaderField(StaticStrings::ReplicationHeaderCheckMore, found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + ": required header " + TRI_REPLICATION_HEADER_CHECKMORE + " is missing"); _masterInfo._endpoint + ": required header " + StaticStrings::ReplicationHeaderCheckMore + " is missing");
} }
bool checkMore = StringUtils::boolean(header); bool checkMore = StringUtils::boolean(header);
// was the specified from value included the result? // was the specified from value included the result?
bool fromIncluded = false; bool fromIncluded = false;
header = response->getHeaderField(TRI_REPLICATION_HEADER_FROMPRESENT, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderFromPresent, found);
if (found) { if (found) {
fromIncluded = StringUtils::boolean(header); fromIncluded = StringUtils::boolean(header);
} }
bool active = false; bool active = false;
header = response->getHeaderField(TRI_REPLICATION_HEADER_ACTIVE, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderActive, found);
if (found) { if (found) {
active = StringUtils::boolean(header); active = StringUtils::boolean(header);
} }
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTINCLUDED, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderLastIncluded, found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + ": required header " + TRI_REPLICATION_HEADER_LASTINCLUDED + _masterInfo._endpoint + ": required header " + StaticStrings::ReplicationHeaderLastIncluded +
" is missing in logger-follow response"); " is missing in logger-follow response");
} }
@ -1428,10 +1428,10 @@ Result TailingSyncer::followMasterLog(TRI_voc_tick_t& fetchTick,
checkMore = false; checkMore = false;
} }
header = response->getHeaderField(TRI_REPLICATION_HEADER_LASTTICK, found); header = response->getHeaderField(StaticStrings::ReplicationHeaderLastTick, found);
if (!found) { if (!found) {
return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") + return Result(TRI_ERROR_REPLICATION_INVALID_RESPONSE, std::string("got invalid response from master at ") +
_masterInfo._endpoint + ": required header " + TRI_REPLICATION_HEADER_LASTTICK + _masterInfo._endpoint + ": required header " + StaticStrings::ReplicationHeaderLastTick +
" is missing in logger-follow response"); " is missing in logger-follow response");
} }

View File

@ -21,13 +21,12 @@
/// @author Jan Steemann /// @author Jan Steemann
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#include "replication-common.h" #include "Replication/common-defines.h"
#include "Basics/tri-strings.h" #include "Basics/tri-strings.h"
//////////////////////////////////////////////////////////////////////////////// namespace arangodb {
/// @brief generate a timestamp string in a target buffer
////////////////////////////////////////////////////////////////////////////////
/// @brief generate a timestamp string in a target buffer
void TRI_GetTimeStampReplication(char* dst, size_t maxLength) { void TRI_GetTimeStampReplication(char* dst, size_t maxLength) {
struct tm tb; struct tm tb;
time_t tt = time(nullptr); time_t tt = time(nullptr);
@ -36,10 +35,7 @@ void TRI_GetTimeStampReplication(char* dst, size_t maxLength) {
strftime(dst, maxLength, "%Y-%m-%dT%H:%M:%SZ", &tb); strftime(dst, maxLength, "%Y-%m-%dT%H:%M:%SZ", &tb);
} }
////////////////////////////////////////////////////////////////////////////////
/// @brief generate a timestamp string in a target buffer /// @brief generate a timestamp string in a target buffer
////////////////////////////////////////////////////////////////////////////////
void TRI_GetTimeStampReplication(double timeStamp, char* dst, void TRI_GetTimeStampReplication(double timeStamp, char* dst,
size_t maxLength) { size_t maxLength) {
struct tm tb; struct tm tb;
@ -49,18 +45,6 @@ void TRI_GetTimeStampReplication(double timeStamp, char* dst,
strftime(dst, maxLength, "%Y-%m-%dT%H:%M:%SZ", &tb); strftime(dst, maxLength, "%Y-%m-%dT%H:%M:%SZ", &tb);
} }
////////////////////////////////////////////////////////////////////////////////
/// @brief determine whether a collection should be included in replication
////////////////////////////////////////////////////////////////////////////////
bool TRI_ExcludeCollectionReplication(char const* name, bool includeSystem) {
if (name == nullptr) {
// name invalid
return true;
}
return TRI_ExcludeCollectionReplication(std::string(name), includeSystem);
}
bool TRI_ExcludeCollectionReplication(std::string const& name, bool includeSystem) { bool TRI_ExcludeCollectionReplication(std::string const& name, bool includeSystem) {
if (name.empty()) { if (name.empty()) {
// should not happen... // should not happen...
@ -88,3 +72,5 @@ bool TRI_ExcludeCollectionReplication(std::string const& name, bool includeSyste
return false; return false;
} }
}

View File

@ -21,46 +21,14 @@
/// @author Jan Steemann /// @author Jan Steemann
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_VOC_BASE_REPLICATION_COMMON_H #ifndef ARANGOD_REPLICATION_COMMON_DEFINES_H
#define ARANGOD_VOC_BASE_REPLICATION_COMMON_H 1 #define ARANGOD_REPLICATION_COMMON_DEFINES_H 1
#include "Basics/Common.h" #include "Basics/Common.h"
#include "VocBase/voc-types.h"
//////////////////////////////////////////////////////////////////////////////// namespace arangodb {
/// @brief HTTP response header for "check for more data?"
////////////////////////////////////////////////////////////////////////////////
#define TRI_REPLICATION_HEADER_CHECKMORE "x-arango-replication-checkmore"
////////////////////////////////////////////////////////////////////////////////
/// @brief HTTP response header for "last included tick"
////////////////////////////////////////////////////////////////////////////////
#define TRI_REPLICATION_HEADER_LASTINCLUDED "x-arango-replication-lastincluded"
////////////////////////////////////////////////////////////////////////////////
/// @brief HTTP response header for "last logged tick"
////////////////////////////////////////////////////////////////////////////////
#define TRI_REPLICATION_HEADER_LASTTICK "x-arango-replication-lasttick"
////////////////////////////////////////////////////////////////////////////////
/// @brief HTTP response header for "from tick present"
////////////////////////////////////////////////////////////////////////////////
#define TRI_REPLICATION_HEADER_FROMPRESENT "x-arango-replication-frompresent"
////////////////////////////////////////////////////////////////////////////////
/// @brief HTTP response header for "replication active"
////////////////////////////////////////////////////////////////////////////////
#define TRI_REPLICATION_HEADER_ACTIVE "x-arango-replication-active"
////////////////////////////////////////////////////////////////////////////////
/// @brief replication operations /// @brief replication operations
////////////////////////////////////////////////////////////////////////////////
typedef enum { typedef enum {
REPLICATION_INVALID = 0, REPLICATION_INVALID = 0,
@ -94,23 +62,15 @@ typedef enum {
REPLICATION_MAX REPLICATION_MAX
} TRI_replication_operation_e; } TRI_replication_operation_e;
////////////////////////////////////////////////////////////////////////////////
/// @brief generate a timestamp string in a target buffer /// @brief generate a timestamp string in a target buffer
////////////////////////////////////////////////////////////////////////////////
void TRI_GetTimeStampReplication(char*, size_t); void TRI_GetTimeStampReplication(char*, size_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief generate a timestamp string in a target buffer /// @brief generate a timestamp string in a target buffer
////////////////////////////////////////////////////////////////////////////////
void TRI_GetTimeStampReplication(double, char*, size_t); void TRI_GetTimeStampReplication(double, char*, size_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief determine whether a collection should be included in replication /// @brief determine whether a collection should be included in replication
////////////////////////////////////////////////////////////////////////////////
bool TRI_ExcludeCollectionReplication(char const* name, bool includeSystem);
bool TRI_ExcludeCollectionReplication(std::string const& name, bool includeSystem); bool TRI_ExcludeCollectionReplication(std::string const& name, bool includeSystem);
}
#endif #endif

View File

@ -27,8 +27,8 @@
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Basics/Result.h" #include "Basics/Result.h"
#include "Replication/common-defines.h"
#include "RestHandler/RestVocbaseBaseHandler.h" #include "RestHandler/RestVocbaseBaseHandler.h"
#include "VocBase/replication-common.h"
namespace arangodb { namespace arangodb {
class ClusterInfo; class ClusterInfo;

View File

@ -22,9 +22,10 @@
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#include "RestWalAccessHandler.h" #include "RestWalAccessHandler.h"
#include "Basics/StaticStrings.h"
#include "Basics/VPackStringBufferAdapter.h" #include "Basics/VPackStringBufferAdapter.h"
#include "Basics/VelocyPackHelper.h" #include "Basics/VelocyPackHelper.h"
#include "Replication/common-defines.h"
#include "Rest/HttpResponse.h" #include "Rest/HttpResponse.h"
#include "Rest/Version.h" #include "Rest/Version.h"
#include "RestServer/DatabaseFeature.h" #include "RestServer/DatabaseFeature.h"
@ -35,7 +36,6 @@
#include "Transaction/Helpers.h" #include "Transaction/Helpers.h"
#include "Utils/CollectionNameResolver.h" #include "Utils/CollectionNameResolver.h"
#include "VocBase/LogicalCollection.h" #include "VocBase/LogicalCollection.h"
#include "VocBase/replication-common.h"
#include <velocypack/Builder.h> #include <velocypack/Builder.h>
#include <velocypack/Iterator.h> #include <velocypack/Iterator.h>
@ -325,15 +325,15 @@ void RestWalAccessHandler::handleCommandTail(WalAccess const* wal) {
// set headers // set headers
bool checkMore = result.lastIncludedTick() > 0 && bool checkMore = result.lastIncludedTick() > 0 &&
result.lastIncludedTick() < result.latestTick(); result.lastIncludedTick() < result.latestTick();
_response->setHeaderNC(TRI_REPLICATION_HEADER_CHECKMORE, _response->setHeaderNC(StaticStrings::ReplicationHeaderCheckMore,
checkMore ? "true" : "false"); checkMore ? "true" : "false");
_response->setHeaderNC( _response->setHeaderNC(
TRI_REPLICATION_HEADER_LASTINCLUDED, StaticStrings::ReplicationHeaderLastIncluded,
StringUtils::itoa(result.lastIncludedTick())); StringUtils::itoa(result.lastIncludedTick()));
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTTICK, _response->setHeaderNC(StaticStrings::ReplicationHeaderLastTick,
StringUtils::itoa(result.latestTick())); StringUtils::itoa(result.latestTick()));
_response->setHeaderNC(TRI_REPLICATION_HEADER_ACTIVE, "true"); _response->setHeaderNC(StaticStrings::ReplicationHeaderActive, "true");
_response->setHeaderNC(TRI_REPLICATION_HEADER_FROMPRESENT, _response->setHeaderNC(StaticStrings::ReplicationHeaderFromPresent,
result.fromTickIncluded() ? "true" : "false"); result.fromTickIncluded() ? "true" : "false");
if (length > 0) { if (length > 0) {
@ -408,9 +408,9 @@ void RestWalAccessHandler::handleCommandDetermineOpenTransactions(
auto cc = r.lastIncludedTick() != 0 ? ResponseCode::OK : ResponseCode::NO_CONTENT; auto cc = r.lastIncludedTick() != 0 ? ResponseCode::OK : ResponseCode::NO_CONTENT;
generateResult(cc, std::move(buffer)); generateResult(cc, std::move(buffer));
_response->setHeaderNC(TRI_REPLICATION_HEADER_FROMPRESENT, _response->setHeaderNC(StaticStrings::ReplicationHeaderFromPresent,
r.fromTickIncluded() ? "true" : "false"); r.fromTickIncluded() ? "true" : "false");
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTINCLUDED, _response->setHeaderNC(StaticStrings::ReplicationHeaderLastIncluded,
StringUtils::itoa(r.lastIncludedTick())); StringUtils::itoa(r.lastIncludedTick()));
} }
} }

View File

@ -27,6 +27,7 @@
#include "Basics/StringRef.h" #include "Basics/StringRef.h"
#include "Basics/VPackStringBufferAdapter.h" #include "Basics/VPackStringBufferAdapter.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "Replication/common-defines.h"
#include "RestServer/DatabaseFeature.h" #include "RestServer/DatabaseFeature.h"
#include "RocksDBEngine/RocksDBCollection.h" #include "RocksDBEngine/RocksDBCollection.h"
#include "RocksDBEngine/RocksDBCommon.h" #include "RocksDBEngine/RocksDBCommon.h"
@ -38,7 +39,6 @@
#include "Transaction/UserTransaction.h" #include "Transaction/UserTransaction.h"
#include "Utils/DatabaseGuard.h" #include "Utils/DatabaseGuard.h"
#include "Utils/ExecContext.h" #include "Utils/ExecContext.h"
#include "VocBase/replication-common.h"
#include "VocBase/ticks.h" #include "VocBase/ticks.h"
#include <velocypack/Dumper.h> #include <velocypack/Dumper.h>

View File

@ -25,12 +25,16 @@
#include "Basics/StaticStrings.h" #include "Basics/StaticStrings.h"
#include "Basics/StringRef.h" #include "Basics/StringRef.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "Replication/common-defines.h"
#include "RocksDBEngine/RocksDBColumnFamily.h" #include "RocksDBEngine/RocksDBColumnFamily.h"
#include "RocksDBEngine/RocksDBCommon.h" #include "RocksDBEngine/RocksDBCommon.h"
#include "RocksDBEngine/RocksDBLogValue.h" #include "RocksDBEngine/RocksDBLogValue.h"
#include "VocBase/replication-common.h"
#include "VocBase/ticks.h" #include "VocBase/ticks.h"
#include <velocypack/Builder.h>
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>
#include <rocksdb/utilities/transaction_db.h> #include <rocksdb/utilities/transaction_db.h>
#include <rocksdb/utilities/write_batch_with_index.h> #include <rocksdb/utilities/write_batch_with_index.h>
#include <rocksdb/write_batch.h> #include <rocksdb/write_batch.h>

View File

@ -25,15 +25,16 @@
#define ARANGO_ROCKSDB_ROCKSDB_REPLICATION_TAILING_H 1 #define ARANGO_ROCKSDB_ROCKSDB_REPLICATION_TAILING_H 1
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Replication/common-defines.h"
#include "RocksDBEngine/RocksDBReplicationCommon.h" #include "RocksDBEngine/RocksDBReplicationCommon.h"
#include "RocksDBEngine/RocksDBTypes.h" #include "RocksDBEngine/RocksDBTypes.h"
#include "VocBase/vocbase.h" #include "VocBase/vocbase.h"
#include "VocBase/replication-common.h"
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>
namespace arangodb { namespace arangodb {
namespace velocypack {
class Builder;
}
namespace rocksutils { namespace rocksutils {
// iterates over WAL starting at 'from' and returns up to 'limit' documents // iterates over WAL starting at 'from' and returns up to 'limit' documents
@ -41,9 +42,9 @@ namespace rocksutils {
RocksDBReplicationResult tailWal(TRI_vocbase_t* vocbase, uint64_t tickStart, RocksDBReplicationResult tailWal(TRI_vocbase_t* vocbase, uint64_t tickStart,
uint64_t tickEnd, size_t chunkSize, uint64_t tickEnd, size_t chunkSize,
bool includeSystem, TRI_voc_cid_t collectionId, bool includeSystem, TRI_voc_cid_t collectionId,
VPackBuilder& builder); arangodb::velocypack::Builder& builder);
TRI_replication_operation_e convertLogType(RocksDBLogType t); arangodb::TRI_replication_operation_e convertLogType(RocksDBLogType t);
} // namespace rocksutils } // namespace rocksutils
} // namespace arangodb } // namespace arangodb

View File

@ -23,6 +23,7 @@
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#include "RocksDBRestReplicationHandler.h" #include "RocksDBRestReplicationHandler.h"
#include "Basics/StaticStrings.h"
#include "Basics/VPackStringBufferAdapter.h" #include "Basics/VPackStringBufferAdapter.h"
#include "Basics/VelocyPackHelper.h" #include "Basics/VelocyPackHelper.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
@ -287,14 +288,14 @@ void RocksDBRestReplicationHandler::handleCommandLoggerFollow() {
_response->setContentType(rest::ContentType::DUMP); _response->setContentType(rest::ContentType::DUMP);
// set headers // set headers
_response->setHeaderNC(TRI_REPLICATION_HEADER_CHECKMORE, _response->setHeaderNC(StaticStrings::ReplicationHeaderCheckMore,
checkMore ? "true" : "false"); checkMore ? "true" : "false");
_response->setHeaderNC( _response->setHeaderNC(
TRI_REPLICATION_HEADER_LASTINCLUDED, StaticStrings::ReplicationHeaderLastIncluded,
StringUtils::itoa((length == 0) ? 0 : result.maxTick())); StringUtils::itoa((length == 0) ? 0 : result.maxTick()));
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTTICK, StringUtils::itoa(latest)); _response->setHeaderNC(StaticStrings::ReplicationHeaderLastTick, StringUtils::itoa(latest));
_response->setHeaderNC(TRI_REPLICATION_HEADER_ACTIVE, "true"); _response->setHeaderNC(StaticStrings::ReplicationHeaderActive, "true");
_response->setHeaderNC(TRI_REPLICATION_HEADER_FROMPRESENT, _response->setHeaderNC(StaticStrings::ReplicationHeaderFromPresent,
result.minTickIncluded() ? "true" : "false"); result.minTickIncluded() ? "true" : "false");
if (length > 0) { if (length > 0) {
@ -340,9 +341,9 @@ void RocksDBRestReplicationHandler::handleCommandDetermineOpenTransactions() {
generateResult(rest::ResponseCode::OK, VPackSlice::emptyArraySlice()); generateResult(rest::ResponseCode::OK, VPackSlice::emptyArraySlice());
// rocksdb only includes finished transactions in the WAL. // rocksdb only includes finished transactions in the WAL.
_response->setContentType(rest::ContentType::DUMP); _response->setContentType(rest::ContentType::DUMP);
_response->setHeaderNC(TRI_REPLICATION_HEADER_LASTTICK, "0"); _response->setHeaderNC(StaticStrings::ReplicationHeaderLastTick, "0");
// always true to satisfy continuous syncer // always true to satisfy continuous syncer
_response->setHeaderNC(TRI_REPLICATION_HEADER_FROMPRESENT, "true"); _response->setHeaderNC(StaticStrings::ReplicationHeaderFromPresent, "true");
} }
void RocksDBRestReplicationHandler::handleCommandInventory() { void RocksDBRestReplicationHandler::handleCommandInventory() {
@ -724,11 +725,11 @@ void RocksDBRestReplicationHandler::handleCommandDump() {
response->setContentType(rest::ContentType::DUMP); response->setContentType(rest::ContentType::DUMP);
// set headers // set headers
_response->setHeaderNC(TRI_REPLICATION_HEADER_CHECKMORE, _response->setHeaderNC(StaticStrings::ReplicationHeaderCheckMore,
(context->more() ? "true" : "false")); (context->more() ? "true" : "false"));
_response->setHeaderNC( _response->setHeaderNC(
TRI_REPLICATION_HEADER_LASTINCLUDED, StaticStrings::ReplicationHeaderLastIncluded,
StringUtils::itoa((dump.length() == 0) ? 0 : result.maxTick())); StringUtils::itoa((dump.length() == 0) ? 0 : result.maxTick()));
// transfer ownership of the buffer contents // transfer ownership of the buffer contents

View File

@ -30,6 +30,7 @@
#include "ApplicationFeatures/ApplicationServer.h" #include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h" #include "Basics/OpenFilesTracker.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Basics/VelocyPackHelper.h" #include "Basics/VelocyPackHelper.h"
#include "Basics/files.h" #include "Basics/files.h"
@ -350,7 +351,7 @@ int DumpFeature::dumpCollection(int fd, std::string const& cid,
// TODO: fix hard-coded headers // TODO: fix hard-coded headers
std::string header = std::string header =
response->getHeaderField("x-arango-replication-checkmore", found); response->getHeaderField(StaticStrings::ReplicationHeaderCheckMore, found);
if (found) { if (found) {
checkMore = StringUtils::boolean(header); checkMore = StringUtils::boolean(header);
@ -358,8 +359,7 @@ int DumpFeature::dumpCollection(int fd, std::string const& cid,
if (checkMore) { if (checkMore) {
// TODO: fix hard-coded headers // TODO: fix hard-coded headers
header = response->getHeaderField("x-arango-replication-lastincluded", header = response->getHeaderField(StaticStrings::ReplicationHeaderLastIncluded, found);
found);
if (found) { if (found) {
uint64_t tick = StringUtils::uint64(header); uint64_t tick = StringUtils::uint64(header);
@ -740,7 +740,7 @@ int DumpFeature::dumpShard(int fd, std::string const& DBserver,
// TODO: fix hard-coded headers // TODO: fix hard-coded headers
std::string header = std::string header =
response->getHeaderField("x-arango-replication-checkmore", found); response->getHeaderField(StaticStrings::ReplicationHeaderCheckMore, found);
if (found) { if (found) {
checkMore = StringUtils::boolean(header); checkMore = StringUtils::boolean(header);
@ -748,8 +748,7 @@ int DumpFeature::dumpShard(int fd, std::string const& DBserver,
if (checkMore) { if (checkMore) {
// TODO: fix hard-coded headers // TODO: fix hard-coded headers
header = response->getHeaderField("x-arango-replication-lastincluded", header = response->getHeaderField(StaticStrings::ReplicationHeaderLastIncluded, found);
found);
if (found) { if (found) {
uint64_t tick = StringUtils::uint64(header); uint64_t tick = StringUtils::uint64(header);

View File

@ -57,6 +57,13 @@ std::string const StaticStrings::SilentString("silent");
std::string const StaticStrings::WaitForSyncString("waitForSync"); std::string const StaticStrings::WaitForSyncString("waitForSync");
std::string const StaticStrings::IsSynchronousReplicationString( std::string const StaticStrings::IsSynchronousReplicationString(
"isSynchronousReplication"); "isSynchronousReplication");
// replication headers
std::string const StaticStrings::ReplicationHeaderCheckMore("x-arango-replication-checkmore");
std::string const StaticStrings::ReplicationHeaderLastIncluded("x-arango-replication-lastincluded");
std::string const StaticStrings::ReplicationHeaderLastTick("x-arango-replication-lasttick");
std::string const StaticStrings::ReplicationHeaderFromPresent("x-arango-replication-frompresent");
std::string const StaticStrings::ReplicationHeaderActive("x-arango-replication-active");
// database and collection names // database and collection names
std::string const StaticStrings::SystemDatabase("_system"); std::string const StaticStrings::SystemDatabase("_system");

View File

@ -62,6 +62,13 @@ class StaticStrings {
static std::string const SilentString; static std::string const SilentString;
static std::string const WaitForSyncString; static std::string const WaitForSyncString;
static std::string const IsSynchronousReplicationString; static std::string const IsSynchronousReplicationString;
// replication headers
static std::string const ReplicationHeaderCheckMore;
static std::string const ReplicationHeaderLastIncluded;
static std::string const ReplicationHeaderLastTick;
static std::string const ReplicationHeaderFromPresent;
static std::string const ReplicationHeaderActive;
// database and collection names // database and collection names
static std::string const SystemDatabase; static std::string const SystemDatabase;