1
0
Fork 0

removed `--recycle-ids` option for arangorestore (#3713)

This commit is contained in:
Jan 2017-11-16 14:25:54 +01:00 committed by Frank Celler
parent 5abf0c1185
commit fce593b724
7 changed files with 99 additions and 177 deletions

View File

@ -1,6 +1,12 @@
devel
-----
* removed `--recycle-ids` option for arangorestore
using that option could have led to problems on the restore, with potential
id conflicts between the originating server (the source dump server) and the
target server (the restore server)
* add readonly mode REST API
* allow compilation of ArangoDB source code with g++7

View File

@ -505,6 +505,7 @@ Result Syncer::createCollection(TRI_vocbase_t* vocbase,
VPackBuilder s;
s.openObject();
s.add("isSystem", VPackValue(true));
s.add("objectId", VPackSlice::nullSlice());
if (uuid.isString() && !simulate32Client()) { // need to use cid for 3.2 master
// if we received a globallyUniqueId from the remote, then we will always use this id
// so we can discard the "cid" and "id" values for the collection

View File

@ -70,7 +70,7 @@ uint64_t const RestReplicationHandler::_maxChunkSize = 128 * 1024 * 1024;
static Result restoreDataParser(char const* ptr, char const* pos,
std::string const& collectionName,
bool useRevision, std::string& key,
std::string& key,
VPackBuilder& builder, VPackSlice& doc,
TRI_replication_operation_e& type) {
builder.clear();
@ -601,8 +601,7 @@ void RestReplicationHandler::handleTrampolineCoordinator() {
httpRequest->body(), *headers, 300.0);
} else {
// do we need to handle multiple payloads here - TODO
// here we switch vorm vst to http?!
// i am not allowed to change cluster comm!
// here we switch from vst to http?!
res = cc->syncRequest("", TRI_NewTickServer(), "server:" + DBserver,
_request->requestType(),
"/_db/" + StringUtils::urlEncode(dbname) +
@ -738,13 +737,6 @@ void RestReplicationHandler::handleCommandRestoreCollection() {
overwrite = StringUtils::boolean(value1);
}
bool recycleIds = false;
std::string const& value2 = _request->value("recycleIds", found);
if (found) {
recycleIds = StringUtils::boolean(value2);
}
bool force = false;
std::string const& value3 = _request->value("force", found);
@ -771,20 +763,19 @@ void RestReplicationHandler::handleCommandRestoreCollection() {
replicationFactor = StringUtils::uint64(value5);
}
std::string errorMsg;
int res;
Result res;
if (ServerState::instance()->isCoordinator()) {
res = processRestoreCollectionCoordinator(
slice, overwrite, recycleIds, force, numberOfShards, errorMsg,
slice, overwrite, force, numberOfShards,
replicationFactor, ignoreDistributeShardsLikeErrors);
} else {
res =
processRestoreCollection(slice, overwrite, recycleIds, force, errorMsg);
processRestoreCollection(slice, overwrite, force);
}
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION_MESSAGE(res, errorMsg);
if (res.fail()) {
THROW_ARANGO_EXCEPTION_MESSAGE(res.errorNumber(), res.errorMessage());
}
VPackBuilder result;
@ -850,14 +841,7 @@ void RestReplicationHandler::handleCommandRestoreData() {
return;
}
bool recycleIds = false;
std::string const& value2 = _request->value("recycleIds");
if (!value2.empty()) {
recycleIds = StringUtils::boolean(value2);
}
Result res = processRestoreData(colName, recycleIds);
Result res = processRestoreData(colName);
if (res.fail()) {
if (res.errorMessage().empty()) {
@ -882,66 +866,39 @@ void RestReplicationHandler::handleCommandRestoreData() {
/// @brief restores the structure of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::processRestoreCollection(
VPackSlice const& collection, bool dropExisting, bool reuseId, bool force,
std::string& errorMsg) {
Result RestReplicationHandler::processRestoreCollection(
VPackSlice const& collection, bool dropExisting, bool force) {
if (!collection.isObject()) {
errorMsg = "collection declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection declaration is invalid");
}
VPackSlice const parameters = collection.get("parameters");
if (!parameters.isObject()) {
errorMsg = "collection parameters declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection parameters declaration is invalid");
}
VPackSlice const indexes = collection.get("indexes");
if (!indexes.isArray()) {
errorMsg = "collection indexes declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection indexes declaration is invalid");
}
std::string const name = arangodb::basics::VelocyPackHelper::getStringValue(
parameters, "name", "");
if (name.empty()) {
errorMsg = "collection name is missing";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection name is missing");
}
if (arangodb::basics::VelocyPackHelper::getBooleanValue(parameters, "deleted",
false)) {
// we don't care about deleted collections
return TRI_ERROR_NO_ERROR;
return Result();
}
grantTemporaryRights();
LogicalCollection* col = nullptr;
if (reuseId) {
TRI_voc_cid_t const cid =
arangodb::basics::VelocyPackHelper::extractIdValue(parameters);
if (cid == 0) {
errorMsg = "collection id is missing";
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
// first look up the collection by the cid
col = _vocbase->lookupCollection(cid);
}
if (col == nullptr) {
// not found, try name next
col = _vocbase->lookupCollection(name);
}
LogicalCollection* col = _vocbase->lookupCollection(name);
// drop an existing collection if it exists
if (col != nullptr) {
@ -957,10 +914,11 @@ int RestReplicationHandler::processRestoreCollection(
AccessMode::Type::EXCLUSIVE);
// to turn off waitForSync!
trx.addHint(transaction::Hints::Hint::RECOVERY);
trx.addHint(transaction::Hints::Hint::READ_OWN_WRITES);
res = trx.begin();
if (!res.ok()) {
return res.errorNumber();
return res;
}
OperationOptions options;
@ -968,73 +926,57 @@ int RestReplicationHandler::processRestoreCollection(
res = trx.finish(opRes.code);
return res.errorNumber();
return res;
}
if (!res.ok()) {
errorMsg =
"unable to drop collection '" + name + "': " + res.errorMessage();
res.reset(res.errorNumber(), errorMsg);
return res.errorNumber();
return Result(res.errorNumber(), std::string("unable to drop collection '") + name + "': " + res.errorMessage());
}
// intentionally falls through
} else {
Result res = TRI_ERROR_ARANGO_DUPLICATE_NAME;
errorMsg =
"unable to create collection '" + name + "': " + res.errorMessage();
res.reset(res.errorNumber(), errorMsg);
return res.errorNumber();
return Result(TRI_ERROR_ARANGO_DUPLICATE_NAME, std::string("unable to create collection '") + name + "': " + TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME));
}
}
// now re-create the collection
int res = createCollection(parameters, &col, reuseId);
int res = createCollection(parameters, &col);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg =
"unable to create collection: " + std::string(TRI_errno_string(res));
return res;
return Result(res, std::string("unable to create collection: ") + TRI_errno_string(res));
}
return TRI_ERROR_NO_ERROR;
return Result();
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the structure of a collection, coordinator case
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::processRestoreCollectionCoordinator(
VPackSlice const& collection, bool dropExisting, bool reuseId, bool force,
uint64_t numberOfShards, std::string& errorMsg, uint64_t replicationFactor,
Result RestReplicationHandler::processRestoreCollectionCoordinator(
VPackSlice const& collection, bool dropExisting, bool force,
uint64_t numberOfShards, uint64_t replicationFactor,
bool ignoreDistributeShardsLikeErrors) {
if (!collection.isObject()) {
errorMsg = "collection declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection declaration is invalid");
}
VPackSlice const parameters = collection.get("parameters");
if (!parameters.isObject()) {
errorMsg = "collection parameters declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection parameters declaration is invalid");
}
std::string const name = arangodb::basics::VelocyPackHelper::getStringValue(
parameters, "name", "");
if (name.empty()) {
errorMsg = "collection name is missing";
return TRI_ERROR_HTTP_BAD_PARAMETER;
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection name is missing");
}
if (arangodb::basics::VelocyPackHelper::getBooleanValue(parameters, "deleted",
false)) {
// we don't care about deleted collections
return TRI_ERROR_NO_ERROR;
return Result();
}
std::string dbName = _vocbase->name();
@ -1047,6 +989,7 @@ int RestReplicationHandler::processRestoreCollectionCoordinator(
// drop an existing collection if it exists
if (dropExisting) {
std::string errorMsg;
int res = ci->dropCollectionCoordinator(dbName, col->cid_as_string(),
errorMsg, 0.0);
if (res == TRI_ERROR_FORBIDDEN ||
@ -1055,29 +998,20 @@ int RestReplicationHandler::processRestoreCollectionCoordinator(
// some collections must not be dropped
res = truncateCollectionOnCoordinator(dbName, name);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg =
"unable to truncate collection (dropping is forbidden): " + name;
return Result(res, std::string("unable to truncate collection (dropping is forbidden): ") + name);
}
return res;
return Result(res, TRI_errno_string(res));
}
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "unable to drop collection '" + name + "': " +
std::string(TRI_errno_string(res));
return res;
return Result(res, std::string("unable to drop collection '") + name + "': " + TRI_errno_string(res));
}
} else {
int res = TRI_ERROR_ARANGO_DUPLICATE_NAME;
errorMsg = "unable to create collection '" + name + "': " +
std::string(TRI_errno_string(res));
return res;
return Result(TRI_ERROR_ARANGO_DUPLICATE_NAME, std::string("unable to create collection '") + name + "': " + TRI_errno_string(TRI_ERROR_ARANGO_DUPLICATE_NAME));
}
} catch (basics::Exception const& e) {
} catch (basics::Exception const& ex) {
LOG_TOPIC(DEBUG, Logger::FIXME) << "processRestoreCollectionCoordinator "
<< "could not drop collection: " << e.message();
<< "could not drop collection: " << ex.what();
} catch (...) {}
// now re-create the collection
@ -1130,14 +1064,12 @@ int RestReplicationHandler::processRestoreCollectionCoordinator(
toMerge.close(); // TopLevel
VPackSlice const type = parameters.get("type");
TRI_col_type_e collectionType;
if (type.isNumber()) {
collectionType = static_cast<TRI_col_type_e>(type.getNumericValue<int>());
} else {
errorMsg = "collection type not given or wrong";
return TRI_ERROR_HTTP_BAD_PARAMETER;
if (!type.isNumber()) {
return Result(TRI_ERROR_HTTP_BAD_PARAMETER, "collection type not given or wrong");
}
TRI_col_type_e collectionType = static_cast<TRI_col_type_e>(type.getNumericValue<int>());
VPackSlice const sliceToMerge = toMerge.slice();
VPackBuilder mergedBuilder =
VPackCollection::merge(parameters, sliceToMerge, false);
@ -1163,26 +1095,26 @@ int RestReplicationHandler::processRestoreCollectionCoordinator(
entry.grantCollection(dbName, col->name(), AuthLevel::RW);
});
}
} catch (basics::Exception const& e) {
} catch (basics::Exception const& ex) {
// Error, report it.
errorMsg = e.message();
return e.code();
return Result(ex.code(), ex.what());
} catch (std::exception const& ex) {
return Result(TRI_ERROR_INTERNAL, ex.what());
}
// All other errors are thrown to the outside.
return TRI_ERROR_NO_ERROR;
return Result();
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the data of a collection
////////////////////////////////////////////////////////////////////////////////
Result RestReplicationHandler::processRestoreData(std::string const& colName,
bool useRevision) {
Result RestReplicationHandler::processRestoreData(std::string const& colName) {
grantTemporaryRights();
if (colName == "_users") {
// We need to handle the _users in a special way
return processRestoreUsersBatch(colName, useRevision);
return processRestoreUsersBatch(colName);
}
auto ctx =
transaction::StandaloneContext::Create(_vocbase);
@ -1199,14 +1131,14 @@ Result RestReplicationHandler::processRestoreData(std::string const& colName,
return res;
}
res = processRestoreDataBatch(trx, colName, useRevision);
res = processRestoreDataBatch(trx, colName);
res = trx.finish(res);
return res;
}
Result RestReplicationHandler::parseBatch(
std::string const& collectionName, bool useRevision,
std::string const& collectionName,
std::unordered_map<std::string, VPackValueLength>& latest,
VPackBuilder& allMarkers) {
VPackBuilder builder;
@ -1245,7 +1177,7 @@ Result RestReplicationHandler::parseBatch(
VPackSlice doc;
TRI_replication_operation_e type = REPLICATION_INVALID;
Result res = restoreDataParser(ptr, pos, collectionName, useRevision,
Result res = restoreDataParser(ptr, pos, collectionName,
key, builder, doc, type);
if (res.fail()) {
return res;
@ -1278,11 +1210,11 @@ Result RestReplicationHandler::parseBatch(
////////////////////////////////////////////////////////////////////////////////
Result RestReplicationHandler::processRestoreUsersBatch(
std::string const& collectionName, bool useRevision) {
std::string const& collectionName) {
std::unordered_map<std::string, VPackValueLength> latest;
VPackBuilder allMarkers;
parseBatch(collectionName, useRevision, latest, allMarkers);
parseBatch(collectionName, latest, allMarkers);
VPackSlice allMarkersSlice = allMarkers.slice();
@ -1344,14 +1276,13 @@ Result RestReplicationHandler::processRestoreUsersBatch(
////////////////////////////////////////////////////////////////////////////////
Result RestReplicationHandler::processRestoreDataBatch(
transaction::Methods& trx, std::string const& collectionName,
bool useRevision) {
transaction::Methods& trx, std::string const& collectionName) {
VPackBuilder builder;
std::unordered_map<std::string, VPackValueLength> latest;
VPackBuilder allMarkers;
parseBatch(collectionName, useRevision, latest, allMarkers);
parseBatch(collectionName, latest, allMarkers);
// First remove all keys of which the last marker we saw was a deletion
// marker:
@ -2437,6 +2368,7 @@ void RestReplicationHandler::handleCommandLoggerFirstTick() {
/// * status
/// * tickMin - tickMax
//////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandLoggerTickRanges() {
StorageEngine* engine = EngineSelectorFeature::ENGINE;
TRI_ASSERT(engine);
@ -2454,11 +2386,8 @@ void RestReplicationHandler::handleCommandLoggerTickRanges() {
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::createCollection(VPackSlice slice,
arangodb::LogicalCollection** dst,
bool reuseId) {
if (dst != nullptr) {
*dst = nullptr;
}
arangodb::LogicalCollection** dst) {
TRI_ASSERT(dst != nullptr);
if (!slice.isObject()) {
return TRI_ERROR_HTTP_BAD_PARAMETER;
@ -2470,28 +2399,24 @@ int RestReplicationHandler::createCollection(VPackSlice slice,
if (name.empty()) {
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
TRI_voc_cid_t cid = 0;
if (reuseId) {
cid = arangodb::basics::VelocyPackHelper::extractIdValue(slice);
if (cid == 0) {
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
}
std::string const uuid =
arangodb::basics::VelocyPackHelper::getStringValue(slice, "globallyUniqueId", "");
TRI_col_type_e const type = static_cast<TRI_col_type_e>(
arangodb::basics::VelocyPackHelper::getNumericValue<int>(
slice, "type", (int)TRI_COL_TYPE_DOCUMENT));
slice, "type", int(TRI_COL_TYPE_DOCUMENT)));
arangodb::LogicalCollection* col = nullptr;
if (cid > 0) {
col = _vocbase->lookupCollection(cid);
if (!uuid.empty()) {
col = _vocbase->lookupCollectionByUuid(uuid);
}
if (col != nullptr) {
col = _vocbase->lookupCollection(name);
}
if (col != nullptr && col->type() == type) {
// TODO
// collection already exists. TODO: compare attributes
return TRI_ERROR_NO_ERROR;
}
@ -2504,13 +2429,17 @@ int RestReplicationHandler::createCollection(VPackSlice slice,
if (!name.empty() && name[0] == '_' && !slice.hasKey("isSystem")) {
// system collection?
patch.add("isSystem", VPackValue(true));
patch.add("objectId", VPackSlice::nullSlice());
patch.add("cid", VPackSlice::nullSlice());
patch.add("id", VPackSlice::nullSlice());
}
StorageEngine* engine = EngineSelectorFeature::ENGINE;
TRI_ASSERT(engine != nullptr);
engine->addParametersForNewCollection(patch, slice);
patch.close();
VPackBuilder builder = VPackCollection::merge(slice, patch.slice(), false);
VPackBuilder builder = VPackCollection::merge(slice, patch.slice(),
/*mergeValues*/true, /*nullMeansRemove*/true);
slice = builder.slice();
col = _vocbase->createCollection(slice);

View File

@ -26,7 +26,7 @@
#define ARANGOD_REST_HANDLER_REST_REPLICATION_HANDLER_H 1
#include "Basics/Common.h"
#include "Basics/Result.h"
#include "RestHandler/RestVocbaseBaseHandler.h"
#include "VocBase/replication-common.h"
@ -237,29 +237,28 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// @brief restores the structure of a collection
//////////////////////////////////////////////////////////////////////////////
int processRestoreCollection(VPackSlice const&, bool, bool, bool,
std::string&);
Result processRestoreCollection(VPackSlice const&, bool overwrite, bool force);
//////////////////////////////////////////////////////////////////////////////
/// @brief restores the structure of a collection, coordinator case
//////////////////////////////////////////////////////////////////////////////
int processRestoreCollectionCoordinator(VPackSlice const&, bool, bool, bool,
uint64_t, std::string&, uint64_t,
bool);
Result processRestoreCollectionCoordinator(VPackSlice const&, bool overwrite, bool force,
uint64_t numberOfShards, uint64_t replicationFactor,
bool ignoreDistributeShardsLikeErrors);
//////////////////////////////////////////////////////////////////////////////
/// @brief restores the data of the _users collection
//////////////////////////////////////////////////////////////////////////////
Result processRestoreUsersBatch(std::string const& colName, bool useRevision);
Result processRestoreUsersBatch(std::string const& colName);
//////////////////////////////////////////////////////////////////////////////
/// @brief restores the data of a collection
//////////////////////////////////////////////////////////////////////////////
Result processRestoreDataBatch(transaction::Methods& trx,
std::string const& colName, bool useRevision);
std::string const& colName);
//////////////////////////////////////////////////////////////////////////////
/// @brief restores the indexes of a collection
@ -277,13 +276,13 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// @brief restores the data of a collection
//////////////////////////////////////////////////////////////////////////////
Result processRestoreData(std::string const& colName, bool useRevision);
Result processRestoreData(std::string const& colName);
//////////////////////////////////////////////////////////////////////////////
/// @brief parse an input batch
//////////////////////////////////////////////////////////////////////////////
Result parseBatch(std::string const& collectionName, bool useRevision,
Result parseBatch(std::string const& collectionName,
std::unordered_map<std::string, VPackValueLength>& latest,
VPackBuilder& allMarkers);
@ -291,7 +290,7 @@ class RestReplicationHandler : public RestVocbaseBaseHandler {
/// @brief creates a collection, based on the VelocyPack provided
//////////////////////////////////////////////////////////////////////////////
int createCollection(VPackSlice, arangodb::LogicalCollection**, bool);
int createCollection(VPackSlice, arangodb::LogicalCollection**);
protected:
//////////////////////////////////////////////////////////////////////////////

View File

@ -1383,25 +1383,16 @@ Result LogicalCollection::compareChecksums(VPackSlice checksumSlice, std::string
std::string LogicalCollection::generateGloballyUniqueId() const {
ServerState::RoleEnum role = ServerState::instance()->getRole();
std::string result;
/*if (_vocbase->isSystem()) {
result.reserve(32);
result.append(StaticStrings::SystemDatabase);
} else {*/
result.reserve(64);
//result.append(_vocbase->name());
//}
//result.push_back('/');
if (ServerState::isCoordinator(role)) {
TRI_ASSERT(_planId != 0);
result.append(std::to_string(_planId));
} else if (ServerState::isDBServer(role)) {
TRI_ASSERT(_planId != 0);
// we add the shard name to the collection. If we every
// replicate shards, we identify them clusterwide
// we add the shard name to the collection. If we ever
// replicate shards, we can identify them cluster-wide
result.append(std::to_string(_planId));
result.push_back('/');
result.append(_name);

View File

@ -63,7 +63,6 @@ RestoreFeature::RestoreFeature(application_features::ApplicationServer* server,
_importStructure(true),
_progress(true),
_overwrite(true),
_recycleIds(false),
_force(false),
_ignoreDistributeShardsLikeErrors(false),
_clusterMode(false),
@ -87,6 +86,9 @@ void RestoreFeature::collectOptions(
"restrict to collection name (can be specified multiple times)",
new VectorParameter<StringParameter>(&_collections));
options->addObsoleteOption("--recycle-ids",
"collection ids are now handled automatically", false);
options->addOption("--batch-size",
"maximum size for individual data batches (in bytes)",
new UInt64Parameter(&_chunkSize));
@ -114,9 +116,6 @@ void RestoreFeature::collectOptions(
options->addOption("--overwrite", "overwrite collections if they exist",
new BooleanParameter(&_overwrite));
options->addOption("--recycle-ids", "recycle collection ids from dump",
new BooleanParameter(&_recycleIds));
options->addOption("--default-number-of-shards",
"default value for numberOfShards if not specified",
new UInt64Parameter(&_defaultNumberOfShards));
@ -230,8 +229,7 @@ int RestoreFeature::sendRestoreCollection(VPackSlice const& slice,
std::string url =
"/_api/replication/restore-collection"
"?overwrite=" +
std::string(_overwrite ? "true" : "false") + "&recycleIds=" +
std::string(_recycleIds ? "true" : "false") + "&force=" +
std::string(_overwrite ? "true" : "false") + "&force=" +
std::string(_force ? "true" : "false") +
"&ignoreDistributeShardsLikeErrors=" +
std::string(_ignoreDistributeShardsLikeErrors ? "true":"false");
@ -317,8 +315,7 @@ int RestoreFeature::sendRestoreData(std::string const& cname,
char const* buffer, size_t bufferSize,
std::string& errorMsg) {
std::string const url = "/_api/replication/restore-data?collection=" +
StringUtils::urlEncode(cname) + "&recycleIds=" +
(_recycleIds ? "true" : "false") + "&force=" +
StringUtils::urlEncode(cname) + "&force=" +
(_force ? "true" : "false");
std::unique_ptr<SimpleHttpResult> response(

View File

@ -58,7 +58,6 @@ class RestoreFeature final
bool _importStructure;
bool _progress;
bool _overwrite;
bool _recycleIds;
bool _force;
bool _ignoreDistributeShardsLikeErrors;
bool _clusterMode;