From 98e9197a3f164825b0554bc80c6b743cbbecc519 Mon Sep 17 00:00:00 2001 From: Jan Date: Fri, 7 Jun 2019 15:45:14 +0200 Subject: [PATCH] =?UTF-8?q?properly=20downgrade=20enterprise=20features=20?= =?UTF-8?q?when=20restoring=20them=20into=20a=20com=E2=80=A6=20(#9161)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RestHandler/RestReplicationHandler.cpp | 78 +++++++++++++++++-- arangosh/Restore/RestoreFeature.cpp | 27 +++++-- 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 26d4782ba7..db9ac7367d 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -775,6 +775,18 @@ void RestReplicationHandler::handleCommandRestoreCollection() { res = processRestoreCollection(slice, overwrite, force); } + if (res.is(TRI_ERROR_ARANGO_DUPLICATE_NAME)) { + VPackSlice name = slice.get("parameters"); + if (name.isObject()) { + name = name.get("name"); + if (name.isString() && !name.copyString().empty() && name.copyString()[0] == '_') { + // system collection, may have been created in the meantime by a background action + // collection should be there now + res.reset(); + } + } + } + if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } @@ -1056,7 +1068,7 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( toMerge.add("id", VPackValue(newId)); // Number of shards. Will be overwritten if not existent - VPackSlice const numberOfShardsSlice = parameters.get("numberOfShards"); + VPackSlice const numberOfShardsSlice = parameters.get(StaticStrings::NumberOfShards); if (!numberOfShardsSlice.isInteger()) { // The information does not contain numberOfShards. Overwrite it. VPackSlice const shards = parameters.get("shards"); @@ -1071,11 +1083,11 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( } } TRI_ASSERT(numberOfShards > 0); - toMerge.add("numberOfShards", VPackValue(numberOfShards)); + toMerge.add(StaticStrings::NumberOfShards, VPackValue(numberOfShards)); } // Replication Factor. Will be overwritten if not existent - VPackSlice const replFactorSlice = parameters.get("replicationFactor"); + VPackSlice const replFactorSlice = parameters.get(StaticStrings::ReplicationFactor); bool isValidReplFactorSlice = replFactorSlice.isInteger() || (replFactorSlice.isString() && replFactorSlice.isEqualString("satellite")); @@ -1084,17 +1096,71 @@ Result RestReplicationHandler::processRestoreCollectionCoordinator( replicationFactor = 1; } TRI_ASSERT(replicationFactor > 0); - toMerge.add("replicationFactor", VPackValue(replicationFactor)); + toMerge.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor)); } // always use current version number when restoring a collection, // because the collection is effectively NEW toMerge.add("version", VPackValue(LogicalCollection::VERSION_33)); - if (!name.empty() && name[0] == '_' && !parameters.hasKey("isSystem")) { + if (!name.empty() && name[0] == '_' && !parameters.hasKey(StaticStrings::DataSourceSystem)) { // system collection? - toMerge.add("isSystem", VPackValue(true)); + toMerge.add(StaticStrings::DataSourceSystem, VPackValue(true)); } +#ifndef USE_ENTERPRISE + std::vector changes; + + // when in the community version, we need to turn off specific attributes + // because they are only supported in enterprise mode + + // watch out for "isSmart" -> we need to set this to false in the community version + VPackSlice s = parameters.get(StaticStrings::GraphIsSmart); + if (s.isBoolean() && s.getBoolean()) { + // isSmart needs to be turned off in the community version + toMerge.add(StaticStrings::GraphIsSmart, VPackValue(false)); + changes.push_back("changed 'isSmart' attribute value to false"); + } + + // "smartGraphAttribute" needs to be set to be removed too + s = parameters.get(StaticStrings::GraphSmartGraphAttribute); + if (s.isString() && !s.copyString().empty()) { + // smartGraphAttribute needs to be removed + toMerge.add(StaticStrings::GraphSmartGraphAttribute, VPackSlice::nullSlice()); + changes.push_back("removed 'smartGraphAttribute' attribute value"); + } + + // same for "smartJoinAttribute" + s = parameters.get(StaticStrings::SmartJoinAttribute); + if (s.isString() && !s.copyString().empty()) { + // smartJoinAttribute needs to be removed + toMerge.add(StaticStrings::SmartJoinAttribute, VPackSlice::nullSlice()); + changes.push_back("removed 'smartJoinAttribute' attribute value"); + } + + // finally rewrite all enterprise sharding strategies to a simple hash-based strategy + s = parameters.get("shardingStrategy"); + if (s.isString() && s.copyString().find("enterprise") != std::string::npos) { + // downgrade sharding strategy to just hash + toMerge.add("shardingStrategy", VPackValue("hash")); + changes.push_back("changed 'shardingStrategy' attribute value to 'hash'"); + } + + s = parameters.get(StaticStrings::ReplicationFactor); + if (s.isString() && s.copyString() == "satellite") { + // set "satellite" replicationFactor to the default replication factor + ClusterFeature* cl = application_features::ApplicationServer::getFeature("Cluster"); + + uint32_t replicationFactor = cl->systemReplicationFactor(); + toMerge.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor)); + changes.push_back(std::string("changed 'replicationFactor' attribute value to ") + std::to_string(replicationFactor));; + } + + if (!changes.empty()) { + LOG_TOPIC("fc359", INFO, Logger::CLUSTER) << "rewrote info for collection '" << name << "' on restore for usage with the community version. the following changes were applied: " << basics::StringUtils::join(changes, ". "); + } +#endif + + // Always ignore `shadowCollections` they were accidentially dumped in // arangodb versions earlier than 3.3.6 toMerge.add("shadowCollections", arangodb::velocypack::Slice::nullSlice()); diff --git a/arangosh/Restore/RestoreFeature.cpp b/arangosh/Restore/RestoreFeature.cpp index 3ce36619a6..f8dc06371c 100644 --- a/arangosh/Restore/RestoreFeature.cpp +++ b/arangosh/Restore/RestoreFeature.cpp @@ -180,16 +180,23 @@ arangodb::Result checkHttpResponse(arangodb::httpclient::SimpleHttpClient& clien bool sortCollections(VPackBuilder const& l, VPackBuilder const& r) { VPackSlice const left = l.slice().get("parameters"); VPackSlice const right = r.slice().get("parameters"); - + + std::string leftName = + arangodb::basics::VelocyPackHelper::getStringValue(left, "name", ""); + std::string rightName = + arangodb::basics::VelocyPackHelper::getStringValue(right, "name", ""); + // First we sort by shard distribution. // We first have to create the collections which have no dependencies. // NB: Dependency graph has depth at most 1, no need to manage complex DAG VPackSlice leftDist = left.get("distributeShardsLike"); VPackSlice rightDist = right.get("distributeShardsLike"); - if (leftDist.isNone() && !rightDist.isNone()) { + if (leftDist.isNone() && rightDist.isString() && + rightDist.copyString() == leftName) { return true; } - if (rightDist.isNone() && !leftDist.isNone()) { + if (rightDist.isNone() && leftDist.isString() && + leftDist.copyString() == rightName) { return false; } @@ -204,11 +211,15 @@ bool sortCollections(VPackBuilder const& l, VPackBuilder const& r) { } // Finally, sort by name so we have stable, reproducible results - std::string leftName = - arangodb::basics::VelocyPackHelper::getStringValue(left, "name", ""); - std::string rightName = - arangodb::basics::VelocyPackHelper::getStringValue(right, "name", ""); - + // Sort system collections first + if (!leftName.empty() && leftName[0] == '_' && + !rightName.empty() && rightName[0] != '_') { + return true; + } + if (!leftName.empty() && leftName[0] != '_' && + !rightName.empty() && rightName[0] == '_') { + return false; + } return strcasecmp(leftName.c_str(), rightName.c_str()) < 0; }