Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/encrypted dump (#3768)

This commit is contained in:
Frank Celler 2017-11-22 15:59:02 +01:00 committed by GitHub
parent 56a2329592
commit 179ae83cbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 439 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arangosh/CMakeLists.txt
View File

@ -77,6 +77,7 @@ add_executable(${BIN_ARANGODUMP}
Shell/ClientFeature.cpp
Shell/ConsoleFeature.cpp
V8Client/ArangoClientHelper.cpp
${ADDITIONAL_BIN_ARANGODUMP_SOURCES}
)
target_link_libraries(${BIN_ARANGODUMP}
@ -229,6 +230,7 @@ add_executable(${BIN_ARANGORESTORE}
Shell/ClientFeature.cpp
Shell/ConsoleFeature.cpp
V8Client/ArangoClientHelper.cpp
${ADDITIONAL_BIN_ARANGORESTORE_SOURCES}
)
target_link_libraries(${BIN_ARANGORESTORE}

169
arangosh/Dump/DumpFeature.cpp
View File

@ -22,6 +22,11 @@
#include "DumpFeature.h"
#include <iostream>
#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
@ -39,10 +44,9 @@
#include "SimpleHttpClient/SimpleHttpResult.h"
#include "Ssl/SslInterface.h"
#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>
#include <iostream>
#ifdef USE_ENTERPRISE
#include "Enterprise/Encryption/EncryptionFeature.h"
#endif
using namespace arangodb;
using namespace arangodb::basics;
@ -68,12 +72,17 @@ DumpFeature::DumpFeature(application_features::ApplicationServer* server,
_result(result),
_batchId(0),
_clusterMode(false),
_encryption(nullptr),
_stats{0, 0, 0} {
requiresElevatedPrivileges(false);
setOptional(false);
startsAfter("Client");
startsAfter("Logger");
#ifdef USE_ENTERPRISE
startsAfter("Encryption");
#endif
_outputDirectory =
FileUtils::buildFilename(FileUtils::currentDirectory().result(), "dump");
}
@ -100,10 +109,10 @@ void DumpFeature::collectOptions(
"--force", "continue dumping even in the face of some server-side errors",
new BooleanParameter(&_force));
options->addOption(
"--ignore-distribute-shards-like-errors",
"continue dump even if sharding prototype collection is not backed up along",
new BooleanParameter(&_ignoreDistributeShardsLikeErrors));
options->addOption("--ignore-distribute-shards-like-errors",
"continue dump even if sharding prototype collection is "
"not backed up along",
new BooleanParameter(&_ignoreDistributeShardsLikeErrors));
options->addOption("--include-system-collections",
"include system collections",
@ -297,16 +306,15 @@ void DumpFeature::endBatch(std::string DBserver) {
// ignore any return value
}
/// @brief dump a single collection
// dump a single collection
int DumpFeature::dumpCollection(int fd, std::string const& cid,
std::string const& name, uint64_t maxTick,
std::string& errorMsg) {
uint64_t chunkSize = _chunkSize;
std::string const baseUrl =
"/_api/replication/dump?collection=" + cid +
"&batchId=" + StringUtils::itoa(_batchId) +
"&ticks=false&flush=false";
std::string const baseUrl = "/_api/replication/dump?collection=" + cid +
"&batchId=" + StringUtils::itoa(_batchId) +
"&ticks=false&flush=false";
uint64_t fromTick = _tickStart;
@ -317,7 +325,7 @@ int DumpFeature::dumpCollection(int fd, std::string const& cid,
if (maxTick > 0) {
url += "&to=" + StringUtils::itoa(maxTick);
}
_stats._totalBatches++;
std::unique_ptr<SimpleHttpResult> response(
@ -373,8 +381,9 @@ int DumpFeature::dumpCollection(int fd, std::string const& cid,
if (res == TRI_ERROR_NO_ERROR) {
StringBuffer const& body = response->getBody();
bool result = writeData(fd, body.c_str(), body.length());
if (!TRI_WritePointer(fd, body.c_str(), body.length())) {
if (!result) {
res = TRI_ERROR_CANNOT_WRITE_FILE;
} else {
_stats._totalWritten += (uint64_t)body.length();
@ -492,6 +501,7 @@ int DumpFeature::runDump(std::string& dbName, std::string& errorMsg) {
meta.openObject();
meta.add("database", VPackValue(dbName));
meta.add("lastTickAtDumpStart", VPackValue(tickString));
meta.close();
// save last tick in file
std::string fileName =
@ -513,16 +523,21 @@ int DumpFeature::runDump(std::string& dbName, std::string& errorMsg) {
return TRI_ERROR_CANNOT_WRITE_FILE;
}
meta.close();
beginEncryption(fd);
std::string const metaString = meta.slice().toJson();
if (!TRI_WritePointer(fd, metaString.c_str(), metaString.size())) {
bool result = writeData(fd, metaString.c_str(), metaString.size());
if (!result) {
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
return TRI_ERROR_CANNOT_WRITE_FILE;
}
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
} catch (...) {
errorMsg = "out of memory";
@ -615,16 +630,21 @@ int DumpFeature::runDump(std::string& dbName, std::string& errorMsg) {
return TRI_ERROR_CANNOT_WRITE_FILE;
}
std::string const collectionInfo = collection.toJson();
beginEncryption(fd);
if (!TRI_WritePointer(fd, collectionInfo.c_str(),
collectionInfo.size())) {
std::string const collectionInfo = collection.toJson();
bool result =
writeData(fd, collectionInfo.c_str(), collectionInfo.size());
if (!result) {
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
return TRI_ERROR_CANNOT_WRITE_FILE;
}
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
}
@ -634,16 +654,14 @@ int DumpFeature::runDump(std::string& dbName, std::string& errorMsg) {
fileName = _outputDirectory + TRI_DIR_SEPARATOR_STR + name + "_" +
hexString + ".data.json";
int fd;
// remove an existing file first
if (TRI_ExistsFile(fileName.c_str())) {
TRI_UnlinkFile(fileName.c_str());
}
fd = TRI_TRACKED_CREATE_FILE(fileName.c_str(),
O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
int fd = TRI_TRACKED_CREATE_FILE(
fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
@ -651,10 +669,13 @@ int DumpFeature::runDump(std::string& dbName, std::string& errorMsg) {
return TRI_ERROR_CANNOT_WRITE_FILE;
}
beginEncryption(fd);
extendBatch("");
int res =
dumpCollection(fd, std::to_string(cid), name, maxTick, errorMsg);
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
if (res != TRI_ERROR_NO_ERROR) {
@ -745,8 +766,9 @@ int DumpFeature::dumpShard(int fd, std::string const& DBserver,
if (res == TRI_ERROR_NO_ERROR) {
StringBuffer const& body = response->getBody();
bool result = writeData(fd, body.c_str(), body.length());
if (!TRI_WritePointer(fd, body.c_str(), body.length())) {
if (!result) {
res = TRI_ERROR_CANNOT_WRITE_FILE;
} else {
_stats._totalWritten += (uint64_t)body.length();
@ -876,18 +898,20 @@ int DumpFeature::runClusterDump(std::string& errorMsg) {
if (!_ignoreDistributeShardsLikeErrors) {
std::string prototypeCollection =
arangodb::basics::VelocyPackHelper::getStringValue(
parameters, "distributeShardsLike", "");
arangodb::basics::VelocyPackHelper::getStringValue(
parameters, "distributeShardsLike", "");
if (!prototypeCollection.empty() && !restrictList.empty()) {
if (std::find(
_collections.begin(), _collections.end(), prototypeCollection) ==
_collections.end()) {
errorMsg = std::string("Collection ") + name
+ "'s shard distribution is based on a that of collection " +
prototypeCollection + ", which is not dumped along. You may "
"dump the collection regardless of the missing prototype collection "
"by using the --ignore-distribute-shards-like-errors parameter.";
if (std::find(_collections.begin(), _collections.end(),
prototypeCollection) == _collections.end()) {
errorMsg =
std::string("Collection ") + name +
"'s shard distribution is based on a that of collection " +
prototypeCollection +
", which is not dumped along. You may "
"dump the collection regardless of the missing prototype "
"collection "
"by using the --ignore-distribute-shards-like-errors parameter.";
return TRI_ERROR_INTERNAL;
}
}
@ -921,16 +945,21 @@ int DumpFeature::runClusterDump(std::string& errorMsg) {
return TRI_ERROR_CANNOT_WRITE_FILE;
}
std::string const collectionInfo = collection.toJson();
beginEncryption(fd);
if (!TRI_WritePointer(fd, collectionInfo.c_str(),
collectionInfo.size())) {
std::string const collectionInfo = collection.toJson();
bool result =
writeData(fd, collectionInfo.c_str(), collectionInfo.size());
if (!result) {
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
errorMsg = "cannot write to file '" + fileName + "'";
return TRI_ERROR_CANNOT_WRITE_FILE;
}
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
}
@ -957,6 +986,8 @@ int DumpFeature::runClusterDump(std::string& errorMsg) {
return TRI_ERROR_CANNOT_WRITE_FILE;
}
beginEncryption(fd);
// First we have to go through all the shards, what are they?
VPackSlice const shards = parameters.get("shards");
@ -968,6 +999,7 @@ int DumpFeature::runClusterDump(std::string& errorMsg) {
if (!it.value.isArray() || it.value.length() == 0 ||
!it.value[0].isString()) {
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
errorMsg = "unexpected value for 'shards' attribute";
@ -980,19 +1012,27 @@ int DumpFeature::runClusterDump(std::string& errorMsg) {
std::cout << "# Dumping shard '" << shardName << "' from DBserver '"
<< DBserver << "' ..." << std::endl;
}
res = startBatch(DBserver, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
return res;
}
res = dumpShard(fd, DBserver, shardName, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
endEncryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
return res;
}
endBatch(DBserver);
}
endEncryption(fd);
res = TRI_TRACKED_CLOSE_FILE(fd);
if (res != TRI_ERROR_NO_ERROR) {
@ -1009,6 +1049,12 @@ int DumpFeature::runClusterDump(std::string& errorMsg) {
}
void DumpFeature::start() {
#ifdef USE_ENTERPRISE
_encryption =
application_features::ApplicationServer::getFeature<EncryptionFeature>(
"Encryption");
#endif
ClientFeature* client =
application_features::ApplicationServer::getFeature<ClientFeature>(
"Client");
@ -1027,8 +1073,9 @@ void DumpFeature::start() {
std::string dbName = client->databaseName();
_httpClient->params().setLocationRewriter(static_cast<void*>(client),
&rewriteLocation);
_httpClient->params().setUserNamePassword("/", client->username(), client->password());
&rewriteLocation);
_httpClient->params().setUserNamePassword("/", client->username(),
client->password());
std::string const versionString = _httpClient->getServerVersion();
@ -1088,6 +1135,12 @@ void DumpFeature::start() {
<< std::endl;
}
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
_encryption->writeEncryptionFile(_outputDirectory);
}
#endif
std::string errorMsg = "";
int res;
@ -1143,3 +1196,37 @@ void DumpFeature::start() {
*_result = ret;
}
bool DumpFeature::writeData(int fd, char const* data, size_t len) {
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
return _encryption->writeData(fd, data, len);
} else {
return TRI_WritePointer(fd, data, len);
}
#else
return TRI_WritePointer(fd, data, len);
#endif
}
void DumpFeature::beginEncryption(int fd) {
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
bool result = _encryption->beginEncryption(fd);
if (!result) {
LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
<< "cannot write prefix, giving up!";
FATAL_ERROR_EXIT();
}
}
#endif
}
void DumpFeature::endEncryption(int fd) {
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
_encryption->endEncryption(fd);
}
#endif
}

12
arangosh/Dump/DumpFeature.h
View File

@ -31,6 +31,8 @@ namespace httpclient {
class SimpleHttpResult;
}
class EncryptionFeature;
class DumpFeature final : public application_features::ApplicationFeature,
public ArangoClientHelper {
public:
@ -70,16 +72,16 @@ class DumpFeature final : public application_features::ApplicationFeature,
std::string& errorMsg);
int runClusterDump(std::string& errorMsg);
bool writeData(int fd, char const* data, size_t len);
void beginEncryption(int fd);
void endEncryption(int fd);
private:
int* _result;
// our batch id
uint64_t _batchId;
// cluster mode flag
bool _clusterMode;
EncryptionFeature* _encryption;
// statistics
struct {
uint64_t _totalBatches;
uint64_t _totalCollections;

24
arangosh/Dump/arangodump.cpp
View File

@ -37,6 +37,10 @@
#include "Shell/ClientFeature.h"
#include "Ssl/SslFeature.h"
#ifdef USE_ENTERPRISE
#include "Enterprise/Encryption/EncryptionFeature.h"
#endif
using namespace arangodb;
using namespace arangodb::application_features;
@ -45,8 +49,10 @@ int main(int argc, char* argv[]) {
ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
context.installHup();
std::shared_ptr<options::ProgramOptions> options(new options::ProgramOptions(
argv[0], "Usage: arangodump [<options>]", "For more information use:", BIN_DIRECTORY));
std::shared_ptr<options::ProgramOptions> options(
new options::ProgramOptions(argv[0], "Usage: arangodump [<options>]",
"For more information use:",
BIN_DIRECTORY));
ApplicationServer server(options, BIN_DIRECTORY);
@ -62,6 +68,10 @@ int main(int argc, char* argv[]) {
server.addFeature(new SslFeature(&server));
server.addFeature(new VersionFeature(&server));
#ifdef USE_ENTERPRISE
server.addFeature(new EncryptionFeature(&server));
#endif
try {
server.run(argc, argv);
if (server.helpShown()) {
@ -69,12 +79,14 @@ int main(int argc, char* argv[]) {
ret = EXIT_SUCCESS;
}
} catch (std::exception const& ex) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "arangodump terminated because of an unhandled exception: "
<< ex.what();
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "arangodump terminated because of an unhandled exception: "
<< ex.what();
ret = EXIT_FAILURE;
} catch (...) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "arangodump terminated because of an unhandled exception of "
"unknown type";
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "arangodump terminated because of an unhandled exception of "
"unknown type";
ret = EXIT_FAILURE;
}

106
arangosh/Restore/RestoreFeature.cpp
View File

@ -22,6 +22,11 @@
#include "RestoreFeature.h"
#include <iostream>
#include <velocypack/Options.h>
#include <velocypack/velocypack-aliases.h>
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
@ -40,10 +45,9 @@
#include "SimpleHttpClient/SimpleHttpResult.h"
#include "Ssl/SslInterface.h"
#include <velocypack/Options.h>
#include <velocypack/velocypack-aliases.h>
#include <iostream>
#ifdef USE_ENTERPRISE
#include "Enterprise/Encryption/EncryptionFeature.h"
#endif
using namespace arangodb;
using namespace arangodb::basics;
@ -69,12 +73,17 @@ RestoreFeature::RestoreFeature(application_features::ApplicationServer* server,
_defaultNumberOfShards(1),
_defaultReplicationFactor(1),
_result(result),
_encryption(nullptr),
_stats{0, 0, 0} {
requiresElevatedPrivileges(false);
setOptional(false);
startsAfter("Client");
startsAfter("Logger");
#ifdef USE_ENTERPRISE
startsAfter("Encryption");
#endif
_inputDirectory =
FileUtils::buildFilename(FileUtils::currentDirectory().result(), "dump");
}
@ -86,8 +95,8 @@ void RestoreFeature::collectOptions(
"restrict to collection name (can be specified multiple times)",
new VectorParameter<StringParameter>(&_collections));
options->addObsoleteOption("--recycle-ids",
"collection ids are now handled automatically", false);
options->addObsoleteOption(
"--recycle-ids", "collection ids are now handled automatically", false);
options->addOption("--batch-size",
"maximum size for individual data batches (in bytes)",
@ -130,8 +139,8 @@ void RestoreFeature::collectOptions(
new BooleanParameter(&_ignoreDistributeShardsLikeErrors));
options->addOption(
"--force", "continue restore even in the face of some server-side errors",
new BooleanParameter(&_force));
"--force", "continue restore even in the face of some server-side errors",
new BooleanParameter(&_force));
}
void RestoreFeature::validateOptions(
@ -227,12 +236,12 @@ int RestoreFeature::sendRestoreCollection(VPackSlice const& slice,
std::string const& name,
std::string& errorMsg) {
std::string url =
"/_api/replication/restore-collection"
"?overwrite=" +
std::string(_overwrite ? "true" : "false") + "&force=" +
std::string(_force ? "true" : "false") +
"&ignoreDistributeShardsLikeErrors=" +
std::string(_ignoreDistributeShardsLikeErrors ? "true":"false");
"/_api/replication/restore-collection"
"?overwrite=" +
std::string(_overwrite ? "true" : "false") + "&force=" +
std::string(_force ? "true" : "false") +
"&ignoreDistributeShardsLikeErrors=" +
std::string(_ignoreDistributeShardsLikeErrors ? "true" : "false");
if (_clusterMode) {
if (!slice.hasKey(std::vector<std::string>({"parameters", "shards"})) &&
@ -409,7 +418,15 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
}
std::string const fqn = _inputDirectory + TRI_DIR_SEPARATOR_STR + file;
VPackBuilder fileContentBuilder = basics::VelocyPackHelper::velocyPackFromFile(fqn);
#ifdef USE_ENTERPRISE
VPackBuilder fileContentBuilder =
(_encryption != nullptr)
? _encryption->velocyPackFromFile(fqn)
: basics::VelocyPackHelper::velocyPackFromFile(fqn);
#else
VPackBuilder fileContentBuilder =
basics::VelocyPackHelper::velocyPackFromFile(fqn);
#endif
VPackSlice const fileContent = fileContentBuilder.slice();
if (!fileContent.isObject()) {
@ -467,7 +484,7 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
}
}
std::sort(collections.begin(), collections.end(), SortCollections);
StringBuffer buffer(true);
// step2: run the actual import
for (VPackBuilder const& b : collections) {
@ -524,7 +541,8 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
<< " collection '" << cname << "'..." << std::endl;
}
int fd = TRI_TRACKED_OPEN_FILE(datafile.c_str(), O_RDONLY | TRI_O_CLOEXEC);
int fd =
TRI_TRACKED_OPEN_FILE(datafile.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
errorMsg = "cannot open collection data file '" + datafile + "'";
@ -532,20 +550,24 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
return TRI_ERROR_INTERNAL;
}
beginDecryption(fd);
buffer.clear();
while (true) {
if (buffer.reserve(16384) != TRI_ERROR_NO_ERROR) {
endDecryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
errorMsg = "out of memory";
return TRI_ERROR_OUT_OF_MEMORY;
}
ssize_t numRead = TRI_READ(fd, buffer.end(), 16384);
ssize_t numRead = readData(fd, buffer.end(), 16384);
if (numRead < 0) {
// error while reading
int res = TRI_errno();
endDecryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
errorMsg = std::string(TRI_errno_string(res));
@ -600,6 +622,8 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
std::cerr << errorMsg << std::endl;
continue;
}
endDecryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
return res;
@ -614,6 +638,7 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
}
}
endDecryption(fd);
TRI_TRACKED_CLOSE_FILE(fd);
}
}
@ -651,6 +676,12 @@ int RestoreFeature::processInputDirectory(std::string& errorMsg) {
}
void RestoreFeature::start() {
#ifdef USE_ENTERPRISE
_encryption =
application_features::ApplicationServer::getFeature<EncryptionFeature>(
"Encryption");
#endif
ClientFeature* client =
application_features::ApplicationServer::getFeature<ClientFeature>(
"Client");
@ -669,8 +700,9 @@ void RestoreFeature::start() {
std::string dbName = client->databaseName();
_httpClient->params().setLocationRewriter(static_cast<void*>(client),
&rewriteLocation);
_httpClient->params().setUserNamePassword("/", client->username(), client->password());
&rewriteLocation);
_httpClient->params().setUserNamePassword("/", client->username(),
client->password());
int err = TRI_ERROR_NO_ERROR;
std::string versionString = _httpClient->getServerVersion(&err);
@ -769,3 +801,37 @@ void RestoreFeature::start() {
*_result = ret;
}
ssize_t RestoreFeature::readData(int fd, char* data, size_t len) {
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
return _encryption->readData(fd, data, len);
} else {
return TRI_READ(fd, data, len);
}
#else
return TRI_READ(fd, data, len);
#endif
}
void RestoreFeature::beginDecryption(int fd) {
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
bool result = _encryption->beginDecryption(fd);
if (!result) {
LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
<< "cannot write prefix, giving up!";
FATAL_ERROR_EXIT();
}
}
#endif
}
void RestoreFeature::endDecryption(int fd) {
#ifdef USE_ENTERPRISE
if (_encryption != nullptr) {
_encryption->endDecryption(fd);
}
#endif
}

14
arangosh/Restore/RestoreFeature.h
View File

@ -34,12 +34,12 @@ namespace httpclient {
class SimpleHttpResult;
}
class RestoreFeature final
: public application_features::ApplicationFeature,
public ArangoClientHelper {
class EncryptionFeature;
class RestoreFeature final : public application_features::ApplicationFeature,
public ArangoClientHelper {
public:
RestoreFeature(application_features::ApplicationServer* server,
int* result);
RestoreFeature(application_features::ApplicationServer* server, int* result);
public:
void collectOptions(std::shared_ptr<options::ProgramOptions>) override;
@ -72,9 +72,13 @@ class RestoreFeature final
int sendRestoreData(std::string const& cname, char const* buffer,
size_t bufferSize, std::string& errorMsg);
int processInputDirectory(std::string& errorMsg);
ssize_t readData(int fd, char* data, size_t len);
void beginDecryption(int fd);
void endDecryption(int fd);
private:
int* _result;
EncryptionFeature* _encryption;
// statistics
struct {

24
arangosh/Restore/arangorestore.cpp
View File

@ -37,6 +37,10 @@
#include "Shell/ClientFeature.h"
#include "Ssl/SslFeature.h"
#ifdef USE_ENTERPRISE
#include "Enterprise/Encryption/EncryptionFeature.h"
#endif
using namespace arangodb;
using namespace arangodb::application_features;
@ -45,8 +49,10 @@ int main(int argc, char* argv[]) {
ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
context.installHup();
std::shared_ptr<options::ProgramOptions> options(new options::ProgramOptions(
argv[0], "Usage: arangorestore [<options>]", "For more information use:", BIN_DIRECTORY));
std::shared_ptr<options::ProgramOptions> options(
new options::ProgramOptions(argv[0], "Usage: arangorestore [<options>]",
"For more information use:",
BIN_DIRECTORY));
ApplicationServer server(options, BIN_DIRECTORY);
@ -63,6 +69,10 @@ int main(int argc, char* argv[]) {
server.addFeature(new TempFeature(&server, "arangorestore"));
server.addFeature(new VersionFeature(&server));
#ifdef USE_ENTERPRISE
server.addFeature(new EncryptionFeature(&server));
#endif
try {
server.run(argc, argv);
if (server.helpShown()) {
@ -70,12 +80,14 @@ int main(int argc, char* argv[]) {
ret = EXIT_SUCCESS;
}
} catch (std::exception const& ex) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "arangorestore terminated because of an unhandled exception: "
<< ex.what();
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "arangorestore terminated because of an unhandled exception: "
<< ex.what();
ret = EXIT_FAILURE;
} catch (...) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "arangorestore terminated because of an unhandled exception of "
"unknown type";
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "arangorestore terminated because of an unhandled exception of "
"unknown type";
ret = EXIT_FAILURE;
}

8
js/client/modules/@arangodb/process-utils.js
View File

@ -580,12 +580,12 @@ function runArangoDumpRestore (options, instanceInfo, which, database, rootDir,
'server.password': options.password,
'server.endpoint': instanceInfo.endpoint,
'server.database': database,
'include-system-collections': includeSystem ? 'true' : 'false'
'include-system-collections': includeSystem ? 'true' : 'false',
};
let exe;
rootDir = rootDir || instanceInfo.rootDir;
if (which === 'dump') {
args['output-directory'] = fs.join(rootDir, dumpDir);
exe = ARANGODUMP_BIN;
@ -594,6 +594,10 @@ function runArangoDumpRestore (options, instanceInfo, which, database, rootDir,
args['input-directory'] = fs.join(rootDir, dumpDir);
exe = ARANGORESTORE_BIN;
}
if (options.encrypted) {
args['encryption.keyfile'] = fs.join(rootDir, 'secret-key');
}
if (options.extremeVerbosity === true) {
print(exe);

5
js/client/modules/@arangodb/testing.js
View File

@ -407,10 +407,7 @@ function loadTestSuites () {
let testSuites = _.filter(fs.list(fs.join(__dirname, 'testsuites')),
function (p) {
return (p.substr(-3) === '.js');
})
.map(function (x) {
return x;
}).sort();
}).sort();
for (let j = 0; j < testSuites.length; j++) {
try {

164
js/client/modules/@arangodb/testsuites/dump_encrypted.js Normal file
View File

@ -0,0 +1,164 @@
/* jshint strict: false, sub: true */
/* global print */
'use strict';
// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// //////////////////////////////////////////////////////////////////////////////
const functionsDocumentation = {
'dump_encrypted': 'encrypted dump tests'
};
const optionsDocumentation = [
' - `skipEncrypted` : if set to true the encryption tests are skipped',
];
const pu = require('@arangodb/process-utils');
const tu = require('@arangodb/test-utils');
const fs = require('fs');
const _ = require('lodash');
const CYAN = require('internal').COLORS.COLOR_CYAN;
const RESET = require('internal').COLORS.COLOR_RESET;
function dumpEncrypted(options) {
let cluster;
if (options.cluster) {
cluster = '-cluster';
} else {
cluster = '';
}
if (options.skipEncrypted === true) {
print('skipping encryption tests!');
return {
failed: 0,
dump_encrypted: {
failed: 0,
status: true,
skipped: true
}
};
} // if
const storageEngine = options.storageEngine;
print(CYAN + 'Encrypted dump tests...' + RESET);
let instanceInfo = pu.startInstance('tcp', options, {}, 'dump_encrypted');
if (instanceInfo === false) {
return {
failed: 1,
dump: {
status: false,
message: 'failed to start server!'
}
};
}
print(CYAN + Date() + ': Setting up' + RESET);
let results = { failed: 1 };
results.setup = tu.runInArangosh(options, instanceInfo,
tu.makePathUnix('js/server/tests/dump/dump-setup' + cluster + '.js'));
results.setup.failed = 1;
let keyFile = fs.join(instanceInfo.rootDir, 'secret-key');
fs.write(keyFile, 'DER-HUND-der-hund-der-hund-der-h'); // must be exactly 32 chars long
if (pu.arangod.check.instanceAlive(instanceInfo, options) &&
(results.setup.status === true)) {
results.setup.failed = 0;
print(CYAN + Date() + ': Encrypted Dump and Restore - dump' + RESET);
let dumpOptions = _.clone(options);
dumpOptions.encrypted = true;
results.dump = pu.run.arangoDumpRestore(dumpOptions, instanceInfo, 'dump',
'UnitTestsDumpSrc');
results.dump.failed = 1;
if (pu.arangod.check.instanceAlive(instanceInfo, options) &&
(results.dump.status === true)) {
results.dump.failed = 0;
print(CYAN + Date() + ': Encrypted Dump and Restore - restore' + RESET);
results.restore = pu.run.arangoDumpRestore(dumpOptions, instanceInfo, 'restore',
'UnitTestsDumpDst');
results.restore.failed = 1;
if (pu.arangod.check.instanceAlive(instanceInfo, options) &&
(results.restore.status === true)) {
results.restore.failed = 0;
print(CYAN + Date() + ': Encrypted Dump and Restore - dump after restore' + RESET);
results.test = tu.runInArangosh(options, instanceInfo,
tu.makePathUnix('js/server/tests/dump/dump-' + storageEngine + cluster + '.js'), {
'server.database': 'UnitTestsDumpDst'
});
results.test.failed = 1;
if (pu.arangod.check.instanceAlive(instanceInfo, options) &&
(results.test.status === true)) {
results.test.failed = 0;
print(CYAN + Date() + ': Encrypted Dump and Restore - teardown' + RESET);
results.tearDown = tu.runInArangosh(options, instanceInfo,
tu.makePathUnix('js/server/tests/dump/dump-teardown' + cluster + '.js'));
results.tearDown.failed = 1;
if (results.tearDown.status) {
results.tearDown.failed = 0;
results.failed = 0;
}
}
}
}
}
print(CYAN + 'Shutting down...' + RESET);
pu.shutdownInstance(instanceInfo, options);
print(CYAN + 'done.' + RESET);
print();
return results;
}
function setup(testFns, defaultFns, opts, fnDocs, optionsDoc) {
// turn off encryption tests by default. only enable them in enterprise version
opts['skipEncrypted'] = true;
let version = {};
if (global.ARANGODB_CLIENT_VERSION) {
version = global.ARANGODB_CLIENT_VERSION(true);
if (version['enterprise-version']) {
opts['skipEncrypted'] = false;
}
}
testFns['dump_encrypted'] = dumpEncrypted;
defaultFns.push('dump_encrypted');
for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; }
for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); }
}
exports.setup = setup;