Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix dumping of views in the cluster (#6024)

This commit is contained in:
Simon 2018-08-02 17:25:49 +02:00 committed by Jan
parent c8660c967c
commit 42cabb858a
14 changed files with 233 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
arangod/Cluster/ClusterInfo.cpp
View File

@ -1221,14 +1221,12 @@ std::vector<std::shared_ptr<LogicalView>> const ClusterInfo::getViews(
// iterate over all collections
DatabaseViews::const_iterator it2 = (*it).second.begin();
while (it2 != (*it).second.end()) {
char c = (*it2).first[0];
if (c < '0' || c > '9') {
// skip collections indexed by id
result.push_back((*it2).second);
while (it2 != it->second.end()) {
char c = it2->first[0];
if (c >= '0' && c <= '9') {
// skip views indexed by name
result.emplace_back(it2->second);
}
++it2;
}

4
arangod/MMFiles/MMFilesCollection.cpp
View File

@ -2114,7 +2114,9 @@ std::shared_ptr<Index> MMFilesCollection::createIndex(transaction::Methods* trx,
idx = engine->indexFactory().prepareIndexFromSlice(
info, true, _logicalCollection, false
);
TRI_ASSERT(idx != nullptr);
if (!idx) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_INDEX_CREATION_FAILED);
}
if (ServerState::instance()->isCoordinator()) {
// In the coordinator case we do not fill the index

17
arangod/RestHandler/RestReplicationHandler.cpp
View File

@ -686,11 +686,11 @@ void RestReplicationHandler::handleCommandClusterInventory() {
ClusterInfo* ci = ClusterInfo::instance();
std::vector<std::shared_ptr<LogicalCollection>> cols =
ci->getCollections(dbName);
auto views = ci->getViews(dbName);
VPackBuilder resultBuilder;
resultBuilder.openObject();
resultBuilder.add(VPackValue("collections"));
resultBuilder.openArray();
resultBuilder.add("collections", VPackValue(VPackValueType::Array));
for (std::shared_ptr<LogicalCollection> const& c : cols) {
// We want to check if the collection is usable and all followers
// are in sync:
@ -714,9 +714,16 @@ void RestReplicationHandler::handleCommandClusterInventory() {
c->toVelocyPackForClusterInventory(resultBuilder, includeSystem, isReady, allInSync);
}
resultBuilder.close(); // collections
resultBuilder.add("views", VPackValue(VPackValueType::Array));
for (auto const& view : views) {
resultBuilder.openObject();
view->toVelocyPack(resultBuilder, /*details*/false, /*forPersistence*/true);
resultBuilder.close();
}
resultBuilder.close(); // views
TRI_voc_tick_t tick = TRI_CurrentTickServer();
auto tickString = std::to_string(tick);
resultBuilder.add("tick", VPackValue(tickString));
resultBuilder.add("tick", VPackValue(std::to_string(tick)));
resultBuilder.add("state", VPackValue("unused"));
resultBuilder.close(); // base
generateResult(rest::ResponseCode::OK, resultBuilder.slice());
@ -1757,6 +1764,8 @@ void RestReplicationHandler::handleCommandRestoreView() {
return;
}
LOG_TOPIC(TRACE, Logger::REPLICATION) << "restoring view: "
<< nameSlice.copyString();
auto view = _vocbase.lookupView(nameSlice.copyString());
if (view) {

7
arangod/RocksDBEngine/RocksDBCollection.cpp
View File

@ -382,13 +382,14 @@ std::shared_ptr<Index> RocksDBCollection::createIndex(
StorageEngine* engine = EngineSelectorFeature::ENGINE;
// We are sure that we do not have an index of this type.
// We also hold the lock.
// Create it
// We also hold the lock. Create it
idx = engine->indexFactory().prepareIndexFromSlice(
info, true, _logicalCollection, false
);
TRI_ASSERT(idx != nullptr);
if (!idx) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_INDEX_CREATION_FAILED);
}
int res = saveIndex(trx, idx);

23
arangod/VocBase/vocbase.cpp
View File

@ -1022,14 +1022,23 @@ void TRI_vocbase_t::inventory(
result.close(); // </collection>
result.add("views", VPackValue(VPackValueType::Array, true));
for (auto const& dataSource : dataSourceById) {
if (dataSource.second->category() != LogicalView::category()) {
continue;
if (ServerState::instance()->isCoordinator()) {
auto views = ClusterInfo::instance()->getViews(name());
for (auto const& view : views) {
result.openObject();
view->toVelocyPack(result, /*details*/false, /*forPersistence*/true);
result.close();
}
} else {
for (auto const& dataSource : dataSourceById) {
if (dataSource.second->category() != LogicalView::category()) {
continue;
}
LogicalView const* view = static_cast<LogicalView*>(dataSource.second.get());
result.openObject();
view->toVelocyPack(result, /*details*/false, /*forPersistence*/true);
result.close();
}
LogicalView const* view = static_cast<LogicalView*>(dataSource.second.get());
result.openObject();
view->toVelocyPack(result, /*details*/false, /*forPersistence*/true);
result.close();
}
result.close(); // </views>
}

187
arangosh/Dump/DumpFeature.cpp
View File

@ -592,7 +592,7 @@ void DumpFeature::validateOptions(
// dump data from server
Result DumpFeature::runDump(httpclient::SimpleHttpClient& client,
std::string& dbName) {
std::string const& dbName) {
Result result;
uint64_t batchId;
std::tie(result, batchId) = ::startBatch(client, "");
@ -640,47 +640,16 @@ Result DumpFeature::runDump(httpclient::SimpleHttpClient& client,
views = VPackSlice::emptyArraySlice();
}
// read the server's max tick value
std::string const tickString =
basics::VelocyPackHelper::getStringValue(body, "tick", "");
if (tickString == "") {
return ::ErrorMalformedJsonResponse;
// Step 1. Store view definition files
Result res = storeDumpJson(body, dbName);
if (res.fail()) {
return res;
}
LOG_TOPIC(INFO, Logger::DUMP)
<< "Last tick provided by server is: " << tickString;
// set the local max tick value
uint64_t maxTick = basics::StringUtils::uint64(tickString);
// check if the user specified a max tick value
if (_options.tickEnd > 0 && maxTick > _options.tickEnd) {
maxTick = _options.tickEnd;
}
// Step 1. write the dump.json file
try {
VPackBuilder meta;
meta.openObject();
meta.add("database", VPackValue(dbName));
meta.add("lastTickAtDumpStart", VPackValue(tickString));
meta.close();
// save last tick in file
auto file = _directory->writableFile("dump.json", true);
if (!::fileOk(file.get())) {
return ::fileError(file.get(), true);
}
std::string const metaString = meta.slice().toJson();
file->write(metaString.c_str(), metaString.size());
if (file->status().fail()) {
return file->status();
}
} catch (basics::Exception const& ex) {
return {ex.code(), ex.what()};
} catch (std::exception const& ex) {
return {TRI_ERROR_INTERNAL, ex.what()};
} catch (...) {
return {TRI_ERROR_OUT_OF_MEMORY, "out of memory"};
// Step 2. Store view definition files
res = storeViews(views);
if (res.fail()) {
return res;
}
// create a lookup table for collections
@ -690,7 +659,7 @@ Result DumpFeature::runDump(httpclient::SimpleHttpClient& client,
std::pair<std::string, bool>(_options.collections[i], true));
}
// Step 2. iterate over collections, queue dump jobs
// Step 3. iterate over collections, queue dump jobs
for (VPackSlice const& collection : VPackArrayIterator(collections)) {
// extract parameters about the individual collection
if (!collection.isObject()) {
@ -749,42 +718,13 @@ Result DumpFeature::runDump(httpclient::SimpleHttpClient& client,
return _workerErrors.front();
}
}
// Step 3. Store view definition files
for (VPackSlice view : VPackArrayIterator(views)) {
auto nameSlice = view.get(StaticStrings::DataSourceName);
if (!nameSlice.isString() || nameSlice.getStringLength() == 0) {
continue; // ignore
}
try {
std::string fname = nameSlice.copyString();
fname.append(".view.json");
// save last tick in file
auto file = _directory->writableFile(fname, true);
if (!::fileOk(file.get())) {
return ::fileError(file.get(), true);
}
std::string const viewString = view.toJson();
file->write(viewString.c_str(), viewString.size());
if (file->status().fail()) {
return file->status();
}
} catch (basics::Exception const& ex) {
return {ex.code(), ex.what()};
} catch (std::exception const& ex) {
return {TRI_ERROR_INTERNAL, ex.what()};
} catch (...) {
return {TRI_ERROR_OUT_OF_MEMORY, "out of memory"};
}
}
return {TRI_ERROR_NO_ERROR};
}
// dump data from cluster via a coordinator
Result DumpFeature::runClusterDump(httpclient::SimpleHttpClient& client) {
Result DumpFeature::runClusterDump(httpclient::SimpleHttpClient& client,
std::string const& dbname) {
// get the cluster inventory
std::string const url =
"/_api/replication/clusterInventory?includeSystem=" +
@ -813,6 +753,24 @@ Result DumpFeature::runClusterDump(httpclient::SimpleHttpClient& client) {
if (!collections.isArray()) {
return ::ErrorMalformedJsonResponse;
}
// get the view list
VPackSlice views = body.get("views");
if (!views.isArray()) {
views = VPackSlice::emptyArraySlice();
}
// Step 1. Store view definition files
Result res = storeDumpJson(body, dbname);
if (res.fail()) {
return res;
}
// Step 2. Store view definition files
res = storeViews(views);
if (res.fail()) {
return res;
}
// create a lookup table for collections
std::map<std::string, bool> restrictList;
@ -821,7 +779,7 @@ Result DumpFeature::runClusterDump(httpclient::SimpleHttpClient& client) {
std::pair<std::string, bool>(_options.collections[i], true));
}
// iterate over collections
// Step 3. iterate over collections
for (auto const& collection : VPackArrayIterator(collections)) {
// extract parameters about the individual collection
if (!collection.isObject()) {
@ -902,6 +860,85 @@ Result DumpFeature::runClusterDump(httpclient::SimpleHttpClient& client) {
return {TRI_ERROR_NO_ERROR};
}
Result DumpFeature::storeDumpJson(VPackSlice const& body,
std::string const& dbName) const {
// read the server's max tick value
std::string const tickString =
basics::VelocyPackHelper::getStringValue(body, "tick", "");
if (tickString == "") {
return ::ErrorMalformedJsonResponse;
}
LOG_TOPIC(INFO, Logger::DUMP)
<< "Last tick provided by server is: " << tickString;
// set the local max tick value
uint64_t maxTick = basics::StringUtils::uint64(tickString);
// check if the user specified a max tick value
if (_options.tickEnd > 0 && maxTick > _options.tickEnd) {
maxTick = _options.tickEnd;
}
try {
VPackBuilder meta;
meta.openObject();
meta.add("database", VPackValue(dbName));
meta.add("lastTickAtDumpStart", VPackValue(tickString));
meta.close();
// save last tick in file
auto file = _directory->writableFile("dump.json", true);
if (!::fileOk(file.get())) {
return ::fileError(file.get(), true);
}
std::string const metaString = meta.slice().toJson();
file->write(metaString.c_str(), metaString.size());
if (file->status().fail()) {
return file->status();
}
} catch (basics::Exception const& ex) {
return {ex.code(), ex.what()};
} catch (std::exception const& ex) {
return {TRI_ERROR_INTERNAL, ex.what()};
} catch (...) {
return {TRI_ERROR_OUT_OF_MEMORY, "out of memory"};
}
return {};
}
Result DumpFeature::storeViews(VPackSlice const& views) const {
for (VPackSlice view : VPackArrayIterator(views)) {
auto nameSlice = view.get(StaticStrings::DataSourceName);
if (!nameSlice.isString() || nameSlice.getStringLength() == 0) {
continue; // ignore
}
try {
std::string fname = nameSlice.copyString();
fname.append(".view.json");
// save last tick in file
auto file = _directory->writableFile(fname, true);
if (!::fileOk(file.get())) {
return ::fileError(file.get(), true);
}
std::string const viewString = view.toJson();
file->write(viewString.c_str(), viewString.size());
if (file->status().fail()) {
return file->status();
}
} catch (basics::Exception const& ex) {
return {ex.code(), ex.what()};
} catch (std::exception const& ex) {
return {TRI_ERROR_INTERNAL, ex.what()};
} catch (...) {
return {TRI_ERROR_OUT_OF_MEMORY, "out of memory"};
}
}
return {};
}
void DumpFeature::reportError(Result const& error) {
try {
@ -986,7 +1023,7 @@ void DumpFeature::start() {
if (!_options.clusterMode) {
res = runDump(*httpClient, dbName);
} else {
res = runClusterDump(*httpClient);
res = runClusterDump(*httpClient, dbName);
}
} catch (std::exception const& ex) {
LOG_TOPIC(ERR, Logger::FIXME) << "caught exception " << ex.what();

7
arangosh/Dump/DumpFeature.h
View File

@ -115,8 +115,11 @@ class DumpFeature : public application_features::ApplicationFeature {
std::queue<Result> _workerErrors;
private:
Result runDump(httpclient::SimpleHttpClient& client, std::string& dbName);
Result runClusterDump(httpclient::SimpleHttpClient& client);
Result runDump(httpclient::SimpleHttpClient& client, std::string const& dbName);
Result runClusterDump(httpclient::SimpleHttpClient& client, std::string const& dbName);
Result storeDumpJson(VPackSlice const& body, std::string const& dbName) const;
Result storeViews(velocypack::Slice const&) const;
};
} // namespace arangodb

3
arangosh/Restore/RestoreFeature.cpp
View File

@ -514,7 +514,6 @@ arangodb::Result restoreView(arangodb::httpclient::SimpleHttpClient& httpClient,
std::string url = "/_api/replication/restore-view?overwrite=" +
std::string(options.overwrite ? "true" : "false") +
"&force=" + std::string(options.force ? "true" : "false");
std::string const body = viewDefinition.toJson();
std::unique_ptr<SimpleHttpResult> response(httpClient.request(arangodb::rest::RequestType::PUT,
@ -642,8 +641,10 @@ arangodb::Result processInputDirectory(
}
std::sort(collections.begin(), collections.end(), ::sortCollections);
LOG_TOPIC(INFO, Logger::RESTORE) << "# Creating views...";
// Step 2: recreate all views
for (VPackBuilder viewDefinition : views) {
LOG_TOPIC(DEBUG, Logger::RESTORE) << "# Creating view: " << viewDefinition.toJson();
Result res = ::restoreView(httpClient, options, viewDefinition.slice());
if (res.fail()) {
return res;

25
js/server/tests/dump/dump-mmfiles-cluster.js
View File

@ -604,6 +604,31 @@ function dumpTestEnterpriseSuite () {
assertEqual("9", res[1].value);
assertEqual("11", res[2].value);
assertEqual("12", res[3].value);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test view restoring
////////////////////////////////////////////////////////////////////////////////
testView : function () {
try {
db._createView("check", "arangosearch", {});
} catch (err) {}
let views = db._views();
if (views.length === 0) {
return; // arangosearch views are not supported
}
let view = db._view("UnitTestsDumpView");
assertTrue(view !== null);
let props = view.properties();
assertEqual(Object.keys(props.links).length, 1);
assertTrue(props.hasOwnProperty("links"));
assertTrue(props.links.hasOwnProperty("UnitTestsDumpViewCollection"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("includeAllFields"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("fields"));
assertTrue(props.links.UnitTestsDumpViewCollection.includeAllFields);
}
};

3
js/server/tests/dump/dump-mmfiles.js
View File

@ -456,6 +456,9 @@ function dumpTestSuite () {
assertEqual(Object.keys(props.links).length, 1);
assertTrue(props.hasOwnProperty("links"));
assertTrue(props.links.hasOwnProperty("UnitTestsDumpViewCollection"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("includeAllFields"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("fields"));
assertTrue(props.links.UnitTestsDumpViewCollection.includeAllFields);
}
};

27
js/server/tests/dump/dump-rocksdb-cluster.js
View File

@ -585,8 +585,35 @@ function dumpTestEnterpriseSuite () {
assertEqual("9", res[1].value);
assertEqual("11", res[2].value);
assertEqual("12", res[3].value);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test view restoring
////////////////////////////////////////////////////////////////////////////////
testView : function () {
try {
db._createView("check", "arangosearch", {});
} catch (err) {}
let views = db._views();
if (views.length === 0) {
return; // arangosearch views are not supported
}
let view = db._view("UnitTestsDumpView");
assertTrue(view !== null);
let props = view.properties();
assertEqual(Object.keys(props.links).length, 1);
assertTrue(props.hasOwnProperty("links"));
assertTrue(props.links.hasOwnProperty("UnitTestsDumpViewCollection"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("includeAllFields"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("fields"));
assertTrue(props.links.UnitTestsDumpViewCollection.includeAllFields);
}
};
}

3
js/server/tests/dump/dump-rocksdb.js
View File

@ -408,6 +408,9 @@ function dumpTestSuite () {
assertEqual(Object.keys(props.links).length, 1);
assertTrue(props.hasOwnProperty("links"));
assertTrue(props.links.hasOwnProperty("UnitTestsDumpViewCollection"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("includeAllFields"));
assertTrue(props.links.UnitTestsDumpViewCollection.hasOwnProperty("fields"));
assertTrue(props.links.UnitTestsDumpViewCollection.includeAllFields);
}
};

14
js/server/tests/dump/dump-setup-cluster.js
View File

@ -221,6 +221,20 @@ function setupSatelliteCollections() {
setupSmartGraph();
setupSatelliteCollections();
// setup a view
try {
db._create("UnitTestsDumpViewCollection");
let view = db._createView("UnitTestsDumpView", "arangosearch", {});
view.properties({ links: {
"UnitTestsDumpViewCollection": {
includeAllFields: true,
fields: {
text: { analyzers: [ "text_en" ] }
}
}
} });
} catch (err) { }
})();
return {

1
js/server/tests/dump/dump-setup.js
View File

@ -222,6 +222,7 @@
c.save({ _key: "test" + i, value: i });
}
// setup a view
try {
db._create("UnitTestsDumpViewCollection");
let view = db._createView("UnitTestsDumpView", "arangosearch", {});