mirror of https://gitee.com/bigwinds/arangodb
issue 511.7.3: restore single-document optimization, implement call to update data-store options, add IResearchView upgrade step (#7918)
* issue 511.7.3: restore single-document optimization, implement call to update data-store options, add IResearchView upgrade step * address merge issue * address another merge issue * and another merge issue * run ./utils/reformat.sh on the source tree * address review comments
This commit is contained in:
Vasiliy
Andrey Abramov
parent
5a0dd57c3a
commit
5959f7f8e8
|
|
@ -335,12 +335,6 @@ FieldIterator::FieldIterator(arangodb::transaction::Methods& trx)
|
|||
// initialize iterator's value
|
||||
}
|
||||
|
||||
FieldIterator::FieldIterator(arangodb::transaction::Methods& trx,
|
||||
VPackSlice const& doc, IResearchLinkMeta const& linkMeta)
|
||||
: FieldIterator(trx) {
|
||||
reset(doc, linkMeta);
|
||||
}
|
||||
|
||||
std::string& FieldIterator::valueBuffer() {
|
||||
if (!_valueBuffer) {
|
||||
_valueBuffer = BufferPool.emplace().release(); // FIXME don't use shared_ptr
|
||||
|
|
|
|||
|
|
@ -124,9 +124,6 @@ class FieldIterator : public std::iterator<std::forward_iterator_tag, Field cons
|
|||
public:
|
||||
explicit FieldIterator(arangodb::transaction::Methods& trx);
|
||||
|
||||
FieldIterator(arangodb::transaction::Methods& trx,
|
||||
arangodb::velocypack::Slice const& doc, IResearchLinkMeta const& linkMeta);
|
||||
|
||||
Field const& operator*() const noexcept { return _value; }
|
||||
|
||||
FieldIterator& operator++() {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
#include "Aql/Function.h"
|
||||
#include "Basics/ConditionLocker.h"
|
||||
#include "Basics/SmallVector.h"
|
||||
#include "Cluster/ClusterInfo.h"
|
||||
#include "Cluster/ServerState.h"
|
||||
#include "Containers.h"
|
||||
#include "IResearchCommon.h"
|
||||
|
|
@ -50,6 +51,8 @@
|
|||
#include "IResearchViewCoordinator.h"
|
||||
#include "Logger/LogMacros.h"
|
||||
#include "MMFiles/MMFilesEngine.h"
|
||||
#include "RestServer/DatabasePathFeature.h"
|
||||
#include "RestServer/UpgradeFeature.h"
|
||||
#include "RestServer/ViewTypesFeature.h"
|
||||
#include "RocksDBEngine/RocksDBEngine.h"
|
||||
#include "StorageEngine/EngineSelectorFeature.h"
|
||||
|
|
@ -156,6 +159,180 @@ size_t computeThreadPoolSize(size_t threads, size_t threadsLimit) {
|
|||
size_t(std::thread::hardware_concurrency()) / 4));
|
||||
}
|
||||
|
||||
bool iresearchViewUpgradeVersion0_1(TRI_vocbase_t& vocbase,
|
||||
arangodb::velocypack::Slice const& upgradeParams) {
|
||||
std::vector<std::shared_ptr<arangodb::LogicalView>> views;
|
||||
|
||||
if (arangodb::ServerState::instance()->isCoordinator()) {
|
||||
auto* ci = arangodb::ClusterInfo::instance();
|
||||
|
||||
if (!ci) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to find 'ClusterInfo' instance while upgrading "
|
||||
"IResearchView from version 0 to version 1";
|
||||
|
||||
return false; // internal error
|
||||
}
|
||||
|
||||
views = ci->getViews(vocbase.name());
|
||||
} else if (arangodb::ServerState::instance()->isSingleServer() ||
|
||||
arangodb::ServerState::instance()->isDBServer()) {
|
||||
views = vocbase.views();
|
||||
} else {
|
||||
return true; // not applicable for other ServerState roles
|
||||
}
|
||||
|
||||
for (auto& view : views) {
|
||||
if (arangodb::ServerState::instance()->isCoordinator()) {
|
||||
if (!arangodb::LogicalView::cast<arangodb::iresearch::IResearchViewCoordinator>(
|
||||
view.get())) {
|
||||
continue; // not an IResearchViewCoordinator
|
||||
}
|
||||
} else { // single-server and db-server
|
||||
if (!arangodb::LogicalView::cast<arangodb::iresearch::IResearchView>(view.get())) {
|
||||
continue; // not an IResearchView
|
||||
}
|
||||
}
|
||||
|
||||
arangodb::velocypack::Builder builder;
|
||||
arangodb::Result res;
|
||||
|
||||
builder.openObject();
|
||||
res = view->properties(builder, true, true); // get JSON with meta + 'version'
|
||||
builder.close();
|
||||
|
||||
if (!res.ok()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to generate persisted definition while upgrading "
|
||||
"IResearchView from version 0 to version 1";
|
||||
|
||||
return false; // definition generation failure
|
||||
}
|
||||
|
||||
auto versionSlice =
|
||||
builder.slice().get(arangodb::iresearch::StaticStrings::VersionField);
|
||||
|
||||
if (!versionSlice.isNumber<uint32_t>()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to find 'version' field while upgrading IResearchView "
|
||||
"from version 0 to version 1";
|
||||
|
||||
return false; // required field is missing
|
||||
}
|
||||
|
||||
auto version = versionSlice.getNumber<uint32_t>();
|
||||
|
||||
if (0 != version) {
|
||||
continue; // no upgrade required
|
||||
}
|
||||
|
||||
builder.clear();
|
||||
builder.openObject();
|
||||
res = view->properties(builder, true, false); // get JSON with end-user definition
|
||||
builder.close();
|
||||
|
||||
if (!res.ok()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to generate persisted definition while upgrading "
|
||||
"IResearchView from version 0 to version 1";
|
||||
|
||||
return false; // definition generation failure
|
||||
}
|
||||
|
||||
irs::utf8_path dataPath;
|
||||
|
||||
if (arangodb::ServerState::instance()->isSingleServer() ||
|
||||
arangodb::ServerState::instance()->isDBServer()) {
|
||||
auto* dbPathFeature =
|
||||
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>(
|
||||
"DatabasePath");
|
||||
|
||||
if (!dbPathFeature) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to find feature 'DatabasePath' while upgrading "
|
||||
"IResearchView from version 0 to version 1";
|
||||
|
||||
return false; // required feature is missing
|
||||
}
|
||||
|
||||
// original algorithm for computing data-store path
|
||||
static const std::string subPath("databases");
|
||||
static const std::string dbPath("database-");
|
||||
|
||||
dataPath = irs::utf8_path(dbPathFeature->directory());
|
||||
dataPath /= subPath;
|
||||
dataPath /= dbPath;
|
||||
dataPath += std::to_string(vocbase.id());
|
||||
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
|
||||
dataPath += "-";
|
||||
dataPath += std::to_string(view->id());
|
||||
}
|
||||
|
||||
res = view->drop(); // drop view (including all links)
|
||||
|
||||
if (!res.ok()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to drop view while upgrading IResearchView from version "
|
||||
"0 to version 1";
|
||||
|
||||
return false; // view drom failure
|
||||
}
|
||||
|
||||
// .........................................................................
|
||||
// non-recoverable state below here
|
||||
// .........................................................................
|
||||
|
||||
// non-version 0 IResearchView implementations no longer drop from vocbase
|
||||
// on db-server, do it explicitly
|
||||
if (arangodb::ServerState::instance()->isDBServer()) {
|
||||
res = arangodb::LogicalViewHelperStorageEngine::drop(*view);
|
||||
|
||||
if (!res.ok()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to drop view from vocbase while upgrading "
|
||||
"IResearchView from version 0 to version 1";
|
||||
|
||||
return false; // view drom failure
|
||||
}
|
||||
}
|
||||
|
||||
if (arangodb::ServerState::instance()->isSingleServer() ||
|
||||
arangodb::ServerState::instance()->isDBServer()) {
|
||||
bool exists;
|
||||
|
||||
// remove any stale data-store
|
||||
if (!dataPath.exists(exists) || (exists && !dataPath.remove())) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to remove old data-store path while upgrading "
|
||||
"IResearchView from version 0 to version 1, view definition: "
|
||||
<< builder.slice().toString();
|
||||
|
||||
return false; // data-store removal failure
|
||||
}
|
||||
}
|
||||
|
||||
if (arangodb::ServerState::instance()->isDBServer()) {
|
||||
continue; // no need to recreate per-cid view
|
||||
}
|
||||
|
||||
// recreate view
|
||||
res = arangodb::iresearch::IResearchView::factory().create(view, vocbase,
|
||||
builder.slice());
|
||||
|
||||
if (!res.ok()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failure to recreate view while upgrading IResearchView from "
|
||||
"version 0 to version 1, error: "
|
||||
<< res.errorNumber() << " " << res.errorMessage()
|
||||
<< ", view definition: " << builder.slice().toString();
|
||||
|
||||
return false; // data-store removal failure
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void registerFunctions(arangodb::aql::AqlFunctionFeature& /*functions*/) {
|
||||
#if 0
|
||||
arangodb::iresearch::addFunction(functions, {
|
||||
|
|
@ -277,6 +454,44 @@ void registerRecoveryHelper() {
|
|||
}
|
||||
}
|
||||
|
||||
void registerUpgradeTasks() {
|
||||
auto* upgrade =
|
||||
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::UpgradeFeature>(
|
||||
"Upgrade");
|
||||
|
||||
if (!upgrade) {
|
||||
return; // nothing to register with (OK if no tasks actually need to be applied)
|
||||
}
|
||||
|
||||
// move IResearch data-store from IResearchView to IResearchLink
|
||||
{
|
||||
arangodb::methods::Upgrade::Task task;
|
||||
|
||||
task.name = "IResearhView version 0->1";
|
||||
task.description =
|
||||
"move IResearch data-store from IResearchView to IResearchLink";
|
||||
task.systemFlag = arangodb::methods::Upgrade::Flags::DATABASE_ALL;
|
||||
task.clusterFlags = arangodb::methods::Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL // any 1 single coordinator
|
||||
| arangodb::methods::Upgrade::Flags::CLUSTER_DB_SERVER_LOCAL // db-server
|
||||
| arangodb::methods::Upgrade::Flags::CLUSTER_LOCAL // any cluster node (filtred out in task) FIXME TODO without this db-server never ges invoked
|
||||
| arangodb::methods::Upgrade::Flags::CLUSTER_NONE // local server
|
||||
;
|
||||
task.databaseFlags = arangodb::methods::Upgrade::Flags::DATABASE_UPGRADE;
|
||||
task.action = &iresearchViewUpgradeVersion0_1;
|
||||
upgrade->addTask(std::move(task));
|
||||
|
||||
// FIXME TODO find out why CLUSTER_COORDINATOR_GLOBAL will only work with DATABASE_INIT (hardcoded in Upgrade::clusterBootstrap(...))
|
||||
task.name = "IResearhView version 0->1";
|
||||
task.description =
|
||||
"move IResearch data-store from IResearchView to IResearchLink";
|
||||
task.systemFlag = arangodb::methods::Upgrade::Flags::DATABASE_ALL;
|
||||
task.clusterFlags = arangodb::methods::Upgrade::Flags::CLUSTER_COORDINATOR_GLOBAL; // any 1 single coordinator
|
||||
task.databaseFlags = arangodb::methods::Upgrade::Flags::DATABASE_INIT;
|
||||
task.action = &iresearchViewUpgradeVersion0_1;
|
||||
upgrade->addTask(std::move(task));
|
||||
}
|
||||
}
|
||||
|
||||
void registerViewFactory() {
|
||||
auto& viewType = arangodb::iresearch::DATA_SOURCE_TYPE;
|
||||
auto* viewTypes =
|
||||
|
|
@ -762,6 +977,8 @@ void IResearchFeature::start() {
|
|||
}
|
||||
}
|
||||
|
||||
registerUpgradeTasks(); // register tasks after UpgradeFeature::prepare() has finished
|
||||
|
||||
_running.store(true);
|
||||
}
|
||||
|
||||
|
|
@ -790,6 +1007,6 @@ void IResearchFeature::validateOptions(std::shared_ptr<arangodb::options::Progra
|
|||
NS_END // iresearch
|
||||
NS_END // arangodb
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// --SECTION-- END-OF-FILE
|
||||
// -----------------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------------
|
||||
// --SECTION-- END-OF-FILE
|
||||
// -----------------------------------------------------------------------------
|
||||
|
|
@ -158,18 +158,24 @@ irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeatu
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// @brief inserts ArangoDB document into an IResearch data store
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
inline void insertDocument(irs::segment_writer::document& doc,
|
||||
arangodb::iresearch::FieldIterator& body, TRI_voc_cid_t cid,
|
||||
arangodb::LocalDocumentId const& docPk) {
|
||||
using namespace arangodb::iresearch;
|
||||
inline arangodb::Result insertDocument(irs::index_writer::documents_context& ctx,
|
||||
arangodb::iresearch::FieldIterator& body,
|
||||
arangodb::velocypack::Slice const& document,
|
||||
arangodb::LocalDocumentId const& documentId,
|
||||
arangodb::iresearch::IResearchLinkMeta const& meta,
|
||||
TRI_idx_iid_t id) {
|
||||
body.reset(document, meta); // reset reusable container to doc
|
||||
|
||||
// reuse the 'Field' instance stored
|
||||
// inside the 'FieldIterator' after
|
||||
auto& field = const_cast<Field&>(*body);
|
||||
if (!body.valid()) {
|
||||
return arangodb::Result(); // no fields to index
|
||||
}
|
||||
|
||||
auto doc = ctx.insert();
|
||||
auto& field = *body;
|
||||
|
||||
// User fields
|
||||
while (body.valid()) {
|
||||
if (ValueStorage::NONE == field._storeValues) {
|
||||
if (arangodb::iresearch::ValueStorage::NONE == field._storeValues) {
|
||||
doc.insert(irs::action::index, field);
|
||||
} else {
|
||||
doc.insert(irs::action::index_store, field);
|
||||
|
|
@ -181,9 +187,21 @@ inline void insertDocument(irs::segment_writer::document& doc,
|
|||
// System fields
|
||||
|
||||
// Indexed and Stored: LocalDocumentId
|
||||
auto const primaryKey = DocumentPrimaryKey::encode(docPk);
|
||||
Field::setPkValue(field, primaryKey);
|
||||
auto docPk = arangodb::iresearch::DocumentPrimaryKey::encode(documentId);
|
||||
|
||||
// reuse the 'Field' instance stored inside the 'FieldIterator'
|
||||
arangodb::iresearch::Field::setPkValue(const_cast<arangodb::iresearch::Field&>(field), docPk);
|
||||
doc.insert(irs::action::index_store, field);
|
||||
|
||||
if (!doc) {
|
||||
return arangodb::Result(
|
||||
TRI_ERROR_INTERNAL,
|
||||
std::string("failed to insert document into arangosearch link '") +
|
||||
std::to_string(id) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "'");
|
||||
}
|
||||
|
||||
return arangodb::Result();
|
||||
}
|
||||
|
||||
NS_END
|
||||
|
|
@ -357,20 +375,12 @@ void IResearchLink::batchInsert(
|
|||
|
||||
try {
|
||||
for (FieldIterator body(trx); begin != end; ++begin) {
|
||||
body.reset(begin->second, _meta);
|
||||
auto res =
|
||||
insertDocument(ctx->_ctx, body, begin->second, begin->first, _meta, id());
|
||||
|
||||
if (!body.valid()) {
|
||||
continue; // find first valid document
|
||||
}
|
||||
|
||||
auto doc = ctx->_ctx.insert();
|
||||
|
||||
insertDocument(doc, body, _collection.id(), begin->first);
|
||||
|
||||
if (!doc) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "failed inserting batch into arangosearch link '" << id() << "'";
|
||||
queue->setStatus(TRI_ERROR_INTERNAL);
|
||||
if (!res.ok()) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC) << res.errorMessage();
|
||||
queue->setStatus(res.errorNumber());
|
||||
|
||||
return;
|
||||
}
|
||||
|
|
@ -972,6 +982,33 @@ arangodb::Result IResearchLink::insert(arangodb::transaction::Methods& trx,
|
|||
std::to_string(id()) + "'");
|
||||
}
|
||||
|
||||
auto insertImpl = [this, &trx, &doc, &documentId](
|
||||
irs::index_writer::documents_context& ctx) -> arangodb::Result {
|
||||
try {
|
||||
FieldIterator body(trx);
|
||||
|
||||
return insertDocument(ctx, body, doc, documentId, _meta, id());
|
||||
} catch (arangodb::basics::Exception const& e) {
|
||||
return arangodb::Result(e.code(),
|
||||
std::string("caught exception while inserting "
|
||||
"document into arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "': " + e.what());
|
||||
} catch (std::exception const& e) {
|
||||
return arangodb::Result(TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while inserting "
|
||||
"document into arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "': " + e.what());
|
||||
} catch (...) {
|
||||
return arangodb::Result(TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while inserting "
|
||||
"document into arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "'");
|
||||
}
|
||||
};
|
||||
|
||||
auto& state = *(trx.state());
|
||||
auto* key = this;
|
||||
|
||||
|
|
@ -998,6 +1035,13 @@ arangodb::Result IResearchLink::insert(arangodb::transaction::Methods& trx,
|
|||
|
||||
TRI_ASSERT(_dataStore); // must be valid if _asyncSelf->get() is valid
|
||||
|
||||
// optimization for single-document insert-only transactions
|
||||
if (trx.isSingleOperationTransaction() && !_inRecovery) {
|
||||
auto ctx = _dataStore._writer->documents();
|
||||
|
||||
return insertImpl(ctx);
|
||||
}
|
||||
|
||||
auto ptr = irs::memory::make_unique<LinkTrxState>(std::move(lock),
|
||||
*(_dataStore._writer));
|
||||
|
||||
|
|
@ -1018,45 +1062,7 @@ arangodb::Result IResearchLink::insert(arangodb::transaction::Methods& trx,
|
|||
ctx->remove(documentId);
|
||||
}
|
||||
|
||||
try {
|
||||
FieldIterator body(trx, doc, _meta);
|
||||
|
||||
if (!body.valid()) {
|
||||
return arangodb::Result(); // nothing to index
|
||||
}
|
||||
|
||||
auto doc = ctx->_ctx.insert();
|
||||
|
||||
insertDocument(doc, body, _collection.id(), documentId);
|
||||
|
||||
if (!doc) {
|
||||
return arangodb::Result(
|
||||
TRI_ERROR_INTERNAL,
|
||||
std::string("failed to insert document into arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "'");
|
||||
}
|
||||
} catch (arangodb::basics::Exception const& e) {
|
||||
return arangodb::Result(
|
||||
e.code(), std::string("caught exception while inserting document into "
|
||||
"arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "': " + e.what());
|
||||
} catch (std::exception const& e) {
|
||||
return arangodb::Result(TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while inserting "
|
||||
"document into arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "': " + e.what());
|
||||
} catch (...) {
|
||||
return arangodb::Result(TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while inserting "
|
||||
"document into arangosearch link '") +
|
||||
std::to_string(id()) + "', revision '" +
|
||||
std::to_string(documentId.id()) + "'");
|
||||
}
|
||||
|
||||
return arangodb::Result();
|
||||
return insertImpl(ctx->_ctx);
|
||||
}
|
||||
|
||||
bool IResearchLink::isSorted() const {
|
||||
|
|
@ -1125,9 +1131,36 @@ size_t IResearchLink::memory() const {
|
|||
return size;
|
||||
}
|
||||
|
||||
bool IResearchLink::properties(irs::index_writer::segment_options const& properties) {
|
||||
// FIXME TODO update the data-store options
|
||||
return true;
|
||||
arangodb::Result IResearchLink::properties(irs::index_writer::segment_options const& properties) {
|
||||
SCOPED_LOCK(_asyncSelf->mutex()); // '_dataStore' can be asynchronously modified
|
||||
|
||||
if (!*_asyncSelf) {
|
||||
return arangodb::Result(TRI_ERROR_ARANGO_INDEX_HANDLE_BAD, // the current link is no longer valid (checked after ReadLock aquisition)
|
||||
std::string(
|
||||
"failed to lock arangosearch link while "
|
||||
"modifying properties of arangosearch link '") +
|
||||
std::to_string(id()) + "'");
|
||||
}
|
||||
|
||||
TRI_ASSERT(_dataStore); // must be valid if _asyncSelf->get() is valid
|
||||
|
||||
try {
|
||||
_dataStore._writer->options(properties);
|
||||
} catch (std::exception const& e) {
|
||||
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
|
||||
<< "caught exception while modifying properties of arangosearch link '"
|
||||
<< id() << "': " << e.what();
|
||||
IR_LOG_EXCEPTION();
|
||||
throw;
|
||||
} catch (...) {
|
||||
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
|
||||
<< "caught exception while modifying properties of arangosearch link '"
|
||||
<< id() << "'";
|
||||
IR_LOG_EXCEPTION();
|
||||
throw;
|
||||
}
|
||||
|
||||
return arangodb::Result();
|
||||
}
|
||||
|
||||
arangodb::Result IResearchLink::remove(arangodb::transaction::Methods& trx,
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ class IResearchLink {
|
|||
/// @brief update runtine data processing properties (not persisted)
|
||||
/// @return success
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
bool properties(irs::index_writer::segment_options const& properties);
|
||||
arangodb::Result properties(irs::index_writer::segment_options const& properties);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// @brief remove an ArangoDB document from an iResearch View
|
||||
|
|
@ -264,4 +264,4 @@ class IResearchLink {
|
|||
NS_END // iresearch
|
||||
NS_END // arangodb
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -243,31 +243,30 @@ struct IResearchView::ViewFactory : public arangodb::ViewFactory {
|
|||
TRI_vocbase_t& vocbase,
|
||||
arangodb::velocypack::Slice const& definition,
|
||||
uint64_t planVersion) const override {
|
||||
auto* databaseFeature =
|
||||
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabaseFeature>(
|
||||
"Database");
|
||||
std::string error;
|
||||
auto impl = std::shared_ptr<IResearchView>(
|
||||
new IResearchView(vocbase, definition, planVersion));
|
||||
bool inUpgrade = databaseFeature ? databaseFeature->upgrade() : false; // check if DB is currently being upgraded (skip validation checks)
|
||||
IResearchViewMeta meta;
|
||||
IResearchViewMetaState metaState;
|
||||
|
||||
{
|
||||
WriteMutex mutex(impl->_mutex);
|
||||
SCOPED_LOCK(mutex);
|
||||
if (!impl->_meta.init(definition, error) ||
|
||||
impl->_meta._version == 0 // version 0 must be upgraded to split
|
||||
// data-store on a per-link basis
|
||||
|| impl->_meta._version > LATEST_VERSION ||
|
||||
(ServerState::instance()->isSingleServer() // init metaState for SingleServer
|
||||
&& !metaState.init(definition, error))) {
|
||||
return arangodb::Result(
|
||||
TRI_ERROR_BAD_PARAMETER,
|
||||
error.empty()
|
||||
? (std::string("failed to initialize arangosearch View '") +
|
||||
impl->name() + "' from definition: " + definition.toString())
|
||||
: (std::string("failed to initialize arangosearch View '") +
|
||||
impl->name() + "' from definition, error in attribute '" +
|
||||
error + "': " + definition.toString()));
|
||||
}
|
||||
if (!meta.init(definition, error) || (meta._version == 0 && !inUpgrade) // version 0 must be upgraded to split data-store on a per-link basis
|
||||
|| meta._version > LATEST_VERSION ||
|
||||
(ServerState::instance()->isSingleServer() // init metaState for SingleServer
|
||||
&& !metaState.init(definition, error))) {
|
||||
return arangodb::Result(TRI_ERROR_BAD_PARAMETER, error.empty() ? (std::string("failed to initialize arangosearch View from definition: ") +
|
||||
definition
|
||||
.toString())
|
||||
: (std::string("failed to initialize arangosearch View from definition, error in attribute '") +
|
||||
error + "': " +
|
||||
definition
|
||||
.toString()));
|
||||
}
|
||||
|
||||
auto impl = std::shared_ptr<IResearchView>(
|
||||
new IResearchView(vocbase, definition, planVersion, std::move(meta)));
|
||||
|
||||
// NOTE: for single-server must have full list of collections to lock
|
||||
// for cluster the shards to lock come from coordinator and are not in
|
||||
// the definition
|
||||
|
|
@ -289,12 +288,13 @@ struct IResearchView::ViewFactory : public arangodb::ViewFactory {
|
|||
}
|
||||
};
|
||||
|
||||
IResearchView::IResearchView(TRI_vocbase_t& vocbase,
|
||||
arangodb::velocypack::Slice const& info, uint64_t planVersion)
|
||||
IResearchView::IResearchView(TRI_vocbase_t& vocbase, arangodb::velocypack::Slice const& info,
|
||||
uint64_t planVersion, IResearchViewMeta&& meta)
|
||||
: LogicalView(vocbase, info, planVersion),
|
||||
FlushTransaction(toString(*this)),
|
||||
_asyncFeature(nullptr),
|
||||
_asyncSelf(irs::memory::make_unique<AsyncViewPtr::element_type>(this)),
|
||||
_meta(std::move(meta)),
|
||||
_asyncTerminate(false),
|
||||
_inRecovery(false) {
|
||||
// set up in-recovery insertion hooks
|
||||
|
|
@ -1107,20 +1107,20 @@ arangodb::Result IResearchView::unlink(TRI_voc_cid_t cid) noexcept {
|
|||
return res;
|
||||
}
|
||||
} catch (arangodb::basics::Exception const& e) {
|
||||
return arangodb::Result(e.code(),
|
||||
std::string("caught exception while collection '") +
|
||||
std::to_string(cid) +
|
||||
"' from arangosearch view '" + name() + "': " + e.what());
|
||||
return arangodb::Result(
|
||||
e.code(), std::string("caught exception while unlinking collection '") +
|
||||
std::to_string(cid) + "' from arangosearch view '" +
|
||||
name() + "': " + e.what());
|
||||
} catch (std::exception const& e) {
|
||||
return arangodb::Result(TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while collection '") +
|
||||
std::to_string(cid) +
|
||||
"' from arangosearch view '" + name() + "': " + e.what());
|
||||
return arangodb::Result(
|
||||
TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while unlinking collection '") + std::to_string(cid) +
|
||||
"' from arangosearch view '" + name() + "': " + e.what());
|
||||
} catch (...) {
|
||||
return arangodb::Result(TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while collection '") +
|
||||
std::to_string(cid) +
|
||||
"' from arangosearch view '" + name() + "'");
|
||||
return arangodb::Result(
|
||||
TRI_ERROR_INTERNAL,
|
||||
std::string("caught exception while unlinking collection '") +
|
||||
std::to_string(cid) + "' from arangosearch view '" + name() + "'");
|
||||
}
|
||||
|
||||
return arangodb::Result();
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ class IResearchView final : public arangodb::LogicalView, public arangodb::Flush
|
|||
typedef std::unique_ptr<arangodb::FlushTransaction, std::function<void(arangodb::FlushTransaction*)>> FlushTransactionPtr;
|
||||
|
||||
IResearchView(TRI_vocbase_t& vocbase, arangodb::velocypack::Slice const& info,
|
||||
uint64_t planVersion);
|
||||
uint64_t planVersion, IResearchViewMeta&& meta);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
/// @brief Called in post-recovery to remove any dangling documents old links
|
||||
|
|
@ -242,4 +242,4 @@ class IResearchView final : public arangodb::LogicalView, public arangodb::Flush
|
|||
} // namespace iresearch
|
||||
} // namespace arangodb
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -50,6 +50,10 @@ UpgradeFeature::UpgradeFeature(application_features::ApplicationServer& server,
|
|||
startsAfter("AQLPhase");
|
||||
}
|
||||
|
||||
void UpgradeFeature::addTask(methods::Upgrade::Task&& task) {
|
||||
_tasks.push_back(std::move(task));
|
||||
}
|
||||
|
||||
void UpgradeFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
|
||||
options->addSection("database", "Configure the database");
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
#define APPLICATION_FEATURES_UPGRADE_FEATURE_H 1
|
||||
|
||||
#include "ApplicationFeatures/ApplicationFeature.h"
|
||||
#include "VocBase/Methods/Upgrade.h"
|
||||
|
||||
namespace arangodb {
|
||||
|
||||
|
|
@ -32,12 +33,15 @@ class UpgradeFeature final : public application_features::ApplicationFeature {
|
|||
UpgradeFeature(application_features::ApplicationServer& server, int* result,
|
||||
std::vector<std::string> const& nonServerFeatures);
|
||||
|
||||
void addTask(methods::Upgrade::Task&& task);
|
||||
void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;
|
||||
void validateOptions(std::shared_ptr<options::ProgramOptions>) override final;
|
||||
void prepare() override final;
|
||||
void start() override final;
|
||||
|
||||
private:
|
||||
friend struct methods::Upgrade; // to allow access to '_tasks'
|
||||
|
||||
bool _upgrade;
|
||||
bool _upgradeCheck;
|
||||
|
||||
|
|
@ -45,8 +49,9 @@ class UpgradeFeature final : public application_features::ApplicationFeature {
|
|||
|
||||
int* _result;
|
||||
std::vector<std::string> _nonServerFeatures;
|
||||
std::vector<methods::Upgrade::Task> _tasks;
|
||||
};
|
||||
|
||||
} // namespace arangodb
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -333,13 +333,12 @@ Result Indexes::ensureIndex(LogicalCollection* collection, VPackSlice const& inp
|
|||
|
||||
VPackBuilder normalized;
|
||||
StorageEngine* engine = EngineSelectorFeature::ENGINE;
|
||||
int res = engine->indexFactory()
|
||||
.enhanceIndexDefinition(input, normalized, create,
|
||||
ServerState::instance()->isCoordinator())
|
||||
.errorNumber();
|
||||
auto res =
|
||||
engine->indexFactory().enhanceIndexDefinition(input, normalized, create,
|
||||
ServerState::instance()->isCoordinator());
|
||||
|
||||
if (res != TRI_ERROR_NO_ERROR) {
|
||||
return Result(res);
|
||||
if (!res.ok()) {
|
||||
return res;
|
||||
}
|
||||
|
||||
TRI_ASSERT(collection);
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@
|
|||
#include "Cluster/ClusterInfo.h"
|
||||
#include "Cluster/ServerState.h"
|
||||
#include "Rest/Version.h"
|
||||
#include "RestServer/UpgradeFeature.h"
|
||||
#include "Utils/ExecContext.h"
|
||||
#include "VocBase/Methods/UpgradeTasks.h"
|
||||
#include "VocBase/Methods/Version.h"
|
||||
|
|
@ -38,11 +39,24 @@
|
|||
#include <velocypack/Iterator.h>
|
||||
#include <velocypack/velocypack-aliases.h>
|
||||
|
||||
namespace {
|
||||
|
||||
void addTask(std::string&& name, std::string&& desc, uint32_t systemFlag, uint32_t clusterFlag,
|
||||
uint32_t dbFlag, arangodb::methods::Upgrade::TaskFunction&& action) {
|
||||
auto* upgradeFeature =
|
||||
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::UpgradeFeature>(
|
||||
"Upgrade");
|
||||
|
||||
TRI_ASSERT(upgradeFeature);
|
||||
upgradeFeature->addTask(arangodb::methods::Upgrade::Task{name, desc, systemFlag, clusterFlag,
|
||||
dbFlag, action});
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
using namespace arangodb;
|
||||
using namespace arangodb::methods;
|
||||
|
||||
std::vector<Upgrade::Task> Upgrade::_tasks;
|
||||
|
||||
/// corresponding to cluster-bootstrap.js
|
||||
UpgradeResult Upgrade::clusterBootstrap(TRI_vocbase_t& system) {
|
||||
uint64_t cc = Version::current(); // not actually used here
|
||||
|
|
@ -192,6 +206,12 @@ UpgradeResult Upgrade::startup(TRI_vocbase_t& vocbase, bool isUpgrade, bool igno
|
|||
|
||||
/// @brief register tasks, only run once on startup
|
||||
void methods::Upgrade::registerTasks() {
|
||||
auto* upgradeFeature =
|
||||
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::UpgradeFeature>(
|
||||
"Upgrade");
|
||||
|
||||
TRI_ASSERT(upgradeFeature);
|
||||
auto& _tasks = upgradeFeature->_tasks;
|
||||
TRI_ASSERT(_tasks.empty());
|
||||
|
||||
addTask("upgradeGeoIndexes", "upgrade legacy geo indexes",
|
||||
|
|
@ -265,16 +285,24 @@ void methods::Upgrade::registerTasks() {
|
|||
/*system*/ Flags::DATABASE_ALL,
|
||||
/*cluster*/ Flags::CLUSTER_NONE | Flags::CLUSTER_DB_SERVER_LOCAL,
|
||||
/*database*/ DATABASE_UPGRADE, &UpgradeTasks::persistLocalDocumentIds);
|
||||
addTask("renameReplicationApplierStateFiles", "rename replication applier state files",
|
||||
addTask("renameReplicationApplierStateFiles",
|
||||
"rename replication applier state files",
|
||||
/*system*/ Flags::DATABASE_ALL,
|
||||
/*cluster*/ Flags::CLUSTER_NONE | Flags::CLUSTER_DB_SERVER_LOCAL,
|
||||
/*database*/ DATABASE_UPGRADE | DATABASE_EXISTING,
|
||||
/*database*/ DATABASE_UPGRADE | DATABASE_EXISTING,
|
||||
&UpgradeTasks::renameReplicationApplierStateFiles);
|
||||
}
|
||||
|
||||
UpgradeResult methods::Upgrade::runTasks(TRI_vocbase_t& vocbase, VersionResult& vinfo,
|
||||
arangodb::velocypack::Slice const& params,
|
||||
uint32_t clusterFlag, uint32_t dbFlag) {
|
||||
auto* upgradeFeature =
|
||||
arangodb::application_features::ApplicationServer::lookupFeature<arangodb::UpgradeFeature>(
|
||||
"Upgrade");
|
||||
|
||||
TRI_ASSERT(upgradeFeature);
|
||||
auto& _tasks = upgradeFeature->_tasks;
|
||||
|
||||
TRI_ASSERT(clusterFlag != 0 && dbFlag != 0);
|
||||
TRI_ASSERT(!_tasks.empty()); // forgot to call registerTask!!
|
||||
// needs to run in superuser scope, otherwise we get errors
|
||||
|
|
@ -323,7 +351,7 @@ UpgradeResult methods::Upgrade::runTasks(TRI_vocbase_t& vocbase, VersionResult&
|
|||
<< "Upgrade: db flag mismatch, skipping " << t.name;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
LOG_TOPIC(DEBUG, Logger::STARTUP) << "Upgrade: Executing " << t.name;
|
||||
try {
|
||||
bool ranTask = t.action(vocbase, params);
|
||||
|
|
|
|||
|
|
@ -89,11 +89,6 @@ struct Upgrade {
|
|||
static UpgradeResult startup(TRI_vocbase_t& vocbase, bool upgrade, bool ignoreFileErrors);
|
||||
|
||||
private:
|
||||
static std::vector<Task> _tasks;
|
||||
static void addTask(std::string&& name, std::string&& desc, uint32_t systemFlag,
|
||||
uint32_t clusterFlag, uint32_t dbFlag, TaskFunction&& action) {
|
||||
_tasks.push_back(Task{name, desc, systemFlag, clusterFlag, dbFlag, action});
|
||||
}
|
||||
|
||||
/// @brief register tasks, only run once on startup
|
||||
static void registerTasks();
|
||||
|
|
|
|||
|
|
@ -317,7 +317,8 @@ SECTION("FieldIterator_traverse_complex_object_custom_nested_delimiter") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
CHECK(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
// default analyzer
|
||||
|
|
@ -395,7 +396,8 @@ SECTION("FieldIterator_traverse_complex_object_all_fields") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
CHECK(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
// default analyzer
|
||||
|
|
@ -498,7 +500,8 @@ SECTION("FieldIterator_traverse_complex_object_ordered_all_fields") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator doc(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator doc(trx);
|
||||
doc.reset(slice, linkMeta);
|
||||
for (;doc.valid(); ++doc) {
|
||||
auto& field = *doc;
|
||||
std::string const actualName = std::string(field.name());
|
||||
|
|
@ -550,7 +553,8 @@ SECTION("FieldIterator_traverse_complex_object_ordered_filtered") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
REQUIRE(it.valid());
|
||||
REQUIRE(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
|
|
@ -601,7 +605,8 @@ SECTION("FieldIterator_traverse_complex_object_ordered_filtered") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
CHECK(!it.valid());
|
||||
CHECK(it == arangodb::iresearch::FieldIterator(trx));
|
||||
}
|
||||
|
|
@ -636,7 +641,8 @@ SECTION("FieldIterator_traverse_complex_object_ordered_empty_analyzers") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
CHECK(!it.valid());
|
||||
CHECK(it == arangodb::iresearch::FieldIterator(trx));
|
||||
}
|
||||
|
|
@ -673,7 +679,8 @@ SECTION("FieldIterator_traverse_complex_object_ordered_check_value_types") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
CHECK(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
// stringValue (with IdentityAnalyzer)
|
||||
|
|
@ -850,7 +857,8 @@ SECTION("FieldIterator_reset") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, json0->slice(), linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(json0->slice(), linkMeta);
|
||||
REQUIRE(it.valid());
|
||||
|
||||
{
|
||||
|
|
@ -971,7 +979,8 @@ SECTION("FieldIterator_traverse_complex_object_ordered_all_fields_custom_list_of
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
CHECK(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
// default analyzer
|
||||
|
|
@ -1043,7 +1052,8 @@ SECTION("FieldIterator_traverse_complex_object_check_meta_inheritance") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
REQUIRE(it.valid());
|
||||
REQUIRE(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
|
|
@ -1401,7 +1411,8 @@ SECTION("FieldIterator_nullptr_analyzer") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
REQUIRE(it.valid());
|
||||
REQUIRE(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
|
|
@ -1459,7 +1470,8 @@ SECTION("FieldIterator_nullptr_analyzer") {
|
|||
arangodb::transaction::Options()
|
||||
);
|
||||
|
||||
arangodb::iresearch::FieldIterator it(trx, slice, linkMeta);
|
||||
arangodb::iresearch::FieldIterator it(trx);
|
||||
it.reset(slice, linkMeta);
|
||||
REQUIRE(it.valid());
|
||||
REQUIRE(it != arangodb::iresearch::FieldIterator(trx));
|
||||
|
||||
|
|
|
|||
|
|
@ -23,43 +23,92 @@
|
|||
|
||||
#include "catch.hpp"
|
||||
#include "common.h"
|
||||
#include "AgencyMock.h"
|
||||
#include "StorageEngineMock.h"
|
||||
|
||||
#include "utils/misc.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/thread_utils.hpp"
|
||||
#include "utils/utf8_path.hpp"
|
||||
#include "utils/version_defines.hpp"
|
||||
|
||||
#include "Agency/Store.h"
|
||||
#include "ApplicationFeatures/CommunicationPhase.h"
|
||||
#include "Aql/AqlFunctionFeature.h"
|
||||
#include "Cluster/ClusterComm.h"
|
||||
#include "Cluster/ClusterFeature.h"
|
||||
#include "Cluster/ClusterInfo.h"
|
||||
|
||||
#if USE_ENTERPRISE
|
||||
#include "Enterprise/Ldap/LdapFeature.h"
|
||||
#endif
|
||||
|
||||
#include "GeneralServer/AuthenticationFeature.h"
|
||||
#include "IResearch/ApplicationServerHelper.h"
|
||||
#include "IResearch/IResearchCommon.h"
|
||||
#include "IResearch/Containers.h"
|
||||
#include "IResearch/IResearchCommon.h"
|
||||
#include "IResearch/IResearchAnalyzerFeature.h"
|
||||
#include "IResearch/IResearchFeature.h"
|
||||
#include "IResearch/IResearchLinkCoordinator.h"
|
||||
#include "IResearch/IResearchLinkHelper.h"
|
||||
#include "IResearch/IResearchMMFilesLink.h"
|
||||
#include "IResearch/IResearchView.h"
|
||||
#include "Rest/Version.h"
|
||||
#include "RestServer/DatabaseFeature.h"
|
||||
#include "RestServer/DatabasePathFeature.h"
|
||||
#include "RestServer/QueryRegistryFeature.h"
|
||||
#include "RestServer/UpgradeFeature.h"
|
||||
#include "RestServer/ViewTypesFeature.h"
|
||||
#include "Sharding/ShardingFeature.h"
|
||||
#include "StorageEngine/EngineSelectorFeature.h"
|
||||
#include "VocBase/LogicalCollection.h"
|
||||
#include "VocBase/Methods/Indexes.h"
|
||||
#include "VocBase/Methods/Upgrade.h"
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// --SECTION-- setup / tear-down
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
struct IResearchFeatureSetup {
|
||||
struct ClusterCommControl : arangodb::ClusterComm {
|
||||
static void reset() {
|
||||
arangodb::ClusterComm::_theInstanceInit.store(0);
|
||||
}
|
||||
};
|
||||
|
||||
arangodb::consensus::Store _agencyStore{nullptr, "arango"};
|
||||
GeneralClientConnectionAgencyMock* agency;
|
||||
StorageEngineMock engine;
|
||||
arangodb::application_features::ApplicationServer server;
|
||||
|
||||
IResearchFeatureSetup(): engine(server), server(nullptr, nullptr) {
|
||||
auto* agencyCommManager = new AgencyCommManagerMock("arango");
|
||||
agency = agencyCommManager->addConnection<GeneralClientConnectionAgencyMock>(_agencyStore);
|
||||
agency = agencyCommManager->addConnection<GeneralClientConnectionAgencyMock>(_agencyStore); // need 2 connections or Agency callbacks will fail
|
||||
arangodb::AgencyCommManager::MANAGER.reset(agencyCommManager); // required for Coordinator tests
|
||||
|
||||
arangodb::EngineSelectorFeature::ENGINE = &engine;
|
||||
|
||||
arangodb::tests::init();
|
||||
|
||||
// suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
|
||||
arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
|
||||
|
||||
// suppress log messages since tests check error conditions
|
||||
arangodb::LogTopic::setLogLevel(arangodb::Logger::AGENCY.name(), arangodb::LogLevel::FATAL);
|
||||
arangodb::LogTopic::setLogLevel(arangodb::Logger::CLUSTER.name(), arangodb::LogLevel::FATAL);
|
||||
arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);
|
||||
}
|
||||
|
||||
~IResearchFeatureSetup() {
|
||||
ClusterCommControl::reset();
|
||||
arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::DEFAULT);
|
||||
arangodb::LogTopic::setLogLevel(arangodb::Logger::CLUSTER.name(), arangodb::LogLevel::DEFAULT);
|
||||
arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::DEFAULT);
|
||||
arangodb::LogTopic::setLogLevel(arangodb::Logger::AGENCY.name(), arangodb::LogLevel::DEFAULT);
|
||||
arangodb::application_features::ApplicationServer::server = nullptr;
|
||||
arangodb::EngineSelectorFeature::ENGINE = nullptr;
|
||||
arangodb::AgencyCommManager::MANAGER.reset();
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -132,6 +181,467 @@ SECTION("test_start") {
|
|||
};
|
||||
}
|
||||
|
||||
SECTION("test_upgrade0_1") {
|
||||
// version 0 data-source path
|
||||
auto getPersistedPath0 = [](arangodb::LogicalView const& view)->irs::utf8_path {
|
||||
auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
|
||||
REQUIRE(dbPathFeature);
|
||||
irs::utf8_path dataPath(dbPathFeature->directory());
|
||||
dataPath /= "databases";
|
||||
dataPath /= "database-";
|
||||
dataPath += std::to_string(view.vocbase().id());
|
||||
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
|
||||
dataPath += "-";
|
||||
dataPath += std::to_string(view.id());
|
||||
return dataPath;
|
||||
};
|
||||
|
||||
// version 1 data-source path
|
||||
auto getPersistedPath1 = [](arangodb::iresearch::IResearchLink const& link)->irs::utf8_path {
|
||||
auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<arangodb::DatabasePathFeature>("DatabasePath");
|
||||
REQUIRE(dbPathFeature);
|
||||
irs::utf8_path dataPath(dbPathFeature->directory());
|
||||
dataPath /= "databases";
|
||||
dataPath /= "database-";
|
||||
dataPath += std::to_string(link.collection().vocbase().id());
|
||||
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
|
||||
dataPath += "-";
|
||||
dataPath += std::to_string(link.collection().id());
|
||||
dataPath += "_";
|
||||
dataPath += std::to_string(link.id());
|
||||
return dataPath;
|
||||
};
|
||||
|
||||
// test single-server (no directory)
|
||||
{
|
||||
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
|
||||
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
|
||||
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
|
||||
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
|
||||
|
||||
// create a new instance of an ApplicationServer and fill it with the required features
|
||||
// cannot use the existing server since its features already have some state
|
||||
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
|
||||
arangodb::application_features::ApplicationServer::server,
|
||||
[](arangodb::application_features::ApplicationServer* ptr)->void {
|
||||
arangodb::application_features::ApplicationServer::server = ptr;
|
||||
}
|
||||
);
|
||||
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
|
||||
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
|
||||
arangodb::iresearch::IResearchFeature feature(server);
|
||||
arangodb::DatabasePathFeature* dbPathFeature;
|
||||
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
|
||||
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
|
||||
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
|
||||
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
|
||||
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
|
||||
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
|
||||
feature.prepare(); // register iresearch view type
|
||||
feature.start(); // register upgrade tasks
|
||||
server.lookupFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
|
||||
|
||||
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
|
||||
auto versionFilename = StorageEngineMock::versionFilenameResult;
|
||||
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
|
||||
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
|
||||
REQUIRE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
|
||||
REQUIRE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
|
||||
|
||||
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
|
||||
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
|
||||
REQUIRE((false == !logicalCollection));
|
||||
auto logicalView0 = vocbase.createView(viewJson->slice());
|
||||
REQUIRE((false == !logicalView0));
|
||||
bool created;
|
||||
auto index = logicalCollection->createIndex(linkJson->slice(), created);
|
||||
REQUIRE((created));
|
||||
REQUIRE((false == !index));
|
||||
auto link0 = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
|
||||
REQUIRE((false == !link0));
|
||||
|
||||
index->unload(); // release file handles
|
||||
bool result;
|
||||
auto linkDataPath = getPersistedPath1(*link0);
|
||||
CHECK((linkDataPath.remove())); // remove link directory
|
||||
auto viewDataPath = getPersistedPath0(*logicalView0);
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure no view directory
|
||||
arangodb::velocypack::Builder builder;
|
||||
builder.openObject();
|
||||
CHECK((logicalView0->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
|
||||
|
||||
CHECK((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
|
||||
auto logicalView1 = vocbase.lookupView(logicalView0->name());
|
||||
CHECK((false == !logicalView1)); // ensure view present after upgrade
|
||||
CHECK((logicalView0->id() == logicalView1->id())); // ensure same id for view
|
||||
auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
|
||||
CHECK((false == !link1)); // ensure link present after upgrade
|
||||
CHECK((link0->id() != link1->id())); // ensure new link
|
||||
linkDataPath = getPersistedPath1(*link1);
|
||||
CHECK((linkDataPath.exists(result) && result)); // ensure link directory created after upgrade
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure view directory not present
|
||||
viewDataPath = getPersistedPath0(*logicalView1);
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure view directory not created
|
||||
builder.clear();
|
||||
builder.openObject();
|
||||
CHECK((logicalView1->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
|
||||
}
|
||||
|
||||
// test single-server (with directory)
|
||||
{
|
||||
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
|
||||
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
|
||||
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
|
||||
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
|
||||
|
||||
// create a new instance of an ApplicationServer and fill it with the required features
|
||||
// cannot use the existing server since its features already have some state
|
||||
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
|
||||
arangodb::application_features::ApplicationServer::server,
|
||||
[](arangodb::application_features::ApplicationServer* ptr)->void {
|
||||
arangodb::application_features::ApplicationServer::server = ptr;
|
||||
}
|
||||
);
|
||||
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
|
||||
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
|
||||
arangodb::iresearch::IResearchFeature feature(server);
|
||||
arangodb::DatabasePathFeature* dbPathFeature;
|
||||
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
|
||||
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
|
||||
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
|
||||
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
|
||||
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
|
||||
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
|
||||
feature.prepare(); // register iresearch view type
|
||||
feature.start(); // register upgrade tasks
|
||||
server.lookupFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
|
||||
|
||||
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
|
||||
auto versionFilename = StorageEngineMock::versionFilenameResult;
|
||||
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
|
||||
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
|
||||
REQUIRE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
|
||||
REQUIRE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
|
||||
|
||||
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
|
||||
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
|
||||
REQUIRE((false == !logicalCollection));
|
||||
auto logicalView0 = vocbase.createView(viewJson->slice());
|
||||
REQUIRE((false == !logicalView0));
|
||||
bool created;
|
||||
auto index = logicalCollection->createIndex(linkJson->slice(), created);
|
||||
REQUIRE((created));
|
||||
REQUIRE((false == !index));
|
||||
auto link0 = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
|
||||
REQUIRE((false == !link0));
|
||||
|
||||
index->unload(); // release file handles
|
||||
bool result;
|
||||
auto linkDataPath = getPersistedPath1(*link0);
|
||||
CHECK((linkDataPath.remove())); // remove link directory
|
||||
auto viewDataPath = getPersistedPath0(*logicalView0);
|
||||
CHECK((viewDataPath.exists(result) && !result));
|
||||
CHECK((viewDataPath.mkdir())); // create view directory
|
||||
CHECK((viewDataPath.exists(result) && result));
|
||||
arangodb::velocypack::Builder builder;
|
||||
builder.openObject();
|
||||
CHECK((logicalView0->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
|
||||
|
||||
CHECK((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
|
||||
auto logicalView1 = vocbase.lookupView(logicalView0->name());
|
||||
CHECK((false == !logicalView1)); // ensure view present after upgrade
|
||||
CHECK((logicalView0->id() == logicalView1->id())); // ensure same id for view
|
||||
auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
|
||||
CHECK((false == !link1)); // ensure link present after upgrade
|
||||
CHECK((link0->id() != link1->id())); // ensure new link
|
||||
linkDataPath = getPersistedPath1(*link1);
|
||||
CHECK((linkDataPath.exists(result) && result)); // ensure link directory created after upgrade
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure view directory removed after upgrade
|
||||
viewDataPath = getPersistedPath0(*logicalView1);
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure view directory not created
|
||||
builder.clear();
|
||||
builder.openObject();
|
||||
CHECK((logicalView1->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
|
||||
}
|
||||
|
||||
// test coordinator
|
||||
{
|
||||
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"id\": \"1\", \"name\": \"testCollection\", \"shards\":{} }");
|
||||
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
|
||||
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"id\": 42, \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
|
||||
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
|
||||
auto collectionId = std::to_string(1);
|
||||
auto viewId = std::to_string(42);
|
||||
|
||||
auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
|
||||
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_COORDINATOR);
|
||||
auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
|
||||
|
||||
// create a new instance of an ApplicationServer and fill it with the required features
|
||||
// cannot use the existing server since its features already have some state
|
||||
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
|
||||
arangodb::application_features::ApplicationServer::server,
|
||||
[](arangodb::application_features::ApplicationServer* ptr)->void {
|
||||
arangodb::application_features::ApplicationServer::server = ptr;
|
||||
}
|
||||
);
|
||||
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
|
||||
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
|
||||
arangodb::DatabaseFeature* database;
|
||||
arangodb::iresearch::IResearchFeature feature(server);
|
||||
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterComm::instance()
|
||||
server.addFeature(new arangodb::ClusterFeature(server)); // required to create ClusterInfo instance
|
||||
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
|
||||
server.addFeature(database = new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
|
||||
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
|
||||
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
|
||||
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
|
||||
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
|
||||
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
|
||||
|
||||
#if USE_ENTERPRISE
|
||||
server.addFeature(new arangodb::LdapFeature(server)); // required for AuthenticationFeature with USE_ENTERPRISE
|
||||
#endif
|
||||
|
||||
feature.prepare(); // register iresearch view type
|
||||
feature.start(); // register upgrade tasks
|
||||
server.lookupFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
|
||||
server.lookupFeature<arangodb::ClusterFeature>("Cluster")->prepare(); // create ClusterInfo instance
|
||||
server.lookupFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
|
||||
server.lookupFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
|
||||
arangodb::AgencyCommManager::MANAGER->start(); // initialize agency
|
||||
arangodb::DatabaseFeature::DATABASE = database; // required for ClusterInfo::createCollectionCoordinator(...)
|
||||
const_cast<arangodb::IndexFactory&>(s.engine.indexFactory()).emplace( // required for Indexes::ensureIndex(...)
|
||||
arangodb::iresearch::DATA_SOURCE_TYPE.name(),
|
||||
arangodb::iresearch::IResearchLinkCoordinator::factory()
|
||||
);
|
||||
auto* ci = arangodb::ClusterInfo::instance();
|
||||
REQUIRE((nullptr != ci));
|
||||
TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature
|
||||
std::string error;
|
||||
|
||||
REQUIRE((TRI_ERROR_NO_ERROR == database->createDatabase(1, "testDatabase", vocbase)));
|
||||
REQUIRE((TRI_ERROR_NO_ERROR == ci->createDatabaseCoordinator(vocbase->name(), arangodb::velocypack::Slice::emptyObjectSlice(), error, 0.0)));
|
||||
CHECK((std::string("no error") == error));
|
||||
error.clear();
|
||||
REQUIRE((TRI_ERROR_NO_ERROR == ci->createCollectionCoordinator(vocbase->name(), collectionId, 0, 1, false, collectionJson->slice(), error, 0.0)));
|
||||
CHECK((error.empty()));
|
||||
auto logicalCollection = ci->getCollection(vocbase->name(), collectionId);
|
||||
REQUIRE((false == !logicalCollection));
|
||||
error.clear();
|
||||
CHECK((TRI_ERROR_NO_ERROR == ci->createViewCoordinator(vocbase->name(), viewId, viewJson->slice(), error)));
|
||||
CHECK((error.empty()));
|
||||
auto logicalView0 = ci->getView(vocbase->name(), viewId);
|
||||
REQUIRE((false == !logicalView0));
|
||||
|
||||
// simulate heartbeat thread (create index in current)
|
||||
{
|
||||
auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id());
|
||||
auto const value = arangodb::velocypack::Parser::fromJson("{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"1\" } ] } }");
|
||||
CHECK(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful());
|
||||
}
|
||||
arangodb::velocypack::Builder tmp;
|
||||
REQUIRE((arangodb::methods::Indexes::ensureIndex(logicalCollection.get(), linkJson->slice(), true, tmp).ok()));
|
||||
logicalCollection = ci->getCollection(vocbase->name(), collectionId);
|
||||
REQUIRE((false == !logicalCollection));
|
||||
auto link0 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView0);
|
||||
REQUIRE((false == !link0));
|
||||
|
||||
arangodb::velocypack::Builder builder;
|
||||
builder.openObject();
|
||||
CHECK((logicalView0->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
|
||||
|
||||
// simulate heartbeat thread (create index in current)
|
||||
{
|
||||
auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id());
|
||||
auto const value = arangodb::velocypack::Parser::fromJson("{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"2\" } ] } }");
|
||||
CHECK(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful());
|
||||
}
|
||||
CHECK((arangodb::methods::Upgrade::clusterBootstrap(*vocbase).ok())); // run upgrade
|
||||
logicalCollection = ci->getCollection(vocbase->name(), collectionId);
|
||||
REQUIRE((false == !logicalCollection));
|
||||
auto logicalView1 = ci->getView(vocbase->name(), viewId);
|
||||
CHECK((false == !logicalView1)); // ensure view present after upgrade
|
||||
CHECK((logicalView0->id() == logicalView1->id())); // ensure same id for view
|
||||
auto link1 = arangodb::iresearch::IResearchLinkHelper::find(*logicalCollection, *logicalView1);
|
||||
CHECK((false == !link1)); // ensure link present after upgrade
|
||||
CHECK((link0->id() != link1->id())); // ensure new link
|
||||
builder.clear();
|
||||
builder.openObject();
|
||||
CHECK((logicalView1->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((1 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 1 after upgrade
|
||||
}
|
||||
|
||||
// test db-server (no directory)
|
||||
{
|
||||
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
|
||||
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
|
||||
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
|
||||
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
|
||||
|
||||
auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
|
||||
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_PRIMARY);
|
||||
auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
|
||||
|
||||
// create a new instance of an ApplicationServer and fill it with the required features
|
||||
// cannot use the existing server since its features already have some state
|
||||
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
|
||||
arangodb::application_features::ApplicationServer::server,
|
||||
[](arangodb::application_features::ApplicationServer* ptr)->void {
|
||||
arangodb::application_features::ApplicationServer::server = ptr;
|
||||
}
|
||||
);
|
||||
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
|
||||
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
|
||||
arangodb::iresearch::IResearchFeature feature(server);
|
||||
arangodb::DatabasePathFeature* dbPathFeature;
|
||||
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
|
||||
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
|
||||
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
|
||||
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
|
||||
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
|
||||
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
|
||||
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
|
||||
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
|
||||
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
|
||||
feature.prepare(); // register iresearch view type
|
||||
feature.start(); // register upgrade tasks
|
||||
server.lookupFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
|
||||
server.lookupFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
|
||||
server.lookupFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
|
||||
|
||||
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
|
||||
auto versionFilename = StorageEngineMock::versionFilenameResult;
|
||||
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
|
||||
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
|
||||
REQUIRE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
|
||||
REQUIRE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
|
||||
|
||||
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
|
||||
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
|
||||
REQUIRE((false == !logicalCollection));
|
||||
auto logicalView = vocbase.createView(viewJson->slice());
|
||||
REQUIRE((false == !logicalView));
|
||||
auto* view = dynamic_cast<arangodb::iresearch::IResearchView*>(logicalView.get());
|
||||
REQUIRE((false == !view));
|
||||
bool created;
|
||||
auto index = logicalCollection->createIndex(linkJson->slice(), created);
|
||||
REQUIRE((created));
|
||||
REQUIRE((false == !index));
|
||||
auto link = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
|
||||
REQUIRE((false == !link));
|
||||
REQUIRE((view->link(link->self()))); // link will not notify view in 'vocbase', hence notify manually
|
||||
|
||||
index->unload(); // release file handles
|
||||
bool result;
|
||||
auto linkDataPath = getPersistedPath1(*link);
|
||||
CHECK((linkDataPath.remove())); // remove link directory
|
||||
auto viewDataPath = getPersistedPath0(*logicalView);
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure no view directory
|
||||
arangodb::velocypack::Builder builder;
|
||||
builder.openObject();
|
||||
CHECK((logicalView->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
|
||||
|
||||
CHECK((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
|
||||
logicalView = vocbase.lookupView(logicalView->name());
|
||||
CHECK((true == !logicalView)); // ensure view removed after upgrade
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure view directory not present
|
||||
}
|
||||
|
||||
// test db-server (with directory)
|
||||
{
|
||||
auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }");
|
||||
auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"type\": \"arangosearch\", \"includeAllFields\": true }");
|
||||
auto viewJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\", \"version\": 0 }");
|
||||
auto versionJson = arangodb::velocypack::Parser::fromJson("{ \"version\": 0, \"tasks\": {} }");
|
||||
|
||||
auto serverRoleBefore = arangodb::ServerState::instance()->getRole();
|
||||
arangodb::ServerState::instance()->setRole(arangodb::ServerState::ROLE_PRIMARY);
|
||||
auto serverRoleRestore = irs::make_finally([&serverRoleBefore]()->void { arangodb::ServerState::instance()->setRole(serverRoleBefore); });
|
||||
|
||||
// create a new instance of an ApplicationServer and fill it with the required features
|
||||
// cannot use the existing server since its features already have some state
|
||||
std::shared_ptr<arangodb::application_features::ApplicationServer> originalServer(
|
||||
arangodb::application_features::ApplicationServer::server,
|
||||
[](arangodb::application_features::ApplicationServer* ptr)->void {
|
||||
arangodb::application_features::ApplicationServer::server = ptr;
|
||||
}
|
||||
);
|
||||
arangodb::application_features::ApplicationServer::server = nullptr; // avoid "ApplicationServer initialized twice"
|
||||
arangodb::application_features::ApplicationServer server(nullptr, nullptr);
|
||||
arangodb::iresearch::IResearchFeature feature(server);
|
||||
arangodb::DatabasePathFeature* dbPathFeature;
|
||||
server.addFeature(new arangodb::AuthenticationFeature(server)); // required for ClusterInfo::loadPlan()
|
||||
server.addFeature(new arangodb::application_features::CommunicationFeaturePhase(server)); // required for SimpleHttpClient::doRequest()
|
||||
server.addFeature(new arangodb::DatabaseFeature(server)); // required to skip IResearchView validation
|
||||
server.addFeature(dbPathFeature = new arangodb::DatabasePathFeature(server)); // required for IResearchLink::initDataStore()
|
||||
server.addFeature(new arangodb::iresearch::IResearchAnalyzerFeature(server)); // required for restoring link analyzers
|
||||
server.addFeature(new arangodb::QueryRegistryFeature(server)); // required for constructing TRI_vocbase_t
|
||||
server.addFeature(new arangodb::ShardingFeature(server)); // required for LogicalCollection::LogicalCollection(...)
|
||||
server.addFeature(new arangodb::UpgradeFeature(server, nullptr, {})); // required for upgrade tasks
|
||||
server.addFeature(new arangodb::ViewTypesFeature(server)); // required for IResearchFeature::prepare()
|
||||
feature.prepare(); // register iresearch view type
|
||||
feature.start(); // register upgrade tasks
|
||||
server.lookupFeature<arangodb::AuthenticationFeature>("Authentication")->prepare(); // create AuthenticationFeature::INSTANCE
|
||||
server.lookupFeature<arangodb::DatabaseFeature>("Database")->enableUpgrade(); // skip IResearchView validation
|
||||
server.lookupFeature<arangodb::ShardingFeature>("Sharding")->prepare(); // register sharding types
|
||||
|
||||
arangodb::tests::setDatabasePath(*dbPathFeature); // ensure test data is stored in a unique directory
|
||||
auto versionFilename = StorageEngineMock::versionFilenameResult;
|
||||
auto versionFilenameRestore = irs::make_finally([&versionFilename]()->void { StorageEngineMock::versionFilenameResult = versionFilename; });
|
||||
StorageEngineMock::versionFilenameResult = (irs::utf8_path(dbPathFeature->directory()) /= "version").utf8();
|
||||
REQUIRE((irs::utf8_path(dbPathFeature->directory()).mkdir()));
|
||||
REQUIRE((arangodb::basics::VelocyPackHelper::velocyPackToFile(StorageEngineMock::versionFilenameResult, versionJson->slice(), false)));
|
||||
|
||||
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
|
||||
auto logicalCollection = vocbase.createCollection(collectionJson->slice());
|
||||
REQUIRE((false == !logicalCollection));
|
||||
auto logicalView = vocbase.createView(viewJson->slice());
|
||||
REQUIRE((false == !logicalView));
|
||||
auto* view = dynamic_cast<arangodb::iresearch::IResearchView*>(logicalView.get());
|
||||
REQUIRE((false == !view));
|
||||
bool created;
|
||||
auto index = logicalCollection->createIndex(linkJson->slice(), created);
|
||||
REQUIRE((created));
|
||||
REQUIRE((false == !index));
|
||||
auto link = std::dynamic_pointer_cast<arangodb::iresearch::IResearchLink>(index);
|
||||
REQUIRE((false == !link));
|
||||
REQUIRE((view->link(link->self()))); // link will not notify view in 'vocbase', hence notify manually
|
||||
|
||||
index->unload(); // release file handles
|
||||
bool result;
|
||||
auto linkDataPath = getPersistedPath1(*link);
|
||||
CHECK((linkDataPath.remove())); // remove link directory
|
||||
auto viewDataPath = getPersistedPath0(*logicalView);
|
||||
CHECK((viewDataPath.exists(result) && !result));
|
||||
CHECK((viewDataPath.mkdir())); // create view directory
|
||||
CHECK((viewDataPath.exists(result) && result));
|
||||
arangodb::velocypack::Builder builder;
|
||||
builder.openObject();
|
||||
CHECK((logicalView->properties(builder, true, true).ok()));
|
||||
builder.close();
|
||||
CHECK((0 == builder.slice().get("version").getNumber<uint32_t>())); // ensure 'version == 0 before upgrade
|
||||
|
||||
CHECK((arangodb::methods::Upgrade::startup(vocbase, true, false).ok())); // run upgrade
|
||||
logicalView = vocbase.lookupView(logicalView->name());
|
||||
CHECK((true == !logicalView)); // ensure view removed after upgrade
|
||||
CHECK((viewDataPath.exists(result) && !result)); // ensure view directory removed after upgrade
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("IResearch_version") {
|
||||
CHECK(IResearch_version == arangodb::rest::Version::getIResearchVersion());
|
||||
CHECK(IResearch_version == arangodb::rest::Version::Values["iresearch-version"]);
|
||||
|
|
|
|||
|
|
@ -1003,6 +1003,7 @@ arangodb::Result PhysicalCollectionMock::updateProperties(arangodb::velocypack::
|
|||
|
||||
std::function<void()> StorageEngineMock::before = []()->void {};
|
||||
bool StorageEngineMock::inRecoveryResult = false;
|
||||
/*static*/ std::string StorageEngineMock::versionFilenameResult;
|
||||
|
||||
StorageEngineMock::StorageEngineMock(
|
||||
arangodb::application_features::ApplicationServer& server
|
||||
|
|
@ -1428,8 +1429,7 @@ void StorageEngineMock::unloadCollection(
|
|||
}
|
||||
|
||||
std::string StorageEngineMock::versionFilename(TRI_voc_tick_t) const {
|
||||
TRI_ASSERT(false);
|
||||
return std::string();
|
||||
return versionFilenameResult;
|
||||
}
|
||||
|
||||
void StorageEngineMock::waitForEstimatorSync(std::chrono::milliseconds) {
|
||||
|
|
|
|||
|
|
@ -163,6 +163,7 @@ class StorageEngineMock: public arangodb::StorageEngine {
|
|||
public:
|
||||
static std::function<void()> before;
|
||||
static bool inRecoveryResult;
|
||||
static std::string versionFilenameResult;
|
||||
std::map<std::pair<TRI_voc_tick_t, TRI_voc_cid_t>, arangodb::velocypack::Builder> views;
|
||||
std::atomic<size_t> vocbaseCount;
|
||||
|
||||
|
|
@ -232,4 +233,4 @@ class StorageEngineMock: public arangodb::StorageEngine {
|
|||
TRI_voc_tick_t _releasedTick;
|
||||
};
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue