Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

issue 383.1: initial stub implementation of IResearchViewDBServer (#5132)

This commit is contained in:
Vasiliy 2018-04-18 13:51:21 +03:00 committed by Andrey Abramov
parent 0972befd87
commit 6053056b6a
11 changed files with 1177 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arangod/CMakeLists.txt
View File

@ -52,6 +52,7 @@ if (USE_IRESEARCH)
IResearch/IResearchAnalyzerFeature.cpp IResearch/IResearchAnalyzerFeature.h
IResearch/IResearchAttributes.cpp IResearch/IResearchAttributes.h
IResearch/IResearchCommon.cpp IResearch/IResearchCommon.h
IResearch/IResearchViewDBServer.cpp IResearch/IResearchViewDBServer.h
IResearch/IResearchKludge.cpp IResearch/IResearchKludge.h
IResearch/IResearchLink.cpp IResearch/IResearchLink.h
IResearch/IResearchLinkMeta.cpp IResearch/IResearchLinkMeta.h

14
arangod/IResearch/IResearchFeature.cpp
View File

@ -39,6 +39,7 @@
#include "IResearchRocksDBRecoveryHelper.h"
#include "IResearchView.h"
#include "IResearchViewCoordinator.h"
#include "IResearchViewDBServer.h"
#include "Aql/AqlValue.h"
#include "Aql/AqlFunctionFeature.h"
#include "Aql/Function.h"
@ -225,11 +226,16 @@ void registerViewFactory() {
);
}
arangodb::Result res;
// DB server in custer or single-server
auto res = arangodb::ServerState::instance()->isCoordinator()
? viewTypes->emplace(viewType, arangodb::iresearch::IResearchViewCoordinator::make)
: viewTypes->emplace(viewType, arangodb::iresearch::IResearchView::make)
;
if (arangodb::ServerState::instance()->isCoordinator()) {
res = viewTypes->emplace(viewType, arangodb::iresearch::IResearchViewCoordinator::make);
} else if (arangodb::ServerState::instance()->isDBServer()) {
res = viewTypes->emplace(viewType, arangodb::iresearch::IResearchViewDBServer::make);
} else {
res = viewTypes->emplace(viewType, arangodb::iresearch::IResearchView::make);
}
if (!res.ok()) {
THROW_ARANGO_EXCEPTION_MESSAGE(

32
arangod/IResearch/IResearchLink.cpp
View File

@ -23,6 +23,7 @@
#include "ApplicationServerHelper.h"
#include "IResearchCommon.h"
#include "IResearchViewDBServer.h"
#include "VelocyPackHelper.h"
#include "Basics/LocalTaskQueue.h"
#include "Logger/Logger.h"
@ -175,6 +176,10 @@ int IResearchLink::drop() {
// if the collection is in the process of being removed then drop it from the view
if (_collection->deleted()) {
if (_wiewDrop) {
_wiewDrop();
}
auto result = _view->updateProperties(emptyObjectSlice(), true, false); // revalidate all links
if (!result.ok()) {
@ -236,6 +241,21 @@ bool IResearchLink::init(arangodb::velocypack::Slice const& definition) {
return false; // no such view
}
IResearchViewDBServer* wiew = nullptr;
// create the IResearchView for the specific collection (on DBServer)
if (arangodb::ServerState::instance()->isDBServer()) {
// TODO FIXME find a better way to look up an iResearch View
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
wiew = dynamic_cast<IResearchViewDBServer*>(logicalView.get());
#else
wiew = static_cast<IResearchViewDBServer*>(logicalView.get());
#endif
// repoint LogicalView at the collection-specific instance
logicalView = wiew ? wiew->create(id()) : nullptr;
}
// TODO FIXME find a better way to look up an iResearch View
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
auto* view = dynamic_cast<IResearchView*>(logicalView.get());
@ -274,6 +294,18 @@ bool IResearchLink::init(arangodb::velocypack::Slice const& definition) {
_meta = std::move(meta);
_view = std::move(view);
// register the IResearchView for the specific collection (on DBServer)
if (wiew) {
_wiewDrop = wiew->emplace(_id, logicalView);
if (!_wiewDrop) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "error registering view: '" << viewId << "' for link '" << _id << "'";
return false;
}
}
// FIXME TODO remove once View::updateProperties(...) will be fixed to write
// the update delta into the WAL marker instead of the full persisted state
{

1
arangod/IResearch/IResearchLink.h
View File

@ -218,6 +218,7 @@ class IResearchLink {
IResearchLinkMeta _meta; // how this collection should be indexed
mutable irs::async_utils::read_write_mutex _mutex; // for use with _view to allow asynchronous disassociation
IResearchView* _view; // effectively the IResearch datastore itself (nullptr == not associated)
std::function<void()> _wiewDrop; // callback for undergistering '_view' from IResearchViewDBServer (valid only on DBServer)
std::unique_lock<irs::async_utils::read_write_mutex::read_mutex> _viewLock; // prevent view deallocation (lock @ AsyncSelf)
}; // IResearchLink

512
arangod/IResearch/IResearchViewDBServer.cpp Normal file
View File

@ -0,0 +1,512 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#include "ApplicationServerHelper.h"
#include "IResearchCommon.h"
#include "IResearchView.h"
#include "IResearchViewDBServer.h"
#include "VelocyPackHelper.h"
#include "Logger/Logger.h"
#include "Logger/LogMacros.h"
#include "RestServer/DatabasePathFeature.h"
#include "RestServer/ViewTypesFeature.h"
#include "VocBase/vocbase.h"
NS_LOCAL
typedef irs::async_utils::read_write_mutex::read_mutex ReadMutex;
typedef irs::async_utils::read_write_mutex::write_mutex WriteMutex;
////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the IResearch View definition denoting the
/// view id (from LogicalView.cpp)
////////////////////////////////////////////////////////////////////////////////
const std::string ID_FIELD("id");
////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the IResearch View definition denoting the
/// view id (from vocbase.cpp)
////////////////////////////////////////////////////////////////////////////////
const std::string IS_SYSTEM_FIELD("isSystem");
////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the iResearch View definition denoting the
/// corresponding link definitions
////////////////////////////////////////////////////////////////////////////////
const std::string LINKS_FIELD("links");
////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the IResearch View definition denoting the
/// view name (from LogicalView.cpp)
////////////////////////////////////////////////////////////////////////////////
const std::string NAME_FIELD("name");
////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the IResearch View definition denoting the
/// view properties (from LogicalView.cpp)
////////////////////////////////////////////////////////////////////////////////
const std::string PROPERTIES_FIELD("properties");
static std::string const VIEW_NAME_PREFIX("_iresearch_");
////////////////////////////////////////////////////////////////////////////////
/// @brief a key in the jSON definition that differentiates a view-cid container
/// from individual per-cid view implementation
/// (view types are identical)
////////////////////////////////////////////////////////////////////////////////
static std::string const VIEW_CONTAINER_MARKER("master");
////////////////////////////////////////////////////////////////////////////////
/// @brief compute the data path to user for iresearch persisted-store
/// get base path from DatabaseServerFeature (similar to MMFilesEngine)
/// the path is hardcoded to reside under:
/// <DatabasePath>/<IResearchView::type()>-<view id>
/// similar to the data path calculation for collections
////////////////////////////////////////////////////////////////////////////////
irs::utf8_path getPersistedPath(
arangodb::DatabasePathFeature const& dbPathFeature, TRI_voc_cid_t id
) {
irs::utf8_path dataPath(dbPathFeature.directory());
static const std::string subPath("databases");
dataPath /= subPath;
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
dataPath += "-";
dataPath += std::to_string(id);
return dataPath;
}
NS_END
NS_BEGIN(arangodb)
NS_BEGIN(iresearch)
IResearchViewDBServer::IResearchViewDBServer(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& info,
arangodb::DatabasePathFeature const& dbPathFeature,
uint64_t planVersion
): LogicalView(vocbase, info, planVersion),
_meta(info),
_persistedPath(getPersistedPath(dbPathFeature, id())) {
}
std::shared_ptr<arangodb::LogicalView> IResearchViewDBServer::create(
TRI_voc_cid_t cid
) {
{
ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously modified
auto itr = _collections.find(cid);
if (itr != _collections.end()) {
return itr->second;
}
}
auto view_name = VIEW_NAME_PREFIX + std::to_string(id()) + "_" + name();
arangodb::velocypack::Builder builder;
builder.openObject();
builder.add(ID_FIELD, arangodb::velocypack::Value("0")); // force unique ID
builder.add(NAME_FIELD, toValuePair(view_name)); // mark the view definition as an internal per-cid instance
builder.add(IS_SYSTEM_FIELD, arangodb::velocypack::Value(true)); // required to for use of VIEW_NAME_PREFIX
if (!mergeSlice(builder, _meta.slice())) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure to generate definition while constructing IResearch View in database '" << vocbase().id() << "'";
return nullptr;
}
builder.close();
auto view = IResearchView::make(vocbase(), builder.slice(), 0);
if (!view) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure while creating an IResearch View for collection '" << cid << "' in database '" << vocbase().name() << "'";
return nullptr;
}
// hold a reference to the original view in the deleter so that the view is still valid inside the deleter
return std::shared_ptr<arangodb::LogicalView>(
view.get(),
[this, cid, view](arangodb::LogicalView* ptr)->void {
ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously read
if (ptr
&& _collections.find(cid) == _collections.end()
&& !ptr->drop().ok()) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure while dropping an IResearch View '" << ptr->id() << "' in database '" << ptr->vocbase().name() << "'";
}
}
);
}
std::function<void()> IResearchViewDBServer::emplace(
TRI_voc_cid_t cid,
std::shared_ptr<arangodb::LogicalView> const& view
) {
if (!view) {
return {}; // invalid argument
}
WriteMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously read
auto itr = _collections.find(cid);
if (itr != _collections.end() && itr->second.get() != view.get()) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "encountered a duplicate definition while registering View for collection '" << cid << "' in database '" << vocbase().name() << "'";
return {}; // duplicate definition
}
_collections.emplace(cid, view);
return [this, cid]()->void {
WriteMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously read
if (!_collections.erase(cid)) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "view not found while removing View for collection '" << cid << "' in database '" << vocbase().name() << "'";
}
};
}
/*static*/ std::shared_ptr<LogicalView> IResearchViewDBServer::make(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& info,
uint64_t planVersion,
LogicalView::PreCommitCallback const& preCommit /*= LogicalView::PreCommitCallback()*/
) {
if (!info.isObject()) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "non-object definition supplied while instantiating IResearch View in database '" << vocbase.name() << "'";
return nullptr;
}
irs::string_ref name;
bool seen;
if (!getString(name, info, NAME_FIELD, seen, std::string()) || !seen) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "definition supplied without a 'name' while instantiating IResearch View in database '" << vocbase.name() << "'";
return nullptr;
}
// not a per-cid view instance (get here from ClusterInfo)
if (!irs::starts_with(name, VIEW_NAME_PREFIX)) {
auto wiew = vocbase.lookupView(name);
// DBServer view already exists, treat as an update
if (wiew) {
return wiew->updateProperties(info.get(PROPERTIES_FIELD), false, true).ok() // 'false' because full view definition
? wiew : nullptr;
}
// if creation request not coming from the vocbase
if (!info.hasKey(VIEW_CONTAINER_MARKER)) {
arangodb::velocypack::Builder builder;
builder.openObject();
builder.add(
VIEW_CONTAINER_MARKER,
arangodb::velocypack::Value(arangodb::velocypack::ValueType::Null)
);
if (!mergeSlice(builder, info)) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure to generate definition while constructing IResearch View in database '" << vocbase.id() << "'";
return nullptr;
}
return vocbase.createView(builder.close().slice());
}
auto* feature =
arangodb::iresearch::getFeature<arangodb::DatabasePathFeature>("DatabasePath");
if (!feature) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure to find feature 'DatabasePath' while constructing IResearch View in database '" << vocbase.id() << "'";
return nullptr;
}
PTR_NAMED(IResearchViewDBServer, view, vocbase, info, *feature, planVersion);
if (preCommit && !preCommit(view)) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "failure during pre-commit while constructing IResearch View in database '" << vocbase.id() << "'";
return nullptr;
}
return view;
}
// ...........................................................................
// a per-cid view instance (get here only from StorageEngine startup or WAL recovery)
// ...........................................................................
auto* begin = name.c_str() + VIEW_NAME_PREFIX.size();
auto* end = (char*)::memchr(begin, '_', name.size() - VIEW_NAME_PREFIX.size());
if (!end) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failed to determine view ID while constructing IResearch View in database '" << vocbase.name() << "'";
return nullptr;
}
irs::string_ref view_id(begin, end - begin);
irs::string_ref view_name(end + 1, name.c_str() + name.size() - (end + 1)); // +1 for '_'
IResearchViewMeta meta;
std::string error;
if (!meta.init(info.get(PROPERTIES_FIELD), error)
|| 1 != meta._collections.size()) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failed to determine collection ID while constructing IResearch View in database '" << vocbase.name() << "'";
return nullptr;
}
auto cid = *(meta._collections.begin());
auto wiew = vocbase.lookupView(view_id);
// create DBServer view
if (!wiew) {
arangodb::velocypack::Builder builder;
builder.openObject();
builder.add(
VIEW_CONTAINER_MARKER,
arangodb::velocypack::Value(arangodb::velocypack::ValueType::Null)
);
if (!mergeSlice(builder, info)) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure to generate definition while constructing IResearch View in database '" << vocbase.name() << "'";
return nullptr;
}
// mark the view definition as a DBServer instance
builder.add(ID_FIELD, arangodb::velocypack::Value(view_id));
builder.add(NAME_FIELD, arangodb::velocypack::Value(view_name));
builder.close();
wiew = vocbase.createView(builder.slice());
if (!wiew) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failure while creating an IResearch View '" << std::string(name) << "' in database '" << vocbase.name() << "'";
return nullptr;
}
}
// TODO FIXME find a better way to look up an iResearch View
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
auto* impl = dynamic_cast<IResearchViewDBServer*>(wiew.get());
#else
auto* impl = static_cast<IResearchViewDBServer*>(wiew.get());
#endif
auto view = impl->create(cid);
auto result = impl->emplace(cid, view);
return result ? view : nullptr;
}
arangodb::Result IResearchViewDBServer::drop() {
ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously modified
for (auto& entry: _collections) {
auto res = entry.second->drop();
if (!res.ok()) {
return res; // fail on first failure
}
}
return arangodb::Result();
}
void IResearchViewDBServer::open() {
ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously modified
for (auto& entry: _collections) {
entry.second->open();
}
}
arangodb::Result IResearchViewDBServer::rename(
std::string&& newName,
bool doSync
) {
ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously modified
for (auto& entry: _collections) {
auto res = entry.second->rename(std::string(newName), doSync);
if (!res.ok()) {
return res; // fail on first failure
}
}
arangodb::velocypack::Builder builder;
builder.openObject();
if (!mergeSlice(builder, _meta.slice())) {
return arangodb::Result(
TRI_ERROR_INTERNAL,
std::string("failure to generate definition while renaming IResearch View in database '") + vocbase().name() + "'"
);
}
builder.add(NAME_FIELD, arangodb::velocypack::Value(std::move(newName)));
builder.close();
_meta = std::move(builder);
return arangodb::Result();
}
void IResearchViewDBServer::toVelocyPack(
arangodb::velocypack::Builder& result,
bool /*includeProperties*/,
bool /*includeSystem*/
) const {
TRI_ASSERT(result.isOpenObject());
mergeSlice(result, _meta.slice());
}
arangodb::Result IResearchViewDBServer::updateProperties(
arangodb::velocypack::Slice const& properties,
bool partialUpdate,
bool doSync
) {
if (!properties.isObject() || properties.hasKey(LINKS_FIELD)) {
return arangodb::Result(
TRI_ERROR_BAD_PARAMETER,
std::string("invalid properties supplied while updating IResearch View in database '") + vocbase().name() + "'"
);
}
IResearchViewMeta meta;
std::string error;
if (partialUpdate) {
IResearchViewMeta oldMeta;
if (!oldMeta.init(_meta.slice().get(PROPERTIES_FIELD), error)
|| !meta.init(properties, error, oldMeta)) {
return arangodb::Result(
TRI_ERROR_BAD_PARAMETER,
std::string("failure parsing properties while updating IResearch View in database '") + vocbase().name() + "'"
);
}
} else if (!meta.init(properties, error)) {
return arangodb::Result(
TRI_ERROR_BAD_PARAMETER,
std::string("failure parsing properties while updating IResearch View in database '") + vocbase().name() + "'"
);
}
arangodb::velocypack::Builder builder;
builder.openObject();
if (!mergeSlice(builder, _meta.slice())) {
return arangodb::Result(
TRI_ERROR_INTERNAL,
std::string("failure to generate definition while updating IResearch View in database '") + vocbase().name() + "'"
);
}
builder.add(
PROPERTIES_FIELD,
arangodb::velocypack::Value(arangodb::velocypack::ValueType::Object)
);
if (!meta.json(builder)) {
return arangodb::Result(
TRI_ERROR_INTERNAL,
std::string("failure to generate 'properties' definition while updating IResearch View in database '") + vocbase().name() + "'"
);
}
builder.close();
WriteMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously read
for (auto& entry: _collections) {
auto res = entry.second->updateProperties(properties, partialUpdate, doSync);
if (!res.ok()) {
return res; // fail on first failure
}
}
_meta = std::move(builder);
return arangodb::Result();
}
bool IResearchViewDBServer::visitCollections(
CollectionVisitor const& visitor
) const {
ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // 'collections_' can be asynchronously modified
for (auto& entry: _collections) {
if (!visitor(entry.first)) {
return false;
}
}
return true;
}
NS_END // iresearch
NS_END // arangodb
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

109
arangod/IResearch/IResearchViewDBServer.h Normal file
View File

@ -0,0 +1,109 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_IRESEARCH__IRESEARCH_VIEW_DBSERVER_H
#define ARANGOD_IRESEARCH__IRESEARCH_VIEW_DBSERVER_H 1
#include "utils/async_utils.hpp"
#include "utils/memory.hpp"
#include "utils/utf8_path.hpp"
#include "VocBase/LogicalView.h"
NS_BEGIN(arangodb)
class DatabasePathFeature; // forward declaration
NS_END // arangodb
NS_BEGIN(arangodb)
NS_BEGIN(iresearch)
class IResearchViewDBServer final: public arangodb::LogicalView {
public:
//////////////////////////////////////////////////////////////////////////////
/// @brief ensure there is a view instance for the specified 'cid'
/// @return an existing instance or create a new instance if none is registred
//////////////////////////////////////////////////////////////////////////////
std::shared_ptr<arangodb::LogicalView> create(TRI_voc_cid_t cid);
virtual arangodb::Result drop() override;
//////////////////////////////////////////////////////////////////////////////
/// @brief register a view instance for the specified 'cid'
/// @return functer used for dropping the association or 'false' on error or
/// duplicate definition
//////////////////////////////////////////////////////////////////////////////
std::function<void()> emplace(
TRI_voc_cid_t cid,
std::shared_ptr<arangodb::LogicalView> const& view
);
///////////////////////////////////////////////////////////////////////////////
/// @brief view factory
/// @returns initialized view object
///////////////////////////////////////////////////////////////////////////////
static std::shared_ptr<LogicalView> make(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& info,
uint64_t planVersion,
LogicalView::PreCommitCallback const& preCommit = LogicalView::PreCommitCallback()
);
virtual void open() override;
virtual arangodb::Result rename(std::string&& newName, bool doSync) override;
virtual void toVelocyPack(
arangodb::velocypack::Builder& result,
bool includeProperties,
bool includeSystem
) const override;
virtual arangodb::Result updateProperties(
arangodb::velocypack::Slice const& properties,
bool partialUpdate,
bool doSync
) override;
virtual bool visitCollections(
CollectionVisitor const& visitor
) const override;
private:
DECLARE_SPTR(LogicalView);
std::map<TRI_voc_cid_t, std::shared_ptr<arangodb::LogicalView>> _collections;
arangodb::velocypack::Builder _meta; // the view definition
mutable irs::async_utils::read_write_mutex _mutex; // for use with members
irs::utf8_path const _persistedPath;
IResearchViewDBServer(
TRI_vocbase_t& vocbase,
arangodb::velocypack::Slice const& info,
arangodb::DatabasePathFeature const& dbPathFeature,
uint64_t planVersion
);
};
NS_END // iresearch
NS_END // arangodb
#endif

8
arangod/VocBase/LogicalView.cpp
View File

@ -73,11 +73,15 @@ TRI_voc_cid_t ReadPlanId(VPackSlice info, TRI_voc_cid_t vid) {
}
if (arangodb::ServerState::instance()->isDBServer()) {
return arangodb::ClusterInfo::instance()->uniqid(1);
auto* ci = arangodb::ClusterInfo::instance();
return ci ? ci->uniqid(1) : 0;
}
if (arangodb::ServerState::instance()->isCoordinator()) {
return arangodb::ClusterInfo::instance()->uniqid(1);
auto* ci = arangodb::ClusterInfo::instance();
return ci ? ci->uniqid(1) : 0;
}
return TRI_NewTickServer();

2
tests/CMakeLists.txt
View File

@ -9,12 +9,14 @@ endforeach()
if (USE_IRESEARCH)
set(IRESEARCH_TESTS_SOURCES
IResearch/common.cpp
IResearch/AgencyCommManagerMock.cpp
# IResearch/AttributeScorer-test.cpp
IResearch/Containers-test.cpp
IResearch/IResearchAnalyzerFeature-test.cpp
IResearch/IResearchFeature-test.cpp
IResearch/IResearchOrder-test.cpp
IResearch/IResearchView-test.cpp
IResearch/IResearchViewDBServer-test.cpp
IResearch/IResearchViewMeta-test.cpp
IResearch/IResearchDocument-test.cpp
IResearch/IResearchFilter-test.cpp

185
tests/IResearch/AgencyCommManagerMock.cpp Normal file
View File

@ -0,0 +1,185 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#include "AgencyCommManagerMock.h"
AgencyCommManagerMock::AgencyCommManagerMock(): AgencyCommManager("") {
}
void AgencyCommManagerMock::addConnection(
std::unique_ptr<arangodb::httpclient::GeneralClientConnection>&& connection
) {
std::string endpoint(""); // must be empty string to match addEndpoint(...) normalization
addEndpoint(endpoint);
release(std::move(connection), endpoint);
}
EndpointMock::EndpointMock()
: arangodb::Endpoint(
arangodb::Endpoint::DomainType:: UNKNOWN,
arangodb::Endpoint::EndpointType::CLIENT,
arangodb::Endpoint::TransportType::HTTP,
arangodb::Endpoint::EncryptionType::NONE,
"",
0
) {
}
TRI_socket_t EndpointMock::connect(
double connectTimeout, double requestTimeout
) {
TRI_ASSERT(false);
return TRI_socket_t();
}
void EndpointMock::disconnect() {
}
int EndpointMock::domain() const {
TRI_ASSERT(false);
return 0;
}
std::string EndpointMock::host() const {
return ""; // empty host for testing
}
std::string EndpointMock::hostAndPort() const {
TRI_ASSERT(false);
return "";
}
bool EndpointMock::initIncoming(TRI_socket_t socket) {
TRI_ASSERT(false);
return 0;
}
int EndpointMock::port() const {
TRI_ASSERT(false);
return 0;
}
GeneralClientConnectionMock::GeneralClientConnectionMock()
: GeneralClientConnection(&endpoint, 0, 0, 0),
nil(file_open((file_path_t)nullptr, "rw")) {
_socket.fileDescriptor = file_no(nil.get()); // must be a readable/writable descriptor
}
bool GeneralClientConnectionMock::connectSocket() {
_isConnected = true;
return true;
}
void GeneralClientConnectionMock::disconnectSocket() {
}
void GeneralClientConnectionMock::getValue(
arangodb::basics::StringBuffer& buffer
) {
buffer.appendChar('\n');
}
bool GeneralClientConnectionMock::readable() {
TRI_ASSERT(false);
return false;
}
bool GeneralClientConnectionMock::readClientConnection(
arangodb::basics::StringBuffer& buffer, bool& connectionClosed
) {
getValue(buffer);
connectionClosed = true;
return true;
}
void GeneralClientConnectionMock::setKey(char const* data, size_t length) {
// NOOP
}
bool GeneralClientConnectionMock::writeClientConnection(
void const* buffer, size_t length, size_t* bytesWritten
) {
setKey(static_cast<char const*>(buffer), length);
*bytesWritten = length; // assume wrote the entire buffer
return true;
}
void GeneralClientConnectionListMock::getValue(
arangodb::basics::StringBuffer& buffer
) {
if (responses.empty()) {
GeneralClientConnectionMock::getValue(buffer);
return;
}
buffer.appendText(responses.front());
responses.pop_front();
}
void GeneralClientConnectionMapMock::getValue(
arangodb::basics::StringBuffer& buffer
) {
auto itr = responses.find(lastKey);
// try to search by just the header
if (itr == responses.end()) {
auto pos = lastKey.find("\r\n");
if (pos != std::string::npos) {
itr = responses.find(lastKey.substr(0, pos));
}
}
if (itr == responses.end()) {
GeneralClientConnectionMock::getValue(buffer);
return;
}
buffer.appendText(itr->second);
}
void GeneralClientConnectionMapMock::setKey(char const* data, size_t length) {
lastKey.assign(data, length);
auto pos = lastKey.find("\r\n");
if (pos == std::string::npos) {
return; // use full string as key
}
auto head = lastKey.substr(0, pos);
pos = lastKey.find("\r\n\r\n", pos);
lastKey = pos == std::string::npos
? head // first line of header (no body in request)
: head.append(lastKey.c_str() + pos); // first line of header with body
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

122
tests/IResearch/AgencyCommManagerMock.h Normal file
View File

@ -0,0 +1,122 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_IRESEARCH__IRESEARCH_AGENCY_COMM_MANAGER_MOCK_H
#define ARANGODB_IRESEARCH__IRESEARCH_AGENCY_COMM_MANAGER_MOCK_H 1
#include "utils/file_utils.hpp"
#include "Agency/AgencyComm.h"
////////////////////////////////////////////////////////////////////////////////
/// @brief mock of AgencyCommManager for use with tests
////////////////////////////////////////////////////////////////////////////////
class AgencyCommManagerMock: public arangodb::AgencyCommManager {
public:
AgencyCommManagerMock();
void addConnection(
std::unique_ptr<arangodb::httpclient::GeneralClientConnection>&& connection
);
template <typename T, typename... Args>
T* addConnection(Args&&... args) {
auto* con = new T(std::forward<Args>(args)...);
addConnection(
std::unique_ptr<arangodb::httpclient::GeneralClientConnection>(con)
);
return con;
}
};
////////////////////////////////////////////////////////////////////////////////
/// @brief mock of Endpoint for use with GeneralClientConnectionMock
////////////////////////////////////////////////////////////////////////////////
class EndpointMock: public arangodb::Endpoint {
public:
EndpointMock();
virtual TRI_socket_t connect(
double connectTimeout, double requestTimeout
) override;
virtual void disconnect() override;
virtual int domain() const override;
virtual std::string host() const override;
virtual std::string hostAndPort() const override;
virtual bool initIncoming(TRI_socket_t socket) override;
virtual int port() const override;
};
////////////////////////////////////////////////////////////////////////////////
/// @brief mock of GeneralClientConnection for use with AgencyCommManagerMock
////////////////////////////////////////////////////////////////////////////////
class GeneralClientConnectionMock
: public arangodb::httpclient::GeneralClientConnection {
public:
EndpointMock endpoint;
irs::file_utils::handle_t nil;
GeneralClientConnectionMock();
virtual bool connectSocket() override;
virtual void disconnectSocket() override;
virtual bool readable() override;
virtual bool readClientConnection(
arangodb::basics::StringBuffer& buffer, bool& connectionClosed
) override;
virtual bool writeClientConnection(
void const* buffer, size_t length, size_t* bytesWritten
) override;
protected:
virtual void getValue(arangodb::basics::StringBuffer& buffer); // override by specializations
virtual void setKey(char const* data, size_t length); // override by specializations
};
////////////////////////////////////////////////////////////////////////////////
/// @brief specialization of GeneralClientConnectionMock returning results from
/// a list
////////////////////////////////////////////////////////////////////////////////
class GeneralClientConnectionListMock: public GeneralClientConnectionMock {
public:
std::deque<std::string> responses;
virtual void getValue(arangodb::basics::StringBuffer& buffer) override;
};
////////////////////////////////////////////////////////////////////////////////
/// @brief specialization of GeneralClientConnectionMock returning results from
/// a map keyed by request
////////////////////////////////////////////////////////////////////////////////
class GeneralClientConnectionMapMock: public GeneralClientConnectionMock {
public:
std::string lastKey;
std::map<std::string, std::string> responses;
virtual void getValue(arangodb::basics::StringBuffer& buffer) override;
virtual void setKey(char const* data, size_t length) override;
};
#endif

197
tests/IResearch/IResearchViewDBServer-test.cpp Normal file
View File

@ -0,0 +1,197 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#include "catch.hpp"
#include "../IResearch/AgencyCommManagerMock.h"
#include "../IResearch/StorageEngineMock.h"
#include "Basics/files.h"
#include "Cluster/ClusterInfo.h"
#include "GeneralServer/AuthenticationFeature.h"
#include "IResearch/IResearchCommon.h"
#include "IResearch/IResearchFeature.h"
#include "IResearch/IResearchViewDBServer.h"
#include "RestServer/DatabasePathFeature.h"
#include "RestServer/QueryRegistryFeature.h"
#include "RestServer/ViewTypesFeature.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "velocypack/Parser.h"
// -----------------------------------------------------------------------------
// --SECTION-- setup / tear-down
// -----------------------------------------------------------------------------
struct IResearchViewDBServerSetup {
GeneralClientConnectionMapMock* agency;
StorageEngineMock engine;
arangodb::application_features::ApplicationServer server;
std::vector<std::pair<arangodb::application_features::ApplicationFeature*, bool>> features;
std::string testFilesystemPath;
IResearchViewDBServerSetup(): server(nullptr, nullptr) {
auto* agencyCommManager = new AgencyCommManagerMock();
agency = agencyCommManager->addConnection<GeneralClientConnectionMapMock>();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::RoleEnum::ROLE_PRIMARY);
arangodb::EngineSelectorFeature::ENGINE = &engine;
arangodb::AgencyCommManager::MANAGER.reset(agencyCommManager);
// suppress INFO {authentication} Authentication is turned on (system only), authentication for unix sockets is turned on
arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::WARN);
// suppress log messages since tests check error conditions
arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::FATAL);
irs::logger::output_le(iresearch::logger::IRL_FATAL, stderr);
// setup required application features
features.emplace_back(new arangodb::AuthenticationFeature(&server), false); // required for AgencyComm::send(...)
features.emplace_back(new arangodb::DatabasePathFeature(&server), false);
features.emplace_back(new arangodb::QueryRegistryFeature(&server), false); // required for TRI_vocbase_t instantiation
features.emplace_back(new arangodb::ViewTypesFeature(&server), false); // required for TRI_vocbase_t::createView(...)
features.emplace_back(new arangodb::iresearch::IResearchFeature(&server), false); // required for instantiating IResearchView*
for (auto& f: features) {
arangodb::application_features::ApplicationServer::server->addFeature(f.first);
}
for (auto& f: features) {
f.first->prepare();
}
for (auto& f: features) {
if (f.second) {
f.first->start();
}
}
arangodb::ClusterInfo::createInstance(nullptr); // required for generating view id
testFilesystemPath = (
(irs::utf8_path()/=
TRI_GetTempPath())/=
(std::string("arangodb_tests.") + std::to_string(TRI_microtime()))
).utf8();
auto* dbPathFeature = arangodb::application_features::ApplicationServer::getFeature<arangodb::DatabasePathFeature>("DatabasePath");
const_cast<std::string&>(dbPathFeature->directory()) = testFilesystemPath;
long systemError;
std::string systemErrorStr;
TRI_CreateDirectory(testFilesystemPath.c_str(), systemError, systemErrorStr);
}
~IResearchViewDBServerSetup() {
TRI_RemoveDirectory(testFilesystemPath.c_str());
arangodb::LogTopic::setLogLevel(arangodb::iresearch::TOPIC.name(), arangodb::LogLevel::DEFAULT);
arangodb::EngineSelectorFeature::ENGINE = nullptr;
arangodb::application_features::ApplicationServer::server = nullptr;
arangodb::AgencyCommManager::MANAGER.reset();
arangodb::ServerState::instance()->setRole(arangodb::ServerState::RoleEnum::ROLE_SINGLE);
// destroy application features
for (auto& f: features) {
if (f.second) {
f.first->stop();
}
}
for (auto& f: features) {
f.first->unprepare();
}
arangodb::LogTopic::setLogLevel(arangodb::Logger::AUTHENTICATION.name(), arangodb::LogLevel::DEFAULT);
}
};
// -----------------------------------------------------------------------------
// --SECTION-- test suite
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief setup
////////////////////////////////////////////////////////////////////////////////
TEST_CASE("IResearchViewDBServerTest", "[cluster][iresearch][iresearch-view]") {
IResearchViewDBServerSetup s;
(void)(s);
SECTION("test_create") {
s.agency->responses["POST /_api/agency/read HTTP/1.1\r\n\r\n[[\"/Sync/LatestID\"]]"] = "http/1.0 200\n\n[ { \"\": { \"Sync\": { \"LatestID\" : 1 } } } ]";
s.agency->responses["POST /_api/agency/write HTTP/1.1"] = "http/1.0 200\n\n{\"results\": []}";
auto json = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testView\", \"type\": \"arangosearch\" }");
TRI_vocbase_t vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase");
auto wiew = arangodb::iresearch::IResearchViewDBServer::make(vocbase, json->slice(), 42);
CHECK((false == !wiew));
auto* impl = dynamic_cast<arangodb::iresearch::IResearchViewDBServer*>(wiew.get());
CHECK((nullptr != impl));
auto view = impl->create(123);
CHECK((false == !view));
CHECK((std::string("testView") == view->name()));
CHECK((false == view->deleted()));
CHECK((wiew->id() != view->id())); // must have unique ID
CHECK((view->id() == view->planId())); // same as view ID
CHECK((0 == view->planVersion()));
CHECK((arangodb::iresearch::DATA_SOURCE_TYPE == view->type()));
CHECK((&vocbase == &(view->vocbase())));
}
SECTION("test_drop") {
// FIXME TODO implemet
}
SECTION("test_emplace") {
// FIXME TODO implemet
}
SECTION("test_make") {
// FIXME TODO implemet
}
SECTION("test_open") {
// FIXME TODO implemet
}
SECTION("test_rename") {
// FIXME TODO implemet
}
SECTION("test_toVelocyPack") {
// FIXME TODO implemet
}
SECTION("test_updateProperties") {
// FIXME TODO implemet
}
SECTION("test_visitCollections") {
// FIXME TODO implemet
}
////////////////////////////////////////////////////////////////////////////////
/// @brief generate tests
////////////////////////////////////////////////////////////////////////////////
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------