Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

reduce number of heap allocations for removals (#7157)

This commit is contained in:
Andrey Abramov 2018-10-31 01:21:25 +03:00 committed by GitHub
parent b2748eb1e5
commit 6a4f462fa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 324 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arangod/CMakeLists.txt
View File

@ -72,6 +72,7 @@ if (USE_IRESEARCH)
IResearch/IResearchViewMeta.cpp IResearch/IResearchViewMeta.h
IResearch/IResearchFeature.cpp IResearch/IResearchFeature.h
IResearch/IResearchDocument.cpp IResearch/IResearchDocument.h
IResearch/IResearchPrimaryKeyFilter.cpp IResearch/IResearchPrimaryKeyFilter.h
IResearch/IResearchFilterFactory.cpp IResearch/IResearchFilterFactory.h
IResearch/IResearchOrderFactory.cpp IResearch/IResearchOrderFactory.h
IResearch/VelocyPackHelper.cpp IResearch/VelocyPackHelper.h

131
arangod/IResearch/IResearchFilterFactory.cpp
View File

@ -47,11 +47,11 @@
#include "IResearchFilterFactory.h"
#include "IResearchDocument.h"
#include "IResearchKludge.h"
#include "IResearchPrimaryKeyFilter.h"
#include "Aql/Function.h"
#include "Aql/Ast.h"
#include "Logger/LogMacros.h"
#include "utils/hash_utils.hpp"
using namespace arangodb::iresearch;
namespace {
@ -2057,135 +2057,6 @@ bool filter(
}
}
class PrimaryKeyFilter
: public irs::filter,
public irs::filter::prepared {
private:
struct PrimaryKeyIterator : public irs::doc_iterator {
PrimaryKeyIterator() = default;
virtual bool next() noexcept override {
_doc = _next;
_next = irs::type_limits<irs::type_t::doc_id_t>::eof();
return !irs::type_limits<irs::type_t::doc_id_t>::eof(_doc);
}
virtual irs::doc_id_t seek(irs::doc_id_t target) noexcept override {
_doc = target <= _next
? _next
: irs::type_limits<irs::type_t::doc_id_t>::eof();
return _doc;
}
virtual irs::doc_id_t value() const noexcept override {
return _doc;
}
virtual irs::attribute_view const& attributes() const noexcept override {
return irs::attribute_view::empty_instance();
}
void reset(irs::sub_reader const& segment, irs::doc_id_t doc) noexcept {
_pkSegment = &segment;
_doc = irs::type_limits<irs::type_t::doc_id_t>::invalid();
_next = doc;
}
mutable irs::sub_reader const* _pkSegment{};
mutable irs::doc_id_t _doc{ irs::type_limits<irs::type_t::doc_id_t>::invalid() };
mutable irs::doc_id_t _next{ irs::type_limits<irs::type_t::doc_id_t>::eof() };
}; // PrimaryKeyIterator
public:
DECLARE_FILTER_TYPE();
PrimaryKeyFilter(TRI_voc_cid_t cid, TRI_voc_rid_t id) noexcept
: irs::filter(PrimaryKeyFilter::type()), _pk(cid, id) {
}
// ----------------------------------------------------------------------------
// --SECTION-- irs::filter::prepared
// ----------------------------------------------------------------------------
virtual irs::doc_iterator::ptr execute(
irs::sub_reader const& segment,
irs::order::prepared const& /*order*/,
irs::attribute_view const& /*ctx*/
) const override {
if (&segment != _pkIterator._pkSegment) {
return irs::doc_iterator::empty();
}
// aliasing constructor
return irs::doc_iterator::ptr(
irs::doc_iterator::ptr(),
const_cast<PrimaryKeyIterator*>(&_pkIterator)
);
}
// ----------------------------------------------------------------------------
// --SECTION-- irs::filter
// ----------------------------------------------------------------------------
virtual size_t hash() const noexcept override {
size_t seed = 0;
irs::hash_combine(seed, filter::hash());
irs::hash_combine(seed, _pk.cid());
irs::hash_combine(seed, _pk.rid());
return seed;
}
virtual filter::prepared::ptr prepare(
irs::index_reader const& index,
irs::order::prepared const& /*ord*/,
boost_t /*boost*/,
irs::attribute_view const& /*ctx*/
) const override {
auto const pkRef = static_cast<irs::bytes_ref>(_pk);
for (auto& segment : index) {
auto* pkField = segment.field(arangodb::iresearch::DocumentPrimaryKey::PK());
if (!pkField) {
continue;
}
auto term = pkField->iterator();
if (!term->seek(pkRef)) {
continue;
}
auto docs = term->postings(irs::flags::empty_instance());
if (docs->next()) {
_pkIterator.reset(segment, docs->value());
}
break;
}
// aliasing constructor
return irs::filter::prepared::ptr(irs::filter::prepared::ptr(), this);
}
protected:
bool equals(filter const& rhs) const noexcept override {
auto const& trhs = static_cast<PrimaryKeyFilter const&>(rhs);
return filter::equals(rhs)
&& _pk.cid() == trhs._pk.cid()
&& _pk.rid() == trhs._pk.rid();
}
private:
DocumentPrimaryKey _pk;
mutable PrimaryKeyIterator _pkIterator;
}; // PrimaryKeyFilter
DEFINE_FILTER_TYPE(PrimaryKeyFilter);
}
namespace arangodb {

105
arangod/IResearch/IResearchPrimaryKeyFilter.cpp Normal file
View File

@ -0,0 +1,105 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#include "IResearchPrimaryKeyFilter.h"
#include "index/index_reader.hpp"
#include "utils/hash_utils.hpp"
namespace arangodb {
namespace iresearch {
// ----------------------------------------------------------------------------
// --SECTION-- PrimaryKeyFilter implementation
// ----------------------------------------------------------------------------
irs::doc_iterator::ptr PrimaryKeyFilter::execute(
irs::sub_reader const& segment,
irs::order::prepared const& /*order*/,
irs::attribute_view const& /*ctx*/
) const {
if (&segment != _pkIterator._pkSegment) {
return irs::doc_iterator::empty();
}
// aliasing constructor
return irs::doc_iterator::ptr(
irs::doc_iterator::ptr(),
const_cast<PrimaryKeyIterator*>(&_pkIterator)
);
}
size_t PrimaryKeyFilter::hash() const noexcept {
size_t seed = 0;
irs::hash_combine(seed, filter::hash());
irs::hash_combine(seed, _pk.cid());
irs::hash_combine(seed, _pk.rid());
return seed;
}
irs::filter::prepared::ptr PrimaryKeyFilter::prepare(
irs::index_reader const& index,
irs::order::prepared const& /*ord*/,
irs::boost::boost_t /*boost*/,
irs::attribute_view const& /*ctx*/
) const {
auto const pkRef = static_cast<irs::bytes_ref>(_pk);
for (auto& segment : index) {
auto* pkField = segment.field(arangodb::iresearch::DocumentPrimaryKey::PK());
if (!pkField) {
continue;
}
auto term = pkField->iterator();
if (!term->seek(pkRef)) {
continue;
}
auto docs = term->postings(irs::flags::empty_instance());
if (docs->next()) {
_pkIterator.reset(segment, docs->value());
}
break;
}
// aliasing constructor
return irs::filter::prepared::ptr(irs::filter::prepared::ptr(), this);
}
bool PrimaryKeyFilter::equals(filter const& rhs) const noexcept {
auto const& trhs = static_cast<PrimaryKeyFilter const&>(rhs);
return filter::equals(rhs)
&& _pk.cid() == trhs._pk.cid()
&& _pk.rid() == trhs._pk.rid();
}
DEFINE_FILTER_TYPE(PrimaryKeyFilter);
} // iresearch
} // arangodb

149
arangod/IResearch/IResearchPrimaryKeyFilter.h Normal file
View File

@ -0,0 +1,149 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_IRESEARCH__IRESEARCH_PRIMARY_KEY_FILTER_H
#define ARANGOD_IRESEARCH__IRESEARCH_PRIMARY_KEY_FILTER_H 1
#include "VocBase/voc-types.h"
#include "IResearchDocument.h"
#include "search/filter.hpp"
#include "utils/type_limits.hpp"
namespace arangodb {
namespace iresearch {
///////////////////////////////////////////////////////////////////////////////
/// @class PrimaryKeyFilter
/// @brief iresearch filter optimized for filtering on primary keys
///////////////////////////////////////////////////////////////////////////////
class PrimaryKeyFilter final
: public irs::filter,
public irs::filter::prepared {
public:
DECLARE_FILTER_TYPE();
PrimaryKeyFilter(TRI_voc_cid_t cid, TRI_voc_rid_t id) noexcept
: irs::filter(PrimaryKeyFilter::type()), _pk(cid, id) {
}
// ----------------------------------------------------------------------------
// --SECTION-- irs::filter::prepared
// ----------------------------------------------------------------------------
virtual irs::doc_iterator::ptr execute(
irs::sub_reader const& segment,
irs::order::prepared const& /*order*/,
irs::attribute_view const& /*ctx*/
) const override;
// ----------------------------------------------------------------------------
// --SECTION-- irs::filter
// ----------------------------------------------------------------------------
virtual size_t hash() const noexcept override;
virtual filter::prepared::ptr prepare(
irs::index_reader const& index,
irs::order::prepared const& /*ord*/,
irs::boost::boost_t /*boost*/,
irs::attribute_view const& /*ctx*/
) const override;
protected:
bool equals(filter const& rhs) const noexcept override;
private:
struct PrimaryKeyIterator final : public irs::doc_iterator {
PrimaryKeyIterator() = default;
virtual bool next() noexcept override {
_doc = _next;
_next = irs::type_limits<irs::type_t::doc_id_t>::eof();
return !irs::type_limits<irs::type_t::doc_id_t>::eof(_doc);
}
virtual irs::doc_id_t seek(irs::doc_id_t target) noexcept override {
_doc = target <= _next
? _next
: irs::type_limits<irs::type_t::doc_id_t>::eof();
return _doc;
}
virtual irs::doc_id_t value() const noexcept override {
return _doc;
}
virtual irs::attribute_view const& attributes() const noexcept override {
return irs::attribute_view::empty_instance();
}
void reset(irs::sub_reader const& segment, irs::doc_id_t doc) noexcept {
_pkSegment = &segment;
_doc = irs::type_limits<irs::type_t::doc_id_t>::invalid();
_next = doc;
}
mutable irs::sub_reader const* _pkSegment{};
mutable irs::doc_id_t _doc{ irs::type_limits<irs::type_t::doc_id_t>::invalid() };
mutable irs::doc_id_t _next{ irs::type_limits<irs::type_t::doc_id_t>::eof() };
}; // PrimaryKeyIterator
DocumentPrimaryKey _pk;
mutable PrimaryKeyIterator _pkIterator;
}; // PrimaryKeyFilter
///////////////////////////////////////////////////////////////////////////////
/// @class PrimaryKeyFilterContainer
/// @brief container for storing 'PrimaryKeyFilter's, does nothing as a filter
///////////////////////////////////////////////////////////////////////////////
class PrimaryKeyFilterContainer final : public irs::empty {
public:
DECLARE_FILTER_TYPE();
PrimaryKeyFilterContainer() = default;
PrimaryKeyFilterContainer(PrimaryKeyFilterContainer&&) = default;
PrimaryKeyFilterContainer& operator=(PrimaryKeyFilterContainer&&) = default;
PrimaryKeyFilter& emplace(TRI_voc_cid_t cid, TRI_voc_rid_t rid) {
_filters.emplace_back(cid, rid);
return _filters.back();
}
bool empty() const noexcept {
return _filters.empty();
}
void clear() noexcept {
_filters.clear();
}
private:
std::deque<PrimaryKeyFilter> _filters; // pointers remain valid
}; // PrimaryKeyFilterContainer
} // iresearch
} // arangodb
#endif // ARANGOD_IRESEARCH__IRESEARCH_PRIMARY_KEY_FILTER_H

102
arangod/IResearch/IResearchView.cpp
View File

@ -37,6 +37,7 @@
#include "IResearchFilterFactory.h"
#include "IResearchLink.h"
#include "IResearchLinkHelper.h"
#include "IResearchPrimaryKeyFilter.h"
#include "Aql/AstNode.h"
#include "Aql/QueryCache.h"
@ -644,8 +645,10 @@ struct IResearchView::ViewFactory: public arangodb::ViewFactory {
////////////////////////////////////////////////////////////////////////////////
/// @brief container storing the view 'read' state for a given TransactionState
////////////////////////////////////////////////////////////////////////////////
struct IResearchView::ViewStateRead: public arangodb::TransactionState::Cookie {
struct IResearchView::ViewStateRead final
: public arangodb::TransactionState::Cookie {
CompoundReader _snapshot;
explicit ViewStateRead(ReadMutex& mutex) noexcept
: _snapshot(mutex) {
}
@ -654,17 +657,53 @@ struct IResearchView::ViewStateRead: public arangodb::TransactionState::Cookie {
////////////////////////////////////////////////////////////////////////////////
/// @brief container storing the view 'write' state for a given TransactionState
////////////////////////////////////////////////////////////////////////////////
struct IResearchView::ViewStateWrite
: public arangodb::TransactionState::Cookie,
public irs::index_writer::documents_context {
class IResearchView::ViewStateWrite final
: public arangodb::TransactionState::Cookie {
public:
std::lock_guard<ReadMutex> _viewLock; // prevent data-store deallocation (lock @ AsyncSelf)
irs::index_writer::documents_context _ctx;
PrimaryKeyFilterContainer _removals; // list of document removals
explicit ViewStateWrite(
ReadMutex& viewMutex,
irs::index_writer& writer
) noexcept
: irs::index_writer::documents_context(writer.documents()),
_viewLock(viewMutex) {
: _viewLock(viewMutex),
_ctx(writer.documents()) {
}
virtual ~ViewStateWrite() noexcept {
if (_removals.empty()) {
// nothing to do
return;
}
try {
// hold references even after transaction
_ctx.remove(
irs::filter::make<PrimaryKeyFilterContainer>(std::move(_removals))
);
} catch (std::exception const& e) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "Failed to apply accumulated removals, error '" << e.what() << "'";
} catch (...) {
// NOOP
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "Failed to apply accumulated removals";
}
}
operator irs::index_writer::documents_context&() noexcept {
return _ctx;
}
void remove(TRI_voc_cid_t cid, TRI_voc_rid_t rid) {
_ctx.remove(_removals.emplace(cid, rid));
}
void reset() noexcept {
_removals.clear();
_ctx.reset();
}
};
@ -751,10 +790,12 @@ class IResearchView::ViewStateHelper {
if (rollback && prev) {
// TODO FIXME find a better way to look up a ViewState
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
dynamic_cast<IResearchView::ViewStateWrite&>(*prev).reset();
auto& ctx = dynamic_cast<IResearchView::ViewStateWrite&>(*prev);
#else
static_cast<IResearchView::ViewStateWrite&>(*prev).reset();
auto& ctx = static_cast<IResearchView::ViewStateWrite&>(*prev);
#endif
ctx.reset();
}
prev.reset();
@ -1117,7 +1158,8 @@ arangodb::Result IResearchView::drop(
TRI_voc_cid_t cid,
bool unlink /*= true*/
) {
std::shared_ptr<irs::filter> shared_filter(iresearch::FilterFactory::filter(cid));
auto filter = iresearch::FilterFactory::filter(cid);
WriteMutex rmutex(_mutex); // '_meta' and '_storeByTid' can be asynchronously updated
WriteMutex wmutex(_mutex); // '_meta' and '_storeByTid' can be asynchronously updated
DEFER_SCOPED_LOCK_NAMED(rmutex, rlock);
@ -1163,7 +1205,7 @@ arangodb::Result IResearchView::drop(
try {
if (_storePersisted) {
_storePersisted._writer->documents().remove(shared_filter);
_storePersisted._writer->documents().remove(std::move(filter));
}
} catch (arangodb::basics::Exception& e) {
IR_LOG_EXCEPTION();
@ -1420,15 +1462,15 @@ arangodb::Result IResearchView::commit() {
return {};
} catch (arangodb::basics::Exception const& e) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "caught exception while committing memory store for arangosearch view '" << name() << "': " << e.code() << " " << e.what();
<< "caught exception while committing store for arangosearch view '" << name() << "': " << e.code() << " " << e.what();
IR_LOG_EXCEPTION();
} catch (std::exception const& e) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "caught exception while committing memory store for arangosearch view '" << name() << "': " << e.what();
<< "caught exception while committing store for arangosearch view '" << name() << "': " << e.what();
IR_LOG_EXCEPTION();
} catch (...) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "caught exception while committing memory store for arangosearch view '" << name() << "'";
<< "caught exception while committing store for arangosearch view '" << name() << "'";
IR_LOG_EXCEPTION();
}
@ -1482,12 +1524,6 @@ int IResearchView::insert(
return TRI_ERROR_INTERNAL;
};
if (_inRecovery) {
auto ctx = _storePersisted._writer->documents();
ctx.remove(FilterFactory::filter(cid, documentId.id()));
return insertImpl(ctx);
}
if (!trx.state()) {
return TRI_ERROR_BAD_PARAMETER; // 'trx' and transaction state required
}
@ -1523,6 +1559,10 @@ int IResearchView::insert(
TRI_ASSERT(ctx);
if (_inRecovery) {
ctx->remove(cid, documentId.id());
}
return insertImpl(*ctx);
}
@ -1575,14 +1615,6 @@ int IResearchView::insert(
return TRI_ERROR_NO_ERROR;
};
if (_inRecovery) {
auto ctx = _storePersisted._writer->documents();
for (auto& doc : batch) {
ctx.remove(FilterFactory::filter(cid, doc.first.id()));
}
return insertImpl(ctx);
}
if (!trx.state()) {
return TRI_ERROR_BAD_PARAMETER; // 'trx' and transaction state required
}
@ -1613,6 +1645,12 @@ int IResearchView::insert(
TRI_ASSERT(ctx);
if (_inRecovery) {
for (auto const& doc : batch) {
ctx->remove(cid, doc.first.id());
}
}
return insertImpl(*ctx);
}
@ -1741,14 +1779,6 @@ int IResearchView::remove(
) {
TRI_ASSERT(_storePersisted);
std::shared_ptr<irs::filter> shared_filter(FilterFactory::filter(cid, documentId.id()));
if (_inRecovery) {
_storePersisted._writer->documents().remove(shared_filter);
return TRI_ERROR_NO_ERROR;
}
if (!trx.state()) {
return TRI_ERROR_BAD_PARAMETER; // 'trx' and transaction state required
}
@ -1784,7 +1814,7 @@ int IResearchView::remove(
// all of its fid stores, no impact to iResearch View data integrity
// ...........................................................................
try {
ctx->remove(shared_filter);
ctx->remove(cid, documentId.id());
return TRI_ERROR_NO_ERROR;
} catch (arangodb::basics::Exception const& e) {

5
arangod/IResearch/IResearchView.h
View File

@ -311,7 +311,6 @@ class IResearchView final
struct DataStore {
irs::directory::ptr _directory;
irs::directory_reader _reader;
irs::index_reader::ptr _readerImpl; // need this for 'std::atomic_exchange_strong'
irs::index_writer::ptr _writer;
DataStore() = default;
@ -331,7 +330,7 @@ class IResearchView final
struct ViewFactory; // forward declaration
class ViewStateHelper; // forward declaration
struct ViewStateRead; // forward declaration
struct ViewStateWrite; // forward declaration
class ViewStateWrite; // forward declaration
struct FlushCallbackUnregisterer {
void operator()(IResearchView* view) const noexcept;
@ -377,4 +376,4 @@ class IResearchView final
} // iresearch
} // arangodb
#endif
#endif