Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

stored value draft

This commit is contained in:
Yuriy Popov 2019-11-28 17:50:01 +01:00
parent d9cb0cd6e3
commit c4a9c88b9a
23 changed files with 648 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
arangod/Aql/IResearchViewExecutor.cpp
View File

@ -95,6 +95,12 @@ inline irs::columnstore_reader::values_reader_f sortColumn(irs::sub_reader const
return reader ? reader->values() : irs::columnstore_reader::values_reader_f{};
}
inline irs::columnstore_reader::values_reader_f storedValueColumn(irs::sub_reader const& segment, irs::string_ref const& name) {
auto const* reader = segment.column_reader(name);
return reader ? reader->values() : irs::columnstore_reader::values_reader_f{};
}
} // namespace
///////////////////////////////////////////////////////////////////////////////
@ -106,6 +112,7 @@ IResearchViewExecutorInfos::IResearchViewExecutorInfos(
RegisterId firstOutputRegister, RegisterId numScoreRegisters, Query& query,
std::vector<Scorer> const& scorers,
std::pair<arangodb::iresearch::IResearchViewSort const*, size_t> const& sort,
iresearch::IResearchViewStoredValue const& storedValue,
ExecutionPlan const& plan, Variable const& outVariable,
aql::AstNode const& filterCondition, std::pair<bool, bool> volatility,
IResearchViewExecutorInfos::VarInfoMap const& varInfoMap, int depth,
@ -117,6 +124,7 @@ IResearchViewExecutorInfos::IResearchViewExecutorInfos(
_query(query),
_scorers(scorers),
_sort(sort),
_storedValue(storedValue),
_plan(plan),
_outVariable(outVariable),
_filterCondition(filterCondition),
@ -192,6 +200,10 @@ const std::pair<const arangodb::iresearch::IResearchViewSort*, size_t>& IResearc
return _sort;
}
arangodb::iresearch::IResearchViewStoredValue const& IResearchViewExecutorInfos::storedValue() const noexcept {
return _storedValue;
}
bool IResearchViewExecutorInfos::isScoreReg(RegisterId reg) const noexcept {
return getNumScoreRegisters() > 0 &&
getFirstScoreRegister() <= reg && reg < getFirstScoreRegister() + getNumScoreRegisters();
@ -285,7 +297,7 @@ std::vector<AqlValue>::iterator IResearchViewExecutorBase<Impl, Traits>::IndexRe
template <typename Impl, typename Traits>
template <typename ValueType>
IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::IndexReadBuffer(std::size_t const numScoreRegisters)
: _keyBuffer(), _scoreBuffer(), _sortValueBuffer(), _numScoreRegisters(numScoreRegisters), _keyBaseIdx(0) {}
: _keyBuffer(), _scoreBuffer(), _storedValueBuffer(), _numScoreRegisters(numScoreRegisters), _keyBaseIdx(0) {}
template <typename Impl, typename Traits>
template <typename ValueType>
@ -314,16 +326,16 @@ void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::pushVa
template <typename Impl, typename Traits>
template <typename ValueType>
void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::pushSortValue(irs::bytes_ref&& sortValue) {
_sortValueBuffer.emplace_back(std::move(sortValue));
void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::pushStoredValue(std::unordered_map<int, irs::bytes_ref>&& storedValue) {
_storedValueBuffer.emplace_back(std::move(storedValue));
}
template <typename Impl, typename Traits>
template <typename ValueType>
irs::bytes_ref IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::getSortValue(
std::unordered_map<int, irs::bytes_ref> const& IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::getStoredValue(
const IResearchViewExecutorBase::IndexReadBufferEntry bufferEntry) const noexcept {
TRI_ASSERT(bufferEntry._keyIdx < _sortValueBuffer.size());
return _sortValueBuffer[bufferEntry._keyIdx];
TRI_ASSERT(bufferEntry._keyIdx < _storedValueBuffer.size());
return _storedValueBuffer[bufferEntry._keyIdx];
}
template <typename Impl, typename Traits>
@ -346,7 +358,7 @@ void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::reset(
_keyBaseIdx = 0;
_keyBuffer.clear();
_scoreBuffer.clear();
_sortValueBuffer.clear();
_storedValueBuffer.clear();
}
template <typename Impl, typename Traits>
@ -666,18 +678,17 @@ bool IResearchViewExecutorBase<Impl, Traits>::writeRow(ReadContext& ctx,
if constexpr (Traits::MaterializeType == MaterializeType::LateMaterializedWithVars) {
auto const& outNonMaterializedViewRegs = infos().getOutNonMaterializedViewRegs();
TRI_ASSERT(!outNonMaterializedViewRegs.empty());
if (ADB_LIKELY(!outNonMaterializedViewRegs.empty())) {
auto sortValue = _indexReadBuffer.getSortValue(bufferEntry);
TRI_ASSERT(!sortValue.empty());
if (ADB_UNLIKELY(sortValue.empty())) {
return false;
}
auto s = sortValue.c_str();
auto totalSize = sortValue.size();
auto const& storedValue = _indexReadBuffer.getStoredValue(bufferEntry);
for (auto const& [columnNum, fieldsRegs] : outNonMaterializedViewRegs) {
auto it = storedValue.find(columnNum);
TRI_ASSERT(it != storedValue.cend());
TRI_ASSERT(!it->second.empty());
auto s = it->second.c_str();
auto totalSize = it->second.size();
auto slice = VPackSlice(s);
size_t size = 0;
size_t i = 0;
for (auto const& [fieldNum, registerId] : outNonMaterializedViewRegs) {
for (auto const& [fieldNum, registerId] : fieldsRegs) {
while (i < fieldNum) {
size += slice.byteSize();
TRI_ASSERT(size <= totalSize);
@ -834,14 +845,23 @@ void IResearchViewExecutor<ordered, materializeType>::fillBuffer(IResearchViewEx
}
if constexpr (materializeType == MaterializeType::LateMaterializedWithVars) {
TRI_ASSERT((!this->_infos.getOutNonMaterializedViewRegs().empty()));
if (ADB_LIKELY(!this->_infos.getOutNonMaterializedViewRegs().empty())) {
TRI_ASSERT(_sortReader);
irs::bytes_ref sortValue{irs::bytes_ref::NIL}; // sort column value
auto ok = _sortReader(_doc->value, sortValue);
auto const& columnsFieldsRegs = this->_infos.getOutNonMaterializedViewRegs();
TRI_ASSERT(!columnsFieldsRegs.empty());
std::unordered_map<int, irs::bytes_ref> storedValue;
irs::columnstore_reader::values_reader_f reader;
for (auto const& columnFieldsRegs : columnsFieldsRegs) {
if (columnFieldsRegs.first >= 0) { // not IResearchViewNode::SortColumnNumber
reader = _storedValueReaders[columnFieldsRegs.first];
} else { // IResearchViewNode::SortColumnNumber
reader = _sortReader;
}
TRI_ASSERT(reader);
irs::bytes_ref value{irs::bytes_ref::NIL}; // column value
auto ok = reader(_doc->value, value);
TRI_ASSERT(ok);
this->_indexReadBuffer.pushSortValue(std::move(sortValue));
storedValue[columnFieldsRegs.first] = std::move(value);
}
this->_indexReadBuffer.pushStoredValue(std::move(storedValue));
}
// doc and scores are both pushed, sizes must now be coherent
this->_indexReadBuffer.assertSizeCoherence();
@ -879,14 +899,33 @@ bool IResearchViewExecutor<ordered, materializeType>::resetIterator() {
}
if constexpr (materializeType == MaterializeType::LateMaterializedWithVars) {
if (!this->_infos.getOutNonMaterializedViewRegs().empty()) {
_sortReader = ::sortColumn(segmentReader);
if (!_sortReader) {
LOG_TOPIC("bc5bd", WARN, arangodb::iresearch::TOPIC)
<< "encountered a sub-reader without a sort column while "
"executing a query, ignoring";
return false;
auto const& columnsfieldsRegs = this->_infos.getOutNonMaterializedViewRegs();
if (!columnsfieldsRegs.empty()) {
auto storedValue = this->_infos.storedValue();
for (auto const& columnfieldsRegs : columnsfieldsRegs) {
if (columnfieldsRegs.first >= 0) { // not IResearchViewNode::SortColumnNumber
TRI_ASSERT(!storedValue.empty());
auto const& columns = storedValue.columns();
auto const storedColumnNumber = static_cast<decltype(columns.size())>(columnfieldsRegs.first);
TRI_ASSERT(storedColumnNumber < columns.size());
// column name is equal to the first field name (TODO: two can have the same)
auto storedValueColumn = ::storedValueColumn(segmentReader, columns[storedColumnNumber].back().first);
if (!storedValueColumn) {
LOG_TOPIC("af7ec", WARN, arangodb::iresearch::TOPIC)
<< "encountered a sub-reader without a stored value column while "
"executing a query, ignoring";
return false;
}
_storedValueReaders[columnfieldsRegs.first] = storedValueColumn;
} else { // IResearchViewNode::SortColumnNumber
_sortReader = ::sortColumn(segmentReader);
if (!_sortReader) {
LOG_TOPIC("bc5bd", WARN, arangodb::iresearch::TOPIC)
<< "encountered a sub-reader without a sort column while "
"executing a query, ignoring";
return false;
}
}
}
}
}

11
arangod/Aql/IResearchViewExecutor.h
View File

@ -66,6 +66,7 @@ class IResearchViewExecutorInfos : public ExecutorInfos {
RegisterId firstOutputRegister, RegisterId numScoreRegisters,
Query& query, std::vector<iresearch::Scorer> const& scorers,
std::pair<iresearch::IResearchViewSort const*, size_t> const& sort,
iresearch::IResearchViewStoredValue const& storedValue,
ExecutionPlan const& plan,
Variable const& outVariable,
aql::AstNode const& filterCondition,
@ -92,6 +93,8 @@ class IResearchViewExecutorInfos : public ExecutorInfos {
// second - number of sort conditions to take into account
std::pair<iresearch::IResearchViewSort const*, size_t> const& sort() const noexcept;
iresearch::IResearchViewStoredValue const& storedValue() const noexcept;
bool isScoreReg(RegisterId reg) const noexcept;
private:
@ -101,6 +104,7 @@ class IResearchViewExecutorInfos : public ExecutorInfos {
Query& _query;
std::vector<iresearch::Scorer> const& _scorers;
std::pair<iresearch::IResearchViewSort const*, size_t> _sort;
iresearch::IResearchViewStoredValue const& _storedValue;
ExecutionPlan const& _plan;
Variable const& _outVariable;
aql::AstNode const& _filterCondition;
@ -244,9 +248,9 @@ class IResearchViewExecutorBase {
// before and after.
void assertSizeCoherence() const noexcept;
void pushSortValue(irs::bytes_ref&& sortValue);
void pushStoredValue(std::unordered_map<int, irs::bytes_ref>&& storedValue);
irs::bytes_ref getSortValue(IndexReadBufferEntry bufferEntry) const noexcept;
std::unordered_map<int, irs::bytes_ref> const& getStoredValue(IndexReadBufferEntry bufferEntry) const noexcept;
private:
// _keyBuffer, _scoreBuffer, _sortValueBuffer together hold all the
@ -260,7 +264,7 @@ class IResearchViewExecutorBase {
// .
std::vector<ValueType> _keyBuffer;
std::vector<AqlValue> _scoreBuffer;
std::vector<irs::bytes_ref> _sortValueBuffer;
std::vector<std::unordered_map<int, irs::bytes_ref>> _storedValueBuffer;
std::size_t _numScoreRegisters;
std::size_t _keyBaseIdx;
};
@ -344,6 +348,7 @@ class IResearchViewExecutor : public IResearchViewExecutorBase<IResearchViewExec
irs::columnstore_reader::values_reader_f _pkReader; // current primary key reader
irs::columnstore_reader::values_reader_f _sortReader; // current sort reader
std::unordered_map<int, irs::columnstore_reader::values_reader_f> _storedValueReaders; // current stored value readers
irs::doc_iterator::ptr _itr;
irs::document const* _doc{};
size_t _readerOffset;

198
arangod/Aql/IResearchViewNode.cpp
View File

@ -650,6 +650,16 @@ inline IResearchViewSort const& primarySort(arangodb::LogicalView const& view) {
return viewImpl.primarySort();
}
inline IResearchViewStoredValue const& storedValue(arangodb::LogicalView const& view) {
if (arangodb::ServerState::instance()->isCoordinator()) {
auto& viewImpl = arangodb::LogicalView::cast<IResearchViewCoordinator>(view);
return viewImpl.storedValue();
}
auto& viewImpl = arangodb::LogicalView::cast<IResearchView>(view);
return viewImpl.storedValue();
}
const char* NODE_DATABASE_PARAM = "database";
const char* NODE_VIEW_NAME_PARAM = "view";
const char* NODE_VIEW_ID_PARAM = "viewId";
@ -664,13 +674,52 @@ const char* NODE_VOLATILITY_PARAM = "volatility";
const char* NODE_PRIMARY_SORT_PARAM = "primarySort";
const char* NODE_PRIMARY_SORT_BUCKETS_PARAM = "primarySortBuckets";
const char* NODE_VIEW_VALUES_VARS = "ViewValuesVars";
const char* NODE_VIEW_STORED_VALUES_VARS = "viewStoredValuesVars";
const char* NODE_VIEW_VALUES_VAR_COLUMN_NUMBER = "columnNumber";
const char* NODE_VIEW_VALUES_VAR_FIELD_NUMBER = "fieldNumber";
const char* NODE_VIEW_VALUES_VAR_ID = "id";
const char* NODE_VIEW_VALUES_VAR_NAME = "name";
const char* NODE_VIEW_VALUES_VAR_FIELD = "field";
std::array<std::unique_ptr<aql::ExecutionBlock> (*)(
aql::ExecutionEngine*, IResearchViewNode const*, aql::IResearchViewExecutorInfos&&), 10> executors {
void addViewValuesVar(VPackBuilder& nodes, std::string const& fieldName,
std::pair<size_t, aql::Variable const*> const& fieldVar) {
nodes.add(NODE_VIEW_VALUES_VAR_FIELD_NUMBER, VPackValue(fieldVar.first));
nodes.add(NODE_VIEW_VALUES_VAR_ID, VPackValue(fieldVar.second->id));
nodes.add(NODE_VIEW_VALUES_VAR_NAME, VPackValue(fieldVar.second->name)); // for explainer.js
nodes.add(NODE_VIEW_VALUES_VAR_FIELD, VPackValue(fieldName)); // for explainer.js
}
void extractViewValuesVar(aql::VariableGenerator const* vars,
IResearchViewNode::ViewValuesVars& viewValuesVars,
int const columnNumber,
velocypack::Slice const& fieldVar) {
auto const fieldNumberSlice = fieldVar.get(NODE_VIEW_VALUES_VAR_FIELD_NUMBER);
if (!fieldNumberSlice.isNumber<size_t>()) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].fieldNumber\" %s should be a number",
fieldNumberSlice.toString().c_str());
}
auto const fieldNumber = fieldNumberSlice.getNumber<size_t>();
auto const varIdSlice = fieldVar.get(NODE_VIEW_VALUES_VAR_ID);
if (!varIdSlice.isNumber<aql::VariableId>()) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" variable id %s should be a number",
varIdSlice.toString().c_str());
}
auto const varId = varIdSlice.getNumber<aql::VariableId>();
auto const* var = vars->getVariable(varId);
if (!var) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" unable to find variable by id %d",
varId);
}
viewValuesVars[columnNumber].emplace_back(fieldNumber, var);
}
std::unique_ptr<aql::ExecutionBlock> (*executors[])(aql::ExecutionEngine*, IResearchViewNode const*, aql::IResearchViewExecutorInfos&&) = {
[](aql::ExecutionEngine* engine, IResearchViewNode const* viewNode, aql::IResearchViewExecutorInfos&& infos) -> std::unique_ptr<aql::ExecutionBlock> {
return std::make_unique<aql::ExecutionBlockImpl<aql::IResearchViewExecutor<false, MaterializeType::Materialized>>>(
engine, viewNode, std::move(infos));
@ -713,10 +762,10 @@ std::array<std::unique_ptr<aql::ExecutionBlock> (*)(
}
};
inline decltype(executors)::size_type getExecutorIndex(bool sorted, bool ordered, MaterializeType materializeType) {
inline int getExecutorIndex(bool sorted, bool ordered, MaterializeType materializeType) {
auto index = static_cast<int>(materializeType) + 3 * static_cast<int>(ordered) + 6 * static_cast<int>(sorted);
TRI_ASSERT(static_cast<decltype(executors)::size_type>(index) <= executors.size());
return static_cast<decltype(executors)::size_type>(index < 9 ? index : index - 1);
TRI_ASSERT(static_cast<std::size_t>(index) <= IRESEARCH_COUNTOF(executors));
return index < 9 ? index : index - 1;
}
} // namespace
@ -926,33 +975,28 @@ IResearchViewNode::IResearchViewNode(aql::ExecutionPlan& plan, velocypack::Slice
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"\"ViewValuesVars\" attribute should be an array");
}
std::unordered_map<size_t, aql::Variable const*> viewValuesVars;
viewValuesVars.reserve(viewValuesVarsSlice.length());
for (auto const indVar : velocypack::ArrayIterator(viewValuesVarsSlice)) {
auto const fieldNumberSlice = indVar.get(NODE_VIEW_VALUES_VAR_FIELD_NUMBER);
if (!fieldNumberSlice.isNumber<size_t>()) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].fieldNumber\" %s should be a number",
fieldNumberSlice.toString().c_str());
ViewValuesVars viewValuesVars;
for (auto const columnFieldsVars : velocypack::ArrayIterator(viewValuesVarsSlice)) {
if (columnFieldsVars.hasKey(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER)) { // not SortColumnNumber
auto const columnNumberSlice = columnFieldsVars.get(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER);
if (!columnNumberSlice.isNumber<size_t>()) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].columnNumber\" %s should be a number",
columnNumberSlice.toString().c_str());
}
auto const columnNumber = columnNumberSlice.getNumber<int>();
auto const viewStoredValuesVarsSlice = columnFieldsVars.get(NODE_VIEW_STORED_VALUES_VARS);
if (!viewStoredValuesVarsSlice.isArray()) {
THROW_ARANGO_EXCEPTION_MESSAGE(
TRI_ERROR_BAD_PARAMETER,
"\"ViewValuesVars[*].viewStoredValuesVars\" attribute should be an array");
}
for (auto const fieldVar : velocypack::ArrayIterator(viewStoredValuesVarsSlice)) {
extractViewValuesVar(vars, viewValuesVars, columnNumber, fieldVar);
}
} else { // SortColumnNumber
extractViewValuesVar(vars, viewValuesVars, SortColumnNumber, columnFieldsVars);
}
auto const fieldNumber = fieldNumberSlice.getNumber<size_t>();
auto const varIdSlice = indVar.get(NODE_VIEW_VALUES_VAR_ID);
if (!varIdSlice.isNumber<aql::VariableId>()) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" variable id %s should be a number",
varIdSlice.toString().c_str());
}
auto const varId = varIdSlice.getNumber<aql::VariableId>();
auto const* var = vars->getVariable(varId);
if (!var) {
THROW_ARANGO_EXCEPTION_FORMAT(
TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" unable to find variable by id %d",
varId);
}
viewValuesVars.emplace(fieldNumber, var);
}
_outNonMaterializedViewVars = std::move(viewValuesVars);
}
@ -985,10 +1029,12 @@ void IResearchViewNode::planNodeRegisters(
++nrRegsHere[depth];
++nrRegs[depth];
varInfo.try_emplace(_outNonMaterializedDocId->id, aql::VarInfo(depth, totalNrRegs++));
for (auto const& viewVar : _outNonMaterializedViewVars) {
++nrRegsHere[depth];
++nrRegs[depth];
varInfo.try_emplace(viewVar.second->id, aql::VarInfo(depth, totalNrRegs++));
for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
for (auto const& fieldVar : columnFieldsVars.second) {
++nrRegsHere[depth];
++nrRegs[depth];
varInfo.try_emplace(fieldVar.second->id, aql::VarInfo(depth, totalNrRegs++));
}
}
} else {
++nrRegsHere[depth];
@ -1006,8 +1052,6 @@ std::pair<bool, bool> IResearchViewNode::volatility(bool force /*=false*/) const
irs::check_bit<1>(_volatilityMask)); // sort
}
/// @brief toVelocyPack, for EnumerateViewNode
void IResearchViewNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags,
std::unordered_set<ExecutionNode const*>& seen) const {
@ -1034,19 +1078,38 @@ void IResearchViewNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags,
_outNonMaterializedColPtr->toVelocyPack(nodes);
}
// stored value
{
auto const& primarySort = ::primarySort(*_view);
auto const& storedValue = ::storedValue(*_view);
VPackArrayBuilder arrayScope(&nodes, NODE_VIEW_VALUES_VARS);
std::string fieldName;
for (auto const& viewVar : _outNonMaterializedViewVars) {
VPackObjectBuilder objectScope(&nodes);
nodes.add(NODE_VIEW_VALUES_VAR_FIELD_NUMBER, VPackValue(viewVar.first));
nodes.add(NODE_VIEW_VALUES_VAR_ID, VPackValue(viewVar.second->id));
nodes.add(NODE_VIEW_VALUES_VAR_NAME, VPackValue(viewVar.second->name)); // for explainer.js
TRI_ASSERT(viewVar.first < primarySort.fields().size());
fieldName.clear();
basics::TRI_AttributeNamesToString(primarySort.fields()[viewVar.first], fieldName, true);
nodes.add(NODE_VIEW_VALUES_VAR_FIELD, VPackValue(fieldName)); // for explainer.js
for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
if (columnFieldsVars.first >= 0) { // not SortColumnNumber
VPackObjectBuilder objectScope(&nodes);
auto const& columns = storedValue.columns();
auto const storedColumnNumber = static_cast<decltype(columns.size())>(columnFieldsVars.first);
TRI_ASSERT(storedColumnNumber < columns.size());
nodes.add(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER, VPackValue(columnFieldsVars.first));
VPackArrayBuilder arrayScope(&nodes, NODE_VIEW_STORED_VALUES_VARS);
for (auto const& fieldVar : columnFieldsVars.second) {
VPackObjectBuilder objectScope(&nodes);
fieldName.clear();
TRI_ASSERT(fieldVar.first < columns[storedColumnNumber].size());
nodes.add(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER, VPackValue(columnFieldsVars.first));
fieldName = columns[storedColumnNumber][fieldVar.first].first;
addViewValuesVar(nodes, fieldName, fieldVar);
}
} else { // SortColumnNumber
TRI_ASSERT(SortColumnNumber == columnFieldsVars.first);
for (auto const& fieldVar : columnFieldsVars.second) {
VPackObjectBuilder objectScope(&nodes);
fieldName.clear();
TRI_ASSERT(fieldVar.first < primarySort.fields().size());
basics::TRI_AttributeNamesToString(primarySort.fields()[fieldVar.first], fieldName, true);
addViewValuesVar(nodes, fieldName, fieldVar);
}
}
}
}
@ -1143,8 +1206,10 @@ aql::ExecutionNode* IResearchViewNode::clone(aql::ExecutionPlan* plan, bool with
TRI_ASSERT(_outNonMaterializedDocId != nullptr);
outNonMaterializedColId = plan->getAst()->variables()->createVariable(outNonMaterializedColId);
}
for (auto& viewVar : outNonMaterializedViewVars) {
viewVar.second = plan->getAst()->variables()->createVariable(viewVar.second);
for (auto& columnFieldsVars : outNonMaterializedViewVars) {
for (auto& fieldVar : columnFieldsVars.second) {
fieldVar.second = plan->getAst()->variables()->createVariable(fieldVar.second);
}
}
}
@ -1202,10 +1267,12 @@ std::vector<arangodb::aql::Variable const*> IResearchViewNode::getVariablesSetHe
if (isLateMaterialized()) {
vars.emplace_back(_outNonMaterializedColPtr);
vars.emplace_back(_outNonMaterializedDocId);
std::transform(_outNonMaterializedViewVars.cbegin(),
_outNonMaterializedViewVars.cend(),
std::back_inserter(vars),
[](auto const& viewVar) { return viewVar.second; });
for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
std::transform(columnFieldsVars.second.cbegin(),
columnFieldsVars.second.cend(),
std::back_inserter(vars),
[](auto const& fieldVar) { return fieldVar.second; });
}
} else {
vars.emplace_back(_outVariable);
}
@ -1332,7 +1399,11 @@ std::unique_ptr<aql::ExecutionBlock> IResearchViewNode::createBlock(
// the input registers.
auto const firstOutputRegister = getNrInputRegisters();
auto numScoreRegisters = static_cast<aql::RegisterCount>(_scorers.size());
auto numViewVarsRegisters = static_cast<aql::RegisterCount>(_outNonMaterializedViewVars.size());
auto numViewVarsRegisters = std::accumulate(_outNonMaterializedViewVars.cbegin(), _outNonMaterializedViewVars.cend(),
static_cast<aql::RegisterCount>(0),
[](aql::RegisterCount const sum, auto const& columnFieldsVars) {
return sum + static_cast<aql::RegisterCount>(columnFieldsVars.second.size());
});
if (numViewVarsRegisters > 0) {
TRI_ASSERT(materializeType == MaterializeType::LateMaterialized);
materializeType = MaterializeType::LateMaterializedWithVars;
@ -1369,14 +1440,14 @@ std::unique_ptr<aql::ExecutionBlock> IResearchViewNode::createBlock(
auto const& varInfos = getRegisterPlan()->varInfo;
ViewValuesRegisters outNonMaterializedViewRegs;
std::transform(_outNonMaterializedViewVars.cbegin(), _outNonMaterializedViewVars.cend(),
std::inserter(outNonMaterializedViewRegs, outNonMaterializedViewRegs.end()),
[&varInfos](auto const& viewVar) {
auto it = varInfos.find(viewVar.second->id);
TRI_ASSERT(it != varInfos.cend());
return std::make_pair(viewVar.first, it->second.registerId);
});
for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
for (auto const& fieldsVars : columnFieldsVars.second) {
auto& fields = outNonMaterializedViewRegs[columnFieldsVars.first];
auto it = varInfos.find(fieldsVars.second->id);
TRI_ASSERT(it != varInfos.cend());
fields.insert({fieldsVars.first, it->second.registerId});
}
}
aql::IResearchViewExecutorInfos executorInfos{std::move(infos),
reader,
@ -1385,6 +1456,7 @@ std::unique_ptr<aql::ExecutionBlock> IResearchViewNode::createBlock(
*engine.getQuery(),
scorers(),
_sort,
::storedValue(*_view),
*plan(),
outVariable(),
filterCondition(),
@ -1402,7 +1474,7 @@ bool IResearchViewNode::OptimizationState::canVariablesBeReplaced(aql::Calculati
return _nodesToChange.find(calclulationNode) != _nodesToChange.cend(); // contains()
}
void IResearchViewNode::OptimizationState::saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs> const& nodesToChange) {
void IResearchViewNode::OptimizationState::saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs<aql::latematerialized::AstAndColumnFieldData>> const& nodesToChange) {
TRI_ASSERT(!nodesToChange.empty());
TRI_ASSERT(_nodesToChange.empty());
_nodesToChange.clear();
@ -1424,7 +1496,7 @@ IResearchViewNode::ViewVarsInfo IResearchViewNode::OptimizationState::replaceVie
auto const& calcNodeData = _nodesToChange[calcNode];
std::transform(calcNodeData.cbegin(), calcNodeData.cend(), std::inserter(uniqueVariables, uniqueVariables.end()),
[ast](auto const& afData) {
return std::make_pair(afData.field, ViewVariable{afData.number,
return std::make_pair(afData.field, ViewVariable{afData.columnNumber, afData.number,
ast->variables()->createTemporaryVariable()});
});
}

16
arangod/Aql/IResearchViewNode.h
View File

@ -29,6 +29,7 @@
#include "Aql/types.h"
#include "IResearch/IResearchOrderFactory.h"
#include "IResearch/IResearchViewSort.h"
#include "IResearch/IResearchViewStoredValue.h"
namespace arangodb {
class LogicalView;
@ -181,33 +182,36 @@ class IResearchViewNode final : public arangodb::aql::ExecutionNode {
_outNonMaterializedColPtr != nullptr;
}
static constexpr int SortColumnNumber = -1;
struct ViewVariable {
size_t viewFieldNum;
int columnNum;
size_t fieldNum;
aql::Variable const* var;
};
using ViewValuesVars = std::unordered_map<size_t, aql::Variable const*>;
using ViewValuesVars = std::unordered_map<int, std::vector<std::pair<size_t, aql::Variable const*>>>;
using ViewValuesRegisters = std::map<size_t, aql::RegisterId>;
using ViewValuesRegisters = std::map<int, std::map<size_t, aql::RegisterId>>;
using ViewVarsInfo = std::unordered_map<std::vector<arangodb::basics::AttributeName> const*, ViewVariable>;
void setViewVariables(ViewVarsInfo const& viewVariables) {
_outNonMaterializedViewVars.clear();
for (auto& viewVars : viewVariables) {
_outNonMaterializedViewVars[viewVars.second.viewFieldNum] = viewVars.second.var;
_outNonMaterializedViewVars[viewVars.second.columnNum].emplace_back(viewVars.second.fieldNum, viewVars.second.var);
}
}
// The structure is used for temporary saving of optimization rule data.
// It contains document references that could be replaced in late materialization rule.
struct OptimizationState {
using ViewVarsToBeReplaced = std::vector<aql::latematerialized::AstAndFieldData>;
using ViewVarsToBeReplaced = std::vector<aql::latematerialized::AstAndColumnFieldData>;
/// @brief calculation node with ast nodes that can be replaced by view values (e.g. primary sort)
std::unordered_map<aql::CalculationNode*, ViewVarsToBeReplaced> _nodesToChange;
void saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs> const& nodesToChange);
void saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs<aql::latematerialized::AstAndColumnFieldData>> const& nodesToChange);
bool canVariablesBeReplaced(aql::CalculationNode* calclulationNode) const;

55
arangod/Aql/IResearchViewOptimizerRules.cpp
View File

@ -64,6 +64,16 @@ inline IResearchViewSort const& primarySort(arangodb::LogicalView const& view) {
return viewImpl.primarySort();
}
inline IResearchViewStoredValue const& storedValue(arangodb::LogicalView const& view) {
if (arangodb::ServerState::instance()->isCoordinator()) {
auto& viewImpl = arangodb::LogicalView::cast<IResearchViewCoordinator>(view);
return viewImpl.storedValue();
}
auto& viewImpl = arangodb::LogicalView::cast<IResearchView>(view);
return viewImpl.storedValue();
}
bool addView(arangodb::LogicalView const& view, Query& query) {
auto* collections = query.collections();
@ -263,19 +273,45 @@ bool optimizeSort(IResearchViewNode& viewNode, ExecutionPlan* plan) {
}
}
bool attributesMatch(IResearchViewSort const& primarySort, latematerialized::NodeWithAttrs& node) {
bool attributesMatch(IResearchViewSort const& primarySort, IResearchViewStoredValue const& storedValue,
latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData>& node) {
// TODO: choose the best variant!
// check all node attributes to be in sort
for (auto& nodeAttr : node.attrs) {
size_t sortFieldNum = 0;
nodeAttr.afData.field = nullptr;
// try to find in the sort column
size_t fieldNum = 0;
for (auto const& field : primarySort.fields()) {
if (arangodb::basics::AttributeName::isIdentical(nodeAttr.attr, field, false)) {
nodeAttr.afData.number = sortFieldNum;
nodeAttr.afData.number = fieldNum;
nodeAttr.afData.columnNumber = IResearchViewNode::SortColumnNumber;
nodeAttr.afData.field = &field;
nodeAttr.attr.clear(); // we do not need later
nodeAttr.attr.shrink_to_fit();
break;
}
++sortFieldNum;
++fieldNum;
}
if (nodeAttr.afData.field != nullptr) {
continue;
}
// try to find in other columns
int columnNum = 0;
fieldNum = 0;
for (auto const& column : storedValue.columns()) {
for (auto const& field : column) {
if (arangodb::basics::AttributeName::isIdentical(nodeAttr.attr, field.second, false)) {
nodeAttr.afData.number = fieldNum;
nodeAttr.afData.field = &field.second;
nodeAttr.afData.columnNumber = columnNum;
nodeAttr.attr.clear(); // we do not need later
nodeAttr.attr.shrink_to_fit();
break;
}
++fieldNum;
}
++columnNum;
}
// not found
if (nodeAttr.afData.field == nullptr) {
@ -287,13 +323,14 @@ bool attributesMatch(IResearchViewSort const& primarySort, latematerialized::Nod
void keepReplacementViewVariables(arangodb::containers::SmallVector<ExecutionNode*> const& calcNodes,
arangodb::containers::SmallVector<ExecutionNode*> const& viewNodes) {
std::vector<latematerialized::NodeWithAttrs> nodesToChange;
std::vector<latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData>> nodesToChange;
for (auto* vNode : viewNodes) {
TRI_ASSERT(vNode && ExecutionNode::ENUMERATE_IRESEARCH_VIEW == vNode->getType());
auto& viewNode = *ExecutionNode::castTo<IResearchViewNode*>(vNode);
auto const& primarySort = ::primarySort(*viewNode.view());
if (primarySort.empty()) {
// no primary sort
auto const& storedValue = ::storedValue(*viewNode.view());
if (primarySort.empty() && storedValue.empty()) {
// neither primary sort nor stored value
continue;
}
auto const& var = viewNode.outVariable();
@ -302,11 +339,11 @@ void keepReplacementViewVariables(arangodb::containers::SmallVector<ExecutionNod
TRI_ASSERT(cNode && ExecutionNode::CALCULATION == cNode->getType());
auto& calcNode = *ExecutionNode::castTo<CalculationNode*>(cNode);
auto astNode = calcNode.expression()->nodeForModification();
latematerialized::NodeWithAttrs node;
latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData> node;
node.node = &calcNode;
// find attributes referenced to view node out variable
if (latematerialized::getReferencedAttributes(astNode, &var, node) &&
!node.attrs.empty() && attributesMatch(primarySort, node)) {
!node.attrs.empty() && attributesMatch(primarySort, storedValue, node)) {
nodesToChange.emplace_back(std::move(node));
}
}

22
arangod/Aql/IndexNode.cpp
View File

@ -135,7 +135,7 @@ IndexNode::IndexNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
"\"IndexValuesVars\" attribute should be an array");
}
std::unordered_map<size_t, Variable const*> indexValuesVars;
std::vector<std::pair<size_t, Variable const*>> indexValuesVars;
indexValuesVars.reserve(indexValuesVarsSlice.length());
for (auto const indVar : velocypack::ArrayIterator(indexValuesVarsSlice)) {
auto const fieldNumberSlice = indVar.get("fieldNumber");
@ -161,7 +161,7 @@ IndexNode::IndexNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
TRI_ERROR_BAD_PARAMETER, "\"IndexValuesVars[*].id\" unable to find variable by id %d",
varId);
}
indexValuesVars.emplace(fieldNumber, var);
indexValuesVars.emplace_back(fieldNumber, var);
}
_outNonMaterializedIndVars.first = indexId;
_outNonMaterializedIndVars.second = std::move(indexValuesVars);
@ -240,10 +240,10 @@ void IndexNode::planNodeRegisters(
if (isLateMaterialized()) {
varInfo.emplace(_outNonMaterializedDocId->id, aql::VarInfo(depth, totalNrRegs++));
// plan registers for index references
for (auto const& var : _outNonMaterializedIndVars.second) {
for (auto const& fieldVar : _outNonMaterializedIndVars.second) {
++nrRegsHere[depth];
++nrRegs[depth];
varInfo.emplace(var.second->id, aql::VarInfo(depth, totalNrRegs++));
varInfo.emplace(fieldVar.second->id, aql::VarInfo(depth, totalNrRegs++));
}
} else {
varInfo.emplace(_outVariable->id, aql::VarInfo(depth, totalNrRegs++));
@ -295,14 +295,14 @@ void IndexNode::toVelocyPackHelper(VPackBuilder& builder, unsigned flags,
TRI_ASSERT(indIt != _indexes.cend());
auto const& fields = (*indIt)->fields();
VPackArrayBuilder arrayScope(&builder, "IndexValuesVars");
for (auto const& indVar : _outNonMaterializedIndVars.second) {
for (auto const& fieldVar : _outNonMaterializedIndVars.second) {
VPackObjectBuilder objectScope(&builder);
builder.add("fieldNumber", VPackValue(indVar.first));
builder.add("id", VPackValue(indVar.second->id));
builder.add("name", VPackValue(indVar.second->name)); // for explainer.js
builder.add("fieldNumber", VPackValue(fieldVar.first));
builder.add("id", VPackValue(fieldVar.second->id));
builder.add("name", VPackValue(fieldVar.second->name)); // for explainer.js
std::string fieldName;
TRI_ASSERT(indVar.first < fields.size());
basics::TRI_AttributeNamesToString(fields[indVar.first], fieldName, true);
TRI_ASSERT(fieldVar.first < fields.size());
basics::TRI_AttributeNamesToString(fields[fieldVar.first], fieldName, true);
builder.add("field", VPackValue(fieldName)); // for explainer.js
}
}
@ -653,7 +653,7 @@ void IndexNode::setLateMaterialized(aql::Variable const* docIdVariable,
_outNonMaterializedIndVars.first = commonIndexId;
_outNonMaterializedDocId = docIdVariable;
for (auto& indVars : indexVariables) {
_outNonMaterializedIndVars.second[indVars.second.indexFieldNum] = indVars.second.var;
_outNonMaterializedIndVars.second.emplace_back(indVars.second.indexFieldNum, indVars.second.var);
}
}

2
arangod/Aql/IndexNode.h
View File

@ -132,7 +132,7 @@ class IndexNode : public ExecutionNode, public DocumentProducingNode, public Col
Variable const* var;
};
using IndexValuesVars = std::pair<TRI_idx_iid_t, std::unordered_map<size_t, Variable const*>>;
using IndexValuesVars = std::pair<TRI_idx_iid_t, std::vector<std::pair<size_t, Variable const*>>>;
using IndexValuesRegisters = std::pair<TRI_idx_iid_t, std::unordered_map<size_t, RegisterId>>;

7
arangod/Aql/IndexNodeOptimizerRules.cpp
View File

@ -34,9 +34,10 @@
using namespace arangodb::aql;
namespace {
bool attributesMatch(TRI_idx_iid_t& commonIndexId, IndexNode const* indexNode, latematerialized::NodeWithAttrs& node) {
bool attributesMatch(TRI_idx_iid_t& commonIndexId, IndexNode const* indexNode, latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData>& node) {
// check all node attributes to be in index
for (auto& nodeAttr : node.attrs) {
nodeAttr.afData.field = nullptr;
for (auto& index : indexNode->getIndexes()) {
if (!index->hasCoveringIterator()) {
continue;
@ -105,7 +106,7 @@ void arangodb::aql::lateDocumentMaterializationRule(Optimizer* opt,
// this node could be appended with materializer
bool stopSearch = false;
bool stickToSortNode = false;
std::vector<latematerialized::NodeWithAttrs> nodesToChange;
std::vector<latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData>> nodesToChange;
TRI_idx_iid_t commonIndexId = 0; // use one index only
while (current != loop) {
auto type = current->getType();
@ -118,7 +119,7 @@ void arangodb::aql::lateDocumentMaterializationRule(Optimizer* opt,
case ExecutionNode::CALCULATION: {
auto calculationNode = ExecutionNode::castTo<CalculationNode*>(current);
auto astNode = calculationNode->expression()->nodeForModification();
latematerialized::NodeWithAttrs node;
latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData> node;
node.node = calculationNode;
// find attributes referenced to index node out variable
if (!latematerialized::getReferencedAttributes(astNode, indexNode->outVariable(), node)) {

28
arangod/Aql/LateMaterializedOptimizerRulesCommon.cpp
View File

@ -50,19 +50,21 @@ void traverseReadOnly(AstNode* node, AstNode* parentNode, size_t childNumber,
}
// traversal state
template<typename T>
struct TraversalState {
Variable const* variable;
latematerialized::NodeWithAttrs& nodeAttrs;
latematerialized::NodeWithAttrs<T>& nodeAttrs;
bool optimize;
bool wasAccess;
};
}
// determines attributes referenced in an expression for the specified out variable
template<typename T>
bool latematerialized::getReferencedAttributes(AstNode* node,
Variable const* variable,
NodeWithAttrs& nodeAttrs) {
TraversalState state{variable, nodeAttrs, true, false};
NodeWithAttrs<T>& nodeAttrs) {
TraversalState<T> state{variable, nodeAttrs, true, false};
auto preVisitor = [&state](AstNode const* node,
AstNode* parentNode, size_t childNumber) {
@ -73,9 +75,12 @@ bool latematerialized::getReferencedAttributes(AstNode* node,
switch (node->type) {
case NODE_TYPE_ATTRIBUTE_ACCESS:
if (!state.wasAccess) {
T afData;
afData.parentNode = parentNode;
afData.childNumber = childNumber;
state.nodeAttrs.attrs.emplace_back(
NodeWithAttrs::AttributeAndField{std::vector<arangodb::basics::AttributeName>{
{std::string(node->getStringValue(), node->getStringLength()), false}}, {parentNode, childNumber, nullptr, 0}});
typename NodeWithAttrs<T>::AttributeAndField{std::vector<arangodb::basics::AttributeName>{
{std::string(node->getStringValue(), node->getStringLength()), false}}, std::move(afData)});
state.wasAccess = true;
} else {
state.nodeAttrs.attrs.back().attr.emplace_back(std::string(node->getStringValue(), node->getStringLength()), false);
@ -118,3 +123,16 @@ bool latematerialized::getReferencedAttributes(AstNode* node,
return state.optimize;
}
template struct latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData>;
template struct latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData>;
template bool latematerialized::getReferencedAttributes(
AstNode* node,
Variable const* variable,
NodeWithAttrs<latematerialized::AstAndFieldData>& nodeAttrs);
template bool latematerialized::getReferencedAttributes(
AstNode* node,
Variable const* variable,
NodeWithAttrs<latematerialized::AstAndColumnFieldData>& nodeAttrs);

10
arangod/Aql/LateMaterializedOptimizerRulesCommon.h
View File

@ -49,17 +49,23 @@ struct AstAndFieldData {
size_t number;
};
struct AstAndColumnFieldData : AstAndFieldData {
int columnNumber;
};
template<typename T>
struct NodeWithAttrs {
struct AttributeAndField {
std::vector<arangodb::basics::AttributeName> attr;
AstAndFieldData afData;
T afData;
};
std::vector<AttributeAndField> attrs;
CalculationNode* node;
};
bool getReferencedAttributes(AstNode* node, Variable const* variable, NodeWithAttrs& nodeAttrs);
template<typename T>
bool getReferencedAttributes(AstNode* node, Variable const* variable, NodeWithAttrs<T>& nodeAttrs);
} // latematerialized
} // namespace aql

1
arangod/CMakeLists.txt
View File

@ -65,6 +65,7 @@ add_library(arango_iresearch
IResearch/IResearchView.cpp IResearch/IResearchView.h
IResearch/IResearchVPackComparer.cpp IResearch/IResearchVPackComparer.h
IResearch/IResearchViewSort.cpp IResearch/IResearchViewSort.h
IResearch/IResearchViewStoredValue.cpp IResearch/IResearchViewStoredValue.h
IResearch/IResearchViewCoordinator.cpp IResearch/IResearchViewCoordinator.h
IResearch/IResearchExpressionContext.cpp IResearch/IResearchExpressionContext.h
IResearch/IResearchViewMeta.cpp IResearch/IResearchViewMeta.h

33
arangod/IResearch/IResearchLink.cpp
View File

@ -155,6 +155,35 @@ inline arangodb::Result insertDocument(irs::index_writer::documents_context& ctx
}
}
// Stored value field
{
struct StoredValue {
bool write(irs::data_output& out) const {
out.write_bytes(slice.start(), slice.byteSize());
return true;
}
irs::string_ref const& name() {
return fieldName;
}
VPackSlice slice;
irs::string_ref fieldName;
} field; // StoredValue
for (auto const& column : meta._storedValue.columns()) {
field.fieldName = irs::string_ref{};
for (auto const& storedValue : column) {
// column name is equal to the first field name (TODO: two can have the same)
if (field.fieldName.empty()) {
field.fieldName = irs::string_ref(storedValue.first);
}
field.slice = arangodb::iresearch::get(document, storedValue.second, VPackSlice::nullSlice());
}
doc.insert<irs::Action::STORE>(field);
}
}
// System fields
// Indexed and Stored: LocalDocumentId
@ -1804,6 +1833,10 @@ void IResearchLink::toVelocyPackStats(VPackBuilder& builder) const {
builder.add("indexSize", VPackValue(stats.indexSize));
}
IResearchViewStoredValue const& IResearchLink::storedValue() const {
return _meta._storedValue;
}
} // namespace iresearch
} // namespace arangodb

5
arangod/IResearch/IResearchLink.h
View File

@ -224,6 +224,11 @@ class IResearchLink {
////////////////////////////////////////////////////////////////////////////////
AnalyzerPool::ptr findAnalyzer(AnalyzerPool const& analyzer) const;
////////////////////////////////////////////////////////////////////////////////
/// @brief get stored value
////////////////////////////////////////////////////////////////////////////////
IResearchViewStoredValue const& storedValue() const;
protected:
//////////////////////////////////////////////////////////////////////////////
/// @brief index stats

31
arangod/IResearch/IResearchLinkMeta.cpp
View File

@ -545,6 +545,10 @@ bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcep
return false;
}
if (_storedValue != other._storedValue) {
return false;
}
return true;
}
@ -577,10 +581,20 @@ bool IResearchLinkMeta::init(velocypack::Slice const& slice,
auto const field = slice.get(fieldName);
mask->_sort = field.isArray();
if (readAnalyzerDefinition && mask->_sort) {
if (!_sort.fromVelocyPack(field, errorField)) {
return false;
}
if (readAnalyzerDefinition && mask->_sort && !_sort.fromVelocyPack(field, errorField)) {
return false;
}
}
{
// optional stored values
static VPackStringRef const fieldName("storedFields");
auto const field = slice.get(fieldName);
mask->_storedValue = field.isArray();
if (readAnalyzerDefinition && mask->_storedValue && !_storedValue.fromVelocyPack(field, errorField)) {
return false;
}
}
@ -756,6 +770,14 @@ bool IResearchLinkMeta::json(velocypack::Builder& builder,
}
}
if (writeAnalyzerDefinition
&& (!mask || mask->_storedValue)) {
velocypack::ArrayBuilder arrayScope(&builder, "storedFields");
if (!_storedValue.toVelocyPack(builder)) {
return false;
}
}
// output definitions if 'writeAnalyzerDefinition' requested and not maked
// this should be the case for the default top-most call
if (writeAnalyzerDefinition && (!mask || mask->_analyzerDefinitions)) {
@ -775,6 +797,7 @@ size_t IResearchLinkMeta::memory() const noexcept {
size += _analyzers.size() * sizeof(decltype(_analyzers)::value_type);
size += _sort.memory();
size += _storedValue.memory();
size += FieldMeta::memory();
return size;

6
arangod/IResearch/IResearchLinkMeta.h
View File

@ -34,6 +34,7 @@
#include "Containers.h"
#include "IResearchAnalyzerFeature.h"
#include "IResearchViewSort.h"
#include "IResearchViewStoredValue.h"
namespace arangodb {
namespace velocypack {
@ -180,15 +181,18 @@ struct IResearchLinkMeta : public FieldMeta {
explicit Mask(bool mask = false) noexcept
: FieldMeta::Mask(mask),
_analyzerDefinitions(mask),
_sort(mask) {
_sort(mask),
_storedValue(mask) {
}
bool _analyzerDefinitions;
bool _sort;
bool _storedValue;
};
std::set<AnalyzerPool::ptr, FieldMeta::AnalyzerComparer> _analyzerDefinitions;
IResearchViewSort _sort; // sort condition associated with the link
IResearchViewStoredValue _storedValue; // stored value associated with the link
// NOTE: if adding fields don't forget to modify the comparison operator !!!
// NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask !!!
// NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask constructor !!!

1
arangod/IResearch/IResearchView.cpp
View File

@ -134,6 +134,7 @@ void ensureImmutableProperties(
dst._writebufferIdle = src._writebufferIdle;
dst._writebufferSizeMax = src._writebufferSizeMax;
dst._primarySort = src._primarySort;
dst._storedValue = src._storedValue;
}
}

9
arangod/IResearch/IResearchView.h
View File

@ -105,7 +105,7 @@ class IResearchView final: public arangodb::LogicalView {
///////////////////////////////////////////////////////////////////////////////
/// @brief destructor to clean up resources
///////////////////////////////////////////////////////////////////////////////
virtual ~IResearchView();
virtual ~IResearchView() override;
using arangodb::LogicalView::name;
@ -178,6 +178,13 @@ class IResearchView final: public arangodb::LogicalView {
return _meta._primarySort;
}
///////////////////////////////////////////////////////////////////////////////
/// @return stored value from links collections
///////////////////////////////////////////////////////////////////////////////
IResearchViewStoredValue const& storedValue() const noexcept {
return _meta._storedValue;
}
protected:
//////////////////////////////////////////////////////////////////////////////

1
arangod/IResearch/IResearchViewCoordinator.cpp
View File

@ -57,6 +57,7 @@ void ensureImmutableProperties(
dst._writebufferIdle = src._writebufferIdle;
dst._writebufferSizeMax = src._writebufferSizeMax;
dst._primarySort = src._primarySort;
dst._storedValue = src._storedValue;
}
} // namespace

7
arangod/IResearch/IResearchViewCoordinator.h
View File

@ -88,6 +88,13 @@ class IResearchViewCoordinator final : public arangodb::LogicalView {
return _meta._primarySort;
}
///////////////////////////////////////////////////////////////////////////////
/// @return stored value from links collections
///////////////////////////////////////////////////////////////////////////////
IResearchViewStoredValue const& storedValue() const noexcept {
return _meta._storedValue;
}
protected:
virtual Result appendVelocyPackImpl(arangodb::velocypack::Builder& builder,
Serialization context) const override;

58
arangod/IResearch/IResearchViewMeta.cpp
View File

@ -252,6 +252,7 @@ IResearchViewMeta& IResearchViewMeta::operator=(IResearchViewMeta&& other) noexc
_writebufferIdle = std::move(other._writebufferIdle);
_writebufferSizeMax = std::move(other._writebufferSizeMax);
_primarySort = std::move(other._primarySort);
_storedValue = std::move(other._storedValue);
}
return *this;
@ -269,6 +270,7 @@ IResearchViewMeta& IResearchViewMeta::operator=(IResearchViewMeta const& other)
_writebufferIdle = other._writebufferIdle;
_writebufferSizeMax = other._writebufferSizeMax;
_primarySort = other._primarySort;
_storedValue = other._storedValue;
}
return *this;
@ -298,27 +300,16 @@ bool IResearchViewMeta::operator==(IResearchViewMeta const& other) const noexcep
if (irs::locale_utils::language(_locale) != irs::locale_utils::language(other._locale) ||
irs::locale_utils::country(_locale) != irs::locale_utils::country(other._locale) ||
irs::locale_utils::encoding(_locale) != irs::locale_utils::encoding(other._locale)) {
return false; // values do not match
return false;
}
if (_version != other._version) {
return false; // values do not match
}
if (_writebufferActive != other._writebufferActive) {
return false; // values do not match
}
if (_writebufferIdle != other._writebufferIdle) {
return false; // values do not match
}
if (_writebufferSizeMax != other._writebufferSizeMax) {
return false; // values do not match
}
if (_primarySort != other._primarySort) {
return false; // values do not match
if (_version != other._version ||
_writebufferActive != other._writebufferActive ||
_writebufferIdle != other._writebufferIdle ||
_writebufferSizeMax != other._writebufferSizeMax ||
_primarySort != other._primarySort ||
_storedValue != other._storedValue) {
return false;
}
return true;
@ -588,7 +579,27 @@ bool IResearchViewMeta::init(arangodb::velocypack::Slice const& slice, std::stri
} else if (!_primarySort.fromVelocyPack(field, errorSubField)) {
errorField = fieldName.toString();
if (!errorSubField.empty()) {
errorField += errorSubField;
errorField += errorSubField;
}
return false;
}
}
{
// optional object
static VPackStringRef const fieldName("storedFields");
std::string errorSubField;
auto const field = slice.get(fieldName);
mask->_storedValue = field.isArray();
if (!mask->_storedValue) {
_storedValue = defaults._storedValue;
} else if (!_storedValue.fromVelocyPack(field, errorSubField)) {
errorField = fieldName.toString();
if (!errorSubField.empty()) {
errorField += errorSubField;
}
return false;
@ -662,6 +673,13 @@ bool IResearchViewMeta::json(arangodb::velocypack::Builder& builder,
}
}
if ((!ignoreEqual || _storedValue != ignoreEqual->_storedValue) && (!mask || mask->_storedValue)) {
velocypack::ArrayBuilder arrayScope(&builder, "storedFields");
if (!_storedValue.toVelocyPack(builder)) {
return false;
}
}
return true;
}

3
arangod/IResearch/IResearchViewMeta.h
View File

@ -28,6 +28,7 @@
#include <unordered_set>
#include "IResearchViewSort.h"
#include "IResearchViewStoredValue.h"
#include <velocypack/Builder.h>
@ -86,6 +87,7 @@ struct IResearchViewMeta {
bool _writebufferIdle;
bool _writebufferSizeMax;
bool _primarySort;
bool _storedValue;
explicit Mask(bool mask = false) noexcept;
};
@ -99,6 +101,7 @@ struct IResearchViewMeta {
size_t _writebufferIdle; // maximum number of segments cached in the pool
size_t _writebufferSizeMax; // maximum memory byte size per segment before a segment flush is triggered (0 == unlimited)
IResearchViewSort _primarySort;
IResearchViewStoredValue _storedValue;
// NOTE: if adding fields don't forget to modify the default constructor !!!
// NOTE: if adding fields don't forget to modify the copy constructor !!!
// NOTE: if adding fields don't forget to modify the move constructor !!!

119
arangod/IResearch/IResearchViewStoredValue.cpp Normal file
View File

@ -0,0 +1,119 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Yuriy Popov
////////////////////////////////////////////////////////////////////////////////
#include "IResearchViewStoredValue.h"
#include <velocypack/Builder.h>
#include <velocypack/Iterator.h>
#include "VelocyPackHelper.h"
namespace arangodb {
namespace iresearch {
/*
{"links" : {
"mycol1" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value",
"storedFields": [["obj.foo.val1", "obj.foo.val2"], ["obj.bar.val1", "obj.bar.val2"]]}, // string
"mycol2" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value"}
}
}
*/
bool IResearchViewStoredValue::toVelocyPack(velocypack::Builder& builder) const {
if (!builder.isOpenArray()) {
return false;
}
for (auto const& column : _storedColumns) {
velocypack::ArrayBuilder arrayScope(&builder);
for (auto const& field : column) {
builder.add(VPackValue(field.first));
}
}
return true;
}
bool IResearchViewStoredValue::fromVelocyPack(
velocypack::Slice slice, std::string& error) {
clear();
if (slice.isArray()) {
_storedColumns.reserve(slice.length());
for (auto columnSlice : VPackArrayIterator(slice)) {
if (columnSlice.isArray()) {
StoredColumn sc;
sc.reserve(columnSlice.length());
for (auto fieldSlice : VPackArrayIterator(columnSlice)) {
if (!fieldSlice.isString()) {
clear();
return false;
}
auto fieldName = arangodb::iresearch::getStringRef(slice);
std::vector<basics::AttributeName> field;
try {
arangodb::basics::TRI_ParseAttributeString(fieldName, field, false);
} catch (...) {
error = "." + std::string(fieldName);
clear();
return false;
}
sc.emplace_back(fieldName, std::move(field));
}
_storedColumns.emplace_back(std::move(sc));
} else if (columnSlice.isString()) {
auto fieldName = arangodb::iresearch::getStringRef(slice);
std::vector<basics::AttributeName> field;
try {
arangodb::basics::TRI_ParseAttributeString(fieldName, field, false);
} catch (...) {
error = "." + std::string(fieldName);
return false;
}
_storedColumns.emplace_back(StoredColumn{{fieldName, std::move(field)}});
} else {
clear();
return false;
}
}
return true;
}
return false;
}
size_t IResearchViewStoredValue::memory() const noexcept {
size_t size = sizeof(IResearchViewStoredValue);
size += sizeof(StoredColumn)*_storedColumns.size();
for (auto const& column : _storedColumns) {
size += sizeof(std::pair<std::string, std::vector<basics::AttributeName>>)*column.size();
for (auto const& field : column) {
size += field.first.size();
size += sizeof(basics::AttributeName)*field.second.size();
for (auto const& attribute : field.second) {
size += attribute.name.size();
}
}
}
return size;
}
} // iresearch
} // arangodb

85
arangod/IResearch/IResearchViewStoredValue.h Normal file
View File

@ -0,0 +1,85 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Yuriy Popov
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_IRESEARCH__IRESEARCH_VIEW_STORED_VALUE_H
#define ARANGODB_IRESEARCH__IRESEARCH_VIEW_STORED_VALUE_H 1
#include "Basics/AttributeNameParser.h"
#include "Basics/debugging.h"
#include <velocypack/Slice.h>
namespace arangodb {
namespace velocypack {
class Builder;
}
/*
{"links" : {
"mycol1" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value",
"storedFields": [["obj.foo.val1", "obj.foo.val2"], ["obj.bar.val1", "obj.bar.val2"]]}, // or string
"mycol2" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value"}
}
}
*/
namespace iresearch {
class IResearchViewStoredValue {
public:
using StoredColumn = std::vector<std::pair<std::string, std::vector<basics::AttributeName>>>;
bool operator==(IResearchViewStoredValue const& rhs) const noexcept {
return _storedColumns == rhs._storedColumns;
}
bool operator!=(IResearchViewStoredValue const& rhs) const noexcept {
return !(*this == rhs);
}
std::vector<StoredColumn> const& columns() const noexcept {
return _storedColumns;
}
size_t memory() const noexcept;
bool empty() const noexcept {
return _storedColumns.empty();
}
bool toVelocyPack(velocypack::Builder& builder) const;
bool fromVelocyPack(velocypack::Slice, std::string& error);
private:
void clear() noexcept {
_storedColumns.clear();
}
std::vector<StoredColumn> _storedColumns;
}; // IResearchViewStoredValue
} // iresearch
} // arangodb
#endif // ARANGODB_IRESEARCH__IRESEARCH_VIEW_STORED_VALUE_H