stored value draft

arangod/Aql/IResearchViewExecutor.cpp

@@ -95,6 +95,12 @@ inline irs::columnstore_reader::values_reader_f sortColumn(irs::sub_reader const
   return reader ? reader->values() : irs::columnstore_reader::values_reader_f{};
 }
 
+inline irs::columnstore_reader::values_reader_f storedValueColumn(irs::sub_reader const& segment, irs::string_ref const& name) {
+  auto const* reader = segment.column_reader(name);
+
+  return reader ? reader->values() : irs::columnstore_reader::values_reader_f{};
+}
+
 }  // namespace
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -106,6 +112,7 @@ IResearchViewExecutorInfos::IResearchViewExecutorInfos(
     RegisterId firstOutputRegister, RegisterId numScoreRegisters, Query& query,
     std::vector<Scorer> const& scorers,
     std::pair<arangodb::iresearch::IResearchViewSort const*, size_t> const& sort,
+    iresearch::IResearchViewStoredValue const& storedValue,
     ExecutionPlan const& plan, Variable const& outVariable,
     aql::AstNode const& filterCondition, std::pair<bool, bool> volatility,
     IResearchViewExecutorInfos::VarInfoMap const& varInfoMap, int depth,
@@ -117,6 +124,7 @@ IResearchViewExecutorInfos::IResearchViewExecutorInfos(
       _query(query),
       _scorers(scorers),
       _sort(sort),
+      _storedValue(storedValue),
       _plan(plan),
       _outVariable(outVariable),
       _filterCondition(filterCondition),
@@ -192,6 +200,10 @@ const std::pair<const arangodb::iresearch::IResearchViewSort*, size_t>& IResearc
   return _sort;
 }
 
+arangodb::iresearch::IResearchViewStoredValue const& IResearchViewExecutorInfos::storedValue() const noexcept {
+  return _storedValue;
+}
+
 bool IResearchViewExecutorInfos::isScoreReg(RegisterId reg) const noexcept {
   return getNumScoreRegisters() > 0 &&
          getFirstScoreRegister() <= reg && reg < getFirstScoreRegister() + getNumScoreRegisters();
@@ -285,7 +297,7 @@ std::vector<AqlValue>::iterator IResearchViewExecutorBase<Impl, Traits>::IndexRe
 template <typename Impl, typename Traits>
 template <typename ValueType>
 IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::IndexReadBuffer(std::size_t const numScoreRegisters)
-  : _keyBuffer(), _scoreBuffer(), _sortValueBuffer(), _numScoreRegisters(numScoreRegisters), _keyBaseIdx(0) {}
+  : _keyBuffer(), _scoreBuffer(), _storedValueBuffer(), _numScoreRegisters(numScoreRegisters), _keyBaseIdx(0) {}
 
 template <typename Impl, typename Traits>
 template <typename ValueType>
@@ -314,16 +326,16 @@ void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::pushVa
 
 template <typename Impl, typename Traits>
 template <typename ValueType>
-void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::pushSortValue(irs::bytes_ref&& sortValue) {
+void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::pushStoredValue(std::unordered_map<int, irs::bytes_ref>&& storedValue) {
-  _sortValueBuffer.emplace_back(std::move(sortValue));
+  _storedValueBuffer.emplace_back(std::move(storedValue));
 }
 
 template <typename Impl, typename Traits>
 template <typename ValueType>
-irs::bytes_ref IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::getSortValue(
+std::unordered_map<int, irs::bytes_ref> const& IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::getStoredValue(
     const IResearchViewExecutorBase::IndexReadBufferEntry bufferEntry) const noexcept {
-  TRI_ASSERT(bufferEntry._keyIdx < _sortValueBuffer.size());
+  TRI_ASSERT(bufferEntry._keyIdx < _storedValueBuffer.size());
-  return _sortValueBuffer[bufferEntry._keyIdx];
+  return _storedValueBuffer[bufferEntry._keyIdx];
 }
 
 template <typename Impl, typename Traits>
@@ -346,7 +358,7 @@ void IResearchViewExecutorBase<Impl, Traits>::IndexReadBuffer<ValueType>::reset(
   _keyBaseIdx = 0;
   _keyBuffer.clear();
   _scoreBuffer.clear();
-  _sortValueBuffer.clear();
+  _storedValueBuffer.clear();
 }
 
 template <typename Impl, typename Traits>
@@ -666,18 +678,17 @@ bool IResearchViewExecutorBase<Impl, Traits>::writeRow(ReadContext& ctx,
     if constexpr (Traits::MaterializeType == MaterializeType::LateMaterializedWithVars) {
       auto const& outNonMaterializedViewRegs = infos().getOutNonMaterializedViewRegs();
       TRI_ASSERT(!outNonMaterializedViewRegs.empty());
-      if (ADB_LIKELY(!outNonMaterializedViewRegs.empty())) {
+      auto const& storedValue = _indexReadBuffer.getStoredValue(bufferEntry);
-        auto sortValue = _indexReadBuffer.getSortValue(bufferEntry);
+      for (auto const& [columnNum, fieldsRegs] : outNonMaterializedViewRegs) {
-        TRI_ASSERT(!sortValue.empty());
+        auto it = storedValue.find(columnNum);
-        if (ADB_UNLIKELY(sortValue.empty())) {
+        TRI_ASSERT(it != storedValue.cend());
-          return false;
+        TRI_ASSERT(!it->second.empty());
-        }
+        auto s = it->second.c_str();
-        auto s = sortValue.c_str();
+        auto totalSize = it->second.size();
-        auto totalSize = sortValue.size();
         auto slice = VPackSlice(s);
         size_t size = 0;
         size_t i = 0;
-        for (auto const& [fieldNum, registerId] : outNonMaterializedViewRegs) {
+        for (auto const& [fieldNum, registerId] : fieldsRegs) {
           while (i < fieldNum) {
             size += slice.byteSize();
             TRI_ASSERT(size <= totalSize);
@@ -834,14 +845,23 @@ void IResearchViewExecutor<ordered, materializeType>::fillBuffer(IResearchViewEx
     }
 
     if constexpr (materializeType == MaterializeType::LateMaterializedWithVars) {
-      TRI_ASSERT((!this->_infos.getOutNonMaterializedViewRegs().empty()));
+      auto const& columnsFieldsRegs = this->_infos.getOutNonMaterializedViewRegs();
-      if (ADB_LIKELY(!this->_infos.getOutNonMaterializedViewRegs().empty())) {
+      TRI_ASSERT(!columnsFieldsRegs.empty());
-        TRI_ASSERT(_sortReader);
+      std::unordered_map<int, irs::bytes_ref> storedValue;
-        irs::bytes_ref sortValue{irs::bytes_ref::NIL}; // sort column value
+      irs::columnstore_reader::values_reader_f reader;
-        auto ok = _sortReader(_doc->value, sortValue);
+      for (auto const& columnFieldsRegs : columnsFieldsRegs) {
+        if (columnFieldsRegs.first >= 0) { // not IResearchViewNode::SortColumnNumber
+          reader = _storedValueReaders[columnFieldsRegs.first];
+        } else { // IResearchViewNode::SortColumnNumber
+          reader = _sortReader;
+        }
+        TRI_ASSERT(reader);
+        irs::bytes_ref value{irs::bytes_ref::NIL}; // column value
+        auto ok = reader(_doc->value, value);
         TRI_ASSERT(ok);
-        this->_indexReadBuffer.pushSortValue(std::move(sortValue));
+        storedValue[columnFieldsRegs.first] = std::move(value);
       }
+      this->_indexReadBuffer.pushStoredValue(std::move(storedValue));
     }
     // doc and scores are both pushed, sizes must now be coherent
     this->_indexReadBuffer.assertSizeCoherence();
@@ -879,14 +899,33 @@ bool IResearchViewExecutor<ordered, materializeType>::resetIterator() {
   }
 
   if constexpr (materializeType == MaterializeType::LateMaterializedWithVars) {
-    if (!this->_infos.getOutNonMaterializedViewRegs().empty()) {
+    auto const& columnsfieldsRegs = this->_infos.getOutNonMaterializedViewRegs();
-      _sortReader = ::sortColumn(segmentReader);
+    if (!columnsfieldsRegs.empty()) {
-
+      auto storedValue = this->_infos.storedValue();
-      if (!_sortReader) {
+      for (auto const& columnfieldsRegs : columnsfieldsRegs) {
-        LOG_TOPIC("bc5bd", WARN, arangodb::iresearch::TOPIC)
+        if (columnfieldsRegs.first >= 0) { // not IResearchViewNode::SortColumnNumber
-            << "encountered a sub-reader without a sort column while "
+          TRI_ASSERT(!storedValue.empty());
-               "executing a query, ignoring";
+          auto const& columns = storedValue.columns();
-        return false;
+          auto const storedColumnNumber = static_cast<decltype(columns.size())>(columnfieldsRegs.first);
+          TRI_ASSERT(storedColumnNumber < columns.size());
+          // column name is equal to the first field name (TODO: two can have the same)
+          auto storedValueColumn = ::storedValueColumn(segmentReader, columns[storedColumnNumber].back().first);
+          if (!storedValueColumn) {
+            LOG_TOPIC("af7ec", WARN, arangodb::iresearch::TOPIC)
+                << "encountered a sub-reader without a stored value column while "
+                   "executing a query, ignoring";
+            return false;
+          }
+          _storedValueReaders[columnfieldsRegs.first] = storedValueColumn;
+        } else { // IResearchViewNode::SortColumnNumber
+          _sortReader = ::sortColumn(segmentReader);
+          if (!_sortReader) {
+            LOG_TOPIC("bc5bd", WARN, arangodb::iresearch::TOPIC)
+                << "encountered a sub-reader without a sort column while "
+                   "executing a query, ignoring";
+            return false;
+          }
+        }
       }
     }
   }

arangod/Aql/IResearchViewExecutor.h

@@ -66,6 +66,7 @@ class IResearchViewExecutorInfos : public ExecutorInfos {
       RegisterId firstOutputRegister, RegisterId numScoreRegisters,
       Query& query, std::vector<iresearch::Scorer> const& scorers,
       std::pair<iresearch::IResearchViewSort const*, size_t> const& sort,
+      iresearch::IResearchViewStoredValue const& storedValue,
       ExecutionPlan const& plan,
       Variable const& outVariable,
       aql::AstNode const& filterCondition,
@@ -92,6 +93,8 @@ class IResearchViewExecutorInfos : public ExecutorInfos {
   // second - number of sort conditions to take into account
   std::pair<iresearch::IResearchViewSort const*, size_t> const& sort() const noexcept;
 
+  iresearch::IResearchViewStoredValue const& storedValue() const noexcept;
+
   bool isScoreReg(RegisterId reg) const noexcept;
 
  private:
@@ -101,6 +104,7 @@ class IResearchViewExecutorInfos : public ExecutorInfos {
   Query& _query;
   std::vector<iresearch::Scorer> const& _scorers;
   std::pair<iresearch::IResearchViewSort const*, size_t> _sort;
+  iresearch::IResearchViewStoredValue const& _storedValue;
   ExecutionPlan const& _plan;
   Variable const& _outVariable;
   aql::AstNode const& _filterCondition;
@@ -244,9 +248,9 @@ class IResearchViewExecutorBase {
     // before and after.
     void assertSizeCoherence() const noexcept;
 
-    void pushSortValue(irs::bytes_ref&& sortValue);
+    void pushStoredValue(std::unordered_map<int, irs::bytes_ref>&& storedValue);
 
-    irs::bytes_ref getSortValue(IndexReadBufferEntry bufferEntry) const noexcept;
+    std::unordered_map<int, irs::bytes_ref> const& getStoredValue(IndexReadBufferEntry bufferEntry) const noexcept;
 
    private:
     // _keyBuffer, _scoreBuffer, _sortValueBuffer together hold all the
@@ -260,7 +264,7 @@ class IResearchViewExecutorBase {
     // .
     std::vector<ValueType> _keyBuffer;
     std::vector<AqlValue> _scoreBuffer;
-    std::vector<irs::bytes_ref> _sortValueBuffer;
+    std::vector<std::unordered_map<int, irs::bytes_ref>> _storedValueBuffer;
     std::size_t _numScoreRegisters;
     std::size_t _keyBaseIdx;
   };
@@ -344,6 +348,7 @@ class IResearchViewExecutor : public IResearchViewExecutorBase<IResearchViewExec
 
   irs::columnstore_reader::values_reader_f _pkReader;   // current primary key reader
   irs::columnstore_reader::values_reader_f _sortReader; // current sort reader
+  std::unordered_map<int, irs::columnstore_reader::values_reader_f> _storedValueReaders; // current stored value readers
   irs::doc_iterator::ptr _itr;
   irs::document const* _doc{};
   size_t _readerOffset;

arangod/Aql/IResearchViewNode.cpp

@@ -650,6 +650,16 @@ inline IResearchViewSort const& primarySort(arangodb::LogicalView const& view) {
   return viewImpl.primarySort();
 }
 
+inline IResearchViewStoredValue const& storedValue(arangodb::LogicalView const& view) {
+  if (arangodb::ServerState::instance()->isCoordinator()) {
+    auto& viewImpl = arangodb::LogicalView::cast<IResearchViewCoordinator>(view);
+    return viewImpl.storedValue();
+  }
+
+  auto& viewImpl = arangodb::LogicalView::cast<IResearchView>(view);
+  return viewImpl.storedValue();
+}
+
 const char* NODE_DATABASE_PARAM = "database";
 const char* NODE_VIEW_NAME_PARAM = "view";
 const char* NODE_VIEW_ID_PARAM = "viewId";
@@ -664,13 +674,52 @@ const char* NODE_VOLATILITY_PARAM = "volatility";
 const char* NODE_PRIMARY_SORT_PARAM = "primarySort";
 const char* NODE_PRIMARY_SORT_BUCKETS_PARAM = "primarySortBuckets";
 const char* NODE_VIEW_VALUES_VARS = "ViewValuesVars";
+const char* NODE_VIEW_STORED_VALUES_VARS = "viewStoredValuesVars";
+const char* NODE_VIEW_VALUES_VAR_COLUMN_NUMBER = "columnNumber";
 const char* NODE_VIEW_VALUES_VAR_FIELD_NUMBER = "fieldNumber";
 const char* NODE_VIEW_VALUES_VAR_ID = "id";
 const char* NODE_VIEW_VALUES_VAR_NAME = "name";
 const char* NODE_VIEW_VALUES_VAR_FIELD = "field";
 
-std::array<std::unique_ptr<aql::ExecutionBlock> (*)(
+void addViewValuesVar(VPackBuilder& nodes, std::string const& fieldName,
-    aql::ExecutionEngine*, IResearchViewNode const*, aql::IResearchViewExecutorInfos&&), 10> executors {
+                      std::pair<size_t, aql::Variable const*> const& fieldVar) {
+  nodes.add(NODE_VIEW_VALUES_VAR_FIELD_NUMBER, VPackValue(fieldVar.first));
+  nodes.add(NODE_VIEW_VALUES_VAR_ID, VPackValue(fieldVar.second->id));
+  nodes.add(NODE_VIEW_VALUES_VAR_NAME, VPackValue(fieldVar.second->name)); // for explainer.js
+  nodes.add(NODE_VIEW_VALUES_VAR_FIELD, VPackValue(fieldName)); // for explainer.js
+}
+
+void extractViewValuesVar(aql::VariableGenerator const* vars,
+                          IResearchViewNode::ViewValuesVars& viewValuesVars,
+                          int const columnNumber,
+                          velocypack::Slice const& fieldVar) {
+  auto const fieldNumberSlice = fieldVar.get(NODE_VIEW_VALUES_VAR_FIELD_NUMBER);
+  if (!fieldNumberSlice.isNumber<size_t>()) {
+    THROW_ARANGO_EXCEPTION_FORMAT(
+        TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].fieldNumber\" %s should be a number",
+        fieldNumberSlice.toString().c_str());
+  }
+  auto const fieldNumber = fieldNumberSlice.getNumber<size_t>();
+
+  auto const varIdSlice = fieldVar.get(NODE_VIEW_VALUES_VAR_ID);
+  if (!varIdSlice.isNumber<aql::VariableId>()) {
+    THROW_ARANGO_EXCEPTION_FORMAT(
+        TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" variable id %s should be a number",
+        varIdSlice.toString().c_str());
+  }
+
+  auto const varId = varIdSlice.getNumber<aql::VariableId>();
+  auto const* var = vars->getVariable(varId);
+
+  if (!var) {
+    THROW_ARANGO_EXCEPTION_FORMAT(
+        TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" unable to find variable by id %d",
+        varId);
+  }
+  viewValuesVars[columnNumber].emplace_back(fieldNumber, var);
+}
+
+std::unique_ptr<aql::ExecutionBlock> (*executors[])(aql::ExecutionEngine*, IResearchViewNode const*, aql::IResearchViewExecutorInfos&&) = {
   [](aql::ExecutionEngine* engine, IResearchViewNode const* viewNode, aql::IResearchViewExecutorInfos&& infos) -> std::unique_ptr<aql::ExecutionBlock> {
     return std::make_unique<aql::ExecutionBlockImpl<aql::IResearchViewExecutor<false, MaterializeType::Materialized>>>(
       engine, viewNode, std::move(infos));
@@ -713,10 +762,10 @@ std::array<std::unique_ptr<aql::ExecutionBlock> (*)(
   }
 };
 
-inline decltype(executors)::size_type getExecutorIndex(bool sorted, bool ordered, MaterializeType materializeType) {
+inline int getExecutorIndex(bool sorted, bool ordered, MaterializeType materializeType) {
   auto index = static_cast<int>(materializeType) + 3 * static_cast<int>(ordered) + 6 * static_cast<int>(sorted);
-  TRI_ASSERT(static_cast<decltype(executors)::size_type>(index) <= executors.size());
+  TRI_ASSERT(static_cast<std::size_t>(index) <= IRESEARCH_COUNTOF(executors));
-  return static_cast<decltype(executors)::size_type>(index < 9 ? index : index - 1);
+  return index < 9 ? index : index - 1;
 }
 
 }  // namespace
@@ -926,33 +975,28 @@ IResearchViewNode::IResearchViewNode(aql::ExecutionPlan& plan, velocypack::Slice
       THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
                                      "\"ViewValuesVars\" attribute should be an array");
     }
-    std::unordered_map<size_t, aql::Variable const*> viewValuesVars;
+    ViewValuesVars viewValuesVars;
-    viewValuesVars.reserve(viewValuesVarsSlice.length());
+    for (auto const columnFieldsVars : velocypack::ArrayIterator(viewValuesVarsSlice)) {
-    for (auto const indVar : velocypack::ArrayIterator(viewValuesVarsSlice)) {
+      if (columnFieldsVars.hasKey(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER)) { // not SortColumnNumber
-      auto const fieldNumberSlice = indVar.get(NODE_VIEW_VALUES_VAR_FIELD_NUMBER);
+        auto const columnNumberSlice = columnFieldsVars.get(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER);
-      if (!fieldNumberSlice.isNumber<size_t>()) {
+        if (!columnNumberSlice.isNumber<size_t>()) {
-        THROW_ARANGO_EXCEPTION_FORMAT(
+          THROW_ARANGO_EXCEPTION_FORMAT(
-            TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].fieldNumber\" %s should be a number",
+              TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].columnNumber\" %s should be a number",
-              fieldNumberSlice.toString().c_str());
+              columnNumberSlice.toString().c_str());
+        }
+        auto const columnNumber = columnNumberSlice.getNumber<int>();
+        auto const viewStoredValuesVarsSlice = columnFieldsVars.get(NODE_VIEW_STORED_VALUES_VARS);
+        if (!viewStoredValuesVarsSlice.isArray()) {
+          THROW_ARANGO_EXCEPTION_MESSAGE(
+                TRI_ERROR_BAD_PARAMETER,
+                "\"ViewValuesVars[*].viewStoredValuesVars\" attribute should be an array");
+        }
+        for (auto const fieldVar : velocypack::ArrayIterator(viewStoredValuesVarsSlice)) {
+          extractViewValuesVar(vars, viewValuesVars, columnNumber, fieldVar);
+        }
+      } else { // SortColumnNumber
+        extractViewValuesVar(vars, viewValuesVars, SortColumnNumber, columnFieldsVars);
       }
-      auto const fieldNumber = fieldNumberSlice.getNumber<size_t>();
-
-      auto const varIdSlice = indVar.get(NODE_VIEW_VALUES_VAR_ID);
-      if (!varIdSlice.isNumber<aql::VariableId>()) {
-        THROW_ARANGO_EXCEPTION_FORMAT(
-            TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" variable id %s should be a number",
-              varIdSlice.toString().c_str());
-      }
-
-      auto const varId = varIdSlice.getNumber<aql::VariableId>();
-      auto const* var = vars->getVariable(varId);
-
-      if (!var) {
-        THROW_ARANGO_EXCEPTION_FORMAT(
-            TRI_ERROR_BAD_PARAMETER, "\"ViewValuesVars[*].id\" unable to find variable by id %d",
-              varId);
-      }
-      viewValuesVars.emplace(fieldNumber, var);
     }
     _outNonMaterializedViewVars = std::move(viewValuesVars);
   }
@@ -985,10 +1029,12 @@ void IResearchViewNode::planNodeRegisters(
     ++nrRegsHere[depth];
     ++nrRegs[depth];
     varInfo.try_emplace(_outNonMaterializedDocId->id, aql::VarInfo(depth, totalNrRegs++));
-    for (auto const& viewVar : _outNonMaterializedViewVars) {
+    for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
-      ++nrRegsHere[depth];
+      for (auto const& fieldVar : columnFieldsVars.second) {
-      ++nrRegs[depth];
+        ++nrRegsHere[depth];
-      varInfo.try_emplace(viewVar.second->id, aql::VarInfo(depth, totalNrRegs++));
+        ++nrRegs[depth];
+        varInfo.try_emplace(fieldVar.second->id, aql::VarInfo(depth, totalNrRegs++));
+      }
     }
   } else {
     ++nrRegsHere[depth];
@@ -1006,8 +1052,6 @@ std::pair<bool, bool> IResearchViewNode::volatility(bool force /*=false*/) const
                         irs::check_bit<1>(_volatilityMask));  // sort
 }
 
-
-
 /// @brief toVelocyPack, for EnumerateViewNode
 void IResearchViewNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags,
                                            std::unordered_set<ExecutionNode const*>& seen) const {
@@ -1034,19 +1078,38 @@ void IResearchViewNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags,
     _outNonMaterializedColPtr->toVelocyPack(nodes);
   }
 
+  // stored value
   {
     auto const& primarySort = ::primarySort(*_view);
+    auto const& storedValue = ::storedValue(*_view);
     VPackArrayBuilder arrayScope(&nodes, NODE_VIEW_VALUES_VARS);
     std::string fieldName;
-    for (auto const& viewVar : _outNonMaterializedViewVars) {
+    for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
-      VPackObjectBuilder objectScope(&nodes);
+      if (columnFieldsVars.first >= 0) { // not SortColumnNumber
-      nodes.add(NODE_VIEW_VALUES_VAR_FIELD_NUMBER, VPackValue(viewVar.first));
+        VPackObjectBuilder objectScope(&nodes);
-      nodes.add(NODE_VIEW_VALUES_VAR_ID, VPackValue(viewVar.second->id));
+        auto const& columns = storedValue.columns();
-      nodes.add(NODE_VIEW_VALUES_VAR_NAME, VPackValue(viewVar.second->name)); // for explainer.js
+        auto const storedColumnNumber = static_cast<decltype(columns.size())>(columnFieldsVars.first);
-      TRI_ASSERT(viewVar.first < primarySort.fields().size());
+        TRI_ASSERT(storedColumnNumber < columns.size());
-      fieldName.clear();
+        nodes.add(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER, VPackValue(columnFieldsVars.first));
-      basics::TRI_AttributeNamesToString(primarySort.fields()[viewVar.first], fieldName, true);
+        VPackArrayBuilder arrayScope(&nodes, NODE_VIEW_STORED_VALUES_VARS);
-      nodes.add(NODE_VIEW_VALUES_VAR_FIELD, VPackValue(fieldName)); // for explainer.js
+        for (auto const& fieldVar : columnFieldsVars.second) {
+          VPackObjectBuilder objectScope(&nodes);
+          fieldName.clear();
+          TRI_ASSERT(fieldVar.first < columns[storedColumnNumber].size());
+          nodes.add(NODE_VIEW_VALUES_VAR_COLUMN_NUMBER, VPackValue(columnFieldsVars.first));
+          fieldName = columns[storedColumnNumber][fieldVar.first].first;
+          addViewValuesVar(nodes, fieldName, fieldVar);
+        }
+      } else { // SortColumnNumber
+        TRI_ASSERT(SortColumnNumber == columnFieldsVars.first);
+        for (auto const& fieldVar : columnFieldsVars.second) {
+          VPackObjectBuilder objectScope(&nodes);
+          fieldName.clear();
+          TRI_ASSERT(fieldVar.first < primarySort.fields().size());
+          basics::TRI_AttributeNamesToString(primarySort.fields()[fieldVar.first], fieldName, true);
+          addViewValuesVar(nodes, fieldName, fieldVar);
+        }
+      }
     }
   }
 
@@ -1143,8 +1206,10 @@ aql::ExecutionNode* IResearchViewNode::clone(aql::ExecutionPlan* plan, bool with
       TRI_ASSERT(_outNonMaterializedDocId != nullptr);
       outNonMaterializedColId = plan->getAst()->variables()->createVariable(outNonMaterializedColId);
     }
-    for (auto& viewVar : outNonMaterializedViewVars) {
+    for (auto& columnFieldsVars : outNonMaterializedViewVars) {
-      viewVar.second = plan->getAst()->variables()->createVariable(viewVar.second);
+      for (auto& fieldVar : columnFieldsVars.second) {
+        fieldVar.second = plan->getAst()->variables()->createVariable(fieldVar.second);
+      }
     }
   }
 
@@ -1202,10 +1267,12 @@ std::vector<arangodb::aql::Variable const*> IResearchViewNode::getVariablesSetHe
   if (isLateMaterialized()) {
     vars.emplace_back(_outNonMaterializedColPtr);
     vars.emplace_back(_outNonMaterializedDocId);
-    std::transform(_outNonMaterializedViewVars.cbegin(),
+    for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
-                   _outNonMaterializedViewVars.cend(),
+      std::transform(columnFieldsVars.second.cbegin(),
-                   std::back_inserter(vars),
+                     columnFieldsVars.second.cend(),
-      [](auto const& viewVar) { return viewVar.second; });
+                     std::back_inserter(vars),
+        [](auto const& fieldVar) { return fieldVar.second; });
+    }
   } else {
     vars.emplace_back(_outVariable);
   }
@@ -1332,7 +1399,11 @@ std::unique_ptr<aql::ExecutionBlock> IResearchViewNode::createBlock(
   // the input registers.
   auto const firstOutputRegister = getNrInputRegisters();
   auto numScoreRegisters = static_cast<aql::RegisterCount>(_scorers.size());
-  auto numViewVarsRegisters = static_cast<aql::RegisterCount>(_outNonMaterializedViewVars.size());
+  auto numViewVarsRegisters = std::accumulate(_outNonMaterializedViewVars.cbegin(), _outNonMaterializedViewVars.cend(),
+    static_cast<aql::RegisterCount>(0),
+    [](aql::RegisterCount const sum, auto const& columnFieldsVars) {
+      return sum + static_cast<aql::RegisterCount>(columnFieldsVars.second.size());
+    });
   if (numViewVarsRegisters > 0) {
     TRI_ASSERT(materializeType == MaterializeType::LateMaterialized);
     materializeType = MaterializeType::LateMaterializedWithVars;
@@ -1369,14 +1440,14 @@ std::unique_ptr<aql::ExecutionBlock> IResearchViewNode::createBlock(
 
   auto const& varInfos = getRegisterPlan()->varInfo;
   ViewValuesRegisters outNonMaterializedViewRegs;
-  std::transform(_outNonMaterializedViewVars.cbegin(), _outNonMaterializedViewVars.cend(),
+  for (auto const& columnFieldsVars : _outNonMaterializedViewVars) {
-                 std::inserter(outNonMaterializedViewRegs, outNonMaterializedViewRegs.end()),
+    for (auto const& fieldsVars : columnFieldsVars.second) {
-                 [&varInfos](auto const& viewVar) {
+      auto& fields = outNonMaterializedViewRegs[columnFieldsVars.first];
-                   auto it = varInfos.find(viewVar.second->id);
+      auto it = varInfos.find(fieldsVars.second->id);
-                   TRI_ASSERT(it != varInfos.cend());
+      TRI_ASSERT(it != varInfos.cend());
-
+      fields.insert({fieldsVars.first, it->second.registerId});
-                   return std::make_pair(viewVar.first, it->second.registerId);
+    }
-                 });
+  }
 
   aql::IResearchViewExecutorInfos executorInfos{std::move(infos),
                                                 reader,
@@ -1385,6 +1456,7 @@ std::unique_ptr<aql::ExecutionBlock> IResearchViewNode::createBlock(
                                                 *engine.getQuery(),
                                                 scorers(),
                                                 _sort,
+                                                ::storedValue(*_view),
                                                 *plan(),
                                                 outVariable(),
                                                 filterCondition(),
@@ -1402,7 +1474,7 @@ bool IResearchViewNode::OptimizationState::canVariablesBeReplaced(aql::Calculati
   return _nodesToChange.find(calclulationNode) != _nodesToChange.cend(); // contains()
 }
 
-void IResearchViewNode::OptimizationState::saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs> const& nodesToChange) {
+void IResearchViewNode::OptimizationState::saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs<aql::latematerialized::AstAndColumnFieldData>> const& nodesToChange) {
   TRI_ASSERT(!nodesToChange.empty());
   TRI_ASSERT(_nodesToChange.empty());
   _nodesToChange.clear();
@@ -1424,7 +1496,7 @@ IResearchViewNode::ViewVarsInfo IResearchViewNode::OptimizationState::replaceVie
     auto const& calcNodeData = _nodesToChange[calcNode];
     std::transform(calcNodeData.cbegin(), calcNodeData.cend(), std::inserter(uniqueVariables, uniqueVariables.end()),
       [ast](auto const& afData) {
-        return std::make_pair(afData.field, ViewVariable{afData.number,
+        return std::make_pair(afData.field, ViewVariable{afData.columnNumber, afData.number,
           ast->variables()->createTemporaryVariable()});
       });
   }

arangod/Aql/IResearchViewNode.h

@@ -29,6 +29,7 @@
 #include "Aql/types.h"
 #include "IResearch/IResearchOrderFactory.h"
 #include "IResearch/IResearchViewSort.h"
+#include "IResearch/IResearchViewStoredValue.h"
 
 namespace arangodb {
 class LogicalView;
@@ -181,33 +182,36 @@ class IResearchViewNode final : public arangodb::aql::ExecutionNode {
            _outNonMaterializedColPtr != nullptr;
   }
 
+  static constexpr int SortColumnNumber = -1;
+
   struct ViewVariable {
-    size_t viewFieldNum;
+    int columnNum;
+    size_t fieldNum;
     aql::Variable const* var;
   };
 
-  using ViewValuesVars = std::unordered_map<size_t, aql::Variable const*>;
+  using ViewValuesVars = std::unordered_map<int, std::vector<std::pair<size_t, aql::Variable const*>>>;
 
-  using ViewValuesRegisters = std::map<size_t, aql::RegisterId>;
+  using ViewValuesRegisters = std::map<int, std::map<size_t, aql::RegisterId>>;
 
   using ViewVarsInfo = std::unordered_map<std::vector<arangodb::basics::AttributeName> const*, ViewVariable>;
 
   void setViewVariables(ViewVarsInfo const& viewVariables) {
     _outNonMaterializedViewVars.clear();
     for (auto& viewVars : viewVariables) {
-      _outNonMaterializedViewVars[viewVars.second.viewFieldNum] = viewVars.second.var;
+      _outNonMaterializedViewVars[viewVars.second.columnNum].emplace_back(viewVars.second.fieldNum, viewVars.second.var);
     }
   }
 
   // The structure is used for temporary saving of optimization rule data.
   // It contains document references that could be replaced in late materialization rule.
   struct OptimizationState {
-    using ViewVarsToBeReplaced = std::vector<aql::latematerialized::AstAndFieldData>;
+    using ViewVarsToBeReplaced = std::vector<aql::latematerialized::AstAndColumnFieldData>;
 
     /// @brief calculation node with ast nodes that can be replaced by view values (e.g. primary sort)
     std::unordered_map<aql::CalculationNode*, ViewVarsToBeReplaced> _nodesToChange;
 
-    void saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs> const& nodesToChange);
+    void saveCalcNodesForViewVariables(std::vector<aql::latematerialized::NodeWithAttrs<aql::latematerialized::AstAndColumnFieldData>> const& nodesToChange);
 
     bool canVariablesBeReplaced(aql::CalculationNode* calclulationNode) const;
 

arangod/Aql/IResearchViewOptimizerRules.cpp

@@ -64,6 +64,16 @@ inline IResearchViewSort const& primarySort(arangodb::LogicalView const& view) {
   return viewImpl.primarySort();
 }
 
+inline IResearchViewStoredValue const& storedValue(arangodb::LogicalView const& view) {
+  if (arangodb::ServerState::instance()->isCoordinator()) {
+    auto& viewImpl = arangodb::LogicalView::cast<IResearchViewCoordinator>(view);
+    return viewImpl.storedValue();
+  }
+
+  auto& viewImpl = arangodb::LogicalView::cast<IResearchView>(view);
+  return viewImpl.storedValue();
+}
+
 bool addView(arangodb::LogicalView const& view, Query& query) {
   auto* collections = query.collections();
 
@@ -263,19 +273,45 @@ bool optimizeSort(IResearchViewNode& viewNode, ExecutionPlan* plan) {
   }
 }
 
-bool attributesMatch(IResearchViewSort const& primarySort, latematerialized::NodeWithAttrs& node) {
+bool attributesMatch(IResearchViewSort const& primarySort, IResearchViewStoredValue const& storedValue,
+                     latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData>& node) {
+  // TODO: choose the best variant!
+
   // check all node attributes to be in sort
   for (auto& nodeAttr : node.attrs) {
-    size_t sortFieldNum = 0;
+    nodeAttr.afData.field = nullptr;
+    // try to find in the sort column
+    size_t fieldNum = 0;
     for (auto const& field : primarySort.fields()) {
       if (arangodb::basics::AttributeName::isIdentical(nodeAttr.attr, field, false)) {
-        nodeAttr.afData.number = sortFieldNum;
+        nodeAttr.afData.number = fieldNum;
+        nodeAttr.afData.columnNumber = IResearchViewNode::SortColumnNumber;
         nodeAttr.afData.field = &field;
         nodeAttr.attr.clear(); // we do not need later
         nodeAttr.attr.shrink_to_fit();
         break;
       }
-      ++sortFieldNum;
+      ++fieldNum;
+    }
+    if (nodeAttr.afData.field != nullptr) {
+      continue;
+    }
+    // try to find in other columns
+    int columnNum = 0;
+    fieldNum = 0;
+    for (auto const& column : storedValue.columns()) {
+      for (auto const& field : column) {
+        if (arangodb::basics::AttributeName::isIdentical(nodeAttr.attr, field.second, false)) {
+          nodeAttr.afData.number = fieldNum;
+          nodeAttr.afData.field = &field.second;
+          nodeAttr.afData.columnNumber = columnNum;
+          nodeAttr.attr.clear(); // we do not need later
+          nodeAttr.attr.shrink_to_fit();
+          break;
+        }
+        ++fieldNum;
+      }
+      ++columnNum;
     }
     // not found
     if (nodeAttr.afData.field == nullptr) {
@@ -287,13 +323,14 @@ bool attributesMatch(IResearchViewSort const& primarySort, latematerialized::Nod
 
 void keepReplacementViewVariables(arangodb::containers::SmallVector<ExecutionNode*> const& calcNodes,
                                   arangodb::containers::SmallVector<ExecutionNode*> const& viewNodes) {
-  std::vector<latematerialized::NodeWithAttrs> nodesToChange;
+  std::vector<latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData>> nodesToChange;
   for (auto* vNode : viewNodes) {
     TRI_ASSERT(vNode && ExecutionNode::ENUMERATE_IRESEARCH_VIEW == vNode->getType());
     auto& viewNode = *ExecutionNode::castTo<IResearchViewNode*>(vNode);
     auto const& primarySort = ::primarySort(*viewNode.view());
-    if (primarySort.empty()) {
+    auto const& storedValue = ::storedValue(*viewNode.view());
-      // no primary sort
+    if (primarySort.empty() && storedValue.empty()) {
+      // neither primary sort nor stored value
       continue;
     }
     auto const& var = viewNode.outVariable();
@@ -302,11 +339,11 @@ void keepReplacementViewVariables(arangodb::containers::SmallVector<ExecutionNod
       TRI_ASSERT(cNode && ExecutionNode::CALCULATION == cNode->getType());
       auto& calcNode = *ExecutionNode::castTo<CalculationNode*>(cNode);
       auto astNode = calcNode.expression()->nodeForModification();
-      latematerialized::NodeWithAttrs node;
+      latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData> node;
       node.node = &calcNode;
       // find attributes referenced to view node out variable
       if (latematerialized::getReferencedAttributes(astNode, &var, node) &&
-          !node.attrs.empty() && attributesMatch(primarySort, node)) {
+          !node.attrs.empty() && attributesMatch(primarySort, storedValue, node)) {
         nodesToChange.emplace_back(std::move(node));
       }
     }

arangod/Aql/IndexNode.cpp

@@ -135,7 +135,7 @@ IndexNode::IndexNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
       THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
                                      "\"IndexValuesVars\" attribute should be an array");
     }
-    std::unordered_map<size_t, Variable const*> indexValuesVars;
+    std::vector<std::pair<size_t, Variable const*>> indexValuesVars;
     indexValuesVars.reserve(indexValuesVarsSlice.length());
     for (auto const indVar : velocypack::ArrayIterator(indexValuesVarsSlice)) {
       auto const fieldNumberSlice = indVar.get("fieldNumber");
@@ -161,7 +161,7 @@ IndexNode::IndexNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
             TRI_ERROR_BAD_PARAMETER, "\"IndexValuesVars[*].id\" unable to find variable by id %d",
               varId);
       }
-      indexValuesVars.emplace(fieldNumber, var);
+      indexValuesVars.emplace_back(fieldNumber, var);
     }
     _outNonMaterializedIndVars.first = indexId;
     _outNonMaterializedIndVars.second = std::move(indexValuesVars);
@@ -240,10 +240,10 @@ void IndexNode::planNodeRegisters(
   if (isLateMaterialized()) {
     varInfo.emplace(_outNonMaterializedDocId->id, aql::VarInfo(depth, totalNrRegs++));
     // plan registers for index references
-    for (auto const& var : _outNonMaterializedIndVars.second) {
+    for (auto const& fieldVar : _outNonMaterializedIndVars.second) {
       ++nrRegsHere[depth];
       ++nrRegs[depth];
-      varInfo.emplace(var.second->id, aql::VarInfo(depth, totalNrRegs++));
+      varInfo.emplace(fieldVar.second->id, aql::VarInfo(depth, totalNrRegs++));
     }
   } else {
     varInfo.emplace(_outVariable->id, aql::VarInfo(depth, totalNrRegs++));
@@ -295,14 +295,14 @@ void IndexNode::toVelocyPackHelper(VPackBuilder& builder, unsigned flags,
     TRI_ASSERT(indIt != _indexes.cend());
     auto const& fields = (*indIt)->fields();
     VPackArrayBuilder arrayScope(&builder, "IndexValuesVars");
-    for (auto const& indVar : _outNonMaterializedIndVars.second) {
+    for (auto const& fieldVar : _outNonMaterializedIndVars.second) {
       VPackObjectBuilder objectScope(&builder);
-      builder.add("fieldNumber", VPackValue(indVar.first));
+      builder.add("fieldNumber", VPackValue(fieldVar.first));
-      builder.add("id", VPackValue(indVar.second->id));
+      builder.add("id", VPackValue(fieldVar.second->id));
-      builder.add("name", VPackValue(indVar.second->name)); // for explainer.js
+      builder.add("name", VPackValue(fieldVar.second->name)); // for explainer.js
       std::string fieldName;
-      TRI_ASSERT(indVar.first < fields.size());
+      TRI_ASSERT(fieldVar.first < fields.size());
-      basics::TRI_AttributeNamesToString(fields[indVar.first], fieldName, true);
+      basics::TRI_AttributeNamesToString(fields[fieldVar.first], fieldName, true);
       builder.add("field", VPackValue(fieldName)); // for explainer.js
     }
   }
@@ -653,7 +653,7 @@ void IndexNode::setLateMaterialized(aql::Variable const* docIdVariable,
   _outNonMaterializedIndVars.first = commonIndexId;
   _outNonMaterializedDocId = docIdVariable;
   for (auto& indVars : indexVariables) {
-    _outNonMaterializedIndVars.second[indVars.second.indexFieldNum] = indVars.second.var;
+    _outNonMaterializedIndVars.second.emplace_back(indVars.second.indexFieldNum, indVars.second.var);
   }
 }
 

arangod/Aql/IndexNode.h

@@ -132,7 +132,7 @@ class IndexNode : public ExecutionNode, public DocumentProducingNode, public Col
     Variable const* var;
   };
 
-  using IndexValuesVars = std::pair<TRI_idx_iid_t, std::unordered_map<size_t, Variable const*>>;
+  using IndexValuesVars = std::pair<TRI_idx_iid_t, std::vector<std::pair<size_t, Variable const*>>>;
 
   using IndexValuesRegisters = std::pair<TRI_idx_iid_t, std::unordered_map<size_t, RegisterId>>;
 

arangod/Aql/IndexNodeOptimizerRules.cpp

@@ -34,9 +34,10 @@
 using namespace arangodb::aql;
 
 namespace {
-  bool attributesMatch(TRI_idx_iid_t& commonIndexId, IndexNode const* indexNode, latematerialized::NodeWithAttrs& node) {
+  bool attributesMatch(TRI_idx_iid_t& commonIndexId, IndexNode const* indexNode, latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData>& node) {
     // check all node attributes to be in index
     for (auto& nodeAttr : node.attrs) {
+      nodeAttr.afData.field = nullptr;
       for (auto& index : indexNode->getIndexes()) {
         if (!index->hasCoveringIterator()) {
           continue;
@@ -105,7 +106,7 @@ void arangodb::aql::lateDocumentMaterializationRule(Optimizer* opt,
       // this node could be appended with materializer
       bool stopSearch = false;
       bool stickToSortNode = false;
-      std::vector<latematerialized::NodeWithAttrs> nodesToChange;
+      std::vector<latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData>> nodesToChange;
       TRI_idx_iid_t commonIndexId = 0; // use one index only
       while (current != loop) {
         auto type = current->getType();
@@ -118,7 +119,7 @@ void arangodb::aql::lateDocumentMaterializationRule(Optimizer* opt,
           case ExecutionNode::CALCULATION: {
             auto calculationNode = ExecutionNode::castTo<CalculationNode*>(current);
             auto astNode = calculationNode->expression()->nodeForModification();
-            latematerialized::NodeWithAttrs node;
+            latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData> node;
             node.node = calculationNode;
             // find attributes referenced to index node out variable
             if (!latematerialized::getReferencedAttributes(astNode, indexNode->outVariable(), node)) {

arangod/Aql/LateMaterializedOptimizerRulesCommon.cpp

@@ -50,19 +50,21 @@ void traverseReadOnly(AstNode* node, AstNode* parentNode, size_t childNumber,
 }
 
 // traversal state
+template<typename T>
 struct TraversalState {
   Variable const* variable;
-  latematerialized::NodeWithAttrs& nodeAttrs;
+  latematerialized::NodeWithAttrs<T>& nodeAttrs;
   bool optimize;
   bool wasAccess;
 };
 }
 
 // determines attributes referenced in an expression for the specified out variable
+template<typename T>
 bool latematerialized::getReferencedAttributes(AstNode* node,
                                                Variable const* variable,
-                                               NodeWithAttrs& nodeAttrs) {
+                                               NodeWithAttrs<T>& nodeAttrs) {
-  TraversalState state{variable, nodeAttrs, true, false};
+  TraversalState<T> state{variable, nodeAttrs, true, false};
 
   auto preVisitor = [&state](AstNode const* node,
       AstNode* parentNode, size_t childNumber) {
@@ -73,9 +75,12 @@ bool latematerialized::getReferencedAttributes(AstNode* node,
     switch (node->type) {
       case NODE_TYPE_ATTRIBUTE_ACCESS:
         if (!state.wasAccess) {
+          T afData;
+          afData.parentNode = parentNode;
+          afData.childNumber = childNumber;
           state.nodeAttrs.attrs.emplace_back(
-            NodeWithAttrs::AttributeAndField{std::vector<arangodb::basics::AttributeName>{
+            typename NodeWithAttrs<T>::AttributeAndField{std::vector<arangodb::basics::AttributeName>{
-              {std::string(node->getStringValue(), node->getStringLength()), false}}, {parentNode, childNumber, nullptr, 0}});
+              {std::string(node->getStringValue(), node->getStringLength()), false}}, std::move(afData)});
           state.wasAccess = true;
         } else {
           state.nodeAttrs.attrs.back().attr.emplace_back(std::string(node->getStringValue(), node->getStringLength()), false);
@@ -118,3 +123,16 @@ bool latematerialized::getReferencedAttributes(AstNode* node,
 
   return state.optimize;
 }
+
+template struct latematerialized::NodeWithAttrs<latematerialized::AstAndFieldData>;
+template struct latematerialized::NodeWithAttrs<latematerialized::AstAndColumnFieldData>;
+
+template bool latematerialized::getReferencedAttributes(
+  AstNode* node,
+  Variable const* variable,
+  NodeWithAttrs<latematerialized::AstAndFieldData>& nodeAttrs);
+
+template bool latematerialized::getReferencedAttributes(
+  AstNode* node,
+  Variable const* variable,
+  NodeWithAttrs<latematerialized::AstAndColumnFieldData>& nodeAttrs);

arangod/Aql/LateMaterializedOptimizerRulesCommon.h

@@ -49,17 +49,23 @@ struct AstAndFieldData {
   size_t number;
 };
 
+struct AstAndColumnFieldData : AstAndFieldData {
+  int columnNumber;
+};
+
+template<typename T>
 struct NodeWithAttrs {
   struct AttributeAndField {
     std::vector<arangodb::basics::AttributeName> attr;
-    AstAndFieldData afData;
+    T afData;
   };
 
   std::vector<AttributeAndField> attrs;
   CalculationNode* node;
 };
 
-bool getReferencedAttributes(AstNode* node, Variable const* variable, NodeWithAttrs& nodeAttrs);
+template<typename T>
+bool getReferencedAttributes(AstNode* node, Variable const* variable, NodeWithAttrs<T>& nodeAttrs);
 
 }  // latematerialized
 }  // namespace aql

arangod/CMakeLists.txt

@@ -65,6 +65,7 @@ add_library(arango_iresearch
   IResearch/IResearchView.cpp IResearch/IResearchView.h
   IResearch/IResearchVPackComparer.cpp IResearch/IResearchVPackComparer.h
   IResearch/IResearchViewSort.cpp IResearch/IResearchViewSort.h
+  IResearch/IResearchViewStoredValue.cpp IResearch/IResearchViewStoredValue.h
   IResearch/IResearchViewCoordinator.cpp IResearch/IResearchViewCoordinator.h
   IResearch/IResearchExpressionContext.cpp IResearch/IResearchExpressionContext.h
   IResearch/IResearchViewMeta.cpp IResearch/IResearchViewMeta.h

arangod/IResearch/IResearchLink.cpp

@@ -155,6 +155,35 @@ inline arangodb::Result insertDocument(irs::index_writer::documents_context& ctx
     }
   }
 
+  // Stored value field
+  {
+    struct StoredValue {
+      bool write(irs::data_output& out) const {
+        out.write_bytes(slice.start(), slice.byteSize());
+        return true;
+      }
+
+      irs::string_ref const& name() {
+        return fieldName;
+      }
+
+      VPackSlice slice;
+      irs::string_ref fieldName;
+    } field; // StoredValue
+
+    for (auto const& column : meta._storedValue.columns()) {
+      field.fieldName = irs::string_ref{};
+      for (auto const& storedValue : column) {
+        // column name is equal to the first field name (TODO: two can have the same)
+        if (field.fieldName.empty()) {
+          field.fieldName = irs::string_ref(storedValue.first);
+        }
+        field.slice = arangodb::iresearch::get(document, storedValue.second, VPackSlice::nullSlice());
+      }
+      doc.insert<irs::Action::STORE>(field);
+    }
+  }
+
   // System fields
 
   // Indexed and Stored: LocalDocumentId
@@ -1804,6 +1833,10 @@ void IResearchLink::toVelocyPackStats(VPackBuilder& builder) const {
   builder.add("indexSize", VPackValue(stats.indexSize));
 }
 
+IResearchViewStoredValue const& IResearchLink::storedValue() const {
+  return _meta._storedValue;
+}
+
 }  // namespace iresearch
 }  // namespace arangodb
 

arangod/IResearch/IResearchLink.h

@@ -224,6 +224,11 @@ class IResearchLink {
   ////////////////////////////////////////////////////////////////////////////////
   AnalyzerPool::ptr findAnalyzer(AnalyzerPool const& analyzer) const;
 
+  ////////////////////////////////////////////////////////////////////////////////
+  /// @brief get stored value
+  ////////////////////////////////////////////////////////////////////////////////
+  IResearchViewStoredValue const& storedValue() const;
+
  protected:
   //////////////////////////////////////////////////////////////////////////////
   /// @brief index stats

arangod/IResearch/IResearchLinkMeta.cpp

@@ -545,6 +545,10 @@ bool IResearchLinkMeta::operator==(IResearchLinkMeta const& other) const noexcep
     return false;
   }
 
+  if (_storedValue != other._storedValue) {
+    return false;
+  }
+
   return true;
 }
 
@@ -577,10 +581,20 @@ bool IResearchLinkMeta::init(velocypack::Slice const& slice,
     auto const field = slice.get(fieldName);
     mask->_sort = field.isArray();
 
-    if (readAnalyzerDefinition && mask->_sort) {
+    if (readAnalyzerDefinition && mask->_sort && !_sort.fromVelocyPack(field, errorField)) {
-      if (!_sort.fromVelocyPack(field, errorField)) {
+      return false;
-        return false;
+    }
-      }
+  }
+
+  {
+    // optional stored values
+    static VPackStringRef const fieldName("storedFields");
+
+    auto const field = slice.get(fieldName);
+    mask->_storedValue = field.isArray();
+
+    if (readAnalyzerDefinition && mask->_storedValue && !_storedValue.fromVelocyPack(field, errorField)) {
+      return false;
     }
   }
 
@@ -756,6 +770,14 @@ bool IResearchLinkMeta::json(velocypack::Builder& builder,
     }
   }
 
+  if (writeAnalyzerDefinition
+      && (!mask || mask->_storedValue)) {
+    velocypack::ArrayBuilder arrayScope(&builder, "storedFields");
+    if (!_storedValue.toVelocyPack(builder)) {
+      return false;
+    }
+  }
+
   // output definitions if 'writeAnalyzerDefinition' requested and not maked
   // this should be the case for the default top-most call
   if (writeAnalyzerDefinition && (!mask || mask->_analyzerDefinitions)) {
@@ -775,6 +797,7 @@ size_t IResearchLinkMeta::memory() const noexcept {
 
   size += _analyzers.size() * sizeof(decltype(_analyzers)::value_type);
   size += _sort.memory();
+  size += _storedValue.memory();
   size += FieldMeta::memory();
 
   return size;

arangod/IResearch/IResearchLinkMeta.h

@@ -34,6 +34,7 @@
 #include "Containers.h"
 #include "IResearchAnalyzerFeature.h"
 #include "IResearchViewSort.h"
+#include "IResearchViewStoredValue.h"
 
 namespace arangodb {
 namespace velocypack {
@@ -180,15 +181,18 @@ struct IResearchLinkMeta : public FieldMeta {
     explicit Mask(bool mask = false) noexcept
       : FieldMeta::Mask(mask),
         _analyzerDefinitions(mask),
-        _sort(mask) {
+        _sort(mask),
+        _storedValue(mask) {
     }
 
     bool _analyzerDefinitions;
     bool _sort;
+    bool _storedValue;
   };
 
   std::set<AnalyzerPool::ptr, FieldMeta::AnalyzerComparer> _analyzerDefinitions;
   IResearchViewSort _sort; // sort condition associated with the link
+  IResearchViewStoredValue _storedValue; // stored value associated with the link
   // NOTE: if adding fields don't forget to modify the comparison operator !!!
   // NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask !!!
   // NOTE: if adding fields don't forget to modify IResearchLinkMeta::Mask constructor !!!

arangod/IResearch/IResearchView.cpp

@@ -134,6 +134,7 @@ void ensureImmutableProperties(
   dst._writebufferIdle = src._writebufferIdle;
   dst._writebufferSizeMax = src._writebufferSizeMax;
   dst._primarySort = src._primarySort;
+  dst._storedValue = src._storedValue;
 }
 
 }

arangod/IResearch/IResearchView.h

@@ -105,7 +105,7 @@ class IResearchView final: public arangodb::LogicalView {
   ///////////////////////////////////////////////////////////////////////////////
   /// @brief destructor to clean up resources
   ///////////////////////////////////////////////////////////////////////////////
-  virtual ~IResearchView();
+  virtual ~IResearchView() override;
 
   using arangodb::LogicalView::name;
 
@@ -178,6 +178,13 @@ class IResearchView final: public arangodb::LogicalView {
     return _meta._primarySort;
   }
 
+  ///////////////////////////////////////////////////////////////////////////////
+  /// @return stored value from links collections
+  ///////////////////////////////////////////////////////////////////////////////
+  IResearchViewStoredValue const& storedValue() const noexcept {
+    return _meta._storedValue;
+  }
+
  protected:
 
   //////////////////////////////////////////////////////////////////////////////

arangod/IResearch/IResearchViewCoordinator.cpp

@@ -57,6 +57,7 @@ void ensureImmutableProperties(
   dst._writebufferIdle = src._writebufferIdle;
   dst._writebufferSizeMax = src._writebufferSizeMax;
   dst._primarySort = src._primarySort;
+  dst._storedValue = src._storedValue;
 }
 
 }  // namespace

arangod/IResearch/IResearchViewCoordinator.h

@@ -88,6 +88,13 @@ class IResearchViewCoordinator final : public arangodb::LogicalView {
     return _meta._primarySort;
   }
 
+  ///////////////////////////////////////////////////////////////////////////////
+  /// @return stored value from links collections
+  ///////////////////////////////////////////////////////////////////////////////
+  IResearchViewStoredValue const& storedValue() const noexcept {
+    return _meta._storedValue;
+  }
+
  protected:
   virtual Result appendVelocyPackImpl(arangodb::velocypack::Builder& builder,
                                       Serialization context) const override;

arangod/IResearch/IResearchViewMeta.cpp

@@ -252,6 +252,7 @@ IResearchViewMeta& IResearchViewMeta::operator=(IResearchViewMeta&& other) noexc
     _writebufferIdle = std::move(other._writebufferIdle);
     _writebufferSizeMax = std::move(other._writebufferSizeMax);
     _primarySort = std::move(other._primarySort);
+    _storedValue = std::move(other._storedValue);
   }
 
   return *this;
@@ -269,6 +270,7 @@ IResearchViewMeta& IResearchViewMeta::operator=(IResearchViewMeta const& other)
     _writebufferIdle = other._writebufferIdle;
     _writebufferSizeMax = other._writebufferSizeMax;
     _primarySort = other._primarySort;
+    _storedValue = other._storedValue;
   }
 
   return *this;
@@ -298,27 +300,16 @@ bool IResearchViewMeta::operator==(IResearchViewMeta const& other) const noexcep
   if (irs::locale_utils::language(_locale) != irs::locale_utils::language(other._locale) ||
       irs::locale_utils::country(_locale) != irs::locale_utils::country(other._locale) ||
       irs::locale_utils::encoding(_locale) != irs::locale_utils::encoding(other._locale)) {
-    return false;  // values do not match
+    return false;
   }
 
-  if (_version != other._version) {
+  if (_version != other._version ||
-    return false;  // values do not match
+      _writebufferActive != other._writebufferActive ||
-  }
+      _writebufferIdle != other._writebufferIdle ||
-
+      _writebufferSizeMax != other._writebufferSizeMax ||
-  if (_writebufferActive != other._writebufferActive) {
+      _primarySort != other._primarySort ||
-    return false;  // values do not match
+      _storedValue != other._storedValue) {
-  }
+    return false;
-
-  if (_writebufferIdle != other._writebufferIdle) {
-    return false;  // values do not match
-  }
-
-  if (_writebufferSizeMax != other._writebufferSizeMax) {
-    return false;  // values do not match
-  }
-
-  if (_primarySort != other._primarySort) {
-    return false;  // values do not match
   }
 
   return true;
@@ -588,7 +579,27 @@ bool IResearchViewMeta::init(arangodb::velocypack::Slice const& slice, std::stri
     } else if (!_primarySort.fromVelocyPack(field, errorSubField)) {
       errorField = fieldName.toString();
       if (!errorSubField.empty()) {
-       errorField += errorSubField;
+        errorField += errorSubField;
+      }
+
+      return false;
+    }
+  }
+
+  {
+    // optional object
+    static VPackStringRef const fieldName("storedFields");
+    std::string errorSubField;
+
+    auto const field = slice.get(fieldName);
+    mask->_storedValue = field.isArray();
+
+    if (!mask->_storedValue) {
+      _storedValue = defaults._storedValue;
+    } else if (!_storedValue.fromVelocyPack(field, errorSubField)) {
+      errorField = fieldName.toString();
+      if (!errorSubField.empty()) {
+        errorField += errorSubField;
       }
 
       return false;
@@ -662,6 +673,13 @@ bool IResearchViewMeta::json(arangodb::velocypack::Builder& builder,
     }
   }
 
+  if ((!ignoreEqual || _storedValue != ignoreEqual->_storedValue) && (!mask || mask->_storedValue)) {
+    velocypack::ArrayBuilder arrayScope(&builder, "storedFields");
+    if (!_storedValue.toVelocyPack(builder)) {
+      return false;
+    }
+  }
+
   return true;
 }
 

arangod/IResearch/IResearchViewMeta.h

@@ -28,6 +28,7 @@
 #include <unordered_set>
 
 #include "IResearchViewSort.h"
+#include "IResearchViewStoredValue.h"
 
 #include <velocypack/Builder.h>
 
@@ -86,6 +87,7 @@ struct IResearchViewMeta {
     bool _writebufferIdle;
     bool _writebufferSizeMax;
     bool _primarySort;
+    bool _storedValue;
     explicit Mask(bool mask = false) noexcept;
   };
 
@@ -99,6 +101,7 @@ struct IResearchViewMeta {
   size_t _writebufferIdle; // maximum number of segments cached in the pool
   size_t _writebufferSizeMax; // maximum memory byte size per segment before a segment flush is triggered (0 == unlimited)
   IResearchViewSort _primarySort;
+  IResearchViewStoredValue _storedValue;
   // NOTE: if adding fields don't forget to modify the default constructor !!!
   // NOTE: if adding fields don't forget to modify the copy constructor !!!
   // NOTE: if adding fields don't forget to modify the move constructor !!!

arangod/IResearch/IResearchViewStoredValue.cpp

@@ -0,0 +1,119 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Yuriy Popov
+////////////////////////////////////////////////////////////////////////////////
+
+#include "IResearchViewStoredValue.h"
+
+#include <velocypack/Builder.h>
+#include <velocypack/Iterator.h>
+
+#include "VelocyPackHelper.h"
+
+namespace arangodb {
+namespace iresearch {
+
+/*
+{"links" : {
+            "mycol1" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value",
+                       "storedFields": [["obj.foo.val1", "obj.foo.val2"], ["obj.bar.val1", "obj.bar.val2"]]}, // string
+
+            "mycol2" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value"}
+           }
+}
+*/
+
+bool IResearchViewStoredValue::toVelocyPack(velocypack::Builder& builder) const {
+  if (!builder.isOpenArray()) {
+    return false;
+  }
+  for (auto const& column : _storedColumns) {
+    velocypack::ArrayBuilder arrayScope(&builder);
+    for (auto const& field : column) {
+      builder.add(VPackValue(field.first));
+    }
+  }
+  return true;
+}
+
+bool IResearchViewStoredValue::fromVelocyPack(
+    velocypack::Slice slice, std::string& error) {
+  clear();
+  if (slice.isArray()) {
+    _storedColumns.reserve(slice.length());
+    for (auto columnSlice : VPackArrayIterator(slice)) {
+      if (columnSlice.isArray()) {
+        StoredColumn sc;
+        sc.reserve(columnSlice.length());
+        for (auto fieldSlice : VPackArrayIterator(columnSlice)) {
+          if (!fieldSlice.isString()) {
+            clear();
+            return false;
+          }
+          auto fieldName = arangodb::iresearch::getStringRef(slice);
+          std::vector<basics::AttributeName> field;
+          try {
+            arangodb::basics::TRI_ParseAttributeString(fieldName, field, false);
+          } catch (...) {
+            error = "." + std::string(fieldName);
+            clear();
+            return false;
+          }
+          sc.emplace_back(fieldName, std::move(field));
+        }
+        _storedColumns.emplace_back(std::move(sc));
+      } else if (columnSlice.isString()) {
+        auto fieldName = arangodb::iresearch::getStringRef(slice);
+        std::vector<basics::AttributeName> field;
+        try {
+          arangodb::basics::TRI_ParseAttributeString(fieldName, field, false);
+        } catch (...) {
+          error = "." + std::string(fieldName);
+          return false;
+        }
+        _storedColumns.emplace_back(StoredColumn{{fieldName, std::move(field)}});
+      } else {
+        clear();
+        return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+size_t IResearchViewStoredValue::memory() const noexcept {
+  size_t size = sizeof(IResearchViewStoredValue);
+  size += sizeof(StoredColumn)*_storedColumns.size();
+  for (auto const& column : _storedColumns) {
+    size += sizeof(std::pair<std::string, std::vector<basics::AttributeName>>)*column.size();
+    for (auto const& field : column) {
+      size += field.first.size();
+      size += sizeof(basics::AttributeName)*field.second.size();
+      for (auto const& attribute : field.second) {
+        size += attribute.name.size();
+      }
+    }
+  }
+  return size;
+}
+
+} // iresearch
+} // arangodb

arangod/IResearch/IResearchViewStoredValue.h

@@ -0,0 +1,85 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Yuriy Popov
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGODB_IRESEARCH__IRESEARCH_VIEW_STORED_VALUE_H
+#define ARANGODB_IRESEARCH__IRESEARCH_VIEW_STORED_VALUE_H 1
+
+#include "Basics/AttributeNameParser.h"
+#include "Basics/debugging.h"
+
+#include <velocypack/Slice.h>
+
+namespace arangodb {
+
+namespace velocypack {
+class Builder;
+}
+
+/*
+{"links" : {
+            "mycol1" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value",
+                       "storedFields": [["obj.foo.val1", "obj.foo.val2"], ["obj.bar.val1", "obj.bar.val2"]]}, // or string
+
+            "mycol2" : {"fields" : {"str" : {"analyzers" : ["text_en"]}}, "includeAllFields" : true, "storeValues" : "value"}
+           }
+}
+*/
+
+namespace iresearch {
+
+class IResearchViewStoredValue {
+ public:
+  using StoredColumn = std::vector<std::pair<std::string, std::vector<basics::AttributeName>>>;
+
+  bool operator==(IResearchViewStoredValue const& rhs) const noexcept {
+    return _storedColumns == rhs._storedColumns;
+  }
+
+  bool operator!=(IResearchViewStoredValue const& rhs) const noexcept {
+    return !(*this == rhs);
+  }
+
+  std::vector<StoredColumn> const& columns() const noexcept {
+    return _storedColumns;
+  }
+
+  size_t memory() const noexcept;
+
+  bool empty() const noexcept {
+    return _storedColumns.empty();
+  }
+
+  bool toVelocyPack(velocypack::Builder& builder) const;
+  bool fromVelocyPack(velocypack::Slice, std::string& error);
+
+ private:
+  void clear() noexcept {
+    _storedColumns.clear();
+  }
+
+  std::vector<StoredColumn> _storedColumns;
+}; // IResearchViewStoredValue
+
+} // iresearch
+} // arangodb
+
+#endif // ARANGODB_IRESEARCH__IRESEARCH_VIEW_STORED_VALUE_H