Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/arangosearch gather node (#5487) 

* integrate ArangoSearch into GatherNode/GatherBlock logic

* fix inconsistency after merge

* add serialization/deserealization logic for IResearchViewNode sorting

* fix optimizer rule order

* fix copy/past typo

* fix tests

* ensure `GatherNode` is properly set up for ArangoSearch views

* adjust optimizer rule order
This commit is contained in:
Andrey Abramov 2018-05-30 02:53:21 +03:00 committed by GitHub
parent f5d7053bb1
commit f708e46973
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 626 additions and 412 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
arangod/Aql/ClusterBlocks.cpp
View File

@ -56,25 +56,103 @@ using namespace arangodb::aql;
using VelocyPackHelper = arangodb::basics::VelocyPackHelper;
using StringBuffer = arangodb::basics::StringBuffer;
namespace {
/// @brief OurLessThan: comparison method for elements of _gatherBlockPos
class OurLessThan {
public:
OurLessThan(
transaction::Methods* trx,
std::vector<std::deque<AqlItemBlock*>>& gatherBlockBuffer,
std::vector<SortRegister>& sortRegisters) noexcept
: _trx(trx),
_gatherBlockBuffer(gatherBlockBuffer),
_sortRegisters(sortRegisters) {
}
bool operator()(
std::pair<size_t, size_t> const& a,
std::pair<size_t, size_t> const& b
) const;
private:
transaction::Methods* _trx;
std::vector<std::deque<AqlItemBlock*>>& _gatherBlockBuffer;
std::vector<SortRegister>& _sortRegisters;
}; // OurLessThan
bool OurLessThan::operator()(
std::pair<size_t, size_t> const& a,
std::pair<size_t, size_t> const& b
) const {
// nothing in the buffer is maximum!
if (_gatherBlockBuffer[a.first].empty()) {
return false;
}
if (_gatherBlockBuffer[b.first].empty()) {
return true;
}
TRI_ASSERT(!_gatherBlockBuffer[a.first].empty());
TRI_ASSERT(!_gatherBlockBuffer[b.first].empty());
for (auto const& reg : _sortRegisters) {
auto const& lhs = _gatherBlockBuffer[a.first].front()->getValueReference(a.second, reg.reg);
auto const& rhs = _gatherBlockBuffer[b.first].front()->getValueReference(b.second, reg.reg);
auto const& attributePath = reg.attributePath;
// Fast path if there is no attributePath:
int cmp;
if (attributePath.empty()) {
#ifdef USE_IRESEARCH
TRI_ASSERT(reg.comparator);
cmp = (*reg.comparator)(reg.scorer.get(), _trx, lhs, rhs);
#else
cmp = AqlValue::Compare(_trx, lhs, rhs, true);
#endif
} else {
// Take attributePath into consideration:
bool mustDestroyA;
AqlValue aa = lhs.get(_trx, attributePath, mustDestroyA, false);
AqlValueGuard guardA(aa, mustDestroyA);
bool mustDestroyB;
AqlValue bb = rhs.get(_trx, attributePath, mustDestroyB, false);
AqlValueGuard guardB(bb, mustDestroyB);
cmp = AqlValue::Compare(_trx, aa, bb, true);
}
if (cmp == -1) {
return reg.asc;
} else if (cmp == 1) {
return !reg.asc;
}
}
return false;
}
}
GatherBlock::GatherBlock(ExecutionEngine* engine, GatherNode const* en)
: ExecutionBlock(engine, en),
_atDep(0),
_sortRegisters(),
_isSimple(en->elements().empty()),
_heap(en->_sortmode == GatherNode::SortMode::Heap ? new Heap : nullptr) {
_isSimple(en->elements().empty()) {
TRI_ASSERT(en && en->plan() && en->getRegisterPlan());
if (!_isSimple) {
for (auto const& p : en->elements()) {
// We know that planRegisters has been run, so
// getPlanNode()->_registerPlan is set up
auto it = en->getRegisterPlan()->varInfo.find(p.var->id);
TRI_ASSERT(it != en->getRegisterPlan()->varInfo.end());
TRI_ASSERT(it->second.registerId < ExecutionNode::MaxRegisterId);
_sortRegisters.emplace_back(it->second.registerId, p.ascending);
if (!p.attributePath.empty()) {
_sortRegisters.back().attributePath = p.attributePath;
}
if (en->_sortmode == GatherNode::SortMode::Heap) {
_heap.reset(new Heap());
}
// We know that planRegisters has been run, so
// getPlanNode()->_registerPlan is set up
SortRegister::fill(
*en->plan(),
*en->getRegisterPlan(),
en->elements(),
_sortRegisters
);
}
}
@ -458,51 +536,6 @@ bool GatherBlock::getBlock(size_t i, size_t atMost) {
DEBUG_END_BLOCK();
}
/// @brief OurLessThan: comparison method for elements of _gatherBlockPos
bool GatherBlock::OurLessThan::operator()(std::pair<size_t, size_t> const& a,
std::pair<size_t, size_t> const& b) {
// nothing in the buffer is maximum!
if (_gatherBlockBuffer[a.first].empty()) {
return false;
}
if (_gatherBlockBuffer[b.first].empty()) {
return true;
}
TRI_ASSERT(!_gatherBlockBuffer[a.first].empty());
TRI_ASSERT(!_gatherBlockBuffer[b.first].empty());
for (auto const& reg : _sortRegisters) {
// Fast path if there is no attributePath:
int cmp;
if (reg.attributePath.empty()) {
cmp = AqlValue::Compare(
_trx,
_gatherBlockBuffer[a.first].front()->getValueReference(a.second, reg.reg),
_gatherBlockBuffer[b.first].front()->getValueReference(b.second, reg.reg),
true);
} else {
// Take attributePath into consideration:
AqlValue const& topA = _gatherBlockBuffer[a.first].front()->getValueReference(a.second, reg.reg);
AqlValue const& topB = _gatherBlockBuffer[b.first].front()->getValueReference(b.second, reg.reg);
bool mustDestroyA;
AqlValue aa = topA.get(_trx, reg.attributePath, mustDestroyA, false);
AqlValueGuard guardA(aa, mustDestroyA);
bool mustDestroyB;
AqlValue bb = topB.get(_trx, reg.attributePath, mustDestroyB, false);
AqlValueGuard guardB(bb, mustDestroyB);
cmp = AqlValue::Compare(_trx, aa, bb, true);
}
if (cmp == -1) {
return reg.ascending;
} else if (cmp == 1) {
return !reg.ascending;
}
}
return false;
}
BlockWithClients::BlockWithClients(ExecutionEngine* engine,
ExecutionNode const* ep,
std::vector<std::string> const& shardIds)

34
arangod/Aql/ClusterBlocks.h
View File

@ -28,6 +28,7 @@
#include "Aql/ClusterNodes.h"
#include "Aql/ExecutionBlock.h"
#include "Aql/ExecutionNode.h"
#include "Aql/SortRegister.h"
#include "Rest/GeneralRequest.h"
#include <velocypack/Builder.h>
@ -44,19 +45,6 @@ class AqlItemBlock;
struct Collection;
class ExecutionEngine;
/// @brief sort element for block, consisting of register, sort direction,
/// and a possible attribute path to dig into the document
struct SortElementBlock {
RegisterId reg;
bool ascending;
std::vector<std::string> attributePath;
SortElementBlock(RegisterId r, bool asc)
: reg(r), ascending(asc) {
}
};
class GatherBlock : public ExecutionBlock {
public:
GatherBlock(ExecutionEngine*, GatherNode const*);
@ -98,29 +86,11 @@ class GatherBlock : public ExecutionBlock {
size_t _atDep = 0;
/// @brief sort elements for this block
std::vector<SortElementBlock> _sortRegisters;
std::vector<SortRegister> _sortRegisters;
/// @brief isSimple: the block is simple if we do not do merge sort . . .
bool const _isSimple;
/// @brief OurLessThan: comparison method for elements of _gatherBlockPos
class OurLessThan {
public:
OurLessThan(transaction::Methods* trx,
std::vector<std::deque<AqlItemBlock*>>& gatherBlockBuffer,
std::vector<SortElementBlock>& sortRegisters)
: _trx(trx),
_gatherBlockBuffer(gatherBlockBuffer),
_sortRegisters(sortRegisters) {}
bool operator()(std::pair<size_t, size_t> const& a,
std::pair<size_t, size_t> const& b);
private:
transaction::Methods* _trx;
std::vector<std::deque<AqlItemBlock*>>& _gatherBlockBuffer;
std::vector<SortElementBlock>& _sortRegisters;
};
using Heap = std::vector<std::pair<std::size_t, std::size_t>>;
std::unique_ptr<Heap> _heap;
};

32
arangod/Aql/ClusterNodes.cpp
View File

@ -47,7 +47,6 @@ bool toSortMode(
) noexcept {
// std::map ~25-30% faster than std::unordered_map for small number of elements
static std::map<arangodb::velocypack::StringRef, GatherNode::SortMode> const NameToValue {
{ SortModeUnset, GatherNode::SortMode::Unset },
{ SortModeMinElement, GatherNode::SortMode::MinElement},
{ SortModeHeap, GatherNode::SortMode::Heap}
};
@ -65,8 +64,6 @@ bool toSortMode(
arangodb::velocypack::StringRef toString(GatherNode::SortMode mode) noexcept {
switch (mode) {
case GatherNode::SortMode::Unset:
return SortModeUnset;
case GatherNode::SortMode::MinElement:
return SortModeMinElement;
case GatherNode::SortMode::Heap:
@ -277,17 +274,6 @@ double DistributeNode::estimateCost(size_t& nrItems) const {
return nullptr;
}
/*static*/ GatherNode::SortMode GatherNode::getSortMode(
Collection const* collection,
std::size_t shardsRequiredForHeapMerge /*= 5*/
) {
return collection
? (collection->numberOfShards() >= shardsRequiredForHeapMerge
? SortMode::Heap
: SortMode::MinElement)
: SortMode::Unset;
}
/// @brief construct a gather node
GatherNode::GatherNode(
ExecutionPlan* plan,
@ -295,12 +281,14 @@ GatherNode::GatherNode(
SortElementVector const& elements)
: ExecutionNode(plan, base),
_elements(elements),
_sortmode(SortMode::Unset) {
auto const sortModeSlice = base.get("sortmode");
_sortmode(SortMode::MinElement) {
if (!_elements.empty()) {
auto const sortModeSlice = base.get("sortmode");
if (!toSortMode(VelocyPackHelper::getStringRef(sortModeSlice, ""), _sortmode)) {
LOG_TOPIC(ERR, Logger::AQL)
<< "invalid sort mode detected while creating 'GatherNode' from vpack";
if (!toSortMode(VelocyPackHelper::getStringRef(sortModeSlice, ""), _sortmode)) {
LOG_TOPIC(ERR, Logger::AQL)
<< "invalid sort mode detected while creating 'GatherNode' from vpack";
}
}
}
@ -317,7 +305,11 @@ void GatherNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags) const {
// call base class method
ExecutionNode::toVelocyPackHelperGeneric(nodes, flags);
nodes.add("sortmode", VPackValue(toString(_sortmode).data()));
if (_elements.empty()) {
nodes.add("sortmode", VPackValue(SortModeUnset.data()));
} else {
nodes.add("sortmode", VPackValue(toString(_sortmode).data()));
}
nodes.add(VPackValue("elements"));
{

15
arangod/Aql/ClusterNodes.h
View File

@ -315,7 +315,6 @@ class GatherNode final : public ExecutionNode {
public:
enum class SortMode : uint32_t {
Unset = 0, // FIXME do we really 'Unset' value?
MinElement,
Heap
};
@ -327,11 +326,15 @@ class GatherNode final : public ExecutionNode {
GatherNode const& node
) noexcept;
/// @returns sort mode for the specified collection
static SortMode getSortMode(
Collection const* collection,
std::size_t shardsRequiredForHeapMerge = 5
);
/// @returns sort mode for the specified number of shards
static SortMode evaluateSortMode(
size_t numberOfShards,
size_t shardsRequiredForHeapMerge = 5
) noexcept {
return numberOfShards >= shardsRequiredForHeapMerge
? SortMode::Heap
: SortMode::MinElement;
}
/// @brief constructor with an id
GatherNode(

2
arangod/Aql/EngineInfoContainerDBServer.cpp
View File

@ -215,7 +215,7 @@ void EngineInfoContainerDBServer::EngineInfo::serializeSnippet(
plan.setVarUsageComputed();
// Always Verbose
const unsigned flags = ExecutionNode::SERIALIZE_DETAILS;
plan.root()->toVelocyPack(infoBuilder, flags, /*keepTopLevelOpen*/true);
plan.root()->toVelocyPack(infoBuilder, flags, /*keepTopLevelOpen*/false);
}
void EngineInfoContainerDBServer::EngineInfo::serializeSnippet(

12
arangod/Aql/OptimizerRule.h
View File

@ -201,6 +201,12 @@ struct OptimizerRule {
// make operations on sharded collections use scatter / gather / remote
scatterInClusterRule_pass10,
#ifdef USE_IRESEARCH
// FIXME order-???
// make operations on sharded IResearch views use scatter / gather / remote
scatterIResearchViewInClusterRule_pass10,
#endif
// move FilterNodes & Calculation nodes in between
// scatter(remote) <-> gather(remote) so they're
// distributed to the cluster nodes.
@ -210,12 +216,6 @@ struct OptimizerRule {
// adjust gathernode to also contain the sort criteria.
distributeSortToClusterRule_pass10,
#ifdef USE_IRESEARCH
// FIXME order-???
// make operations on sharded IResearch views use scatter / gather / remote
scatterIResearchViewInClusterRule_pass10,
#endif
// try to get rid of a RemoteNode->ScatterNode combination which has
// only a SingletonNode and possibly some CalculationNodes as dependencies
removeUnnecessaryRemoteScatterRule_pass10,

46
arangod/Aql/OptimizerRules.cpp
View File

@ -2703,7 +2703,13 @@ void arangodb::aql::optimizeClusterSingleShardRule(Optimizer* opt,
remoteNode->addDependency(rootNode);
// insert a gather node
auto* gatherNode = new GatherNode(plan.get(), plan->nextId(), GatherNode::getSortMode(c));
auto const sortMode = GatherNode::evaluateSortMode(
c->numberOfShards()
);
auto* gatherNode = new GatherNode(
plan.get(), plan->nextId(), sortMode
);
plan->registerNode(gatherNode);
gatherNode->addDependency(remoteNode);
@ -2945,6 +2951,14 @@ void arangodb::aql::scatterInClusterRule(Optimizer* opt,
SmallVector<ExecutionNode*> nodes{a};
plan->findNodesOfType(nodes, types, true);
TRI_ASSERT(
plan->getAst()
&& plan->getAst()->query()
&& plan->getAst()->query()->trx()
);
auto* resolver = plan->getAst()->query()->trx()->resolver();
TRI_ASSERT(resolver);
for (auto& node : nodes) {
// found a node we need to replace in the plan
@ -3050,8 +3064,14 @@ void arangodb::aql::scatterInClusterRule(Optimizer* opt,
remoteNode->addDependency(node);
// insert a gather node
GatherNode* gatherNode =
new GatherNode(plan.get(), plan->nextId(), GatherNode::getSortMode(collection));
auto const sortMode = GatherNode::evaluateSortMode(
collection->numberOfShards()
);
auto* gatherNode = new GatherNode(
plan.get(),
plan->nextId(),
sortMode
);
plan->registerNode(gatherNode);
TRI_ASSERT(remoteNode);
gatherNode->addDependency(remoteNode);
@ -3309,8 +3329,14 @@ void arangodb::aql::distributeInClusterRule(Optimizer* opt,
remoteNode->addDependency(node);
// insert a gather node
ExecutionNode* gatherNode =
new GatherNode(plan.get(), plan->nextId(), GatherNode::getSortMode(collection));
auto const sortMode = GatherNode::evaluateSortMode(
collection->numberOfShards()
);
auto* gatherNode = new GatherNode(
plan.get(),
plan->nextId(),
sortMode
);
plan->registerNode(gatherNode);
gatherNode->addDependency(remoteNode);
@ -3778,13 +3804,17 @@ void arangodb::aql::distributeSortToClusterRule(
if (thisSortNode->_reinsertInCluster) {
plan->insertDependency(rn, inspectNode);
}
// On SmartEdge collections we have 0 shards and we need the elements
// to be injected here as well. So do not replace it with > 1
auto const* collection = GatherNode::findCollection(*gatherNode);
if (collection && collection->numberOfShards() != 1) {
// For views (when 'collection == nullptr') we don't need
// to check number of shards
// On SmartEdge collections we have 0 shards and we need the elements
// to be injected here as well. So do not replace it with > 1
if (!collection || collection->numberOfShards() != 1) {
gatherNode->elements(thisSortNode->elements());
}
modified = true;
// ready to rumble!
break;

162
arangod/Aql/SortBlock.cpp
View File

@ -29,89 +29,8 @@
using namespace arangodb::aql;
#ifdef USE_IRESEARCH
#include "IResearch/IResearchViewNode.h"
#include "IResearch/IResearchOrderFactory.h"
#include "IResearch/AqlHelper.h"
namespace {
int compareIResearchScores(
irs::sort::prepared const* comparer,
arangodb::transaction::Methods*,
AqlValue const& lhs,
AqlValue const& rhs
) {
arangodb::velocypack::ValueLength tmp;
auto const* lhsScore = reinterpret_cast<irs::byte_type const*>(lhs.slice().getString(tmp));
auto const* rhsScore = reinterpret_cast<irs::byte_type const*>(rhs.slice().getString(tmp));
if (comparer->less(lhsScore, rhsScore)) {
return -1;
}
if (comparer->less(rhsScore, lhsScore)) {
return 1;
}
return 0;
}
int compareAqlValues(
irs::sort::prepared const*,
arangodb::transaction::Methods* trx,
AqlValue const& lhs,
AqlValue const& rhs) {
return AqlValue::Compare(
reinterpret_cast<arangodb::transaction::Methods*>(trx), lhs, rhs, true
);
}
void fillSortRegisters(
std::vector<SortRegister>& sortRegisters,
SortNode const& en
) {
TRI_ASSERT(en.plan());
auto& execPlan = *en.plan();
auto const& elements = en.elements();
sortRegisters.reserve(elements.size());
std::unordered_map<ExecutionNode const*, size_t> offsets(sortRegisters.capacity());
irs::sort::ptr comparer;
for (auto const& p : elements) {
auto const varId = p.var->id;
auto it = en.getRegisterPlan()->varInfo.find(varId);
TRI_ASSERT(it != en.getRegisterPlan()->varInfo.end());
TRI_ASSERT(it->second.registerId < ExecutionNode::MaxRegisterId);
sortRegisters.emplace_back(it->second.registerId, p.ascending, nullptr, &compareAqlValues);
auto const* setter = execPlan.getVarSetBy(varId);
if (setter && ExecutionNode::ENUMERATE_IRESEARCH_VIEW == setter->getType()) {
// sort condition is covered by `IResearchViewNode`
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
auto const& viewNode = dynamic_cast<arangodb::iresearch::IResearchViewNode const&>(*setter);
#else
auto const& viewNode = static_cast<arangodb::iresearch::IResearchViewNode const&>(*setter);
#endif // ARANGODB_ENABLE_MAINTAINER_MODE
auto& offset = offsets[&viewNode];
auto* node = viewNode.sortCondition()[offset++].node;
if (arangodb::iresearch::OrderFactory::comparer(&comparer, *node)) {
auto& reg = sortRegisters.back();
std::get<2>(reg) = comparer->prepare(); // set score comparer
std::get<3>(reg) = &compareIResearchScores; // set comparator
}
}
}
}
/// @brief OurLessThan
class OurLessThan {
public:
@ -127,72 +46,20 @@ class OurLessThan {
bool operator()(std::pair<uint32_t, uint32_t> const& a,
std::pair<uint32_t, uint32_t> const& b) const {
for (auto const& reg : _sortRegisters) {
auto const& lhs = _buffer[a.first]->getValueReference(a.second, std::get<0>(reg));
auto const& rhs = _buffer[b.first]->getValueReference(b.second, std::get<0>(reg));
auto const comparator = std::get<3>(reg);
TRI_ASSERT(comparator);
int const cmp = (*comparator)(std::get<2>(reg).get(), _trx, lhs, rhs);
if (cmp < 0) {
return std::get<1>(reg);
} else if (cmp > 0) {
return !std::get<1>(reg);
}
}
return false;
}
private:
arangodb::transaction::Methods* _trx;
std::deque<AqlItemBlock*>& _buffer;
std::vector<SortRegister>& _sortRegisters;
};
}
auto const& lhs = _buffer[a.first]->getValueReference(a.second, reg.reg);
auto const& rhs = _buffer[b.first]->getValueReference(b.second, reg.reg);
#ifdef USE_IRESEARCH
TRI_ASSERT(reg.comparator);
int const cmp = (*reg.comparator)(reg.scorer.get(), _trx, lhs, rhs);
#else
void fillSortRegisters(
std::vector<SortRegister>& sortRegisters,
SortNode const& en
) {
auto const& elements = en.elements();
sortRegisters.reserve(elements.size());
for (auto const& p : elements) {
auto const varId = p.var->id;
auto it = en.getRegisterPlan()->varInfo.find(varId);
TRI_ASSERT(it != en.getRegisterPlan()->varInfo.end());
TRI_ASSERT(it->second.registerId < ExecutionNode::MaxRegisterId);
sortRegisters.emplace_back(it->second.registerId, p.ascending);
}
}
/// @brief OurLessThan
class OurLessThan {
public:
OurLessThan(arangodb::transaction::Methods* trx,
std::deque<AqlItemBlock*>& buffer,
std::vector<SortRegister>& sortRegisters)
: _trx(trx),
_buffer(buffer),
_sortRegisters(sortRegisters) {}
bool operator()(std::pair<size_t, size_t> const& a,
std::pair<size_t, size_t> const& b) const {
for (auto const& reg : _sortRegisters) {
auto const& lhs = _buffer[a.first]->getValueReference(a.second, std::get<0>(reg));
auto const& rhs = _buffer[b.first]->getValueReference(b.second, std::get<0>(reg));
int const cmp = AqlValue::Compare(_trx, lhs, rhs, true);
#endif
if (cmp < 0) {
return std::get<1>(reg);
return reg.asc;
} else if (cmp > 0) {
return !std::get<1>(reg);
return !reg.asc;
}
}
@ -203,16 +70,21 @@ class OurLessThan {
arangodb::transaction::Methods* _trx;
std::deque<AqlItemBlock*>& _buffer;
std::vector<SortRegister>& _sortRegisters;
};
}; // OurLessThan
#endif // USE_IRESEARCH
}
SortBlock::SortBlock(ExecutionEngine* engine, SortNode const* en)
: ExecutionBlock(engine, en),
_stable(en->_stable),
_mustFetchAll(true) {
TRI_ASSERT(en);
fillSortRegisters(_sortRegisters, *en);
TRI_ASSERT(en && en->plan() && en->getRegisterPlan());
SortRegister::fill(
*en->plan(),
*en->getRegisterPlan(),
en->elements(),
_sortRegisters
);
}
SortBlock::~SortBlock() {}

22
arangod/Aql/SortBlock.h
View File

@ -27,35 +27,15 @@
#include "Basics/Common.h"
#include "Aql/ExecutionBlock.h"
#include "Aql/SortNode.h"
#ifdef USE_IRESEARCH
#include "search/sort.hpp"
#endif // USE_IRESEARCH
#include "Aql/SortRegister.h"
namespace arangodb {
namespace transaction {
class Methods;
}
namespace aql {
class AqlItemBlock;
class ExecutionEngine;
#ifdef USE_IRESEARCH
typedef int(*CompareFunc)(
irs::sort::prepared const* comparer,
transaction::Methods* trx,
AqlValue const& lhs,
AqlValue const& rhs
);
typedef std::tuple<RegisterId, bool, irs::sort::prepared::ptr, CompareFunc> SortRegister;
#else
typedef std::tuple<RegisterId, bool> SortRegister;
#endif
class SortBlock final : public ExecutionBlock {
public:
SortBlock(ExecutionEngine*, SortNode const*);

147
arangod/Aql/SortRegister.cpp Normal file
View File

@ -0,0 +1,147 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#include "SortRegister.h"
#include "Aql/AqlValue.h"
#include "Aql/ClusterNodes.h"
#include "Aql/ExecutionPlan.h"
#include "Aql/SortNode.h"
#ifdef USE_IRESEARCH
#include "IResearch/IResearchViewNode.h"
#include "IResearch/IResearchOrderFactory.h"
namespace {
int compareIResearchScores(
irs::sort::prepared const* comparer,
arangodb::transaction::Methods*,
arangodb::aql::AqlValue const& lhs,
arangodb::aql::AqlValue const& rhs
) {
arangodb::velocypack::ValueLength tmp;
auto const* lhsScore = reinterpret_cast<irs::byte_type const*>(lhs.slice().getString(tmp));
auto const* rhsScore = reinterpret_cast<irs::byte_type const*>(rhs.slice().getString(tmp));
if (comparer->less(lhsScore, rhsScore)) {
return -1;
}
if (comparer->less(rhsScore, lhsScore)) {
return 1;
}
return 0;
}
int compareAqlValues(
irs::sort::prepared const*,
arangodb::transaction::Methods* trx,
arangodb::aql::AqlValue const& lhs,
arangodb::aql::AqlValue const& rhs) {
return arangodb::aql::AqlValue::Compare(trx, lhs, rhs, true);
}
}
#endif
namespace arangodb {
namespace aql {
// -----------------------------------------------------------------------------
// -- SECTION -- SortRegister
// -----------------------------------------------------------------------------
SortRegister::SortRegister(
RegisterId reg,
SortElement const& element) noexcept
: attributePath(element.attributePath),
reg(reg),
asc(element.ascending) {
}
#ifdef USE_IRESEARCH
void SortRegister::fill(
ExecutionPlan const& execPlan,
ExecutionNode::RegisterPlan const& regPlan,
std::vector<SortElement> const& elements,
std::vector<SortRegister>& sortRegisters
) {
sortRegisters.reserve(elements.size());
std::unordered_map<ExecutionNode const*, size_t> offsets(sortRegisters.capacity());
irs::sort::ptr comparer;
auto const& vars = regPlan.varInfo;
for (auto const& p : elements) {
auto const varId = p.var->id;
auto const it = vars.find(varId);
TRI_ASSERT(it != vars.end());
TRI_ASSERT(it->second.registerId < ExecutionNode::MaxRegisterId);
sortRegisters.emplace_back(it->second.registerId, p, &compareAqlValues);
auto const* setter = execPlan.getVarSetBy(varId);
if (setter && ExecutionNode::ENUMERATE_IRESEARCH_VIEW == setter->getType()) {
// sort condition is covered by `IResearchViewNode`
auto const* viewNode = ExecutionNode::castTo<iresearch::IResearchViewNode const*>(setter);
auto& offset = offsets[viewNode];
auto* node = viewNode->sortCondition()[offset++].node;
if (arangodb::iresearch::OrderFactory::comparer(&comparer, *node)) {
auto& reg = sortRegisters.back();
reg.scorer = comparer->prepare(); // set score comparer
reg.comparator = &compareIResearchScores; // set comparator
}
}
}
}
#else
void SortRegister::fill(
ExecutionPlan const& /*execPlan*/,
ExecutionNode::RegisterPlan const& regPlan,
std::vector<SortElement> const& elements,
std::vector<SortRegister>& sortRegisters
) {
sortRegisters.reserve(elements.size());
auto const& vars = regPlan.varInfo;
for (auto const& p : elements) {
auto const varId = p.var->id;
auto const it = vars.find(varId);
TRI_ASSERT(it != vars.end());
TRI_ASSERT(it->second.registerId < ExecutionNode::MaxRegisterId);
sortRegisters.emplace_back(it->second.registerId, p);
}
}
#endif // USE_IRESEARCH
} // aql
} // arangodb

81
arangod/Aql/SortRegister.h Normal file
View File

@ -0,0 +1,81 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Andrey Abramov
/// @author Vasiliy Nabatchikov
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_AQL_SORT_REGISTER_H
#define ARANGOD_AQL_SORT_REGISTER_H 1
#include "Aql/ExecutionNode.h"
#include "types.h"
#ifdef USE_IRESEARCH
#include "search/sort.hpp"
#endif
namespace arangodb {
namespace aql {
/// @brief sort element for block, consisting of register, sort direction,
/// and a possible attribute path to dig into the document
struct SortRegister {
#ifdef USE_IRESEARCH
typedef int(*CompareFunc)(
irs::sort::prepared const* scorer,
transaction::Methods* trx,
AqlValue const& lhs,
AqlValue const& rhs
);
irs::sort::prepared::ptr scorer;
CompareFunc comparator;
#endif
std::vector<std::string> const& attributePath;
RegisterId reg;
bool asc;
SortRegister(
RegisterId reg,
SortElement const& element
) noexcept;
#ifdef USE_IRESEARCH
SortRegister(
RegisterId reg,
SortElement const& element,
CompareFunc comparator) noexcept
: SortRegister(reg, element) {
this->comparator = comparator;
}
#endif
static void fill(
ExecutionPlan const& /*execPlan*/,
ExecutionNode::RegisterPlan const& regPlan,
std::vector<SortElement> const& elements,
std::vector<SortRegister>& sortRegisters
);
}; // SortRegister
} // aql
} // arangodb
#endif // ARANGOD_AQL_SORT_REGISTER_H

1
arangod/CMakeLists.txt
View File

@ -243,6 +243,7 @@ SET(ARANGOD_SOURCES
Aql/SortBlock.cpp
Aql/SortCondition.cpp
Aql/SortNode.cpp
Aql/SortRegister.cpp
Aql/SubqueryBlock.cpp
Aql/TraversalBlock.cpp
Aql/TraversalConditionFinder.cpp

213
arangod/IResearch/AqlHelper.h
View File

@ -26,9 +26,11 @@
#include "Aql/SortCondition.h"
#include "VelocyPackHelper.h"
#include "search/sort.hpp"
#include "utils/string.hpp"
#include "utils/noncopyable.hpp"
#ifndef ARANGOD_IRESEARCH__AQL_HELPER_H
#define ARANGOD_IRESEARCH__AQL_HELPER_H 1
@ -39,32 +41,31 @@
#endif
#endif
NS_BEGIN(arangodb)
NS_BEGIN(aql)
namespace arangodb {
namespace aql {
struct Variable; // forward declaration
class ExecutionPlan; // forward declaration
class ExpressionContext; // forward declaration
class Ast; // forward declaration
NS_END // aql
} // aql
NS_BEGIN(transaction)
namespace transaction {
class Methods; // forward declaration
NS_END // transaction
} // transaction
NS_BEGIN(iresearch)
namespace iresearch {
//////////////////////////////////////////////////////////////////////////////
/// @brief extracts string_ref from an AstNode, note that provided 'node'
/// must be an arangodb::aql::VALUE_TYPE_STRING
/// @return extracted string_ref
//////////////////////////////////////////////////////////////////////////////
inline irs::string_ref getStringRef(arangodb::aql::AstNode const& node) {
TRI_ASSERT(arangodb::aql::VALUE_TYPE_STRING == node.value.type);
inline irs::string_ref getStringRef(aql::AstNode const& node) {
TRI_ASSERT(aql::VALUE_TYPE_STRING == node.value.type);
return irs::string_ref(node.getStringValue(), node.getStringLength());
}
@ -73,16 +74,16 @@ inline irs::string_ref getStringRef(arangodb::aql::AstNode const& node) {
/// @brief tries to extract 'size_t' value from the specified AstNode 'node'
/// @returns true on success, false otherwise
////////////////////////////////////////////////////////////////////////////////
inline bool parseValue(size_t& value, arangodb::aql::AstNode const& node) {
inline bool parseValue(size_t& value, aql::AstNode const& node) {
switch (node.value.type) {
case arangodb::aql::VALUE_TYPE_NULL:
case arangodb::aql::VALUE_TYPE_BOOL:
case aql::VALUE_TYPE_NULL:
case aql::VALUE_TYPE_BOOL:
return false;
case arangodb::aql::VALUE_TYPE_INT:
case arangodb::aql::VALUE_TYPE_DOUBLE:
case aql::VALUE_TYPE_INT:
case aql::VALUE_TYPE_DOUBLE:
value = node.getIntValue();
return true;
case arangodb::aql::VALUE_TYPE_STRING:
case aql::VALUE_TYPE_STRING:
return false;
}
@ -97,17 +98,17 @@ inline bool parseValue(size_t& value, arangodb::aql::AstNode const& node) {
template<typename String>
inline bool parseValue(
String& value,
arangodb::aql::AstNode const& node
aql::AstNode const& node
) {
typedef typename String::traits_type traits_t;
switch (node.value.type) {
case arangodb::aql::VALUE_TYPE_NULL:
case arangodb::aql::VALUE_TYPE_BOOL:
case arangodb::aql::VALUE_TYPE_INT:
case arangodb::aql::VALUE_TYPE_DOUBLE:
case aql::VALUE_TYPE_NULL:
case aql::VALUE_TYPE_BOOL:
case aql::VALUE_TYPE_INT:
case aql::VALUE_TYPE_DOUBLE:
return false;
case arangodb::aql::VALUE_TYPE_STRING:
case aql::VALUE_TYPE_STRING:
value = String(
reinterpret_cast<typename traits_t::char_type const*>(node.getStringValue()),
node.getStringLength()
@ -119,7 +120,7 @@ inline bool parseValue(
}
template<typename Visitor>
bool visit(arangodb::aql::SortCondition const& sort, Visitor const& visitor) {
bool visit(aql::SortCondition const& sort, Visitor const& visitor) {
for (size_t i = 0, size = sort.numAttributes(); i < size; ++i) {
auto entry = sort.field(i);
@ -136,7 +137,7 @@ bool visit(arangodb::aql::SortCondition const& sort, Visitor const& visitor) {
/// to the specified visiting strategy (preorder/postorder)
////////////////////////////////////////////////////////////////////////////////
template<bool Preorder, typename Visitor>
bool visit(arangodb::aql::AstNode const& root, Visitor visitor) {
bool visit(aql::AstNode const& root, Visitor visitor) {
if (Preorder && !visitor(root)) {
return false;
}
@ -174,7 +175,7 @@ enum ScopedValueType {
////////////////////////////////////////////////////////////////////////////////
struct AqlValueTraits {
static ScopedValueType type(
arangodb::aql::AqlValue const& value
aql::AqlValue const& value
) noexcept {
typedef typename std::underlying_type<
ScopedValueType
@ -191,26 +192,26 @@ struct AqlValueTraits {
}
static ScopedValueType type(
arangodb::aql::AstNode const& node
aql::AstNode const& node
) noexcept {
switch (node.type) {
case arangodb::aql::NODE_TYPE_VALUE:
case aql::NODE_TYPE_VALUE:
switch (node.value.type) {
case arangodb::aql::VALUE_TYPE_NULL:
case aql::VALUE_TYPE_NULL:
return SCOPED_VALUE_TYPE_NULL;
case arangodb::aql::VALUE_TYPE_BOOL:
case aql::VALUE_TYPE_BOOL:
return SCOPED_VALUE_TYPE_BOOL;
case arangodb::aql::VALUE_TYPE_INT: // all numerics are doubles here
case arangodb::aql::VALUE_TYPE_DOUBLE:
case aql::VALUE_TYPE_INT: // all numerics are doubles here
case aql::VALUE_TYPE_DOUBLE:
return SCOPED_VALUE_TYPE_DOUBLE;
case arangodb::aql::VALUE_TYPE_STRING:
case aql::VALUE_TYPE_STRING:
return SCOPED_VALUE_TYPE_STRING;
default:
return SCOPED_VALUE_TYPE_INVALID;
}
case arangodb::aql::NODE_TYPE_ARRAY:
case aql::NODE_TYPE_ARRAY:
return SCOPED_VALUE_TYPE_ARRAY;
case arangodb::aql::NODE_TYPE_RANGE:
case aql::NODE_TYPE_RANGE:
return SCOPED_VALUE_TYPE_RANGE;
default:
return SCOPED_VALUE_TYPE_INVALID;
@ -222,11 +223,11 @@ struct AqlValueTraits {
/// @struct QueryContext
////////////////////////////////////////////////////////////////////////////////
struct QueryContext {
arangodb::transaction::Methods* trx;
arangodb::aql::ExecutionPlan* plan;
arangodb::aql::Ast* ast;
arangodb::aql::ExpressionContext* ctx;
arangodb::aql::Variable const* ref;
transaction::Methods* trx;
aql::ExecutionPlan* plan;
aql::Ast* ast;
aql::ExpressionContext* ctx;
aql::Variable const* ref;
}; // QueryContext
////////////////////////////////////////////////////////////////////////////////
@ -235,10 +236,10 @@ struct QueryContext {
////////////////////////////////////////////////////////////////////////////////
class ScopedAqlValue : private irs::util::noncopyable {
public:
static arangodb::aql::AstNode const INVALID_NODE;
static aql::AstNode const INVALID_NODE;
explicit ScopedAqlValue(
arangodb::aql::AstNode const& node = INVALID_NODE
aql::AstNode const& node = INVALID_NODE
) noexcept {
reset(node);
}
@ -253,7 +254,7 @@ class ScopedAqlValue : private irs::util::noncopyable {
rhs._executed = true;
}
void reset(arangodb::aql::AstNode const& node = INVALID_NODE) noexcept {
void reset(aql::AstNode const& node = INVALID_NODE) noexcept {
_node = &node;
_type = AqlValueTraits::type(node);
_executed = node.isConstant();
@ -315,7 +316,7 @@ class ScopedAqlValue : private irs::util::noncopyable {
return true;
}
arangodb::aql::Range const* getRange() const {
aql::Range const* getRange() const {
return _node->isConstant()
? nullptr
: _value.range();
@ -327,14 +328,14 @@ class ScopedAqlValue : private irs::util::noncopyable {
: _value.length();
}
void toVelocyPack(arangodb::velocypack::Builder& builder) const {
void toVelocyPack(velocypack::Builder& builder) const {
_node->isConstant()
? _node->toVelocyPackValue(builder)
: _value.toVelocyPack(nullptr, builder, false);
}
private:
ScopedAqlValue(arangodb::aql::AqlValue const& src, size_t i, bool doCopy) {
ScopedAqlValue(aql::AqlValue const& src, size_t i, bool doCopy) {
_value = src.at(nullptr, i, _destroy, doCopy);
_node = &INVALID_NODE;
_executed = true;
@ -347,8 +348,8 @@ class ScopedAqlValue : private irs::util::noncopyable {
}
}
arangodb::aql::AqlValue _value;
arangodb::aql::AstNode const* _node;
aql::AqlValue _value;
aql::AstNode const* _node;
ScopedValueType _type;
bool _destroy{ false };
bool _executed;
@ -365,8 +366,8 @@ class ScopedAqlValue : private irs::util::noncopyable {
////////////////////////////////////////////////////////////////////////////////
template<typename T>
bool visitAttributeAccess(
arangodb::aql::AstNode const*& head,
arangodb::aql::AstNode const* node,
aql::AstNode const*& head,
aql::AstNode const* node,
T& visitor
) {
if (!node) {
@ -374,11 +375,11 @@ bool visitAttributeAccess(
}
switch (node->type) {
case arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS: // .
case aql::NODE_TYPE_ATTRIBUTE_ACCESS: // .
return node->numMembers() >= 1
&& visitAttributeAccess(head, node->getMemberUnchecked(0), visitor)
&& visitor.attributeAccess(*node);
case arangodb::aql::NODE_TYPE_INDEXED_ACCESS: { // [<something>]
case aql::NODE_TYPE_INDEXED_ACCESS: { // [<something>]
if (node->numMembers() < 2) {
// malformed node
return false;
@ -390,7 +391,7 @@ bool visitAttributeAccess(
&& visitAttributeAccess(head, node->getMemberUnchecked(0), visitor)
&& visitor.indexAccess(*offset);
}
case arangodb::aql::NODE_TYPE_EXPANSION: { // [*]
case aql::NODE_TYPE_EXPANSION: { // [*]
if (node->numMembers() < 2) {
// malformed node
return false;
@ -404,14 +405,14 @@ bool visitAttributeAccess(
auto* var = itr->getMemberUnchecked(0);
return ref
&& arangodb::aql::NODE_TYPE_ITERATOR == itr->type
&& arangodb::aql::NODE_TYPE_REFERENCE == ref->type
&& var && arangodb::aql::NODE_TYPE_VARIABLE == var->type
&& aql::NODE_TYPE_ITERATOR == itr->type
&& aql::NODE_TYPE_REFERENCE == ref->type
&& var && aql::NODE_TYPE_VARIABLE == var->type
&& visitAttributeAccess(head, root, visitor) // 1st visit root
&& visitor.expansion(*node); // 2nd visit current node
}
}
case arangodb::aql::NODE_TYPE_REFERENCE: {
case aql::NODE_TYPE_REFERENCE: {
head = node;
return true;
}
@ -431,12 +432,12 @@ bool visitAttributeAccess(
////////////////////////////////////////////////////////////////////////////////
template<typename T>
bool visitAttributePath(
arangodb::aql::AstNode const*& head,
arangodb::aql::AstNode const& node,
aql::AstNode const*& head,
aql::AstNode const& node,
T& visitor
) {
if (node.numMembers() >= 2
&& arangodb::aql::NODE_TYPE_EXPANSION == node.type) { // [*]
&& aql::NODE_TYPE_EXPANSION == node.type) { // [*]
auto* itr = node.getMemberUnchecked(0);
auto* ref = node.getMemberUnchecked(1);
@ -445,15 +446,15 @@ bool visitAttributePath(
auto* var = itr->getMemberUnchecked(0);
return ref
&& arangodb::aql::NODE_TYPE_ITERATOR == itr->type
&& arangodb::aql::NODE_TYPE_REFERENCE == ref->type
&& aql::NODE_TYPE_ITERATOR == itr->type
&& aql::NODE_TYPE_REFERENCE == ref->type
&& root && var
&& arangodb::aql::NODE_TYPE_VARIABLE == var->type
&& aql::NODE_TYPE_VARIABLE == var->type
&& visitAttributePath(head, *root, visitor) // 1st visit root
&& visitor(); // 2nd visit current node
}
} else if (node.numMembers() == 2
&& arangodb::aql::NODE_TYPE_INDEXED_ACCESS == node.type) { // [<something>]
&& aql::NODE_TYPE_INDEXED_ACCESS == node.type) { // [<something>]
auto* root = node.getMemberUnchecked(0);
auto* offset = node.getMemberUnchecked(1);
@ -466,22 +467,22 @@ bool visitAttributePath(
return root && offset && offset->isStringValue()
&& visitAttributePath(head, *root, visitor) // 1st visit root
&& visitor(arangodb::iresearch::getStringRef(*offset)); // 2nd visit current node
&& visitor(iresearch::getStringRef(*offset)); // 2nd visit current node
} else if (node.numMembers() == 1
&& arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS == node.type) {
&& aql::NODE_TYPE_ATTRIBUTE_ACCESS == node.type) {
auto* root = node.getMemberUnchecked(0);
return root
&& arangodb::aql::VALUE_TYPE_STRING == node.value.type
&& aql::VALUE_TYPE_STRING == node.value.type
&& visitAttributePath(head, *root, visitor) // 1st visit root
&& visitor(arangodb::iresearch::getStringRef(node)); // 2nd visit current node
&& visitor(iresearch::getStringRef(node)); // 2nd visit current node
} else if (!node.numMembers()) { // end of attribute path (base case)
head = &node;
return arangodb::aql::NODE_TYPE_REFERENCE == node.type
|| (arangodb::aql::NODE_TYPE_VALUE == node.type
&& arangodb::aql::VALUE_TYPE_STRING == node.value.type
&& visitor(arangodb::iresearch::getStringRef(node))
return aql::NODE_TYPE_REFERENCE == node.type
|| (aql::NODE_TYPE_VALUE == node.type
&& aql::VALUE_TYPE_STRING == node.value.type
&& visitor(iresearch::getStringRef(node))
);
}
@ -489,9 +490,9 @@ bool visitAttributePath(
}
struct NormalizedCmpNode {
arangodb::aql::AstNode const* attribute;
arangodb::aql::AstNode const* value;
arangodb::aql::AstNodeType cmp;
aql::AstNode const* attribute;
aql::AstNode const* value;
aql::AstNodeType cmp;
};
////////////////////////////////////////////////////////////////////////////////
@ -499,11 +500,11 @@ struct NormalizedCmpNode {
/// TypeMap, nullptr otherwise
////////////////////////////////////////////////////////////////////////////////
inline std::string const* getNodeTypeName(
arangodb::aql::AstNodeType type
aql::AstNodeType type
) noexcept {
auto const it = arangodb::aql::AstNode::TypeNames.find(type);
auto const it = aql::AstNode::TypeNames.find(type);
if (arangodb::aql::AstNode::TypeNames.end() == it) {
if (aql::AstNode::TypeNames.end() == it) {
return nullptr;
}
@ -513,10 +514,10 @@ inline std::string const* getNodeTypeName(
////////////////////////////////////////////////////////////////////////////////
/// @returns pointer to 'idx'th member of type 'expectedType', or nullptr
////////////////////////////////////////////////////////////////////////////////
inline arangodb::aql::AstNode const* getNode(
arangodb::aql::AstNode const& node,
inline aql::AstNode const* getNode(
aql::AstNode const& node,
size_t idx,
arangodb::aql::AstNodeType expectedType
aql::AstNodeType expectedType
) {
TRI_ASSERT(idx < node.numMembers());
@ -532,11 +533,11 @@ inline arangodb::aql::AstNode const* getNode(
/// @returns true if the specified node contains variable, false otherwise
////////////////////////////////////////////////////////////////////////////////
inline bool findReference(
arangodb::aql::AstNode const& root,
arangodb::aql::Variable const& ref
aql::AstNode const& root,
aql::Variable const& ref
) noexcept {
auto visitor = [&ref](arangodb::aql::AstNode const& node) noexcept {
return arangodb::aql::NODE_TYPE_REFERENCE != node.type
auto visitor = [&ref](aql::AstNode const& node) noexcept {
return aql::NODE_TYPE_REFERENCE != node.type
|| reinterpret_cast<void const*>(&ref) != node.getData();
};
@ -550,8 +551,8 @@ inline bool findReference(
/// false otherwise
////////////////////////////////////////////////////////////////////////////////
bool normalizeCmpNode(
arangodb::aql::AstNode const& in,
arangodb::aql::Variable const& ref,
aql::AstNode const& in,
aql::Variable const& ref,
NormalizedCmpNode& out
);
@ -560,8 +561,8 @@ bool normalizeCmpNode(
/// @returns true if the specified nodes are equal, false otherwise
////////////////////////////////////////////////////////////////////////////////
bool attributeAccessEqual(
arangodb::aql::AstNode const* lhs,
arangodb::aql::AstNode const* rhs,
aql::AstNode const* lhs,
aql::AstNode const* rhs,
QueryContext const* ctx
);
@ -571,7 +572,7 @@ bool attributeAccessEqual(
////////////////////////////////////////////////////////////////////////////////
bool nameFromAttributeAccess(
std::string& name,
arangodb::aql::AstNode const& node,
aql::AstNode const& node,
QueryContext const& ctx
);
@ -580,13 +581,37 @@ bool nameFromAttributeAccess(
/// treats node of type NODE_TYPE_REFERENCE as invalid
/// @returns the specified node on success, nullptr otherwise
////////////////////////////////////////////////////////////////////////////////
arangodb::aql::AstNode const* checkAttributeAccess(
arangodb::aql::AstNode const* node,
arangodb::aql::Variable const& ref
aql::AstNode const* checkAttributeAccess(
aql::AstNode const* node,
aql::Variable const& ref
) noexcept;
NS_END // iresearch
NS_END // arangodb
namespace compare {
typedef int(*Func)(
irs::sort::prepared const* comparer,
transaction::Methods* trx,
aql::AqlValue const& lhs,
aql::AqlValue const& rhs
);
int compareIResearchScores(
irs::sort::prepared const* comparer,
transaction::Methods*,
aql::AqlValue const& lhs,
aql::AqlValue const& rhs
);
int compareAqlValues(
irs::sort::prepared const*,
transaction::Methods* trx,
aql::AqlValue const& lhs,
aql::AqlValue const& rhs
);
} // compare
} // iresearch
} // arangodb
#endif // ARANGOD_IRESEARCH__AQL_HELPER_H
@ -596,4 +621,4 @@ NS_END // arangodb
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

68
arangod/IResearch/IResearchViewNode.cpp
View File

@ -106,17 +106,71 @@ void toVelocyPack(
std::vector<arangodb::iresearch::IResearchSort> const& sorts,
bool verbose
) {
builder.openObject();
// FIXME todo
builder.close();
VPackArrayBuilder arrayScope(&builder);
for (auto const sort : sorts) {
VPackObjectBuilder objectScope(&builder);
builder.add("varId", VPackValue(sort.var->id));
builder.add("asc", VPackValue(sort.asc));
builder.add(VPackValue("node"));
sort.node->toVelocyPack(builder, verbose);
}
}
std::vector<arangodb::iresearch::IResearchSort> fromVelocyPack(
arangodb::aql::ExecutionPlan* plan,
arangodb::aql::ExecutionPlan& plan,
arangodb::velocypack::Slice const& slice
) {
// FIXME todo
return {};
if (!slice.isArray()) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "invalid json format detected while building IResearchViewNode sorting from velocy pack, array expected";
return {};
}
auto* ast = plan.getAst();
TRI_ASSERT(ast);
auto const* vars = plan.getAst()->variables();
TRI_ASSERT(vars);
std::vector<arangodb::iresearch::IResearchSort> sorts;
size_t i = 0;
for (auto const sortSlice : velocypack::ArrayIterator(slice)) {
auto const varIdSlice = sortSlice.get("varId");
if (!varIdSlice.isNumber()) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "malformed variable identifier at line '" << i << "', number expected";
return {};
}
auto const varId = varIdSlice.getNumber<aql::VariableId>();
auto const* var = vars->getVariable(varId);
if (!var) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "unable to find variable '" << varId << "' at line '" << i
<< "' while building IResearchViewNode sorting from velocy pack";
return {};
}
auto const ascSlice = sortSlice.get("asc");
if (!ascSlice.isBoolean()) {
LOG_TOPIC(ERR, arangodb::iresearch::TOPIC)
<< "malformed order mark at line " << i << "', boolean expected";
return {};
}
bool const asc = ascSlice.getBoolean();
// will be owned by Ast
auto* node = new aql::AstNode(ast, sortSlice.get("node"));
sorts.emplace_back(var, node, asc);
++i;
}
return sorts;
}
}
@ -157,7 +211,7 @@ IResearchViewNode::IResearchViewNode(
// in case if filter is not specified
// set it to surrogate 'RETURN ALL' node
_filterCondition(&ALL),
_sortCondition(fromVelocyPack(&plan, base.get("sortCondition"))) {
_sortCondition(fromVelocyPack(plan, base.get("sortCondition"))) {
// FIXME how to check properly
auto view = _vocbase.lookupView(
basics::StringUtils::uint64(base.get("viewId").copyString())

40
arangod/IResearch/IResearchViewOptimizerRules.cpp
View File

@ -35,6 +35,8 @@
#include "Aql/Optimizer.h"
#include "Aql/WalkerWorker.h"
#include "Cluster/ServerState.h"
#include "Utils/CollectionNameResolver.h"
#include "VocBase/LogicalCollection.h"
using namespace arangodb::iresearch;
using namespace arangodb::aql;
@ -42,6 +44,24 @@ using EN = arangodb::aql::ExecutionNode;
namespace {
size_t numberOfShards(
arangodb::CollectionNameResolver const& resolver,
arangodb::LogicalView const& view
) {
size_t numberOfShards = 0;
auto visitor = [&numberOfShards](
arangodb::LogicalCollection const& collection
) noexcept {
numberOfShards += collection.numberOfShards();
return true;
};
resolver.visitCollections(visitor, view.id());
return numberOfShards;
}
std::vector<arangodb::iresearch::IResearchSort> buildSort(
ExecutionPlan const& plan,
arangodb::aql::Variable const& ref,
@ -525,6 +545,14 @@ void scatterViewInClusterRule(
nodes.clear();
plan->findNodesOfType(nodes, ExecutionNode::ENUMERATE_IRESEARCH_VIEW, true);
TRI_ASSERT(
plan->getAst()
&& plan->getAst()->query()
&& plan->getAst()->query()->trx()
);
auto* resolver = plan->getAst()->query()->trx()->resolver();
TRI_ASSERT(resolver);
for (auto* node : nodes) {
TRI_ASSERT(node);
auto& viewNode = *EN::castTo<IResearchViewNode*>(node);
@ -590,21 +618,19 @@ void scatterViewInClusterRule(
TRI_ASSERT(node);
remoteNode->addDependency(node);
// insert a gather node
// insert gather node
auto const sortMode = GatherNode::evaluateSortMode(
numberOfShards(*resolver, view)
);
auto* gatherNode = plan->registerNode(
std::make_unique<GatherNode>(
plan.get(),
plan->nextId(),
GatherNode::SortMode::Unset
sortMode
));
TRI_ASSERT(remoteNode);
gatherNode->addDependency(remoteNode);
// FIXME
// if (!elements.empty() && gatherNode->collection()->numberOfShards() > 1) {
// gatherNode->setElements(elements);
// }
// and now link the gather node with the rest of the plan
if (parents.size() == 1) {
parents[0]->replaceDependency(deps[0], gatherNode);

2
arangod/VocBase/LogicalCollection.cpp
View File

@ -565,7 +565,7 @@ void LogicalCollection::replicationFactor(int r) {
}
// SECTION: Sharding
int LogicalCollection::numberOfShards() const {
int LogicalCollection::numberOfShards() const noexcept {
return static_cast<int>(_numberOfShards);
}
void LogicalCollection::numberOfShards(int n) {

2
arangod/VocBase/LogicalCollection.h
View File

@ -205,7 +205,7 @@ class LogicalCollection: public LogicalDataSource {
bool isSatellite() const;
// SECTION: Sharding
int numberOfShards() const;
int numberOfShards() const noexcept;
void numberOfShards(int);
bool allowUserKeys() const;
virtual bool usesDefaultShardKeys() const;

8
js/server/tests/shell/shell-cluster-gatherblocks.js
View File

@ -121,10 +121,10 @@ function gatherBlocksTestSuite() {
//s(int) shards
//i(bool) index
let tests = [
{ test : "q1s3if", query: query1, bindvars: bindvars1, collection: colName1, gathernodes: 1, sortmode: 'minelement' , count: documents/2 , sorted: false },
{ test : "q1s9if", query: query1, bindvars: bindvars1, collection: colName2, gathernodes: 1, sortmode: 'heap' , count: documents/2 , sorted: false },
{ test : "q2s3if", query: query2, bindvars: bindvars2, collection: colName1, gathernodes: 1, sortmode: 'minelement' , count: 1 , sorted: false },
{ test : "q2s9if", query: query2, bindvars: bindvars2, collection: colName2, gathernodes: 1, sortmode: 'heap' , count: 1 , sorted: false },
{ test : "q1s3if", query: query1, bindvars: bindvars1, collection: colName1, gathernodes: 1, sortmode: 'unset', count: documents/2 , sorted: false },
{ test : "q1s9if", query: query1, bindvars: bindvars1, collection: colName2, gathernodes: 1, sortmode: 'unset', count: documents/2 , sorted: false },
{ test : "q2s3if", query: query2, bindvars: bindvars2, collection: colName1, gathernodes: 1, sortmode: 'unset', count: 1 , sorted: false },
{ test : "q2s9if", query: query2, bindvars: bindvars2, collection: colName2, gathernodes: 1, sortmode: 'unset', count: 1 , sorted: false },
{ test : "q3s3if", query: query3, bindvars: bindvars3, collection: colName1, gathernodes: 1, sortmode: 'minelement' , count: 4000 , sorted: true },
{ test : "q3s9if", query: query3, bindvars: bindvars3, collection: colName2, gathernodes: 1, sortmode: 'heap' , count: documents , sorted: true },
];