Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

query less documents

This commit is contained in:
jsteemann 2016-05-13 14:26:34 +02:00
parent a259e73923
commit 0828abbb5c
4 changed files with 94 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
3rdParty/velocypack/include/velocypack/Slice.h vendored
View File

@ -69,53 +69,53 @@ class Slice {
public: public:
// constructor for an empty Value of type None // constructor for an empty Value of type None
Slice() : Slice("\x00") {} Slice() noexcept : Slice("\x00") {}
// creates a slice of type None // creates a slice of type None
static Slice noneSlice() { return Slice("\x00"); } static Slice noneSlice() noexcept { return Slice("\x00"); }
// creates a slice of type Illegal // creates a slice of type Illegal
static Slice illegalSlice() { return Slice("\x17"); } static Slice illegalSlice() noexcept { return Slice("\x17"); }
// creates a slice of type Null // creates a slice of type Null
static Slice nullSlice() { return Slice("\x18"); } static Slice nullSlice() noexcept { return Slice("\x18"); }
// creates a slice of type Boolean with false value // creates a slice of type Boolean with false value
static Slice falseSlice() { return Slice("\x19"); } static Slice falseSlice() noexcept { return Slice("\x19"); }
// creates a slice of type Boolean with true value // creates a slice of type Boolean with true value
static Slice trueSlice() { return Slice("\x1a"); } static Slice trueSlice() noexcept { return Slice("\x1a"); }
// creates a slice of type Smallint(0) // creates a slice of type Smallint(0)
static Slice zeroSlice() { return Slice("\x30"); } static Slice zeroSlice() noexcept { return Slice("\x30"); }
// creates a slice of type Array, empty // creates a slice of type Array, empty
static Slice emptyArraySlice() { return Slice("\x01"); } static Slice emptyArraySlice() noexcept { return Slice("\x01"); }
// creates a slice of type Object, empty // creates a slice of type Object, empty
static Slice emptyObjectSlice() { return Slice("\x0a"); } static Slice emptyObjectSlice() noexcept { return Slice("\x0a"); }
// creates a slice of type MinKey // creates a slice of type MinKey
static Slice minKeySlice() { return Slice("\x1e"); } static Slice minKeySlice() noexcept { return Slice("\x1e"); }
// creates a slice of type MaxKey // creates a slice of type MaxKey
static Slice maxKeySlice() { return Slice("\x1f"); } static Slice maxKeySlice() noexcept { return Slice("\x1f"); }
// creates a Slice from Json and adds it to a scope // creates a Slice from Json and adds it to a scope
static Slice fromJson(SliceScope& scope, std::string const& json, static Slice fromJson(SliceScope& scope, std::string const& json,
Options const* options = &Options::Defaults); Options const* options = &Options::Defaults);
// creates a Slice from a pointer to a uint8_t array // creates a Slice from a pointer to a uint8_t array
explicit Slice(uint8_t const* start) explicit Slice(uint8_t const* start) noexcept
: _start(start) {} : _start(start) {}
// creates a Slice from a pointer to a char array // creates a Slice from a pointer to a char array
explicit Slice(char const* start) explicit Slice(char const* start) noexcept
: _start(reinterpret_cast<uint8_t const*>(start)) {} : _start(reinterpret_cast<uint8_t const*>(start)) {}
uint8_t const* begin() { return _start; } uint8_t const* begin() noexcept { return _start; }
uint8_t const* begin() const { return _start; } uint8_t const* begin() const noexcept { return _start; }
uint8_t const* end() { return _start + byteSize(); } uint8_t const* end() { return _start + byteSize(); }
@ -124,12 +124,12 @@ class Slice {
// No destructor, does not take part in memory management, // No destructor, does not take part in memory management,
// get the type for the slice // get the type for the slice
inline ValueType type() const throw() { return TypeMap[head()]; } inline ValueType type() const noexcept { return TypeMap[head()]; }
char const* typeName() const { return valueTypeName(type()); } char const* typeName() const { return valueTypeName(type()); }
// pointer to the head byte // pointer to the head byte
uint8_t const* start() const throw() { return _start; } uint8_t const* start() const noexcept { return _start; }
// Set new memory position // Set new memory position
void set(uint8_t const* s) { _start = s; } void set(uint8_t const* s) { _start = s; }
@ -141,7 +141,7 @@ class Slice {
} }
// value of the head byte // value of the head byte
inline uint8_t head() const throw() { return *_start; } inline uint8_t head() const noexcept { return *_start; }
// hashes the binary representation of a value // hashes the binary representation of a value
inline uint64_t hash(uint64_t seed = 0xdeadbeef) const { inline uint64_t hash(uint64_t seed = 0xdeadbeef) const {
@ -155,85 +155,85 @@ class Slice {
// hashes the binary representation of a String slice. No check // hashes the binary representation of a String slice. No check
// is done if the Slice value is actually of type String // is done if the Slice value is actually of type String
inline uint64_t hashString(uint64_t seed = 0xdeadbeef) const throw() { inline uint64_t hashString(uint64_t seed = 0xdeadbeef) const noexcept {
return VELOCYPACK_HASH(start(), static_cast<size_t>(stringSliceLength()), seed); return VELOCYPACK_HASH(start(), static_cast<size_t>(stringSliceLength()), seed);
} }
// check if slice is of the specified type // check if slice is of the specified type
inline bool isType(ValueType t) const throw() { return TypeMap[*_start] == t; } inline bool isType(ValueType t) const noexcept { return TypeMap[*_start] == t; }
// check if slice is a None object // check if slice is a None object
bool isNone() const throw() { return isType(ValueType::None); } bool isNone() const noexcept { return isType(ValueType::None); }
// check if slice is an Illegal object // check if slice is an Illegal object
bool isIllegal() const throw() { return isType(ValueType::Illegal); } bool isIllegal() const noexcept { return isType(ValueType::Illegal); }
// check if slice is a Null object // check if slice is a Null object
bool isNull() const throw() { return isType(ValueType::Null); } bool isNull() const noexcept { return isType(ValueType::Null); }
// check if slice is a Bool object // check if slice is a Bool object
bool isBool() const throw() { return isType(ValueType::Bool); } bool isBool() const noexcept { return isType(ValueType::Bool); }
// check if slice is a Bool object - this is an alias for isBool() // check if slice is a Bool object - this is an alias for isBool()
bool isBoolean() const throw() { return isBool(); } bool isBoolean() const noexcept { return isBool(); }
// check if slice is the Boolean value true // check if slice is the Boolean value true
bool isTrue() const throw() { return head() == 0x1a; } bool isTrue() const noexcept { return head() == 0x1a; }
// check if slice is the Boolean value false // check if slice is the Boolean value false
bool isFalse() const throw() { return head() == 0x19; } bool isFalse() const noexcept { return head() == 0x19; }
// check if slice is an Array object // check if slice is an Array object
bool isArray() const throw() { return isType(ValueType::Array); } bool isArray() const noexcept { return isType(ValueType::Array); }
// check if slice is an Object object // check if slice is an Object object
bool isObject() const throw() { return isType(ValueType::Object); } bool isObject() const noexcept { return isType(ValueType::Object); }
// check if slice is a Double object // check if slice is a Double object
bool isDouble() const throw() { return isType(ValueType::Double); } bool isDouble() const noexcept { return isType(ValueType::Double); }
// check if slice is a UTCDate object // check if slice is a UTCDate object
bool isUTCDate() const throw() { return isType(ValueType::UTCDate); } bool isUTCDate() const noexcept { return isType(ValueType::UTCDate); }
// check if slice is an External object // check if slice is an External object
bool isExternal() const throw() { return isType(ValueType::External); } bool isExternal() const noexcept { return isType(ValueType::External); }
// check if slice is a MinKey object // check if slice is a MinKey object
bool isMinKey() const throw() { return isType(ValueType::MinKey); } bool isMinKey() const noexcept { return isType(ValueType::MinKey); }
// check if slice is a MaxKey object // check if slice is a MaxKey object
bool isMaxKey() const throw() { return isType(ValueType::MaxKey); } bool isMaxKey() const noexcept { return isType(ValueType::MaxKey); }
// check if slice is an Int object // check if slice is an Int object
bool isInt() const throw() { return isType(ValueType::Int); } bool isInt() const noexcept { return isType(ValueType::Int); }
// check if slice is a UInt object // check if slice is a UInt object
bool isUInt() const throw() { return isType(ValueType::UInt); } bool isUInt() const noexcept { return isType(ValueType::UInt); }
// check if slice is a SmallInt object // check if slice is a SmallInt object
bool isSmallInt() const throw() { return isType(ValueType::SmallInt); } bool isSmallInt() const noexcept { return isType(ValueType::SmallInt); }
// check if slice is a String object // check if slice is a String object
bool isString() const throw() { return isType(ValueType::String); } bool isString() const noexcept { return isType(ValueType::String); }
// check if slice is a Binary object // check if slice is a Binary object
bool isBinary() const throw() { return isType(ValueType::Binary); } bool isBinary() const noexcept { return isType(ValueType::Binary); }
// check if slice is a BCD // check if slice is a BCD
bool isBCD() const throw() { return isType(ValueType::BCD); } bool isBCD() const noexcept { return isType(ValueType::BCD); }
// check if slice is a Custom type // check if slice is a Custom type
bool isCustom() const throw() { return isType(ValueType::Custom); } bool isCustom() const noexcept { return isType(ValueType::Custom); }
// check if a slice is any number type // check if a slice is any number type
bool isInteger() const throw() { bool isInteger() const noexcept {
return (isInt() || isUInt() || isSmallInt()); return (isInt() || isUInt() || isSmallInt());
} }
// check if slice is any Number-type object // check if slice is any Number-type object
bool isNumber() const throw() { return isInteger() || isDouble(); } bool isNumber() const noexcept { return isInteger() || isDouble(); }
bool isSorted() const throw() { bool isSorted() const noexcept {
auto const h = head(); auto const h = head();
return (h >= 0x0b && h <= 0x0e); return (h >= 0x0b && h <= 0x0e);
} }
@ -743,7 +743,7 @@ class Slice {
throw Exception(Exception::InternalError); throw Exception(Exception::InternalError);
} }
ValueLength findDataOffset(uint8_t head) const throw() { ValueLength findDataOffset(uint8_t head) const noexcept {
// Must be called for a nonempty array or object at start(): // Must be called for a nonempty array or object at start():
VELOCYPACK_ASSERT(head <= 0x12); VELOCYPACK_ASSERT(head <= 0x12);
unsigned int fsm = FirstSubMap[head]; unsigned int fsm = FirstSubMap[head];
@ -794,7 +794,7 @@ class Slice {
private: private:
// get the total byte size for a String slice, including the head byte // get the total byte size for a String slice, including the head byte
// not check is done if the type of the slice is actually String // not check is done if the type of the slice is actually String
ValueLength stringSliceLength() const throw() { ValueLength stringSliceLength() const noexcept {
// check if the type has a fixed length first // check if the type has a fixed length first
auto const h = head(); auto const h = head();
if (h == 0xbf) { if (h == 0xbf) {
@ -823,7 +823,7 @@ class Slice {
// get the offset for the nth member from a compact Array or Object type // get the offset for the nth member from a compact Array or Object type
ValueLength getNthOffsetFromCompact(ValueLength index) const; ValueLength getNthOffsetFromCompact(ValueLength index) const;
inline ValueLength indexEntrySize(uint8_t head) const throw() { inline ValueLength indexEntrySize(uint8_t head) const noexcept {
VELOCYPACK_ASSERT(head <= 0x12); VELOCYPACK_ASSERT(head <= 0x12);
return static_cast<ValueLength>(WidthMap[head]); return static_cast<ValueLength>(WidthMap[head]);
} }

90
arangod/Aql/EnumerateCollectionBlock.cpp
View File

@ -35,17 +35,13 @@ EnumerateCollectionBlock::EnumerateCollectionBlock(
ExecutionEngine* engine, EnumerateCollectionNode const* ep) ExecutionEngine* engine, EnumerateCollectionNode const* ep)
: ExecutionBlock(engine, ep), : ExecutionBlock(engine, ep),
_collection(ep->_collection), _collection(ep->_collection),
_scanner(nullptr), _scanner(_trx, _collection->getName(), ep->_random),
_iterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue()), _iterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue()),
_mustStoreResult(true) { _mustStoreResult(true) {}
_scanner = new CollectionScanner(_trx, _collection->getName(), ep->_random);
}
EnumerateCollectionBlock::~EnumerateCollectionBlock() { delete _scanner; }
/// @brief initialize fetching of documents /// @brief initialize fetching of documents
void EnumerateCollectionBlock::initializeDocuments() { void EnumerateCollectionBlock::initializeDocuments() {
_scanner->reset(); _scanner.reset();
_documents = VPackSlice(); _documents = VPackSlice();
_iterator = VPackArrayIterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue()); _iterator = VPackArrayIterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue());
} }
@ -56,7 +52,7 @@ bool EnumerateCollectionBlock::skipDocuments(size_t toSkip, size_t& skipped) {
throwIfKilled(); // check if we were aborted throwIfKilled(); // check if we were aborted
uint64_t skippedHere = 0; uint64_t skippedHere = 0;
int res = _scanner->forward(toSkip, skippedHere); int res = _scanner.forward(toSkip, skippedHere);
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res); THROW_ARANGO_EXCEPTION(res);
@ -80,9 +76,9 @@ bool EnumerateCollectionBlock::skipDocuments(size_t toSkip, size_t& skipped) {
/// @brief continue fetching of documents /// @brief continue fetching of documents
bool EnumerateCollectionBlock::moreDocuments(size_t hint) { bool EnumerateCollectionBlock::moreDocuments(size_t hint) {
DEBUG_BEGIN_BLOCK(); DEBUG_BEGIN_BLOCK();
if (hint < DefaultBatchSize()) { if (hint < DefaultBatchSize()) {
hint = DefaultBatchSize(); // hint = DefaultBatchSize();
} }
throwIfKilled(); // check if we were aborted throwIfKilled(); // check if we were aborted
@ -91,7 +87,7 @@ bool EnumerateCollectionBlock::moreDocuments(size_t hint) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
} }
_documents = _scanner->scan(hint); _documents = _scanner.scan(hint);
TRI_ASSERT(_documents.isArray()); TRI_ASSERT(_documents.isArray());
_iterator = VPackArrayIterator(_documents, true); _iterator = VPackArrayIterator(_documents, true);
@ -148,29 +144,47 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast,
if (_done) { if (_done) {
return nullptr; return nullptr;
} }
bool needMore;
AqlItemBlock* cur = nullptr;
if (_buffer.empty()) { do {
size_t toFetch = (std::min)(DefaultBatchSize(), atMost); needMore = false;
if (!ExecutionBlock::getBlock(toFetch, toFetch)) {
_done = true; if (_buffer.empty()) {
return nullptr; size_t toFetch = (std::min)(DefaultBatchSize(), atMost);
if (!ExecutionBlock::getBlock(toFetch, toFetch)) {
_done = true;
return nullptr;
}
_pos = 0; // this is in the first block
initializeDocuments();
} }
_pos = 0; // this is in the first block
initializeDocuments();
}
// If we get here, we do have _buffer.front() // If we get here, we do have _buffer.front()
AqlItemBlock* cur = _buffer.front(); cur = _buffer.front();
size_t const curRegs = cur->getNrRegs();
// Advance read position:
if (_iterator.index() >= _iterator.size()) {
// we have exhausted our local documents buffer
// fetch more documents into our buffer
if (!moreDocuments(atMost)) {
// nothing more to read, re-initialize fetching of documents
needMore = true;
initializeDocuments();
// Get more documents from collection if _documents is empty: if (++_pos >= cur->size()) {
if (_iterator.index() >= _iterator.size()) { _buffer.pop_front(); // does not throw
if (!moreDocuments(atMost)) { returnBlock(cur);
_done = true; _pos = 0;
return nullptr; }
}
} }
} } while (needMore);
TRI_ASSERT(cur != nullptr);
size_t curRegs = cur->getNrRegs();
size_t available = static_cast<size_t>(_iterator.size() - _iterator.index()); size_t available = static_cast<size_t>(_iterator.size() - _iterator.index());
size_t toSend = (std::min)(atMost, available); size_t toSend = (std::min)(atMost, available);
RegisterId nrRegs = RegisterId nrRegs =
@ -197,29 +211,13 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast,
// The result is in the first variable of this depth, // The result is in the first variable of this depth,
// we do not need to do a lookup in getPlanNode()->_registerPlan->varInfo, // we do not need to do a lookup in getPlanNode()->_registerPlan->varInfo,
// but can just take cur->getNrRegs() as registerId: // but can just take cur->getNrRegs() as registerId:
res->setValue(j, static_cast<arangodb::aql::RegisterId>(curRegs), AqlValue(_iterator.value())); res->setValue(j, static_cast<arangodb::aql::RegisterId>(curRegs), AqlValue(_iterator.value().begin()));
// No harm done, if the setValue throws! // No harm done, if the setValue throws!
} }
_iterator.next(); _iterator.next();
} }
// Advance read position:
if (_iterator.index() >= _iterator.size()) {
// we have exhausted our local documents buffer
// fetch more documents into our buffer
if (!moreDocuments(atMost)) {
// nothing more to read, re-initialize fetching of documents
initializeDocuments();
if (++_pos >= cur->size()) {
_buffer.pop_front(); // does not throw
returnBlock(cur);
_pos = 0;
}
}
}
// Clear out registers no longer needed later: // Clear out registers no longer needed later:
clearRegisters(res.get()); clearRegisters(res.get());

5
arangod/Aql/EnumerateCollectionBlock.h
View File

@ -24,6 +24,7 @@
#ifndef ARANGOD_AQL_ENUMERATE_COLLECTION_BLOCK_H #ifndef ARANGOD_AQL_ENUMERATE_COLLECTION_BLOCK_H
#define ARANGOD_AQL_ENUMERATE_COLLECTION_BLOCK_H 1 #define ARANGOD_AQL_ENUMERATE_COLLECTION_BLOCK_H 1
#include "Aql/CollectionScanner.h"
#include "Aql/ExecutionBlock.h" #include "Aql/ExecutionBlock.h"
#include "Aql/ExecutionNode.h" #include "Aql/ExecutionNode.h"
@ -44,7 +45,7 @@ class EnumerateCollectionBlock : public ExecutionBlock {
EnumerateCollectionBlock(ExecutionEngine* engine, EnumerateCollectionBlock(ExecutionEngine* engine,
EnumerateCollectionNode const* ep); EnumerateCollectionNode const* ep);
~EnumerateCollectionBlock(); ~EnumerateCollectionBlock() = default;
/// @brief initialize fetching of documents /// @brief initialize fetching of documents
void initializeDocuments(); void initializeDocuments();
@ -74,7 +75,7 @@ class EnumerateCollectionBlock : public ExecutionBlock {
Collection* _collection; Collection* _collection;
/// @brief collection scanner /// @brief collection scanner
CollectionScanner* _scanner; CollectionScanner _scanner;
/// @brief iterator over documents /// @brief iterator over documents
arangodb::velocypack::ArrayIterator _iterator; arangodb::velocypack::ArrayIterator _iterator;

9
js/server/tests/aql/aql-failures-noncluster.js
View File

@ -419,15 +419,6 @@ function ahuacatlFailureSuite () {
assertFailingQuery("FOR i IN [1,2,3,4] FILTER i IN [1,2,3,4] RETURN i"); assertFailingQuery("FOR i IN [1,2,3,4] FILTER i IN [1,2,3,4] RETURN i");
}, },
////////////////////////////////////////////////////////////////////////////////
/// @brief test failure
////////////////////////////////////////////////////////////////////////////////
testExecutionBlock6 : function () {
internal.debugSetFailAt("ExecutionBlock::getOrSkipSomeConcatenate");
assertFailingQuery("FOR c IN UnitTestsAhuacatlFailures FILTER c.value >= 40 FILTER c.value <= 9999 LIMIT 50, 5 RETURN c");
},
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/// @brief test failure /// @brief test failure
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////