From 0828abbb5cfe10e93f5cfc08e32ef423d6eac01e Mon Sep 17 00:00:00 2001 From: jsteemann Date: Fri, 13 May 2016 14:26:34 +0200 Subject: [PATCH] query less documents --- .../velocypack/include/velocypack/Slice.h | 94 +++++++++---------- arangod/Aql/EnumerateCollectionBlock.cpp | 90 +++++++++--------- arangod/Aql/EnumerateCollectionBlock.h | 5 +- .../tests/aql/aql-failures-noncluster.js | 9 -- 4 files changed, 94 insertions(+), 104 deletions(-) diff --git a/3rdParty/velocypack/include/velocypack/Slice.h b/3rdParty/velocypack/include/velocypack/Slice.h index e44e5c0bd4..6108853c81 100644 --- a/3rdParty/velocypack/include/velocypack/Slice.h +++ b/3rdParty/velocypack/include/velocypack/Slice.h @@ -69,53 +69,53 @@ class Slice { public: // constructor for an empty Value of type None - Slice() : Slice("\x00") {} + Slice() noexcept : Slice("\x00") {} // creates a slice of type None - static Slice noneSlice() { return Slice("\x00"); } + static Slice noneSlice() noexcept { return Slice("\x00"); } // creates a slice of type Illegal - static Slice illegalSlice() { return Slice("\x17"); } + static Slice illegalSlice() noexcept { return Slice("\x17"); } // creates a slice of type Null - static Slice nullSlice() { return Slice("\x18"); } + static Slice nullSlice() noexcept { return Slice("\x18"); } // creates a slice of type Boolean with false value - static Slice falseSlice() { return Slice("\x19"); } + static Slice falseSlice() noexcept { return Slice("\x19"); } // creates a slice of type Boolean with true value - static Slice trueSlice() { return Slice("\x1a"); } + static Slice trueSlice() noexcept { return Slice("\x1a"); } // creates a slice of type Smallint(0) - static Slice zeroSlice() { return Slice("\x30"); } + static Slice zeroSlice() noexcept { return Slice("\x30"); } // creates a slice of type Array, empty - static Slice emptyArraySlice() { return Slice("\x01"); } + static Slice emptyArraySlice() noexcept { return Slice("\x01"); } // creates a slice of type Object, empty - static Slice emptyObjectSlice() { return Slice("\x0a"); } + static Slice emptyObjectSlice() noexcept { return Slice("\x0a"); } // creates a slice of type MinKey - static Slice minKeySlice() { return Slice("\x1e"); } + static Slice minKeySlice() noexcept { return Slice("\x1e"); } // creates a slice of type MaxKey - static Slice maxKeySlice() { return Slice("\x1f"); } + static Slice maxKeySlice() noexcept { return Slice("\x1f"); } // creates a Slice from Json and adds it to a scope static Slice fromJson(SliceScope& scope, std::string const& json, Options const* options = &Options::Defaults); // creates a Slice from a pointer to a uint8_t array - explicit Slice(uint8_t const* start) + explicit Slice(uint8_t const* start) noexcept : _start(start) {} // creates a Slice from a pointer to a char array - explicit Slice(char const* start) + explicit Slice(char const* start) noexcept : _start(reinterpret_cast(start)) {} - uint8_t const* begin() { return _start; } + uint8_t const* begin() noexcept { return _start; } - uint8_t const* begin() const { return _start; } + uint8_t const* begin() const noexcept { return _start; } uint8_t const* end() { return _start + byteSize(); } @@ -124,12 +124,12 @@ class Slice { // No destructor, does not take part in memory management, // get the type for the slice - inline ValueType type() const throw() { return TypeMap[head()]; } + inline ValueType type() const noexcept { return TypeMap[head()]; } char const* typeName() const { return valueTypeName(type()); } // pointer to the head byte - uint8_t const* start() const throw() { return _start; } + uint8_t const* start() const noexcept { return _start; } // Set new memory position void set(uint8_t const* s) { _start = s; } @@ -141,7 +141,7 @@ class Slice { } // value of the head byte - inline uint8_t head() const throw() { return *_start; } + inline uint8_t head() const noexcept { return *_start; } // hashes the binary representation of a value inline uint64_t hash(uint64_t seed = 0xdeadbeef) const { @@ -155,85 +155,85 @@ class Slice { // hashes the binary representation of a String slice. No check // is done if the Slice value is actually of type String - inline uint64_t hashString(uint64_t seed = 0xdeadbeef) const throw() { + inline uint64_t hashString(uint64_t seed = 0xdeadbeef) const noexcept { return VELOCYPACK_HASH(start(), static_cast(stringSliceLength()), seed); } // check if slice is of the specified type - inline bool isType(ValueType t) const throw() { return TypeMap[*_start] == t; } + inline bool isType(ValueType t) const noexcept { return TypeMap[*_start] == t; } // check if slice is a None object - bool isNone() const throw() { return isType(ValueType::None); } + bool isNone() const noexcept { return isType(ValueType::None); } // check if slice is an Illegal object - bool isIllegal() const throw() { return isType(ValueType::Illegal); } + bool isIllegal() const noexcept { return isType(ValueType::Illegal); } // check if slice is a Null object - bool isNull() const throw() { return isType(ValueType::Null); } + bool isNull() const noexcept { return isType(ValueType::Null); } // check if slice is a Bool object - bool isBool() const throw() { return isType(ValueType::Bool); } + bool isBool() const noexcept { return isType(ValueType::Bool); } // check if slice is a Bool object - this is an alias for isBool() - bool isBoolean() const throw() { return isBool(); } + bool isBoolean() const noexcept { return isBool(); } // check if slice is the Boolean value true - bool isTrue() const throw() { return head() == 0x1a; } + bool isTrue() const noexcept { return head() == 0x1a; } // check if slice is the Boolean value false - bool isFalse() const throw() { return head() == 0x19; } + bool isFalse() const noexcept { return head() == 0x19; } // check if slice is an Array object - bool isArray() const throw() { return isType(ValueType::Array); } + bool isArray() const noexcept { return isType(ValueType::Array); } // check if slice is an Object object - bool isObject() const throw() { return isType(ValueType::Object); } + bool isObject() const noexcept { return isType(ValueType::Object); } // check if slice is a Double object - bool isDouble() const throw() { return isType(ValueType::Double); } + bool isDouble() const noexcept { return isType(ValueType::Double); } // check if slice is a UTCDate object - bool isUTCDate() const throw() { return isType(ValueType::UTCDate); } + bool isUTCDate() const noexcept { return isType(ValueType::UTCDate); } // check if slice is an External object - bool isExternal() const throw() { return isType(ValueType::External); } + bool isExternal() const noexcept { return isType(ValueType::External); } // check if slice is a MinKey object - bool isMinKey() const throw() { return isType(ValueType::MinKey); } + bool isMinKey() const noexcept { return isType(ValueType::MinKey); } // check if slice is a MaxKey object - bool isMaxKey() const throw() { return isType(ValueType::MaxKey); } + bool isMaxKey() const noexcept { return isType(ValueType::MaxKey); } // check if slice is an Int object - bool isInt() const throw() { return isType(ValueType::Int); } + bool isInt() const noexcept { return isType(ValueType::Int); } // check if slice is a UInt object - bool isUInt() const throw() { return isType(ValueType::UInt); } + bool isUInt() const noexcept { return isType(ValueType::UInt); } // check if slice is a SmallInt object - bool isSmallInt() const throw() { return isType(ValueType::SmallInt); } + bool isSmallInt() const noexcept { return isType(ValueType::SmallInt); } // check if slice is a String object - bool isString() const throw() { return isType(ValueType::String); } + bool isString() const noexcept { return isType(ValueType::String); } // check if slice is a Binary object - bool isBinary() const throw() { return isType(ValueType::Binary); } + bool isBinary() const noexcept { return isType(ValueType::Binary); } // check if slice is a BCD - bool isBCD() const throw() { return isType(ValueType::BCD); } + bool isBCD() const noexcept { return isType(ValueType::BCD); } // check if slice is a Custom type - bool isCustom() const throw() { return isType(ValueType::Custom); } + bool isCustom() const noexcept { return isType(ValueType::Custom); } // check if a slice is any number type - bool isInteger() const throw() { + bool isInteger() const noexcept { return (isInt() || isUInt() || isSmallInt()); } // check if slice is any Number-type object - bool isNumber() const throw() { return isInteger() || isDouble(); } + bool isNumber() const noexcept { return isInteger() || isDouble(); } - bool isSorted() const throw() { + bool isSorted() const noexcept { auto const h = head(); return (h >= 0x0b && h <= 0x0e); } @@ -743,7 +743,7 @@ class Slice { throw Exception(Exception::InternalError); } - ValueLength findDataOffset(uint8_t head) const throw() { + ValueLength findDataOffset(uint8_t head) const noexcept { // Must be called for a nonempty array or object at start(): VELOCYPACK_ASSERT(head <= 0x12); unsigned int fsm = FirstSubMap[head]; @@ -794,7 +794,7 @@ class Slice { private: // get the total byte size for a String slice, including the head byte // not check is done if the type of the slice is actually String - ValueLength stringSliceLength() const throw() { + ValueLength stringSliceLength() const noexcept { // check if the type has a fixed length first auto const h = head(); if (h == 0xbf) { @@ -823,7 +823,7 @@ class Slice { // get the offset for the nth member from a compact Array or Object type ValueLength getNthOffsetFromCompact(ValueLength index) const; - inline ValueLength indexEntrySize(uint8_t head) const throw() { + inline ValueLength indexEntrySize(uint8_t head) const noexcept { VELOCYPACK_ASSERT(head <= 0x12); return static_cast(WidthMap[head]); } diff --git a/arangod/Aql/EnumerateCollectionBlock.cpp b/arangod/Aql/EnumerateCollectionBlock.cpp index 1de38423cf..2b99c10123 100644 --- a/arangod/Aql/EnumerateCollectionBlock.cpp +++ b/arangod/Aql/EnumerateCollectionBlock.cpp @@ -35,17 +35,13 @@ EnumerateCollectionBlock::EnumerateCollectionBlock( ExecutionEngine* engine, EnumerateCollectionNode const* ep) : ExecutionBlock(engine, ep), _collection(ep->_collection), - _scanner(nullptr), + _scanner(_trx, _collection->getName(), ep->_random), _iterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue()), - _mustStoreResult(true) { - _scanner = new CollectionScanner(_trx, _collection->getName(), ep->_random); -} - -EnumerateCollectionBlock::~EnumerateCollectionBlock() { delete _scanner; } + _mustStoreResult(true) {} /// @brief initialize fetching of documents void EnumerateCollectionBlock::initializeDocuments() { - _scanner->reset(); + _scanner.reset(); _documents = VPackSlice(); _iterator = VPackArrayIterator(arangodb::basics::VelocyPackHelper::EmptyArrayValue()); } @@ -56,7 +52,7 @@ bool EnumerateCollectionBlock::skipDocuments(size_t toSkip, size_t& skipped) { throwIfKilled(); // check if we were aborted uint64_t skippedHere = 0; - int res = _scanner->forward(toSkip, skippedHere); + int res = _scanner.forward(toSkip, skippedHere); if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); @@ -80,9 +76,9 @@ bool EnumerateCollectionBlock::skipDocuments(size_t toSkip, size_t& skipped) { /// @brief continue fetching of documents bool EnumerateCollectionBlock::moreDocuments(size_t hint) { - DEBUG_BEGIN_BLOCK(); + DEBUG_BEGIN_BLOCK(); if (hint < DefaultBatchSize()) { - hint = DefaultBatchSize(); + // hint = DefaultBatchSize(); } throwIfKilled(); // check if we were aborted @@ -91,7 +87,7 @@ bool EnumerateCollectionBlock::moreDocuments(size_t hint) { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - _documents = _scanner->scan(hint); + _documents = _scanner.scan(hint); TRI_ASSERT(_documents.isArray()); _iterator = VPackArrayIterator(_documents, true); @@ -148,29 +144,47 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast, if (_done) { return nullptr; } + + bool needMore; + AqlItemBlock* cur = nullptr; - if (_buffer.empty()) { - size_t toFetch = (std::min)(DefaultBatchSize(), atMost); - if (!ExecutionBlock::getBlock(toFetch, toFetch)) { - _done = true; - return nullptr; + do { + needMore = false; + + if (_buffer.empty()) { + size_t toFetch = (std::min)(DefaultBatchSize(), atMost); + if (!ExecutionBlock::getBlock(toFetch, toFetch)) { + _done = true; + return nullptr; + } + _pos = 0; // this is in the first block + initializeDocuments(); } - _pos = 0; // this is in the first block - initializeDocuments(); - } - // If we get here, we do have _buffer.front() - AqlItemBlock* cur = _buffer.front(); - size_t const curRegs = cur->getNrRegs(); + // If we get here, we do have _buffer.front() + cur = _buffer.front(); + + // Advance read position: + if (_iterator.index() >= _iterator.size()) { + // we have exhausted our local documents buffer + // fetch more documents into our buffer + if (!moreDocuments(atMost)) { + // nothing more to read, re-initialize fetching of documents + needMore = true; + initializeDocuments(); - // Get more documents from collection if _documents is empty: - if (_iterator.index() >= _iterator.size()) { - if (!moreDocuments(atMost)) { - _done = true; - return nullptr; + if (++_pos >= cur->size()) { + _buffer.pop_front(); // does not throw + returnBlock(cur); + _pos = 0; + } + } } - } + } while (needMore); + TRI_ASSERT(cur != nullptr); + size_t curRegs = cur->getNrRegs(); + size_t available = static_cast(_iterator.size() - _iterator.index()); size_t toSend = (std::min)(atMost, available); RegisterId nrRegs = @@ -197,29 +211,13 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast, // The result is in the first variable of this depth, // we do not need to do a lookup in getPlanNode()->_registerPlan->varInfo, // but can just take cur->getNrRegs() as registerId: - res->setValue(j, static_cast(curRegs), AqlValue(_iterator.value())); + res->setValue(j, static_cast(curRegs), AqlValue(_iterator.value().begin())); // No harm done, if the setValue throws! } _iterator.next(); } - - // Advance read position: - if (_iterator.index() >= _iterator.size()) { - // we have exhausted our local documents buffer - // fetch more documents into our buffer - if (!moreDocuments(atMost)) { - // nothing more to read, re-initialize fetching of documents - initializeDocuments(); - - if (++_pos >= cur->size()) { - _buffer.pop_front(); // does not throw - returnBlock(cur); - _pos = 0; - } - } - } - + // Clear out registers no longer needed later: clearRegisters(res.get()); diff --git a/arangod/Aql/EnumerateCollectionBlock.h b/arangod/Aql/EnumerateCollectionBlock.h index 18aff8a895..29d4bb59df 100644 --- a/arangod/Aql/EnumerateCollectionBlock.h +++ b/arangod/Aql/EnumerateCollectionBlock.h @@ -24,6 +24,7 @@ #ifndef ARANGOD_AQL_ENUMERATE_COLLECTION_BLOCK_H #define ARANGOD_AQL_ENUMERATE_COLLECTION_BLOCK_H 1 +#include "Aql/CollectionScanner.h" #include "Aql/ExecutionBlock.h" #include "Aql/ExecutionNode.h" @@ -44,7 +45,7 @@ class EnumerateCollectionBlock : public ExecutionBlock { EnumerateCollectionBlock(ExecutionEngine* engine, EnumerateCollectionNode const* ep); - ~EnumerateCollectionBlock(); + ~EnumerateCollectionBlock() = default; /// @brief initialize fetching of documents void initializeDocuments(); @@ -74,7 +75,7 @@ class EnumerateCollectionBlock : public ExecutionBlock { Collection* _collection; /// @brief collection scanner - CollectionScanner* _scanner; + CollectionScanner _scanner; /// @brief iterator over documents arangodb::velocypack::ArrayIterator _iterator; diff --git a/js/server/tests/aql/aql-failures-noncluster.js b/js/server/tests/aql/aql-failures-noncluster.js index 0d90377eb2..2eaf42c62a 100644 --- a/js/server/tests/aql/aql-failures-noncluster.js +++ b/js/server/tests/aql/aql-failures-noncluster.js @@ -419,15 +419,6 @@ function ahuacatlFailureSuite () { assertFailingQuery("FOR i IN [1,2,3,4] FILTER i IN [1,2,3,4] RETURN i"); }, -//////////////////////////////////////////////////////////////////////////////// -/// @brief test failure -//////////////////////////////////////////////////////////////////////////////// - - testExecutionBlock6 : function () { - internal.debugSetFailAt("ExecutionBlock::getOrSkipSomeConcatenate"); - assertFailingQuery("FOR c IN UnitTestsAhuacatlFailures FILTER c.value >= 40 FILTER c.value <= 9999 LIMIT 50, 5 RETURN c"); - }, - //////////////////////////////////////////////////////////////////////////////// /// @brief test failure ////////////////////////////////////////////////////////////////////////////////