From 6e7f9ef49c8d3336252c6595c18c26013198d0fd Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 1 Mar 2016 14:37:40 +0100 Subject: [PATCH 1/5] Transaction OperationCursor now implements a skip method. --- arangod/Indexes/IndexIterator.cpp | 3 ++- arangod/Indexes/IndexIterator.h | 2 +- arangod/Utils/OperationCursor.cpp | 29 +++++++++++++++++++++++++++++ arangod/Utils/OperationCursor.h | 11 ++++++++++- arangod/Utils/Transaction.cpp | 3 ++- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/arangod/Indexes/IndexIterator.cpp b/arangod/Indexes/IndexIterator.cpp index b0700e15c5..fa6deb0bab 100644 --- a/arangod/Indexes/IndexIterator.cpp +++ b/arangod/Indexes/IndexIterator.cpp @@ -103,11 +103,12 @@ void IndexIterator::reset() {} /// @brief default implementation for skip //////////////////////////////////////////////////////////////////////////////// -void IndexIterator::skip(uint64_t count) { +void IndexIterator::skip(uint64_t count, uint64_t& skipped) { // Skip the first count-many entries // TODO: Can be improved while (count > 0 && next() != nullptr) { --count; + skipped++; } } diff --git a/arangod/Indexes/IndexIterator.h b/arangod/Indexes/IndexIterator.h index bbb26aa220..b82dd8c018 100644 --- a/arangod/Indexes/IndexIterator.h +++ b/arangod/Indexes/IndexIterator.h @@ -71,7 +71,7 @@ class IndexIterator { virtual void reset(); - virtual void skip(uint64_t count); + virtual void skip(uint64_t count, uint64_t& skipped); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Utils/OperationCursor.cpp b/arangod/Utils/OperationCursor.cpp index 41696f36e9..f884575ed0 100644 --- a/arangod/Utils/OperationCursor.cpp +++ b/arangod/Utils/OperationCursor.cpp @@ -64,3 +64,32 @@ int OperationCursor::getMore(uint64_t batchSize) { } return TRI_ERROR_NO_ERROR; } + +////////////////////////////////////////////////////////////////////////////// +/// @brief Skip the next toSkip many elements. +/// skipped will be increased by the amount of skipped elements afterwards +/// Check hasMore()==true before using this +/// NOTE: This will throw on OUT_OF_MEMORY +////////////////////////////////////////////////////////////////////////////// + +int OperationCursor::skip(uint64_t toSkip, uint64_t& skipped) { + if (!hasMore()) { + TRI_ASSERT(false); + // You requested more even if you should have checked it before. + return TRI_ERROR_FORBIDDEN; + } + + if (toSkip > _limit) { + // Short-cut, we jump to the end + _limit = 0; + _hasMore = false; + return TRI_ERROR_NO_ERROR; + } + + _indexIterator->skip(toSkip, skipped); + _limit -= skipped; + if (skipped != toSkip || _limit == 0) { + _hasMore = false; + } + return TRI_ERROR_NO_ERROR; +} diff --git a/arangod/Utils/OperationCursor.h b/arangod/Utils/OperationCursor.h index 32b0b0edc8..fdf58f8445 100644 --- a/arangod/Utils/OperationCursor.h +++ b/arangod/Utils/OperationCursor.h @@ -100,7 +100,7 @@ struct OperationCursor : public OperationResult { /// NOTE: This will throw on OUT_OF_MEMORY ////////////////////////////////////////////////////////////////////////////// - int getMore(uint64_t batchSize); + int getMore(uint64_t); ////////////////////////////////////////////////////////////////////////////// /// @brief Get next default batchSize many elements. @@ -109,6 +109,15 @@ struct OperationCursor : public OperationResult { ////////////////////////////////////////////////////////////////////////////// int getMore(); + +////////////////////////////////////////////////////////////////////////////// +/// @brief Skip the next toSkip many elements. +/// skipped will be increased by the amount of skipped elements afterwards +/// Check hasMore()==true before using this +/// NOTE: This will throw on OUT_OF_MEMORY +////////////////////////////////////////////////////////////////////////////// + + int skip(uint64_t, uint64_t&); }; } diff --git a/arangod/Utils/Transaction.cpp b/arangod/Utils/Transaction.cpp index a9bfc258bb..8aff64202d 100644 --- a/arangod/Utils/Transaction.cpp +++ b/arangod/Utils/Transaction.cpp @@ -1793,7 +1793,8 @@ OperationCursor Transaction::indexScan( return OperationCursor(TRI_ERROR_OUT_OF_MEMORY); } - iterator->skip(skip); + uint64_t unused = 0; + iterator->skip(skip, unused); return OperationCursor(transactionContext()->orderCustomTypeHandler(), iterator.release(), limit, batchSize); From 03dbc4a7450f180cbef89433b13017e41c0d05c3 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 1 Mar 2016 16:36:02 +0100 Subject: [PATCH 2/5] Added a dummy to insert External VPackSlice references in AqlItemBlock. --- arangod/Aql/AqlItemBlock.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arangod/Aql/AqlItemBlock.h b/arangod/Aql/AqlItemBlock.h index 9dcfb8f813..6e808edad9 100644 --- a/arangod/Aql/AqlItemBlock.h +++ b/arangod/Aql/AqlItemBlock.h @@ -134,6 +134,16 @@ class AqlItemBlock { v._type = AqlValue::SHAPED; } + ////////////////////////////////////////////////////////////////////////////// + /// @brief fill a slot in the item block with an external VelocyPack value. + /// This value should not be freed. + ////////////////////////////////////////////////////////////////////////////// + + void setExternal(size_t index, RegisterId varNr, + arangodb::velocypack::Slice external) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + } + ////////////////////////////////////////////////////////////////////////////// /// @brief eraseValue, erase the current value of a register and freeing it /// if this was the last reference to the value From 922a96fee0553dda00e46f021d9c48f85457240e Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 1 Mar 2016 16:37:21 +0100 Subject: [PATCH 3/5] Added a reset function to OperationCursor. ALso added a switch to getMore to define that all documents are referenced as Externals instead of inplace --- arangod/Utils/OperationCursor.cpp | 15 +++++++++++++-- arangod/Utils/OperationCursor.h | 8 +++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/arangod/Utils/OperationCursor.cpp b/arangod/Utils/OperationCursor.cpp index f884575ed0..d729ca4bfd 100644 --- a/arangod/Utils/OperationCursor.cpp +++ b/arangod/Utils/OperationCursor.cpp @@ -25,6 +25,11 @@ using namespace arangodb; + +void OperationCursor::reset() { + _indexIterator->reset(); +} + ////////////////////////////////////////////////////////////////////////////// /// @brief Get next default batchSize many elements. /// Check hasMore()==true before using this @@ -38,10 +43,12 @@ int OperationCursor::getMore() { ////////////////////////////////////////////////////////////////////////////// /// @brief Get next batchSize many elements. /// Check hasMore()==true before using this +/// If useExternals is set to true all elements in the vpack are +/// externals. Otherwise they are inlined. /// NOTE: This will throw on OUT_OF_MEMORY ////////////////////////////////////////////////////////////////////////////// -int OperationCursor::getMore(uint64_t batchSize) { +int OperationCursor::getMore(uint64_t batchSize, bool useExternals) { // This may throw out of memory if (!hasMore()) { TRI_ASSERT(false); @@ -56,7 +63,11 @@ int OperationCursor::getMore(uint64_t batchSize) { while (batchSize > 0 && _limit > 0 && (mptr = _indexIterator->next()) != nullptr) { --batchSize; --_limit; - _builder.add(VPackSlice(mptr->vpack())); + if (useExternals) { + _builder.add(VPackValue(mptr->vpack(), VPackValueType::External)); + } else { + _builder.add(VPackSlice(mptr->vpack())); + } } if (batchSize > 0 || _limit == 0) { // Iterator empty, there is no more diff --git a/arangod/Utils/OperationCursor.h b/arangod/Utils/OperationCursor.h index fdf58f8445..9c33b6d875 100644 --- a/arangod/Utils/OperationCursor.h +++ b/arangod/Utils/OperationCursor.h @@ -94,13 +94,19 @@ struct OperationCursor : public OperationResult { return _hasMore; } +////////////////////////////////////////////////////////////////////////////// +/// @brief Reset the cursor +////////////////////////////////////////////////////////////////////////////// + + void reset(); + ////////////////////////////////////////////////////////////////////////////// /// @brief Get next batchSize many elements. /// Check hasMore()==true before using this /// NOTE: This will throw on OUT_OF_MEMORY ////////////////////////////////////////////////////////////////////////////// - int getMore(uint64_t); + int getMore(uint64_t, bool useExternals = false); ////////////////////////////////////////////////////////////////////////////// /// @brief Get next default batchSize many elements. From 7e30dab62facb0cd12da53083c82fb081c14b1d6 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 1 Mar 2016 16:38:55 +0100 Subject: [PATCH 4/5] CollectionScanner and EnumerateCollectionBlocks now use VPack instead of Shapes and mptrs. --- arangod/Aql/CollectionScanner.cpp | 73 +++++++----------------- arangod/Aql/CollectionScanner.h | 48 ++++------------ arangod/Aql/EnumerateCollectionBlock.cpp | 62 ++++++++------------ arangod/Aql/EnumerateCollectionBlock.h | 8 ++- 4 files changed, 62 insertions(+), 129 deletions(-) diff --git a/arangod/Aql/CollectionScanner.cpp b/arangod/Aql/CollectionScanner.cpp index a79b6d7616..e4ae55e222 100644 --- a/arangod/Aql/CollectionScanner.cpp +++ b/arangod/Aql/CollectionScanner.cpp @@ -25,64 +25,31 @@ using namespace arangodb::aql; -CollectionScanner::CollectionScanner( - arangodb::AqlTransaction* trx, TRI_transaction_collection_t* trxCollection) - : trx(trx), trxCollection(trxCollection), totalCount(0) {} +CollectionScanner::CollectionScanner(arangodb::AqlTransaction* trx, + std::string const& collection, + bool readRandom) + : _cursor(trx->indexScan(collection, + (readRandom ? Transaction::CursorType::ANY + : Transaction::CursorType::ALL), + "", VPackSlice(), 0, UINT64_MAX, 1000, false)) { + TRI_ASSERT(_cursor.successful()); + } CollectionScanner::~CollectionScanner() {} -RandomCollectionScanner::RandomCollectionScanner( - arangodb::AqlTransaction* trx, TRI_transaction_collection_t* trxCollection) - : CollectionScanner(trx, trxCollection), step(0) {} - -int RandomCollectionScanner::scan(std::vector& docs, - size_t batchSize) { - return trx->any(trxCollection, docs, initialPosition, position, - static_cast(batchSize), step, totalCount); +VPackSlice CollectionScanner::scan(size_t batchSize) { + if (!_cursor.hasMore()) { + VPackSlice empty; + return empty; + } + _cursor.getMore(batchSize, true); + return _cursor.slice(); } -int RandomCollectionScanner::forward(size_t batchSize, size_t& skipped) { - // Basic implementation, no gain - std::vector unusedDocs; - unusedDocs.reserve(batchSize); - int res = scan(unusedDocs, batchSize); - skipped += unusedDocs.size(); - // TRI_doc_mptr_t is never freed - unusedDocs.clear(); - return res; +int CollectionScanner::forward(size_t batchSize, uint64_t& skipped) { + return _cursor.skip(batchSize, skipped); } -void RandomCollectionScanner::reset() { - initialPosition.reset(); - position.reset(); - step = 0; +void CollectionScanner::reset() { + _cursor.reset(); } - -LinearCollectionScanner::LinearCollectionScanner( - arangodb::AqlTransaction* trx, TRI_transaction_collection_t* trxCollection) - : CollectionScanner(trx, trxCollection) {} - -int LinearCollectionScanner::scan(std::vector& docs, - size_t batchSize) { - uint64_t skip = 0; - return trx->readIncremental(trxCollection, docs, position, - static_cast(batchSize), skip, - UINT64_MAX, totalCount); -} - -int LinearCollectionScanner::forward(size_t batchSize, size_t& skipped) { - // Basic implementation, no gain - std::vector unusedDocs; - uint64_t toSkip = static_cast(batchSize); - - int res = trx->readIncremental(trxCollection, unusedDocs, position, 0, - toSkip, // Will be modified. Will reach 0 if - // batchSize many docs have been skipped - UINT64_MAX, totalCount); - uint64_t reallySkipped = static_cast(batchSize) - toSkip; - skipped += static_cast(reallySkipped); - TRI_ASSERT(unusedDocs.empty()); - return res; -} - -void LinearCollectionScanner::reset() { position.reset(); } diff --git a/arangod/Aql/CollectionScanner.h b/arangod/Aql/CollectionScanner.h index 54861b8c40..469b84d9f5 100644 --- a/arangod/Aql/CollectionScanner.h +++ b/arangod/Aql/CollectionScanner.h @@ -26,21 +26,20 @@ #include "Basics/Common.h" #include "Utils/AqlTransaction.h" -#include "VocBase/document-collection.h" -#include "VocBase/transaction.h" -#include "VocBase/vocbase.h" +#include "Utils/OperationCursor.h" namespace arangodb { namespace aql { -struct CollectionScanner { - CollectionScanner(arangodb::AqlTransaction*, TRI_transaction_collection_t*); +class CollectionScanner { + public: + CollectionScanner(arangodb::AqlTransaction*, std::string const&, bool); - virtual ~CollectionScanner(); + ~CollectionScanner(); - virtual int scan(std::vector&, size_t) = 0; + arangodb::velocypack::Slice scan(size_t); - virtual void reset() = 0; + void reset(); ////////////////////////////////////////////////////////////////////////////// /// @brief forwards the cursor n elements. Does not read the data. @@ -49,37 +48,10 @@ struct CollectionScanner { /// really skipped ////////////////////////////////////////////////////////////////////////////// - virtual int forward(size_t, size_t&) = 0; + int forward(size_t, uint64_t&); - arangodb::AqlTransaction* trx; - TRI_transaction_collection_t* trxCollection; - uint64_t totalCount; - arangodb::basics::BucketPosition position; -}; - -struct RandomCollectionScanner final : public CollectionScanner { - RandomCollectionScanner(arangodb::AqlTransaction*, - TRI_transaction_collection_t*); - - int scan(std::vector&, size_t) override; - - void reset() override; - - int forward(size_t, size_t&) override; - - arangodb::basics::BucketPosition initialPosition; - uint64_t step; -}; - -struct LinearCollectionScanner final : public CollectionScanner { - LinearCollectionScanner(arangodb::AqlTransaction*, - TRI_transaction_collection_t*); - - int scan(std::vector&, size_t) override; - - void reset() override; - - int forward(size_t, size_t&) override; + private: + OperationCursor _cursor; }; } } diff --git a/arangod/Aql/EnumerateCollectionBlock.cpp b/arangod/Aql/EnumerateCollectionBlock.cpp index 45601c0418..e05797199c 100644 --- a/arangod/Aql/EnumerateCollectionBlock.cpp +++ b/arangod/Aql/EnumerateCollectionBlock.cpp @@ -41,18 +41,7 @@ EnumerateCollectionBlock::EnumerateCollectionBlock( _posInDocuments(0), _random(ep->_random), _mustStoreResult(true) { - auto trxCollection = _trx->trxCollection(_collection->cid()); - if (trxCollection != nullptr) { - _trx->orderDitch(trxCollection); - } - - if (_random) { - // random scan - _scanner = new RandomCollectionScanner(_trx, trxCollection); - } else { - // default: linear scan - _scanner = new LinearCollectionScanner(_trx, trxCollection); - } + _scanner = new CollectionScanner(_trx, _collection->getName(), _random); } EnumerateCollectionBlock::~EnumerateCollectionBlock() { delete _scanner; } @@ -63,7 +52,8 @@ EnumerateCollectionBlock::~EnumerateCollectionBlock() { delete _scanner; } void EnumerateCollectionBlock::initializeDocuments() { _scanner->reset(); - _documents.clear(); + VPackSlice none; + _documents = none; _posInDocuments = 0; } @@ -73,7 +63,7 @@ void EnumerateCollectionBlock::initializeDocuments() { bool EnumerateCollectionBlock::skipDocuments(size_t toSkip, size_t& skipped) { throwIfKilled(); // check if we were aborted - size_t skippedHere = 0; + uint64_t skippedHere = 0; int res = _scanner->forward(toSkip, skippedHere); @@ -83,7 +73,8 @@ bool EnumerateCollectionBlock::skipDocuments(size_t toSkip, size_t& skipped) { skipped += skippedHere; - _documents.clear(); + VPackSlice none; + _documents = none; _posInDocuments = 0; _engine->_stats.scannedFull += static_cast(skippedHere); @@ -111,22 +102,19 @@ bool EnumerateCollectionBlock::moreDocuments(size_t hint) { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - std::vector newDocs; - newDocs.reserve(hint); + _documents = _scanner->scan(hint); + TRI_ASSERT(_documents.isArray()); + VPackValueLength count = _documents.length(); - int res = _scanner->scan(newDocs, hint); - - if (res != TRI_ERROR_NO_ERROR) { - THROW_ARANGO_EXCEPTION(res); - } - - if (newDocs.empty()) { + if (count == 0) { + VPackSlice none; + _documents = none; return false; } - _engine->_stats.scannedFull += static_cast(newDocs.size()); + _engine->_stats.scannedFull += static_cast(count); - _documents.swap(newDocs); + _documentsSize = static_cast(count); _posInDocuments = 0; return true; @@ -182,14 +170,14 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast, size_t const curRegs = cur->getNrRegs(); // Get more documents from collection if _documents is empty: - if (_posInDocuments >= _documents.size()) { + if (_posInDocuments >= _documentsSize) { if (!moreDocuments(atMost)) { _done = true; return nullptr; } } - size_t available = _documents.size() - _posInDocuments; + size_t available = _documentsSize - _posInDocuments; size_t toSend = (std::min)(atMost, available); RegisterId nrRegs = getPlanNode()->getRegisterPlan()->nrRegs[getPlanNode()->getDepth()]; @@ -219,9 +207,8 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast, // The result is in the first variable of this depth, // we do not need to do a lookup in getPlanNode()->_registerPlan->varInfo, // but can just take cur->getNrRegs() as registerId: - res->setShaped(j, static_cast(curRegs), - reinterpret_cast( - _documents[_posInDocuments].getDataPtr())); + res->setExternal(j, static_cast(curRegs), + _documents.at(_posInDocuments)); // No harm done, if the setValue throws! } @@ -229,7 +216,7 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t, // atLeast, } // Advance read position: - if (_posInDocuments >= _documents.size()) { + if (_posInDocuments >= _documentsSize) { // we have exhausted our local documents buffer // fetch more documents into our buffer if (!moreDocuments(atMost)) { @@ -257,18 +244,19 @@ size_t EnumerateCollectionBlock::skipSome(size_t atLeast, size_t atMost) { return skipped; } - if (!_documents.empty()) { - if (_posInDocuments < _documents.size()) { + if (!_documents.isNone()) { + if (_posInDocuments < _documentsSize) { // We still have unread documents in the _documents buffer // Just skip them - size_t couldSkip = _documents.size() - _posInDocuments; + size_t couldSkip = _documentsSize - _posInDocuments; if (atMost <= couldSkip) { // More in buffer then to skip. _posInDocuments += atMost; return atMost; } // Skip entire buffer - _documents.clear(); + VPackSlice none; + _documents = none; _posInDocuments = 0; skipped += couldSkip; } @@ -276,7 +264,7 @@ size_t EnumerateCollectionBlock::skipSome(size_t atLeast, size_t atMost) { // No _documents buffer. But could Skip more // Fastforward the _scanner - TRI_ASSERT(_documents.empty()); + TRI_ASSERT(_documents.isNone()); while (skipped < atLeast) { if (_buffer.empty()) { diff --git a/arangod/Aql/EnumerateCollectionBlock.h b/arangod/Aql/EnumerateCollectionBlock.h index 2184abe81d..51992faa1f 100644 --- a/arangod/Aql/EnumerateCollectionBlock.h +++ b/arangod/Aql/EnumerateCollectionBlock.h @@ -104,7 +104,13 @@ class EnumerateCollectionBlock : public ExecutionBlock { /// @brief document buffer ////////////////////////////////////////////////////////////////////////////// - std::vector _documents; + arangodb::velocypack::Slice _documents; + + ////////////////////////////////////////////////////////////////////////////// + /// @brief length of documents + ////////////////////////////////////////////////////////////////////////////// + + size_t _documentsSize; ////////////////////////////////////////////////////////////////////////////// /// @brief current position in _documents From 026cb571387a1fce805ce8d20bb79629bd947b3e Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 1 Mar 2016 17:03:44 +0100 Subject: [PATCH 5/5] collection export now using vpack --- arangod/Utils/CollectionExport.cpp | 5 +- arangod/Utils/Cursor.cpp | 91 +++++++--------------------- arangod/Utils/TransactionContext.cpp | 11 +++- arangod/Utils/TransactionContext.h | 11 ++++ 4 files changed, 47 insertions(+), 71 deletions(-) diff --git a/arangod/Utils/CollectionExport.cpp b/arangod/Utils/CollectionExport.cpp index 6181de5b2c..6ee25a3caa 100644 --- a/arangod/Utils/CollectionExport.cpp +++ b/arangod/Utils/CollectionExport.cpp @@ -100,13 +100,14 @@ void CollectionExport::run(uint64_t maxWaitTime, size_t limit) { SingleCollectionTransaction trx(StandaloneTransactionContext::Create(_document->_vocbase), _name, TRI_TRANSACTION_READ); - trx.addHint(TRI_TRANSACTION_HINT_NO_USAGE_LOCK, - true); // already locked by guard above + // already locked by guard above + trx.addHint(TRI_TRANSACTION_HINT_NO_USAGE_LOCK, true); int res = trx.begin(); if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); } + auto idx = _document->primaryIndex(); size_t maxDocuments = idx->size(); if (limit > 0 && limit < maxDocuments) { diff --git a/arangod/Utils/Cursor.cpp b/arangod/Utils/Cursor.cpp index d6477c87af..0e82cfd0dd 100644 --- a/arangod/Utils/Cursor.cpp +++ b/arangod/Utils/Cursor.cpp @@ -26,14 +26,15 @@ #include "Basics/VelocyPackHelper.h" #include "Basics/VPackStringBufferAdapter.h" #include "Utils/CollectionExport.h" +#include "Utils/CollectionNameResolver.h" +#include "Utils/TransactionContext.h" #include "VocBase/document-collection.h" -#include "VocBase/shaped-json.h" #include "VocBase/vocbase.h" -#include "VocBase/VocShaper.h" #include #include #include +#include #include using namespace arangodb; @@ -276,13 +277,20 @@ static bool IncludeAttribute( //////////////////////////////////////////////////////////////////////////////// void ExportCursor::dump(arangodb::basics::StringBuffer& buffer) { - TRI_ASSERT(_ex != nullptr); + auto resolver = std::make_unique(_vocbase); + std::unique_ptr customTypeHandler(TransactionContext::createCustomTypeHandler(_vocbase, resolver.get())); - auto shaper = _ex->_document->getShaper(); + TRI_ASSERT(_ex != nullptr); auto const restrictionType = _ex->_restrictions.type; buffer.appendText("\"result\":["); + // copy original options + VPackOptions options = VPackOptions::Defaults; + options.customTypeHandler = customTypeHandler.get(); + + VPackBuilder result; + size_t const n = batchSize(); for (size_t i = 0; i < n; ++i) { @@ -294,31 +302,17 @@ void ExportCursor::dump(arangodb::basics::StringBuffer& buffer) { buffer.appendChar(','); } - auto marker = - static_cast(_ex->_documents->at(_position++)); + char const* p = reinterpret_cast(_ex->_documents->at(_position++)); + VPackSlice const slice(p + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT)); - TRI_shaped_json_t shaped; - TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, marker); - // Only Temporary wait for Shaped ==> VPack - std::unique_ptr tmp(TRI_JsonShapedJson(shaper, &shaped)); - std::shared_ptr builder = arangodb::basics::JsonHelper::toVelocyPack(tmp.get()); - - if (builder == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); - } - VPackSlice const shapedSlice = builder->slice(); - VPackBuilder result; { + result.clear(); + VPackObjectBuilder b(&result); // Copy over shaped values - for (auto const& entry : VPackObjectIterator(shapedSlice)) { - std::string key = entry.key.copyString(); - if (key == TRI_VOC_ATTRIBUTE_ID || key == TRI_VOC_ATTRIBUTE_KEY || - key == TRI_VOC_ATTRIBUTE_FROM || key == TRI_VOC_ATTRIBUTE_TO || - key == TRI_VOC_ATTRIBUTE_REV) { - // This if excludes all internal values. Just to make sure they are not present when added later. - continue; - } + for (auto const& entry : VPackObjectIterator(slice)) { + std::string key(entry.key.copyString()); + if (!IncludeAttribute(restrictionType, _ex->_restrictions.fields, key)) { // Ignore everything that should be excluded or not included continue; @@ -327,51 +321,12 @@ void ExportCursor::dump(arangodb::basics::StringBuffer& buffer) { result.add(key, entry.value); } // append the internal attributes - - // _id, _key, _rev - char const* key = TRI_EXTRACT_MARKER_KEY(marker); - if (IncludeAttribute(restrictionType, _ex->_restrictions.fields, TRI_VOC_ATTRIBUTE_ID)) { - std::string id( - _ex->_resolver.getCollectionName(_ex->_document->_info.id())); - id.push_back('/'); - id.append(key); - result.add(TRI_VOC_ATTRIBUTE_ID, VPackValue(id)); - } - if (IncludeAttribute(restrictionType, _ex->_restrictions.fields, TRI_VOC_ATTRIBUTE_KEY)) { - result.add(TRI_VOC_ATTRIBUTE_KEY, VPackValue(key)); - } - if (IncludeAttribute(restrictionType, _ex->_restrictions.fields, TRI_VOC_ATTRIBUTE_REV)) { - std::string rev = std::to_string(TRI_EXTRACT_MARKER_RID(marker)); - result.add(TRI_VOC_ATTRIBUTE_REV, VPackValue(rev)); - } - -#if 0 - // TODO - if (TRI_IS_EDGE_MARKER(marker)) { - if (IncludeAttribute(restrictionType, _ex->_restrictions.fields, TRI_VOC_ATTRIBUTE_FROM)) { - // _from - std::string from(_ex->_resolver.getCollectionNameCluster( - TRI_EXTRACT_MARKER_FROM_CID(marker))); - from.push_back('/'); - from.append(TRI_EXTRACT_MARKER_FROM_KEY(marker)); - result.add(TRI_VOC_ATTRIBUTE_FROM, VPackValue(from)); - } - - if (IncludeAttribute(restrictionType, _ex->_restrictions.fields, TRI_VOC_ATTRIBUTE_TO)) { - // _to - std::string to(_ex->_resolver.getCollectionNameCluster( - TRI_EXTRACT_MARKER_TO_CID(marker))); - to.push_back('/'); - to.append(TRI_EXTRACT_MARKER_TO_KEY(marker)); - result.add(TRI_VOC_ATTRIBUTE_FROM, VPackValue(to)); - } - } -#endif } - arangodb::basics::VPackStringBufferAdapter bufferAdapter( - buffer.stringBuffer()); - VPackDumper dumper(&bufferAdapter); + + arangodb::basics::VPackStringBufferAdapter bufferAdapter(buffer.stringBuffer()); + try { + VPackDumper dumper(&bufferAdapter, &options); dumper.dump(result.slice()); } catch (...) { /// TODO correct error Handling! diff --git a/arangod/Utils/TransactionContext.cpp b/arangod/Utils/TransactionContext.cpp index c2e4cc3a0f..eb8646b2c6 100644 --- a/arangod/Utils/TransactionContext.cpp +++ b/arangod/Utils/TransactionContext.cpp @@ -134,6 +134,15 @@ TransactionContext::~TransactionContext() { _resolver = nullptr; } } + +////////////////////////////////////////////////////////////////////////////// +/// @brief factory to create a custom type handler, not managed +////////////////////////////////////////////////////////////////////////////// + +VPackCustomTypeHandler* TransactionContext::createCustomTypeHandler(TRI_vocbase_t* vocbase, + CollectionNameResolver const* resolver) { + return new CustomTypeHandler(vocbase, resolver); +} ////////////////////////////////////////////////////////////////////////////// /// @brief order a document ditch for the collection @@ -141,7 +150,7 @@ TransactionContext::~TransactionContext() { std::shared_ptr TransactionContext::orderCustomTypeHandler() { if (_customTypeHandler == nullptr) { - _customTypeHandler.reset(new CustomTypeHandler(_vocbase, getResolver())); + _customTypeHandler.reset(TransactionContext::createCustomTypeHandler(_vocbase, getResolver())); } TRI_ASSERT(_customTypeHandler != nullptr); diff --git a/arangod/Utils/TransactionContext.h b/arangod/Utils/TransactionContext.h index c8014c4394..a4b54752f0 100644 --- a/arangod/Utils/TransactionContext.h +++ b/arangod/Utils/TransactionContext.h @@ -37,6 +37,7 @@ namespace velocypack { struct CustomTypeHandler; } +class CollectionNameResolver; class DocumentDitch; class TransactionContext { @@ -45,6 +46,7 @@ class TransactionContext { TransactionContext& operator=(TransactionContext const&) = delete; protected: + ////////////////////////////////////////////////////////////////////////////// /// @brief create the context ////////////////////////////////////////////////////////////////////////////// @@ -52,12 +54,21 @@ class TransactionContext { explicit TransactionContext(TRI_vocbase_t* vocbase); public: + ////////////////////////////////////////////////////////////////////////////// /// @brief destroy the context ////////////////////////////////////////////////////////////////////////////// virtual ~TransactionContext(); + ////////////////////////////////////////////////////////////////////////////// + /// @brief factory to create a custom type handler, not managed + ////////////////////////////////////////////////////////////////////////////// + + static arangodb::velocypack::CustomTypeHandler* createCustomTypeHandler( + TRI_vocbase_t*, + arangodb::CollectionNameResolver const*); + ////////////////////////////////////////////////////////////////////////////// /// @brief return the vocbase //////////////////////////////////////////////////////////////////////////////