From e9f8d5ac990c8c71da9116e8f8729f217dd5fb26 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 26 Sep 2014 08:51:11 +0100 Subject: [PATCH 1/6] fixed bug in ScatterBlock::getSomeForClient. --- arangod/Aql/ExecutionBlock.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index 3a002a8104..d7f980d06d 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -3756,7 +3756,6 @@ bool GatherBlock::OurLessThan::operator() (std::pair const& a, // --SECTION-- class ScatterBlock // ----------------------------------------------------------------------------- - bool ScatterBlock::hasMoreForClient (size_t clientId){ TRI_ASSERT(0 <= clientId && clientId < _nrClients); @@ -3833,6 +3832,10 @@ AqlItemBlock* ScatterBlock::getSomeForClient (size_t atLeast, size_t atMost, siz } if (popit) { _buffer.pop_front(); + // update the values in first coord of _posForClient + for (size_t i = 0; i < _nrClients; i++) { + _posForClient.at(i).first--; + } } return res; } From 2070175a5178714b3c2353c24aeadeb0dbef91fa Mon Sep 17 00:00:00 2001 From: James Date: Fri, 26 Sep 2014 09:32:39 +0100 Subject: [PATCH 2/6] getOrSkipSomeForClient method for ScatterBlock. --- arangod/Aql/ExecutionBlock.cpp | 74 ++++++++++++++++++++++++---------- arangod/Aql/ExecutionBlock.h | 7 +++- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index d7f980d06d..c2124964a4 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -3790,11 +3790,16 @@ bool ScatterBlock::hasMore () { return false; } -AqlItemBlock* ScatterBlock::getSomeForClient (size_t atLeast, size_t atMost, size_t clientId){ +int ScatterBlock::getOrSkipSomeForClient (size_t atLeast, + size_t atMost, bool skipping, AqlItemBlock*& result, + size_t& skipped, size_t clientId){ + + TRI_ASSERT(0 < atLeast && atLeast <= atMost); + TRI_ASSERT(result == nullptr && skipped == 0); TRI_ASSERT(0 <= clientId && clientId < _nrClients); if (_doneForClient.at(clientId)) { - return nullptr; + return TRI_ERROR_NO_ERROR; } std::pair pos = _posForClient.at(clientId); @@ -3803,41 +3808,68 @@ AqlItemBlock* ScatterBlock::getSomeForClient (size_t atLeast, size_t atMost, siz if (pos.first > _buffer.size()) { if (!getBlock(atLeast, atMost)) { _doneForClient.at(clientId) = true; - return nullptr; + return TRI_ERROR_NO_ERROR; } } size_t available = _buffer.at(pos.first)->size() - pos.second; // available should be non-zero - size_t toSend = (std::min)(available, atMost); //nr rows in outgoing block + skipped = std::min(available, atMost); //nr rows in outgoing block - AqlItemBlock* res = _buffer.at(pos.first)->slice(pos.second, pos.second + toSend); + if(!skipping){ + result = _buffer.at(pos.first)->slice(pos.second, pos.second + skipped); + } // increment the position . . . - _posForClient.at(clientId).second += toSend; - + _posForClient.at(clientId).second += skipped; + + // check if we're done at current block in buffer . . . if (_posForClient.at(clientId).second == _buffer.at(_posForClient.at(clientId).first)->size()) { _posForClient.at(clientId).first++; _posForClient.at(clientId).second = 0; - } - // check if we can pop the front of the buffer . . . - bool popit = true; - for (size_t i = 0; i < _nrClients; i++) { - if (_posForClient.at(i).first == 0) { - popit = false; - break; - } - } - if (popit) { - _buffer.pop_front(); - // update the values in first coord of _posForClient + // check if we can pop the front of the buffer . . . + bool popit = true; for (size_t i = 0; i < _nrClients; i++) { - _posForClient.at(i).first--; + if (_posForClient.at(i).first == 0) { + popit = false; + break; + } + } + if (popit) { + delete(_buffer.front()); + _buffer.pop_front(); + // update the values in first coord of _posForClient + for (size_t i = 0; i < _nrClients; i++) { + _posForClient.at(i).first--; + } + } } - return res; + return TRI_ERROR_NO_ERROR; +} + +AqlItemBlock* ScatterBlock::getSomeForClient ( + size_t atLeast, size_t atMost, size_t clientId) { + size_t skipped = 0; + AqlItemBlock* result = nullptr; + int out = getOrSkipSome(atLeast, atMost, false, result, skipped); + if (out != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(out); + } + return result; +} + +size_t ScatterBlock::skipSomeForClient (size_t atLeast, size_t atMost, size_t clientId) { + size_t skipped = 0; + AqlItemBlock* result = nullptr; + int out = getOrSkipSome(atLeast, atMost, true, result, skipped); + TRI_ASSERT(result == nullptr); + if (out != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(out); + } + return skipped; } // Local Variables: diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index 51b13254df..7afc1e9ba2 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -1607,8 +1607,13 @@ public: } bool hasMoreForClient (size_t clientId); - AqlItemBlock* getSomeForClient (size_t atLeast, size_t atMost, size_t clientId); + + int getOrSkipSomeForClient (size_t atLeast, size_t atMost, + bool skipping, AqlItemBlock*& result, size_t& skipped, size_t clientId); + size_t skipSomeForClient (size_t atLeast, size_t atMost, size_t clientId); + + AqlItemBlock* getSomeForClient (size_t atLeast, size_t atMost, size_t clientId); private: From ca1ccbab79ce71629756fbf891eafb7cbc77b9c3 Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Fri, 26 Sep 2014 10:37:39 +0200 Subject: [PATCH 3/6] Try to match the jenkins grinder input tools csv grammer --- js/common/modules/reportgenerator.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/common/modules/reportgenerator.js b/js/common/modules/reportgenerator.js index 53abf5af93..6d41c5fd8e 100644 --- a/js/common/modules/reportgenerator.js +++ b/js/common/modules/reportgenerator.js @@ -42,7 +42,7 @@ var generatePerfReportXML = function (reportName, testdata) { var generatePerfReportGrinderCSV = function (reportName, testdata) { "use strict"; - var x = "Thread, Run, Test, Start time (ms since Epoch), Test time, Errors, HTTP response code, HTTP response length, HTTP response errors, Time to resolve host, Time to establish connection, Time to first byte, New connections"; + var x = "Thread, Run, Test 1, Start time (ms since Epoch), Test time, Errors, TPS , HTTP response length, HTTP response errors, Time to resolve host, Time to establish connection, Time to first byte, New connections"; for (var testname in testdata) { @@ -53,7 +53,7 @@ var generatePerfReportGrinderCSV = function (reportName, testdata) { { var s = testdata[testname][testCalculation].duration; if (!isNaN(s)) { - x = x + '\n0, 0, ' + testname + '/' + testCalculation + ',' + require("internal").time() + ', ' + s + ', 0, 0, 0, 0, 0, 0, 0, 0'; + x = x + '\n0, 0, ' + testname + '/' + testCalculation + ',' + Math.floor(require("internal").time()*1000) + ', ' + Math.floor(s*1000) + ', 0, 0, 0, 0, 0, 0, 0, 0'; } } } From 06d039d3a0442ea0a988c40eab09cff65dfa161b Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Fri, 26 Sep 2014 11:18:17 +0200 Subject: [PATCH 4/6] Fix a deserialisation bug for AqlItemBlocks. --- arangod/Aql/AqlItemBlock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/AqlItemBlock.cpp b/arangod/Aql/AqlItemBlock.cpp index 266ec685d6..80f17fa8b5 100644 --- a/arangod/Aql/AqlItemBlock.cpp +++ b/arangod/Aql/AqlItemBlock.cpp @@ -135,7 +135,7 @@ AqlItemBlock::AqlItemBlock (Json const& json) { } } else if (n == 1) { - Json x(data.at(posInRaw++)); + Json x(raw.at(posInRaw++)); AqlValue a(new Json(TRI_UNKNOWN_MEM_ZONE, x.copy().steal())); try { setValue(i, column, a); // if this throws, a is destroyed again From e8dc7694e374f06fd7879306b6672ee1d50c2643 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 26 Sep 2014 10:40:05 +0100 Subject: [PATCH 5/6] shutdown method for GatherBlock and initializeCursor for ScatterBlock --- arangod/Aql/ExecutionBlock.cpp | 39 +++++++++++++++++++++++++++++++++- arangod/Aql/ExecutionBlock.h | 5 ++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index c2124964a4..38c6ace479 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -3492,6 +3492,26 @@ int GatherBlock::initializeCursor (AqlItemBlock* items, size_t pos) { return TRI_ERROR_NO_ERROR; } +int GatherBlock::shutdown() { + //don't call default shutdown method since it does the wrong thing to _buffer + for (auto it = _dependencies.begin(); it != _dependencies.end(); ++it) { + int res = (*it)->shutdown(); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + } + + for (std::deque x: _buffer) { + for (AqlItemBlock* y: x) { + delete y; + } + x.clear(); + } + _buffer.clear(); + return TRI_ERROR_NO_ERROR; +} + int64_t GatherBlock::count () const { int64_t sum = 0; for (auto x: _dependencies) { @@ -3724,7 +3744,7 @@ size_t GatherBlock::skipSome (size_t atLeast, size_t atMost) { // renew the comparison function OurLessThan ourLessThan(_trx, _buffer, _sortRegisters, colls); } - _pos.at(val.first) = make_pair(val.first,0); + _pos.at(val.first) = make_pair(val.first, 0); } } @@ -3756,9 +3776,25 @@ bool GatherBlock::OurLessThan::operator() (std::pair const& a, // --SECTION-- class ScatterBlock // ----------------------------------------------------------------------------- +int ScatterBlock::initializeCursor (AqlItemBlock* items, size_t pos) { + int res = ExecutionBlock::initializeCursor(items, pos); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + for (size_t i = 0; i < _nrClients; i++) { + _posForClient.push_back(std::make_pair(0, 0)); + _doneForClient.push_back(false); + } + + return TRI_ERROR_NO_ERROR; +} + bool ScatterBlock::hasMoreForClient (size_t clientId){ TRI_ASSERT(0 <= clientId && clientId < _nrClients); + if (_doneForClient.at(clientId)) { return false; } @@ -3787,6 +3823,7 @@ bool ScatterBlock::hasMore () { return true; } } + _done = true; return false; } diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index 7afc1e9ba2..59ab44c5c7 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -1516,6 +1516,9 @@ public: AqlItemBlock* getSome (size_t, size_t); size_t skipSome (size_t, size_t); + + // need our own shutdown method since our _buffer is different + int shutdown (); private: @@ -1590,7 +1593,7 @@ public: return ExecutionBlock::initialize(); } - //int initializeCursor (AqlItemBlock* items, size_t pos); + int initializeCursor (AqlItemBlock* items, size_t pos); int64_t remaining () { return _dependencies[0]->remaining(); From 2e151f7ab0012e31fe0e5f103776cedd22b8cbf7 Mon Sep 17 00:00:00 2001 From: Thomas Schmidts Date: Fri, 26 Sep 2014 11:53:44 +0200 Subject: [PATCH 6/6] Added type definition to db._create documentation --- .../Users/Collections/DatabaseMethods.mdpp | 8 +++++- arangod/V8Server/v8-vocindex.cpp | 28 +++++++++---------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Documentation/Books/Users/Collections/DatabaseMethods.mdpp b/Documentation/Books/Users/Collections/DatabaseMethods.mdpp index 9ed667ff40..bd20fdcdad 100644 --- a/Documentation/Books/Users/Collections/DatabaseMethods.mdpp +++ b/Documentation/Books/Users/Collections/DatabaseMethods.mdpp @@ -5,9 +5,15 @@ @startDocuBlock collectionDatabaseName !SUBSECTION Create - + @startDocuBlock collectionDatabaseCreate + +@startDocuBlock collectionCreateEdgeCollection + + +@startDocuBlock collectionCreateDocumentCollection + !SUBSECTION All Collections @startDocuBlock collectionDatabaseNameAll diff --git a/arangod/V8Server/v8-vocindex.cpp b/arangod/V8Server/v8-vocindex.cpp index a3fb3ff008..1085ecb93b 100644 --- a/arangod/V8Server/v8-vocindex.cpp +++ b/arangod/V8Server/v8-vocindex.cpp @@ -1673,7 +1673,7 @@ static v8::Handle CreateVocBase (v8::Arguments const& argv, /// *false*, then the key generator will solely be responsible for /// generating keys and supplying own key values in the *_key* attribute /// of documents is considered an error. -/// * *{increment*: increment value for *autoincrement* key generator. +/// * *increment*: increment value for *autoincrement* key generator. /// Not used for other key generator types. /// * *offset*: initial offset value for *autoincrement* key generator. /// Not used for other key generator types. @@ -1705,6 +1705,12 @@ static v8::Handle CreateVocBase (v8::Arguments const& argv, /// attribute and this can only be done efficiently if this is the /// only shard key by delegating to the individual shards. /// +/// `db._create(collection-name, properties, type)` +/// +/// Specifies the optional *type* of the collection, it can either be *document* +/// or *edge*. On default it is document. Instead of giving a type you can also use +/// *db._createEdgeCollection* or *db._createDocumentCollection*. +/// /// @EXAMPLES /// /// With defaults: @@ -1751,14 +1757,11 @@ static v8::Handle JS_CreateVocbase (v8::Arguments const& argv) { //////////////////////////////////////////////////////////////////////////////// /// @brief creates a new document collection -/// @startDocuBlock collectionCreateDocumentaion +/// @startDocuBlock collectionCreateDocumentCollection /// `db._createDocumentCollection(collection-name)` /// -/// `db._createDocumentCollection(collection-name, properties)` -/// -/// Creates a new document collection named *collection-name*. -/// This is an alias for @ref JS_CreateVocbase, with the difference that the -/// collection type is not automatically detected. +/// Creates a new document collection named *collection-name*. If the +/// document name already exists and error is thrown. /// @endDocuBlock //////////////////////////////////////////////////////////////////////////////// @@ -1766,13 +1769,13 @@ static v8::Handle JS_CreateDocumentCollectionVocbase (v8::Arguments c return CreateVocBase(argv, TRI_COL_TYPE_DOCUMENT); } -//////////////////////////////////////////////////////////////////////////////// +dan//////////////////////////////////////////////////////////////////////////////// /// @brief creates a new edge collection /// @startDocuBlock collectionCreateEdgeCollection /// `db._createEdgeCollection(collection-name)` /// /// Creates a new edge collection named *collection-name*. If the -/// collection name already exists, then an error is thrown. The default value +/// collection name already exists an error is thrown. The default value /// for *waitForSync* is *false*. /// /// `db._createEdgeCollection(collection-name, properties)` @@ -1781,14 +1784,11 @@ static v8::Handle JS_CreateDocumentCollectionVocbase (v8::Arguments c /// /// * *waitForSync* (optional, default *false*): If *true* creating /// a document will only return after the data was synced to disk. -/// * *journalSize* (optional, default is a @ref CommandLineArangod +/// * *journalSize* (optional, default is /// "configuration parameter"): The maximal size of /// a journal or datafile. Note that this also limits the maximal -/// size of a single object. Must be at least 1MB. +/// size of a single object and must be at least 1MB. /// -/// @EXAMPLES -/// -/// See @ref JS_CreateVocbase for example. /// @endDocuBlock ////////////////////////////////////////////////////////////////////////////////