From 4a8146ed021645ceb79d4d8d4c226ad4ac36ec92 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 5 May 2015 00:33:12 +0200 Subject: [PATCH] fixed some potential leaks --- arangod/Aql/ExecutionBlock.cpp | 195 ++++++++------ arangod/Aql/Executor.cpp | 6 +- arangod/Aql/Expression.cpp | 15 +- arangod/Aql/Functions.cpp | 238 +++++++++++------ js/server/tests/aql-failures-noncluster.js | 56 ++++ js/server/tests/aql-functions.js | 281 ++++++++++++++++++++- 6 files changed, 620 insertions(+), 171 deletions(-) diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index ddaa956af6..a78940b7da 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -482,17 +482,13 @@ bool ExecutionBlock::getBlock (size_t atLeast, size_t atMost) { return false; } - try { - TRI_IF_FAILURE("ExecutionBlock::getBlock") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - _buffer.emplace_back(docs.get()); - docs.release(); - } - catch (...) { - throw; + TRI_IF_FAILURE("ExecutionBlock::getBlock") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } + _buffer.emplace_back(docs.get()); + docs.release(); + return true; } @@ -508,6 +504,7 @@ AqlItemBlock* ExecutionBlock::getSomeWithoutRegisterClearout (size_t atLeast, size_t atMost) { TRI_ASSERT(0 < atLeast && atLeast <= atMost); size_t skipped = 0; + AqlItemBlock* result = nullptr; int out = getOrSkipSome(atLeast, atMost, false, result, skipped); @@ -528,12 +525,16 @@ void ExecutionBlock::clearRegisters (AqlItemBlock* result) { size_t ExecutionBlock::skipSome (size_t atLeast, size_t atMost) { TRI_ASSERT(0 < atLeast && atLeast <= atMost); size_t skipped = 0; + AqlItemBlock* result = nullptr; int out = getOrSkipSome(atLeast, atMost, true, result, skipped); + TRI_ASSERT(result == nullptr); + if (out != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(out); } + return skipped; } @@ -582,12 +583,13 @@ int ExecutionBlock::getOrSkipSome (size_t atLeast, size_t& skipped) { TRI_ASSERT(result == nullptr && skipped == 0); + if (_done) { return TRI_ERROR_NO_ERROR; } // if _buffer.size() is > 0 then _pos points to a valid place . . . - vector collector; + std::vector collector; auto freeCollector = [&collector]() { for (auto x : collector) { @@ -635,7 +637,7 @@ int ExecutionBlock::getOrSkipSome (size_t atLeast, // The current block fits into our result, but it is already // half-eaten: if (! skipping) { - unique_ptr more(cur->slice(_pos, cur->size())); + std::unique_ptr more(cur->slice(_pos, cur->size())); TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome2") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -653,6 +655,8 @@ int ExecutionBlock::getOrSkipSome (size_t atLeast, // The current block fits into our result and is fresh: skipped += cur->size(); if (! skipping) { + // if any of the following statements throw, then cur is not lost, + // as it is still contained in _buffer TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome3") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } @@ -671,6 +675,8 @@ int ExecutionBlock::getOrSkipSome (size_t atLeast, throw; } + TRI_ASSERT(result == nullptr); + if (! skipping) { if (collector.size() == 1) { result = collector[0]; @@ -2941,8 +2947,7 @@ void CalculationBlock::doEvaluation (AqlItemBlock* result) { AqlItemBlock* CalculationBlock::getSome (size_t atLeast, size_t atMost) { - unique_ptr res(ExecutionBlock::getSomeWithoutRegisterClearout( - atLeast, atMost)); + std::unique_ptr res(ExecutionBlock::getSomeWithoutRegisterClearout(atLeast, atMost)); if (res.get() == nullptr) { return nullptr; @@ -3023,6 +3028,11 @@ AqlItemBlock* SubqueryBlock::getSome (size_t atLeast, } else { // initial subquery execution or subquery is not constant + + // prevent accidental double-freeing in case of exception + subqueryResults = nullptr; + + // and execute the subquery subqueryResults = executeSubquery(); TRI_ASSERT(subqueryResults != nullptr); @@ -3039,8 +3049,8 @@ AqlItemBlock* SubqueryBlock::getSome (size_t atLeast, } throwIfKilled(); // check if we were aborted - } + // Clear out registers no longer needed later: clearRegisters(res.get()); return res.release(); @@ -3179,7 +3189,8 @@ int FilterBlock::getOrSkipSome (size_t atLeast, } // if _buffer.size() is > 0 then _pos is valid - vector collector; + std::vector collector; + try { while (skipped < atLeast) { if (_buffer.empty()) { @@ -3196,8 +3207,7 @@ int FilterBlock::getOrSkipSome (size_t atLeast, if (_chosen.size() - _pos + skipped > atMost) { // The current block of chosen ones is too large for atMost: if (! skipping) { - unique_ptr more(cur->slice(_chosen, - _pos, _pos + (atMost - skipped))); + std::unique_ptr more(cur->slice(_chosen, _pos, _pos + (atMost - skipped))); TRI_IF_FAILURE("FilterBlock::getOrSkipSome1") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -3213,7 +3223,7 @@ int FilterBlock::getOrSkipSome (size_t atLeast, // The current block fits into our result, but it is already // half-eaten or needs to be copied anyway: if (! skipping) { - unique_ptr more(cur->steal(_chosen, _pos, _chosen.size())); + std::unique_ptr more(cur->steal(_chosen, _pos, _chosen.size())); TRI_IF_FAILURE("FilterBlock::getOrSkipSome2") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -3233,6 +3243,8 @@ int FilterBlock::getOrSkipSome (size_t atLeast, // takes them all, so we can just hand it on: skipped += cur->size(); if (! skipping) { + // if any of the following statements throw, then cur is not lost, + // as it is still contained in _buffer TRI_IF_FAILURE("FilterBlock::getOrSkipSome3") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } @@ -3408,7 +3420,7 @@ int SortedAggregateBlock::getOrSkipSome (size_t atLeast, } bool const isTotalAggregation = _aggregateRegisters.empty(); - unique_ptr res; + std::unique_ptr res; if (_buffer.empty()) { if (! ExecutionBlock::getBlock(atLeast, atMost)) { @@ -3514,7 +3526,17 @@ int SortedAggregateBlock::getOrSkipSome (size_t atLeast, bool hasMore = ! _buffer.empty(); if (! hasMore) { - hasMore = ExecutionBlock::getBlock(atLeast, atMost); + try { + TRI_IF_FAILURE("SortedAggregateBlock::hasMore") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + hasMore = ExecutionBlock::getBlock(atLeast, atMost); + } + catch (...) { + // prevent leak + delete cur; + throw; + } } if (! hasMore) { @@ -3534,14 +3556,12 @@ int SortedAggregateBlock::getOrSkipSome (size_t atLeast, ++skipped; } delete cur; - cur = nullptr; _done = true; result = res.release(); return TRI_ERROR_NO_ERROR; } catch (...) { delete cur; - cur = nullptr; throw; } } @@ -3719,7 +3739,6 @@ int HashedAggregateBlock::getOrSkipSome (size_t atLeast, GroupKeyHash(_trx, colls), GroupKeyEqual(_trx, colls) ); - auto buildResult = [&] (AqlItemBlock const* src) { auto planNode = static_cast(getPlanNode()); @@ -3742,12 +3761,13 @@ int HashedAggregateBlock::getOrSkipSome (size_t atLeast, size_t row = 0; for (auto const& it : allGroups) { - auto const& keys = it.first; + auto& keys = it.first; TRI_ASSERT_EXPENSIVE(keys.size() == n); size_t i = 0; - for (auto const& key : keys) { + for (auto& key : keys) { result->setValue(row, _aggregateRegisters[i++].first, key); + const_cast(&key)->erase(); // to prevent double-freeing later } if (planNode->_count) { @@ -3770,76 +3790,88 @@ int HashedAggregateBlock::getOrSkipSome (size_t atLeast, std::vector group; group.reserve(n); - while (skipped < atMost) { - groupValues.clear(); + try { + while (skipped < atMost) { + groupValues.clear(); - // for hashing simply re-use the aggregate registers, without cloning their contents - for (size_t i = 0; i < n; ++i) { - groupValues.emplace_back(cur->getValueReference(_pos, _aggregateRegisters[i].second)); - } - - // now check if we already know this group - auto it = allGroups.find(groupValues); - - if (it == allGroups.end()) { - // new group - group.clear(); - - // copy the group values before they get invalidated + // for hashing simply re-use the aggregate registers, without cloning their contents for (size_t i = 0; i < n; ++i) { - group.emplace_back(cur->getValueReference(_pos, _aggregateRegisters[i].second).clone()); + groupValues.emplace_back(cur->getValueReference(_pos, _aggregateRegisters[i].second)); } - allGroups.emplace(group, 1); - } - else { - // existing group. simply increase the counter - (*it).second++; - } + // now check if we already know this group + auto it = allGroups.find(groupValues); - if (++_pos >= cur->size()) { - _buffer.pop_front(); - _pos = 0; + if (it == allGroups.end()) { + // new group + group.clear(); - bool hasMore = ! _buffer.empty(); + // copy the group values before they get invalidated + for (size_t i = 0; i < n; ++i) { + group.emplace_back(cur->getValueReference(_pos, _aggregateRegisters[i].second).clone()); + } - if (! hasMore) { - hasMore = ExecutionBlock::getBlock(atLeast, atMost); + allGroups.emplace(group, 1); + } + else { + // existing group. simply increase the counter + (*it).second++; } - if (! hasMore) { - // no more input. we're done - try { - // emit last buffered group - if (! skipping) { - TRI_IF_FAILURE("HashedAggregateBlock::getOrSkipSome") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + if (++_pos >= cur->size()) { + _buffer.pop_front(); + _pos = 0; + + bool hasMore = ! _buffer.empty(); + + if (! hasMore) { + hasMore = ExecutionBlock::getBlock(atLeast, atMost); + } + + if (! hasMore) { + // no more input. we're done + try { + // emit last buffered group + if (! skipping) { + TRI_IF_FAILURE("HashedAggregateBlock::getOrSkipSome") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } } + + ++skipped; + result = buildResult(cur); + + returnBlock(cur); + _done = true; + + allGroups.clear(); + groupValues.clear(); + + return TRI_ERROR_NO_ERROR; + } + catch (...) { + returnBlock(cur); + throw; } - - ++skipped; - result = buildResult(cur); - - returnBlock(cur); - _done = true; - - allGroups.clear(); - groupValues.clear(); - - return TRI_ERROR_NO_ERROR; - } - catch (...) { - returnBlock(cur); - throw; } + + // hasMore + + returnBlock(cur); + cur = _buffer.front(); } - - // hasMore - - returnBlock(cur); - cur = _buffer.front(); } } + catch (...) { + // clean up + for (auto& it : allGroups) { + for (auto& it2 : it.first) { + const_cast(&it2)->destroy(); + } + } + allGroups.clear(); + throw; + } allGroups.clear(); groupValues.clear(); @@ -4198,6 +4230,7 @@ int LimitBlock::getOrSkipSome (size_t atLeast, } ExecutionBlock::getOrSkipSome(atLeast, atMost, skipping, result, skipped); + if (skipped == 0) { return TRI_ERROR_NO_ERROR; } @@ -4222,10 +4255,12 @@ int LimitBlock::getOrSkipSome (size_t atLeast, skipped = 0; AqlItemBlock* ignore = nullptr; ExecutionBlock::getOrSkipSome(atLeast, atMost, skipping, ignore, skipped); + if (ignore != nullptr) { _engine->_stats.fullCount += static_cast(ignore->size()); delete ignore; } + if (skipped == 0) { break; } diff --git a/arangod/Aql/Executor.cpp b/arangod/Aql/Executor.cpp index e358458231..b8cf9d0892 100644 --- a/arangod/Aql/Executor.cpp +++ b/arangod/Aql/Executor.cpp @@ -141,10 +141,10 @@ std::unordered_map const Executor::FunctionNames{ // list functions { "RANGE", Function("RANGE", "AQL_RANGE", "n,n|n", true, false, true) }, - { "UNION", Function("UNION", "AQL_UNION", "l,l|+",true, false, true) }, - { "UNION_DISTINCT", Function("UNION_DISTINCT", "AQL_UNION_DISTINCT", "l,l|+", true, false, true) }, + { "UNION", Function("UNION", "AQL_UNION", "l,l|+",true, false, true, &Functions::Union) }, + { "UNION_DISTINCT", Function("UNION_DISTINCT", "AQL_UNION_DISTINCT", "l,l|+", true, false, true, &Functions::UnionDistinct) }, { "MINUS", Function("MINUS", "AQL_MINUS", "l,l|+", true, false, true) }, - { "INTERSECTION", Function("INTERSECTION", "AQL_INTERSECTION", "l,l|+", true, false, true) }, + { "INTERSECTION", Function("INTERSECTION", "AQL_INTERSECTION", "l,l|+", true, false, true, &Functions::Intersection) }, { "FLATTEN", Function("FLATTEN", "AQL_FLATTEN", "l|n", true, false, true) }, { "LENGTH", Function("LENGTH", "AQL_LENGTH", "las", true, false, true, &Functions::Length) }, { "MIN", Function("MIN", "AQL_MIN", "l", true, false, true, &Functions::Min) }, diff --git a/arangod/Aql/Expression.cpp b/arangod/Aql/Expression.cpp index 33a59f12a0..6ee627442c 100644 --- a/arangod/Aql/Expression.cpp +++ b/arangod/Aql/Expression.cpp @@ -620,10 +620,17 @@ AqlValue Expression::executeSimpleExpression (AstNode const* node, TRI_ASSERT(member->type == NODE_TYPE_ARRAY); AqlValue result = executeSimpleExpression(member, &myCollection, trx, argv, startPos, vars, regs); - - auto res2 = func->implementation(_ast->query(), trx, myCollection, result); - result.destroy(); - return res2; + + try { + auto res2 = func->implementation(_ast->query(), trx, myCollection, result); + result.destroy(); + return res2; + } + catch (...) { + // prevent leak and rethrow error + result.destroy(); + throw; + } } else if (node->type == NODE_TYPE_RANGE) { diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index 1cab4924b6..ee447ef03c 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -831,7 +831,7 @@ AqlValue Functions::Unique (triagens::aql::Query* query, } TRI_json_t const* valueJson = value.json(); - size_t const n = valueJson->_value._objects._length; + size_t const n = TRI_LengthArrayJson(valueJson); std::unordered_set values( 512, @@ -888,27 +888,35 @@ AqlValue Functions::Union (triagens::aql::Query* query, } TRI_json_t const* valueJson = value.json(); + size_t const nrValues = TRI_LengthArrayJson(valueJson); - std::unique_ptr copy(TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, valueJson)); - - if (copy == nullptr) { + if (TRI_ReserveVector(&(result.get()->_value._objects), nrValues) != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); } - size_t const toAppend = TRI_LengthArrayJson(copy.get()); - - if (TRI_ReserveVector(&(result.get()->_value._objects), toAppend) != TRI_ERROR_NO_ERROR) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + TRI_IF_FAILURE("AqlFunctions::OutOfMemory1") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } // this passes ownership for the JSON contens into result - for (size_t j = 0; j < toAppend; ++j) { - TRI_PushBack2ArrayJson(result.get(), static_cast(TRI_AddressVector(&(copy.get()->_value._objects), j))); - } + for (size_t j = 0; j < nrValues; ++j) { + TRI_json_t* copy = TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, TRI_LookupArrayJson(valueJson, j)); - // free the top-level pointer of copy (but not its internals as they have been handed over into result) - TRI_Free(TRI_UNKNOWN_MEM_ZONE, copy.release()); + if (copy == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + } + + TRI_PushBack3ArrayJson(TRI_UNKNOWN_MEM_ZONE, result.get(), copy); + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory2") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + } } + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory3") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } auto jr = new Json(TRI_UNKNOWN_MEM_ZONE, result.get()); result.release(); @@ -923,47 +931,77 @@ AqlValue Functions::UnionDistinct (triagens::aql::Query* query, triagens::arango::AqlTransaction* trx, TRI_document_collection_t const* collection, AqlValue const parameters) { - std::unordered_set values( + std::unordered_set values( 512, triagens::basics::JsonHash(), triagens::basics::JsonEqual() ); + auto freeValues = [&values] () -> void { + for (auto& it : values) { + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, it); + } + }; + + std::unique_ptr result; size_t const n = parameters.arraySize(); - for (size_t i = 0; i < n; ++i) { - Json value(parameters.extractArrayMember(trx, collection, i, false)); + try { + for (size_t i = 0; i < n; ++i) { + Json value(parameters.extractArrayMember(trx, collection, i, false)); - if (! value.isArray()) { - // not an array - RegisterWarning(query, "UNION_DISTINCT", TRI_ERROR_QUERY_ARRAY_EXPECTED); - return AqlValue(new Json(Json::Null)); - } - - TRI_json_t const* valueJson = value.json(); - size_t const nrValues = valueJson->_value._objects._length; - - for (size_t j = 0; j < nrValues; ++j) { - auto value = static_cast(TRI_AddressVector(&valueJson->_value._objects, j)); - - if (value == nullptr) { - continue; + if (! value.isArray()) { + // not an array + freeValues(); + RegisterWarning(query, "UNION_DISTINCT", TRI_ERROR_QUERY_ARRAY_EXPECTED); + return AqlValue(new Json(Json::Null)); } - values.emplace(value); + TRI_json_t const* valueJson = value.json(); + size_t const nrValues = TRI_LengthArrayJson(valueJson); + + for (size_t j = 0; j < nrValues; ++j) { + auto value = static_cast(TRI_AddressVector(&valueJson->_value._objects, j)); + + if (values.find(value) == values.end()) { + std::unique_ptr copy(TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, value)); + + if (copy == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + } + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory1") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + values.emplace(copy.get()); + copy.release(); + } + } } - } - std::unique_ptr result(TRI_CreateArrayJson(TRI_UNKNOWN_MEM_ZONE, values.size())); - - for (auto const& it : values) { - auto copy = TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, it); + result.reset(TRI_CreateArrayJson(TRI_UNKNOWN_MEM_ZONE, values.size())); - if (copy == nullptr) { + if (result == nullptr) { THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); } - - TRI_PushBack3ArrayJson(TRI_UNKNOWN_MEM_ZONE, result.get(), copy); + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory2") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + for (auto const& it : values) { + TRI_PushBack3ArrayJson(TRI_UNKNOWN_MEM_ZONE, result.get(), it); + } + + } + catch (...) { + freeValues(); + throw; + } + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory3") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } auto jr = new Json(TRI_UNKNOWN_MEM_ZONE, result.get()); @@ -979,62 +1017,108 @@ AqlValue Functions::Intersection (triagens::aql::Query* query, triagens::arango::AqlTransaction* trx, TRI_document_collection_t const* collection, AqlValue const parameters) { - std::unordered_set values( + std::unordered_map values( 512, triagens::basics::JsonHash(), triagens::basics::JsonEqual() ); + auto freeValues = [&values] () -> void { + for (auto& it : values) { + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, it.first); + } + values.clear(); + }; + + std::unique_ptr result; size_t const n = parameters.arraySize(); - for (size_t i = 0; i < n; ++i) { - Json value(parameters.extractArrayMember(trx, collection, i, false)); + try { + for (size_t i = 0; i < n; ++i) { + Json value(parameters.extractArrayMember(trx, collection, i, false)); - if (! value.isArray()) { - // not an array - RegisterWarning(query, "INTERSECTION", TRI_ERROR_QUERY_ARRAY_EXPECTED); - return AqlValue(new Json(Json::Null)); - } - - // create a copy of the previous set - auto old(values); - values.clear(); - - TRI_json_t const* valueJson = value.json(); - size_t const nrValues = valueJson->_value._objects._length; - - for (size_t j = 0; j < nrValues; ++j) { - auto value = static_cast(TRI_AddressVector(&valueJson->_value._objects, j)); - - if (value == nullptr) { - continue; + if (! value.isArray()) { + // not an array + freeValues(); + RegisterWarning(query, "INTERSECTION", TRI_ERROR_QUERY_ARRAY_EXPECTED); + return AqlValue(new Json(Json::Null)); } - if (i == 0) { - // round one - values.emplace(value); - } - else { - // check if we have seen the same element before - auto it = old.find(value); + TRI_json_t const* valueJson = value.json(); + size_t const nrValues = TRI_LengthArrayJson(valueJson); - if (it != old.end()) { - values.emplace(value); + for (size_t j = 0; j < nrValues; ++j) { + auto value = static_cast(TRI_AddressVector(&valueJson->_value._objects, j)); + + if (i == 0) { + // round one + std::unique_ptr copy(TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, value)); + + if (copy == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + } + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory1") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + auto r = values.emplace(copy.get(), 1); + + if (r.second) { + // successfully inserted + copy.release(); + } + } + else { + // check if we have seen the same element before + auto it = values.find(const_cast(value)); + + if (it != values.end()) { + // already seen + TRI_ASSERT((*it).second > 0); + ++((*it).second); + } } } } - } - - std::unique_ptr result(TRI_CreateArrayJson(TRI_UNKNOWN_MEM_ZONE, values.size())); - for (auto const& it : values) { - auto copy = TRI_CopyJson(TRI_UNKNOWN_MEM_ZONE, it); + // count how many valid we have + size_t total = 0; - if (copy == nullptr) { + for (auto const& it : values) { + if (it.second == n) { + ++total; + } + } + + result.reset(TRI_CreateArrayJson(TRI_UNKNOWN_MEM_ZONE, total)); + + if (result == nullptr) { THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); } - - TRI_PushBack3ArrayJson(TRI_UNKNOWN_MEM_ZONE, result.get(), copy); + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory2") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + + for (auto& it : values) { + if (it.second == n) { + TRI_PushBack3ArrayJson(TRI_UNKNOWN_MEM_ZONE, result.get(), it.first); + } + else { + TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, it.first); + } + } + values.clear(); + + } + catch (...) { + freeValues(); + throw; + } + + TRI_IF_FAILURE("AqlFunctions::OutOfMemory3") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } auto jr = new Json(TRI_UNKNOWN_MEM_ZONE, result.get()); diff --git a/js/server/tests/aql-failures-noncluster.js b/js/server/tests/aql-failures-noncluster.js index 1eb4baf5c3..8874ea18ca 100644 --- a/js/server/tests/aql-failures-noncluster.js +++ b/js/server/tests/aql-failures-noncluster.js @@ -74,6 +74,51 @@ function ahuacatlFailureSuite () { c = null; }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test UNION for memleaks +//////////////////////////////////////////////////////////////////////////////// + + testAqlFunctionUnion : function () { + var where = [ "AqlFunctions::OutOfMemory1", "AqlFunctions::OutOfMemory2", "AqlFunctions::OutOfMemory3" ]; + + where.forEach(function(w) { + internal.debugClearFailAt(); + + internal.debugSetFailAt(w); + assertFailingQuery("RETURN NOOPT(UNION([ 1, 2, 3, 4 ], [ 5, 6, 7, 8 ], [ 9, 10, 11 ]))"); + }); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test UNION_DISTINCT for memleaks +//////////////////////////////////////////////////////////////////////////////// + + testAqlFunctionUnionDistinct : function () { + var where = [ "AqlFunctions::OutOfMemory1", "AqlFunctions::OutOfMemory2", "AqlFunctions::OutOfMemory3" ]; + + where.forEach(function(w) { + internal.debugClearFailAt(); + + internal.debugSetFailAt(w); + assertFailingQuery("RETURN NOOPT(UNION_DISTINCT([ 1, 2, 3, 4 ], [ 5, 6, 7, 8 ], [ 9, 10, 11 ]))"); + }); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test INTERSECTION for memleaks +//////////////////////////////////////////////////////////////////////////////// + + testAqlFunctionIntersection : function () { + var where = [ "AqlFunctions::OutOfMemory1", "AqlFunctions::OutOfMemory2", "AqlFunctions::OutOfMemory3" ]; + + where.forEach(function(w) { + internal.debugClearFailAt(); + + internal.debugSetFailAt(w); + assertFailingQuery("RETURN NOOPT(INTERSECTION([ 1, 2, 3, 4 ], [ 5, 6, 7, 8 ], [ 9, 10, 11 ]))"); + }); + }, + //////////////////////////////////////////////////////////////////////////////// /// @brief test failure //////////////////////////////////////////////////////////////////////////////// @@ -95,6 +140,17 @@ function ahuacatlFailureSuite () { assertFailingQuery("FOR i IN 1..10000 COLLECT key = i INTO g RETURN [ key, g ]"); }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test failure +//////////////////////////////////////////////////////////////////////////////// + + testSortedAggregateBlock3 : function () { + internal.debugSetFailAt("SortedAggregateBlock::hasMore"); + assertFailingQuery("FOR i IN " + c.name() + " COLLECT key = i.value INTO g RETURN [ key, g ]"); + assertFailingQuery("FOR i IN " + c.name() + " COLLECT key = i.value2 INTO g RETURN [ key, g ]"); + assertFailingQuery("FOR i IN 1..10000 COLLECT key = i INTO g RETURN [ key, g ]"); + }, + //////////////////////////////////////////////////////////////////////////////// /// @brief test failure //////////////////////////////////////////////////////////////////////////////// diff --git a/js/server/tests/aql-functions.js b/js/server/tests/aql-functions.js index 629a0d4178..942cc37833 100644 --- a/js/server/tests/aql-functions.js +++ b/js/server/tests/aql-functions.js @@ -910,7 +910,6 @@ function ahuacatlFunctionsTestSuite () { } } ], actual); }, - //////////////////////////////////////////////////////////////////////////////// /// @brief test merge_recursive function //////////////////////////////////////////////////////////////////////////////// @@ -1149,7 +1148,7 @@ function ahuacatlFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testUnionDistinct1 : function () { - var expected = [ [ 1, 2, 3, ] ]; + var expected = [ [ 1, 2, 3 ] ]; var actual = getQueryResults("RETURN UNION_DISTINCT([ 1, 2, 3 ], [ 1, 2, 3 ])"); assertEqual(expected, actual); }, @@ -1228,6 +1227,174 @@ function ahuacatlFunctionsTestSuite () { assertQueryWarningAndNull(errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH.code, "RETURN UNION_DISTINCT({ }, [ ])"); }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxx1 : function () { + var expected = [ [ 1, 2, 3, 1, 2, 3 ] ]; + var actual = getQueryResults("RETURN NOOPT(UNION([ 1, 2, 3 ], [ 1, 2, 3 ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxx2 : function () { + var expected = [ [ 1, 2, 3, 3, 2, 1 ] ]; + var actual = getQueryResults("RETURN NOOPT(UNION([ 1, 2, 3 ], [ 3, 2, 1 ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxx3 : function () { + var expected = [ "Fred", "John", "John", "Amy" ]; + var actual = getQueryResults("FOR u IN NOOPT(UNION([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"])) RETURN u"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxxInvalid : function () { + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN NOOPT(UNION())"); + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN NOOPT(UNION([ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], null))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], true))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], 3))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], \"yes\"))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], { }))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], [ ], null))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], [ ], true))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], [ ], 3))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], [ ], \"yes\"))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION([ ], [ ], { }))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION(null, [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION(true, [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION(3, [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION(\"yes\", [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION({ }, [ ]))"); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function indexed access +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxxIndexedAccess1 : function () { + var expected = [ "Fred" ]; + var actual = getQueryResults("RETURN NOOPT(UNION([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"]))[0]"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function indexed access +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxxIndexedAccess2 : function () { + var expected = [ "John" ]; + var actual = getQueryResults("RETURN NOOPT(UNION([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"]))[1]"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union function indexed access +//////////////////////////////////////////////////////////////////////////////// + + testUnionCxxIndexedAccess3 : function () { + var expected = [ "bar" ]; + var actual = getQueryResults("RETURN NOOPT(UNION([ { title : \"foo\" } ], [ { title : \"bar\" } ]))[1].title"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxx1 : function () { + var expected = [ 1, 2, 3 ]; + var actual = getQueryResults("RETURN NOOPT(UNION_DISTINCT([ 1, 2, 3 ], [ 1, 2, 3 ]))"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxx2 : function () { + var expected = [ 1, 2, 3 ]; + var actual = getQueryResults("RETURN NOOPT(UNION_DISTINCT([ 1, 2, 3 ], [ 3, 2, 1 ]))"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxx3 : function () { + var expected = [ "Amy", "Fred", "John" ]; + var actual = getQueryResults("FOR u IN NOOPT(UNION_DISTINCT([ \"Fred\", \"John\" ], [ \"John\", \"Amy\"])) RETURN u"); + assertEqual(expected, actual.sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxx4 : function () { + var expected = [ 1, 2, 3, 4, 5, 6 ]; + var actual = getQueryResults("RETURN NOOPT(UNION_DISTINCT([ 1, 2, 3 ], [ 3, 2, 1 ], [ 4 ], [ 5, 6, 1 ]))"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxx5 : function () { + var expected = [ [ ] ]; + var actual = getQueryResults("RETURN NOOPT(UNION_DISTINCT([ ], [ ], [ ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxx6 : function () { + var expected = [ false, true ]; + var actual = getQueryResults("RETURN NOOPT(UNION_DISTINCT([ ], [ false ], [ ], [ true ]))"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test union_distinct function +//////////////////////////////////////////////////////////////////////////////// + + testUnionDistinctCxxInvalid : function () { + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN NOOPT(UNION_DISTINCT())"); + assertQueryError(errors.ERROR_QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH.code, "RETURN NOOPT(UNION_DISTINCT([ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], null))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], true))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], 3))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], \"yes\"))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], { }))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], [ ], null))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], [ ], true))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], [ ], 3))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], [ ], \"yes\"))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT([ ], [ ], { }))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT(null, [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT(true, [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT(3, [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT(\"yes\", [ ]))"); + assertQueryWarningAndNull(errors.ERROR_QUERY_ARRAY_EXPECTED.code, "RETURN NOOPT(UNION_DISTINCT({ }, [ ]))"); + }, + //////////////////////////////////////////////////////////////////////////////// /// @brief test range function //////////////////////////////////////////////////////////////////////////////// @@ -1405,9 +1572,9 @@ function ahuacatlFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testIntersection1 : function () { - var expected = [ [ 1, -3 ] ]; + var expected = [ -3, 1 ]; var actual = getQueryResults("RETURN INTERSECTION([ 1, -3 ], [ -3, 1 ])"); - assertEqual(expected, actual); + assertEqual(expected, actual[0].sort()); }, //////////////////////////////////////////////////////////////////////////////// @@ -1445,9 +1612,9 @@ function ahuacatlFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testIntersection5 : function () { - var expected = [ [ 2, 4 ] ]; + var expected = [ 2, 4 ]; var actual = getQueryResults("RETURN INTERSECTION([ 1, 3, 2, 4 ], [ 2, 3, 1, 4 ], [ 4, 5, 6, 2 ])"); - assertEqual(expected, actual); + assertEqual(expected, actual[0].sort()); }, //////////////////////////////////////////////////////////////////////////////// @@ -1495,11 +1662,111 @@ function ahuacatlFunctionsTestSuite () { //////////////////////////////////////////////////////////////////////////////// testIntersection10 : function () { - var expected = [ [ 2, 4, 5 ] ]; + var expected = [ 2, 4, 5 ]; var actual = getQueryResults("RETURN INTERSECTION([ 1, 2, 3, 3, 4, 4, 5, 1 ], [ 2, 4, 5 ])"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersection function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx1 : function () { + var expected = [ -3, 1 ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ 1, -3 ], [ -3, 1 ]))"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx2 : function () { + var expected = [ [ ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ ], [ 1 ]))"); assertEqual(expected, actual); }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx3 : function () { + var expected = [ [ ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ 1 ], [ ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx4 : function () { + var expected = [ [ ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ 1 ], [ 2, 3, 1 ], [ 4, 5, 6 ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx5 : function () { + var expected = [ 2, 4 ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ 1, 3, 2, 4 ], [ 2, 3, 1, 4 ], [ 4, 5, 6, 2 ]))"); + assertEqual(expected, actual[0].sort()); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx6 : function () { + var expected = [ [ ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ [ 1, 2 ] ], [ 2, 1 ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx7 : function () { + var expected = [ [ ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ [ 1, 2 ] ], [ 1, 2 ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx8 : function () { + var expected = [ [ [ 1, 2 ] ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ [ 1, 2 ] ], [ [ 1, 2 ] ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx9 : function () { + var expected = [ [ { foo: 'test' } ] ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ { foo: 'bar' }, { foo: 'test' } ], [ { foo: 'test' } ]))"); + assertEqual(expected, actual); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test intersect function +//////////////////////////////////////////////////////////////////////////////// + + testIntersectionCxx10 : function () { + var expected = [ 2, 4, 5 ]; + var actual = getQueryResults("RETURN NOOPT(INTERSECTION([ 1, 2, 3, 3, 4, 4, 5, 1 ], [ 2, 4, 5 ]))"); + assertEqual(expected, actual[0].sort()); + }, + //////////////////////////////////////////////////////////////////////////////// /// @brief test flatten function ////////////////////////////////////////////////////////////////////////////////