From 5e1166e38ce66fdcc40e60c659b8cc106d8da4bd Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 10 Oct 2014 16:14:10 +0200 Subject: [PATCH 1/3] Removed debug output --- .../system/aardvark/frontend/js/graphViewer/graph/edgeShaper.js | 1 - 1 file changed, 1 deletion(-) diff --git a/js/apps/system/aardvark/frontend/js/graphViewer/graph/edgeShaper.js b/js/apps/system/aardvark/frontend/js/graphViewer/graph/edgeShaper.js index da51ec5580..e6b09be716 100644 --- a/js/apps/system/aardvark/frontend/js/graphViewer/graph/edgeShaper.js +++ b/js/apps/system/aardvark/frontend/js/graphViewer/graph/edgeShaper.js @@ -150,7 +150,6 @@ function EdgeShaper(parent, config, idfunc) { bindEvent = function (type, func) { if (type === "update") { - console.log("Overwriting"); addUpdate = func; } else if (events[type] === undefined) { throw "Sorry Unknown Event " + type + " cannot be bound."; From 3a7a03a51f883534c541652b22432ffbc4c934b4 Mon Sep 17 00:00:00 2001 From: Willi Goesgens Date: Fri, 10 Oct 2014 17:00:49 +0200 Subject: [PATCH 2/3] Don't add intermediate calculationnodes for LET-statements to forward the results of subqueries. Instead adjust the outvariable of the subquerynode. --- arangod/Aql/ExecutionNode.cpp | 115 +++++++++++++++++--------------- arangod/Aql/ExecutionNode.h | 7 +- arangod/Aql/ExecutionPlan.cpp | 8 +++ arangod/Aql/ExecutionPlan.h | 6 ++ js/server/tests/aql-subquery.js | 10 +++ 5 files changed, 91 insertions(+), 55 deletions(-) diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index c9b6db642c..624cc7df6c 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -375,7 +375,7 @@ void ExecutionNode::CloneHelper (ExecutionNode* other, other->_varsValid.insert(var); } if (_varOverview.get() != nullptr) { - auto othervarOverview = std::shared_ptr(_varOverview->clone(plan)); + auto othervarOverview = std::shared_ptr(_varOverview->clone(plan, _plan)); other->_varOverview = othervarOverview; } } @@ -388,6 +388,7 @@ void ExecutionNode::CloneHelper (ExecutionNode* other, other->_varsValid = _varsValid; other->_varOverview = _varOverview; } + plan->registerNode(other); if (withDependencies) { cloneDependencies(plan, other, withProperties); @@ -619,62 +620,63 @@ triagens::basics::Json ExecutionNode::toJsonHelperGeneric (triagens::basics::Jso json("id", triagens::basics::Json(static_cast(id()))); json("estimatedCost", triagens::basics::Json(_estimatedCost)); - json("depth", triagens::basics::Json(static_cast(_depth))); + if (verbose) { + json("depth", triagens::basics::Json(static_cast(_depth))); - if (_varOverview) { - triagens::basics::Json jsonVarInfoList(triagens::basics::Json::List, _varOverview->varInfo.size()); - for (auto oneVarInfo: _varOverview->varInfo) { - triagens::basics::Json jsonOneVarInfoArray(triagens::basics::Json::Array, 2); - jsonOneVarInfoArray( - "VariableId", - triagens::basics::Json(static_cast(oneVarInfo.first))) - ("depth", triagens::basics::Json(static_cast(oneVarInfo.second.depth))) - ("RegisterId", triagens::basics::Json(static_cast(oneVarInfo.second.registerId))) - ; - jsonVarInfoList(jsonOneVarInfoArray); - } - json("varInfoList", jsonVarInfoList); + if (_varOverview) { + triagens::basics::Json jsonVarInfoList(triagens::basics::Json::List, _varOverview->varInfo.size()); + for (auto oneVarInfo: _varOverview->varInfo) { + triagens::basics::Json jsonOneVarInfoArray(triagens::basics::Json::Array, 2); + jsonOneVarInfoArray( + "VariableId", + triagens::basics::Json(static_cast(oneVarInfo.first))) + ("depth", triagens::basics::Json(static_cast(oneVarInfo.second.depth))) + ("RegisterId", triagens::basics::Json(static_cast(oneVarInfo.second.registerId))) + ; + jsonVarInfoList(jsonOneVarInfoArray); + } + json("varInfoList", jsonVarInfoList); - triagens::basics::Json jsonNRRegsList(triagens::basics::Json::List, _varOverview->nrRegs.size()); - for (auto oneRegisterID: _varOverview->nrRegs) { - jsonNRRegsList(triagens::basics::Json(static_cast(oneRegisterID))); - } - json("nrRegs", jsonNRRegsList); + triagens::basics::Json jsonNRRegsList(triagens::basics::Json::List, _varOverview->nrRegs.size()); + for (auto oneRegisterID: _varOverview->nrRegs) { + jsonNRRegsList(triagens::basics::Json(static_cast(oneRegisterID))); + } + json("nrRegs", jsonNRRegsList); - triagens::basics::Json jsonNRRegsHereList(triagens::basics::Json::List, _varOverview->nrRegsHere.size()); - for (auto oneRegisterID: _varOverview->nrRegsHere) { - jsonNRRegsHereList(triagens::basics::Json(static_cast(oneRegisterID))); + triagens::basics::Json jsonNRRegsHereList(triagens::basics::Json::List, _varOverview->nrRegsHere.size()); + for (auto oneRegisterID: _varOverview->nrRegsHere) { + jsonNRRegsHereList(triagens::basics::Json(static_cast(oneRegisterID))); + } + json("nrRegsHere", jsonNRRegsHereList); + json("totalNrRegs", triagens::basics::Json(static_cast(_varOverview->totalNrRegs))); } - json("nrRegsHere", jsonNRRegsHereList); - json("totalNrRegs", triagens::basics::Json(static_cast(_varOverview->totalNrRegs))); + else { + json("varInfoList", triagens::basics::Json(triagens::basics::Json::List)); + json("nrRegs", triagens::basics::Json(triagens::basics::Json::List)); + json("nrRegsHere", triagens::basics::Json(triagens::basics::Json::List)); + json("totalNrRegs", triagens::basics::Json(0.0)); + } + + triagens::basics::Json jsonRegsToClearList(triagens::basics::Json::List, _regsToClear.size()); + for (auto oneRegisterID : _regsToClear) { + jsonRegsToClearList(triagens::basics::Json(static_cast(oneRegisterID))); + } + json("regsToClear", jsonRegsToClearList); + + triagens::basics::Json jsonVarsUsedLaterList(triagens::basics::Json::List, _varsUsedLater.size()); + for (auto oneVarUsedLater: _varsUsedLater) { + jsonVarsUsedLaterList.add(oneVarUsedLater->toJson()); + } + + json("varsUsedLater", jsonVarsUsedLaterList); + + triagens::basics::Json jsonvarsValidList(triagens::basics::Json::List, _varsValid.size()); + for (auto oneVarUsedLater: _varsValid) { + jsonvarsValidList.add(oneVarUsedLater->toJson()); + } + + json("varsValid", jsonvarsValidList); } - else { - json("varInfoList", triagens::basics::Json(triagens::basics::Json::List)); - json("nrRegs", triagens::basics::Json(triagens::basics::Json::List)); - json("nrRegsHere", triagens::basics::Json(triagens::basics::Json::List)); - json("totalNrRegs", triagens::basics::Json(0.0)); - } - - triagens::basics::Json jsonRegsToClearList(triagens::basics::Json::List, _regsToClear.size()); - for (auto oneRegisterID : _regsToClear) { - jsonRegsToClearList(triagens::basics::Json(static_cast(oneRegisterID))); - } - json("regsToClear", jsonRegsToClearList); - - triagens::basics::Json jsonVarsUsedLaterList(triagens::basics::Json::List, _varsUsedLater.size()); - for (auto oneVarUsedLater: _varsUsedLater) { - jsonVarsUsedLaterList.add(oneVarUsedLater->toJson()); - } - - json("varsUsedLater", jsonVarsUsedLaterList); - - triagens::basics::Json jsonvarsValidList(triagens::basics::Json::List, _varsValid.size()); - for (auto oneVarUsedLater: _varsValid) { - jsonvarsValidList.add(oneVarUsedLater->toJson()); - } - - json("varsValid", jsonvarsValidList); - return json; } @@ -782,7 +784,7 @@ void ExecutionNode::VarOverview::clear () { totalNrRegs = 0; } -ExecutionNode::VarOverview* ExecutionNode::VarOverview::clone (ExecutionPlan* plan) { +ExecutionNode::VarOverview* ExecutionNode::VarOverview::clone (ExecutionPlan* otherPlan, ExecutionPlan* plan) { VarOverview* other = new VarOverview(); other->nrRegsHere = nrRegsHere; @@ -794,7 +796,7 @@ ExecutionNode::VarOverview* ExecutionNode::VarOverview::clone (ExecutionPlan* pl for (auto en: subQueryNodes) { auto otherId = en->id(); - auto otherEN = plan->getNodeById(otherId); + auto otherEN = otherPlan->getNodeById(otherId); other->subQueryNodes.push_back(otherEN); } return other; @@ -1564,6 +1566,11 @@ ExecutionNode* SubqueryNode::clone (ExecutionPlan* plan, return static_cast(c); } + +void SubqueryNode::replaceOutVariable(Variable const* var) { + _outVariable = var; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief helper struct to find all (outer) variables used in a SubqueryNode //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Aql/ExecutionNode.h b/arangod/Aql/ExecutionNode.h index b8bfb392b7..57ecadada7 100644 --- a/arangod/Aql/ExecutionNode.h +++ b/arangod/Aql/ExecutionNode.h @@ -561,7 +561,7 @@ namespace triagens { virtual void after (ExecutionNode *eb); - VarOverview* clone(ExecutionPlan* plan); + VarOverview* clone(ExecutionPlan* otherPlan, ExecutionPlan* plan); }; @@ -1555,6 +1555,11 @@ namespace triagens { return v; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief replace the out variable, so we can adjust the name. +//////////////////////////////////////////////////////////////////////////////// + void replaceOutVariable(Variable const* var); + //////////////////////////////////////////////////////////////////////////////// /// @brief can the node throw? Note that this means that an exception can /// *originate* from this node. That is, this method does not need to diff --git a/arangod/Aql/ExecutionPlan.cpp b/arangod/Aql/ExecutionPlan.cpp index adb23074c8..7e0b8a58b4 100644 --- a/arangod/Aql/ExecutionPlan.cpp +++ b/arangod/Aql/ExecutionPlan.cpp @@ -58,6 +58,7 @@ ExecutionPlan::ExecutionPlan (Ast* ast) _varUsageComputed(false), _nextId(0), _ast(ast) { + _lastSubqueryNodeId = (size_t)-1; } @@ -426,8 +427,15 @@ ExecutionNode* ExecutionPlan::fromNodeLet (ExecutionNode* previous, } en = registerNode(new SubqueryNode(this, nextId(), subquery, v)); + _lastSubqueryNodeId = en->id(); } else { + if ((expression->type == NODE_TYPE_REFERENCE) && + (_lastSubqueryNodeId == _nextId)) { + auto sn = static_cast(getNodeById(_lastSubqueryNodeId)); + sn->replaceOutVariable(v); + return sn; + } // operand is some misc expression, including references to other variables auto expr = new Expression(_ast, const_cast(expression)); diff --git a/arangod/Aql/ExecutionPlan.h b/arangod/Aql/ExecutionPlan.h index 7d77046d0e..0b31edb417 100644 --- a/arangod/Aql/ExecutionPlan.h +++ b/arangod/Aql/ExecutionPlan.h @@ -456,6 +456,12 @@ namespace triagens { size_t _nextId; +//////////////////////////////////////////////////////////////////////////////// +/// @brief id of last Subquerynode. +//////////////////////////////////////////////////////////////////////////////// + + size_t _lastSubqueryNodeId; + //////////////////////////////////////////////////////////////////////////////// /// @brief the ast //////////////////////////////////////////////////////////////////////////////// diff --git a/js/server/tests/aql-subquery.js b/js/server/tests/aql-subquery.js index 5780fca2e2..162c74d9d2 100644 --- a/js/server/tests/aql-subquery.js +++ b/js/server/tests/aql-subquery.js @@ -1,3 +1,4 @@ +/*global require, assertTrue, assertEqual, AQL_EXECUTE, AQL_EXPLAIN */ //////////////////////////////////////////////////////////////////////////////// /// @brief tests for Ahuacatl, subqueries /// @@ -28,6 +29,7 @@ var jsunity = require("jsunity"); var helper = require("org/arangodb/aql-helper"); var getQueryResults = helper.getQueryResults2; +var findExecutionNodes = helper.findExecutionNodes; //////////////////////////////////////////////////////////////////////////////// /// @brief test suite @@ -126,6 +128,14 @@ function ahuacatlSubqueryTestSuite () { var actual = getQueryResults("LET a = (FOR i IN [ 1, 2, 3 ] LET s = (FOR j IN [ 1, 2 ] RETURN j) RETURN i * s[1]) RETURN a"); assertEqual(expected, actual); + }, + + testSubqueryOutVariableName : function () { + var XPResult = AQL_EXPLAIN('FOR u IN _users LET theLetVariable = (FOR j IN _users RETURN j) RETURN theLetVariable'); + + var SubqueryNode = findExecutionNodes(XPResult, "SubqueryNode")[0]; + + assertEqual(SubqueryNode.outVariable.name, "theLetVariable"); } }; From 4c366dad632a348a8e65d08fe4e0647c932f8350 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Fri, 10 Oct 2014 18:05:23 +0200 Subject: [PATCH 3/3] issue #1038: use linked lists for collisions --- arangod/CMakeLists.txt | 1 + arangod/HashIndex/hash-array-multi.cpp | 577 +++++++++++++++++++++++++ arangod/HashIndex/hash-array-multi.h | 143 ++++++ arangod/HashIndex/hash-array.cpp | 178 +------- arangod/HashIndex/hash-array.h | 33 +- arangod/HashIndex/hash-index.cpp | 180 +++++--- arangod/HashIndex/hash-index.h | 13 +- arangod/Makefile.files | 1 + 8 files changed, 866 insertions(+), 260 deletions(-) create mode 100644 arangod/HashIndex/hash-array-multi.cpp create mode 100644 arangod/HashIndex/hash-array-multi.h diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 2520c9e4e5..81c2604e12 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -108,6 +108,7 @@ add_executable( GeoIndex/GeoIndex.cpp GeoIndex/geo-index.cpp HashIndex/hash-array.cpp + HashIndex/hash-array-multi.cpp HashIndex/hash-index.cpp IndexOperators/index-operator.cpp Replication/ContinuousSyncer.cpp diff --git a/arangod/HashIndex/hash-array-multi.cpp b/arangod/HashIndex/hash-array-multi.cpp new file mode 100644 index 0000000000..415fc9f927 --- /dev/null +++ b/arangod/HashIndex/hash-array-multi.cpp @@ -0,0 +1,577 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief multi-hash array implementation, using a linked-list for collisions +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2014 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Dr. Frank Celler +/// @author Dr. Oreste Costa-Panaia +/// @author Martin Schoenert +/// @author Jan Steemann +/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany +/// @author Copyright 2004-2013, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "hash-array-multi.h" + +#include "Basics/fasthash.h" +#include "HashIndex/hash-index.h" +#include "VocBase/document-collection.h" + +// ----------------------------------------------------------------------------- +// --SECTION-- COMPARISON +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroys an element, removing any allocated memory +//////////////////////////////////////////////////////////////////////////////// + +static void DestroyElement (TRI_hash_array_multi_t* array, + TRI_hash_index_element_multi_t* element) { + TRI_ASSERT_EXPENSIVE(element != nullptr); + TRI_ASSERT_EXPENSIVE(element->_document != nullptr); + + TRI_Free(TRI_UNKNOWN_MEM_ZONE, element->_subObjects); + element->_document = nullptr; + element->_next = nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief determines if a key corresponds to an element +//////////////////////////////////////////////////////////////////////////////// + +static bool IsEqualKeyElement (TRI_hash_array_multi_t const* array, + TRI_index_search_value_t const* left, + TRI_hash_index_element_multi_t const* right) { + TRI_ASSERT_EXPENSIVE(right->_document != nullptr); + + for (size_t j = 0; j < array->_numFields; ++j) { + TRI_shaped_json_t* leftJson = &left->_values[j]; + TRI_shaped_sub_t* rightSub = &right->_subObjects[j]; + + if (leftJson->_sid != rightSub->_sid) { + return false; + } + + auto length = leftJson->_data.length; + + if (length != rightSub->_length) { + return false; + } + + if (0 < length) { + char const* ptr = right->_document->getShapedJsonPtr() + rightSub->_offset; // ONLY IN INDEX + + if (memcmp(leftJson->_data.data, ptr, length) != 0) { + return false; + } + } + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief given a key generates a hash integer +//////////////////////////////////////////////////////////////////////////////// + +static uint64_t HashKey (TRI_hash_array_multi_t const* array, + TRI_index_search_value_t const* key) { + uint64_t hash = 0x0123456789abcdef; + + for (size_t j = 0; j < array->_numFields; ++j) { + // ignore the sid for hashing + hash = fasthash64(key->_values[j]._data.data, key->_values[j]._data.length, hash); + } + + return hash; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief given an element generates a hash integer +//////////////////////////////////////////////////////////////////////////////// + +static uint64_t HashElement (TRI_hash_array_multi_t const* array, + TRI_hash_index_element_multi_t const* element) { + uint64_t hash = 0x0123456789abcdef; + char const* ptr = element->_document->getShapedJsonPtr(); // ONLY IN INDEX + + for (size_t j = 0; j < array->_numFields; j++) { + // ignore the sid for hashing + // only hash the data block + hash = fasthash64(ptr + element->_subObjects[j]._offset, element->_subObjects[j]._length, hash); + } + + return hash; +} + +// ----------------------------------------------------------------------------- +// --SECTION-- HASH ARRAY +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// --SECTION-- private defines +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initial preallocation size of the hash table when the table is +/// first created +/// setting this to a high value will waste memory but reduce the number of +/// reallocations/repositionings necessary when the table grows +//////////////////////////////////////////////////////////////////////////////// + +static inline uint64_t InitialSize () { + return 251; +} + +// ----------------------------------------------------------------------------- +// --SECTION-- private functions +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the size of a single entry +//////////////////////////////////////////////////////////////////////////////// + +static inline size_t TableEntrySize () { + return sizeof(TRI_hash_index_element_multi_t); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief allocate memory for the hash table +/// +/// the hash table memory will be aligned on a cache line boundary +//////////////////////////////////////////////////////////////////////////////// + +static int AllocateTable (TRI_hash_array_multi_t* array, + uint64_t numElements) { + size_t const size = (size_t) (TableEntrySize() * numElements + 64); + + TRI_hash_index_element_multi_t* table = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, size, true)); + + if (table == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + array->_tablePtr = table; + array->_table = static_cast(TRI_Align64(table)); + array->_nrAlloc = numElements; + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief resizes the array +//////////////////////////////////////////////////////////////////////////////// + +static int ResizeHashArray (TRI_hash_array_multi_t* array, + uint64_t targetSize, + bool allowShrink) { + if (array->_nrAlloc >= targetSize && ! allowShrink) { + return TRI_ERROR_NO_ERROR; + } + + TRI_hash_index_element_multi_t* oldTable = array->_table; + TRI_hash_index_element_multi_t* oldTablePtr = array->_tablePtr; + uint64_t oldAlloc = array->_nrAlloc; + + TRI_ASSERT(targetSize > 0); + + int res = AllocateTable(array, targetSize); + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + if (array->_nrUsed > 0) { + uint64_t const n = array->_nrAlloc; + + for (uint64_t j = 0; j < oldAlloc; j++) { + TRI_hash_index_element_multi_t* element = &oldTable[j]; + + if (element->_document != nullptr) { + uint64_t i, k; + i = k = HashElement(array, element) % n; + + for (; i < n && array->_table[i]._document != nullptr; ++i); + if (i == n) { + for (i = 0; i < k && array->_table[i]._document != nullptr; ++i); + } + + TRI_ASSERT_EXPENSIVE(i < n); + + // ........................................................................... + // add a new element to the associative array + // memcpy ok here since are simply moving array items internally + // ........................................................................... + + memcpy(&array->_table[i], element, TableEntrySize()); + } + } + } + + TRI_Free(TRI_UNKNOWN_MEM_ZONE, oldTablePtr); + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief triggers a resize if necessary +//////////////////////////////////////////////////////////////////////////////// + +static bool CheckResize (TRI_hash_array_multi_t* array) { + if (array->_nrAlloc < 2 * array->_nrUsed) { + int res = ResizeHashArray(array, 2 * array->_nrAlloc + 1, false); + + if (res != TRI_ERROR_NO_ERROR) { + return false; + } + } + + return true; +} + +// ----------------------------------------------------------------------------- +// --SECTION-- constructors and destructors +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initialises an array +//////////////////////////////////////////////////////////////////////////////// + +int TRI_InitHashArrayMulti (TRI_hash_array_multi_t* array, + size_t numFields) { + + TRI_ASSERT(numFields > 0); + + array->_numFields = numFields; + array->_tablePtr = nullptr; + array->_table = nullptr; + array->_nrUsed = 0; + array->_nrAlloc = 0; + array->_nrOverflow = 0; + + return AllocateTable(array, InitialSize()); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroys an array, but does not free the pointer +//////////////////////////////////////////////////////////////////////////////// + +void TRI_DestroyHashArrayMulti (TRI_hash_array_multi_t* array) { + if (array == nullptr) { + return; + } + + // ........................................................................... + // Go through each item in the array and remove any internal allocated memory + // ........................................................................... + + // array->_table might be NULL if array initialisation fails + if (array->_table != nullptr) { + TRI_hash_index_element_multi_t* p; + TRI_hash_index_element_multi_t* e; + + p = array->_table; + e = p + array->_nrAlloc; + + for (; p < e; ++p) { + auto current = p; + + while (current != nullptr && current->_document != nullptr) { + auto ptr = current->_next; + DestroyElement(array, current); + if (current != p) { + TRI_Free(TRI_UNKNOWN_MEM_ZONE, current); + } + current = ptr; + } + } + + TRI_Free(TRI_UNKNOWN_MEM_ZONE, array->_tablePtr); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroys an array and frees the pointer +//////////////////////////////////////////////////////////////////////////////// + +void TRI_FreeHashArrayMulti (TRI_hash_array_multi_t* array) { + if (array != nullptr) { + TRI_DestroyHashArrayMulti(array); + TRI_Free(TRI_UNKNOWN_MEM_ZONE, array); + } +} + +// ----------------------------------------------------------------------------- +// --SECTION-- public functions +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the hash array's memory usage +//////////////////////////////////////////////////////////////////////////////// + +size_t TRI_MemoryUsageHashArrayMulti (TRI_hash_array_multi_t const* array) { + if (array == nullptr) { + return 0; + } + + size_t tableSize = (size_t) (array->_nrAlloc * TableEntrySize() + 64); + size_t memberSize = (size_t) (array->_nrUsed * array->_numFields * sizeof(TRI_shaped_sub_t)); + size_t overflowSize = (size_t) (array->_nrOverflow * array->_numFields * sizeof(TRI_shaped_sub_t)); + + return (size_t) (tableSize + memberSize + overflowSize); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief resizes the hash table +//////////////////////////////////////////////////////////////////////////////// + +int TRI_ResizeHashArrayMulti (TRI_hash_array_multi_t* array, + size_t size) { + return ResizeHashArray(array, (uint64_t) (2 * size + 1), false); +} + +// ----------------------------------------------------------------------------- +// --SECTION-- public functions +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookups an element given a key +//////////////////////////////////////////////////////////////////////////////// + +TRI_vector_pointer_t TRI_LookupByKeyHashArrayMulti (TRI_hash_array_multi_t const* array, + TRI_index_search_value_t const* key) { + TRI_ASSERT_EXPENSIVE(array->_nrUsed < array->_nrAlloc); + + // ........................................................................... + // initialise the vector which will hold the result if any + // ........................................................................... + + TRI_vector_pointer_t result; + TRI_InitVectorPointer(&result, TRI_UNKNOWN_MEM_ZONE); + + uint64_t const n = array->_nrAlloc; + uint64_t i, k; + + i = k = HashKey(array, key) % n; + + for (; i < n && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); + if (i == n) { + for (i = 0; i < k && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); + } + + TRI_ASSERT_EXPENSIVE(i < n); + + if (array->_table[i]._document != nullptr) { + auto current = &array->_table[i]; + while (current != nullptr) { + if (IsEqualKeyElement(array, key, current)) { + TRI_PushBackVectorPointer(&result, current); + } + current = current->_next; + } + } + + // ........................................................................... + // return whatever we found -- which could be an empty vector list if nothing + // matches. + // ........................................................................... + + return result; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief adds an element to the array +/// +/// This function claims the owenship of the sub-objects in the inserted +/// element. +//////////////////////////////////////////////////////////////////////////////// + +int TRI_InsertElementHashArrayMulti (TRI_hash_array_multi_t* array, + TRI_index_search_value_t const* key, + TRI_hash_index_element_multi_t* element, + bool isRollback) { + if (! CheckResize(array)) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + element->_next = nullptr; + + uint64_t const n = array->_nrAlloc; + uint64_t i, k; + + i = k = HashKey(array, key) % n; + + for (; i < n && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); + if (i == n) { + for (i = 0; i < k && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); + } + + TRI_ASSERT_EXPENSIVE(i < n); + + TRI_hash_index_element_multi_t* arrayElement = &array->_table[i]; + + // ........................................................................... + // If we found an element, return. While we allow duplicate entries in the + // hash table, we do not allow duplicate elements. Elements would refer to the + // (for example) an actual row in memory. This is different from the + // TRI_InsertElementMultiArray function below where we only have keys to + // differentiate between elements. + // ........................................................................... + + bool found = (arrayElement->_document != nullptr); + + if (found) { + if (isRollback) { + auto current = arrayElement; + while (current != nullptr) { + if (current->_document == element->_document) { + DestroyElement(array, element); + + return TRI_RESULT_ELEMENT_EXISTS; + } + current = current->_next; + } + } + + auto ptr = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_hash_index_element_multi_t), true)); + if (ptr == nullptr) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + element->_next = arrayElement->_next; + *ptr = *element; + arrayElement->_next = ptr; + array->_nrOverflow++; + + return TRI_ERROR_NO_ERROR; + } + + // ........................................................................... + // add a new element to the associative array + // ........................................................................... + + *arrayElement = *element; + array->_nrUsed++; + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief removes an element from the array +//////////////////////////////////////////////////////////////////////////////// + +int TRI_RemoveElementHashArrayMulti (TRI_hash_array_multi_t* array, + TRI_index_search_value_t const* key, + TRI_hash_index_element_multi_t* element) { + uint64_t const n = array->_nrAlloc; + uint64_t i, k; + + i = k = HashKey(array, key) % n; + + for (; i < n && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); + if (i == n) { + for (i = 0; i < k && array->_table[i]._document != nullptr && ! IsEqualKeyElement(array, key, &array->_table[i]); ++i); + } + + TRI_ASSERT_EXPENSIVE(i < n); + + TRI_hash_index_element_multi_t* arrayElement = &array->_table[i]; + + // ........................................................................... + // if we did not find such an item return false + // ........................................................................... + + bool found = (arrayElement->_document != nullptr); + + if (! found) { + return TRI_RESULT_ELEMENT_NOT_FOUND; + } + + if (arrayElement->_document != element->_document) { + auto current = arrayElement; + while (current->_next != nullptr) { + if (current->_next->_document == element->_document) { + auto ptr = current->_next->_next; + DestroyElement(array, current->_next); + TRI_Free(TRI_UNKNOWN_MEM_ZONE, current->_next); + + current->_next = ptr; + array->_nrOverflow--; + + return TRI_ERROR_NO_ERROR; + } + current = current->_next; + } + } + + if (arrayElement->_next != nullptr) { + auto ptr = arrayElement->_next; + DestroyElement(array, arrayElement); + + *arrayElement = *ptr; + TRI_Free(TRI_UNKNOWN_MEM_ZONE, ptr); + + array->_nrOverflow--; + + return TRI_ERROR_NO_ERROR; + } + + // ........................................................................... + // remove item + // ........................................................................... + + DestroyElement(array, arrayElement); + array->_nrUsed--; + + // ........................................................................... + // and now check the following places for items to move here + // ........................................................................... + + k = TRI_IncModU64(i, n); + + while (array->_table[k]._document != nullptr) { + uint64_t j = HashElement(array, &array->_table[k]) % n; + + if ((i < k && ! (i < j && j <= k)) || (k < i && ! (i < j || j <= k))) { + array->_table[i] = array->_table[k]; + array->_table[k]._document = nullptr; + i = k; + } + + k = TRI_IncModU64(k, n); + } + + if (array->_nrUsed == 0) { + ResizeHashArray(array, InitialSize(), true); + } + + return TRI_ERROR_NO_ERROR; +} + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + +// Local Variables: +// mode: outline-minor +// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" +// End: diff --git a/arangod/HashIndex/hash-array-multi.h b/arangod/HashIndex/hash-array-multi.h new file mode 100644 index 0000000000..ee56d740b8 --- /dev/null +++ b/arangod/HashIndex/hash-array-multi.h @@ -0,0 +1,143 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief multi-hash array implementation, using a linked-list for collisions +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2014 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Dr. Frank Celler +/// @author Dr. Oreste Costa-Panaia +/// @author Martin Schoenert +/// @author Jan Steemann +/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany +/// @author Copyright 2006-2013, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_HASH_INDEX_HASH__ARRAY_MULTI_H +#define ARANGODB_HASH_INDEX_HASH__ARRAY_MULTI_H 1 + +#include "Basics/Common.h" +#include "Basics/vector.h" + +// ----------------------------------------------------------------------------- +// --SECTION-- forward declarations +// ----------------------------------------------------------------------------- + +struct TRI_hash_index_element_multi_s; +struct TRI_index_search_value_s; + +// ----------------------------------------------------------------------------- +// --SECTION-- public types +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief associative array +//////////////////////////////////////////////////////////////////////////////// + +typedef struct TRI_hash_array_multi_s { + size_t _numFields; // the number of fields indexes + + uint64_t _nrAlloc; // the size of the table + uint64_t _nrUsed; // the number of used entries + uint64_t _nrOverflow; // the number of overflow entries + + struct TRI_hash_index_element_multi_s* _table; // the table itself, aligned to a cache line boundary + struct TRI_hash_index_element_multi_s* _tablePtr; // the table itself +} +TRI_hash_array_multi_t; + +// ----------------------------------------------------------------------------- +// --SECTION-- HASH ARRAY +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// --SECTION-- constructors and destructors +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief initialises an array +//////////////////////////////////////////////////////////////////////////////// + +int TRI_InitHashArrayMulti (TRI_hash_array_multi_t*, + size_t); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroys an array, but does not free the pointer +//////////////////////////////////////////////////////////////////////////////// + +void TRI_DestroyHashArrayMulti (TRI_hash_array_multi_t*); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroys an array and frees the pointer +//////////////////////////////////////////////////////////////////////////////// + +void TRI_FreeHashArrayMulti (TRI_hash_array_multi_t*); + +// ----------------------------------------------------------------------------- +// --SECTION-- public functions +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief get the hash array's memory usage +//////////////////////////////////////////////////////////////////////////////// + +size_t TRI_MemoryUsageHashArrayMulti (TRI_hash_array_multi_t const*); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief resizes the hash table +//////////////////////////////////////////////////////////////////////////////// + +int TRI_ResizeHashArrayMulti (TRI_hash_array_multi_t*, + size_t); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief lookups an element given a key +//////////////////////////////////////////////////////////////////////////////// + +TRI_vector_pointer_t TRI_LookupByKeyHashArrayMulti (TRI_hash_array_multi_t const*, + struct TRI_index_search_value_s const*); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief adds an element to the array +//////////////////////////////////////////////////////////////////////////////// + +int TRI_InsertElementHashArrayMulti (TRI_hash_array_multi_t*, + struct TRI_index_search_value_s const*, + struct TRI_hash_index_element_multi_s*, + bool); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief removes an element from the array +//////////////////////////////////////////////////////////////////////////////// + +int TRI_RemoveElementHashArrayMulti (TRI_hash_array_multi_t*, + struct TRI_index_search_value_s const*, + struct TRI_hash_index_element_multi_s*); + +#endif + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + +// Local Variables: +// mode: outline-minor +// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" +// End: diff --git a/arangod/HashIndex/hash-array.cpp b/arangod/HashIndex/hash-array.cpp index ffd1779f51..0434d31f23 100644 --- a/arangod/HashIndex/hash-array.cpp +++ b/arangod/HashIndex/hash-array.cpp @@ -390,7 +390,8 @@ TRI_hash_index_element_t* TRI_FindByKeyHashArray (TRI_hash_array_t* array, int TRI_InsertKeyHashArray (TRI_hash_array_t* array, TRI_index_search_value_t* key, TRI_hash_index_element_t* element, - bool overwrite) { + bool overwrite, + bool isRollback) { // ........................................................................... // we are adding and the table is more than half full, extend it @@ -502,181 +503,6 @@ int TRI_RemoveElementHashArray (TRI_hash_array_t* array, return TRI_ERROR_NO_ERROR; } -// ----------------------------------------------------------------------------- -// --SECTION-- HASH ARRAY MULTI -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// --SECTION-- public functions -// ----------------------------------------------------------------------------- - -//////////////////////////////////////////////////////////////////////////////// -/// @brief lookups an element given a key -//////////////////////////////////////////////////////////////////////////////// - -TRI_vector_pointer_t TRI_LookupByKeyHashArrayMulti (TRI_hash_array_t* array, - TRI_index_search_value_t* key) { - TRI_ASSERT_EXPENSIVE(array->_nrUsed < array->_nrAlloc); - - // ........................................................................... - // initialise the vector which will hold the result if any - // ........................................................................... - - TRI_vector_pointer_t result; - TRI_InitVectorPointer(&result, TRI_UNKNOWN_MEM_ZONE); - - uint64_t const n = array->_nrAlloc; - uint64_t i, k; - - i = k = HashKey(array, key) % n; - - for (; i < n && array->_table[i]._document != nullptr; ++i) { - if (IsEqualKeyElement(array, key, &array->_table[i])) { - TRI_PushBackVectorPointer(&result, &array->_table[i]); - } - } - - if (i == n) { - for (i = 0; i < k && array->_table[i]._document != nullptr; ++i) { - if (IsEqualKeyElement(array, key, &array->_table[i])) { - TRI_PushBackVectorPointer(&result, &array->_table[i]); - } - } - } - - TRI_ASSERT_EXPENSIVE(i < n); - - // ........................................................................... - // return whatever we found -- which could be an empty vector list if nothing - // matches. - // ........................................................................... - - return result; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief adds an element to the array -/// -/// This function claims the owenship of the sub-objects in the inserted -/// element. -//////////////////////////////////////////////////////////////////////////////// - -int TRI_InsertElementHashArrayMulti (TRI_hash_array_t* array, - TRI_hash_index_element_t* element, - bool overwrite) { - if (! CheckResize(array)) { - return TRI_ERROR_OUT_OF_MEMORY; - } - - uint64_t const n = array->_nrAlloc; - uint64_t i, k; - - i = k = HashElement(array, element) % n; - - for (; i < n && array->_table[i]._document != nullptr && element->_document != array->_table[i]._document; ++i); - if (i == n) { - for (i = 0; i < k && array->_table[i]._document != nullptr && element->_document != array->_table[i]._document; ++i); - } - - TRI_ASSERT_EXPENSIVE(i < n); - - TRI_hash_index_element_t* arrayElement = &array->_table[i]; - - // ........................................................................... - // If we found an element, return. While we allow duplicate entries in the - // hash table, we do not allow duplicate elements. Elements would refer to the - // (for example) an actual row in memory. This is different from the - // TRI_InsertElementMultiArray function below where we only have keys to - // differentiate between elements. - // ........................................................................... - - bool found = (arrayElement->_document != nullptr); - - if (found) { - if (overwrite) { - // destroy the underlying element since we are going to stomp on top if it - DestroyElement(array, arrayElement); - *arrayElement = *element; - } - else { - DestroyElement(array, element); - } - - return TRI_RESULT_ELEMENT_EXISTS; - } - - // ........................................................................... - // add a new element to the associative array - // ........................................................................... - - *arrayElement = *element; - array->_nrUsed++; - - return TRI_ERROR_NO_ERROR; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief removes an element from the array -//////////////////////////////////////////////////////////////////////////////// - -int TRI_RemoveElementHashArrayMulti (TRI_hash_array_t* array, - TRI_hash_index_element_t* element) { - uint64_t const n = array->_nrAlloc; - uint64_t i, k; - - i = k = HashElement(array, element) % n; - - for (; i < n && array->_table[i]._document != nullptr && element->_document != array->_table[i]._document; ++i); - if (i == n) { - for (i = 0; i < k && array->_table[i]._document != nullptr && element->_document != array->_table[i]._document; ++i); - } - - TRI_ASSERT_EXPENSIVE(i < n); - - TRI_hash_index_element_t* arrayElement = &array->_table[i]; - - // ........................................................................... - // if we did not find such an item return false - // ........................................................................... - - bool found = (arrayElement->_document != nullptr); - - if (! found) { - return TRI_RESULT_ELEMENT_NOT_FOUND; - } - - // ........................................................................... - // remove item - // ........................................................................... - - DestroyElement(array, arrayElement); - array->_nrUsed--; - - // ........................................................................... - // and now check the following places for items to move here - // ........................................................................... - - k = TRI_IncModU64(i, n); - - while (array->_table[k]._document != nullptr) { - uint64_t j = HashElement(array, &array->_table[k]) % n; - - if ((i < k && ! (i < j && j <= k)) || (k < i && ! (i < j || j <= k))) { - array->_table[i] = array->_table[k]; - array->_table[k]._document = nullptr; - i = k; - } - - k = TRI_IncModU64(k, n); - } - - if (array->_nrUsed == 0) { - ResizeHashArray(array, InitialSize(), true); - } - - return TRI_ERROR_NO_ERROR; -} - // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // ----------------------------------------------------------------------------- diff --git a/arangod/HashIndex/hash-array.h b/arangod/HashIndex/hash-array.h index e3f7c3cdac..c132f75b79 100644 --- a/arangod/HashIndex/hash-array.h +++ b/arangod/HashIndex/hash-array.h @@ -125,7 +125,8 @@ struct TRI_hash_index_element_s* TRI_FindByKeyHashArray (TRI_hash_array_t*, int TRI_InsertKeyHashArray (TRI_hash_array_t*, struct TRI_index_search_value_s* key, struct TRI_hash_index_element_s* element, - bool overwrite); + bool overwrite, + bool isRollback); //////////////////////////////////////////////////////////////////////////////// /// @brief removes an element from the array @@ -134,36 +135,6 @@ int TRI_InsertKeyHashArray (TRI_hash_array_t*, int TRI_RemoveElementHashArray (TRI_hash_array_t*, struct TRI_hash_index_element_s* element); -// ----------------------------------------------------------------------------- -// --SECTION-- MULTI HASH ARRAY -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// --SECTION-- public functions -// ----------------------------------------------------------------------------- - -//////////////////////////////////////////////////////////////////////////////// -/// @brief lookups an element given a key -//////////////////////////////////////////////////////////////////////////////// - -TRI_vector_pointer_t TRI_LookupByKeyHashArrayMulti (TRI_hash_array_t*, - struct TRI_index_search_value_s* key); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief adds an element to the array -//////////////////////////////////////////////////////////////////////////////// - -int TRI_InsertElementHashArrayMulti (TRI_hash_array_t*, - struct TRI_hash_index_element_s* element, - bool overwrite); - -//////////////////////////////////////////////////////////////////////////////// -/// @brief removes an element from the array -//////////////////////////////////////////////////////////////////////////////// - -int TRI_RemoveElementHashArrayMulti (TRI_hash_array_t*, - struct TRI_hash_index_element_s* element); - #endif // ----------------------------------------------------------------------------- diff --git a/arangod/HashIndex/hash-index.cpp b/arangod/HashIndex/hash-index.cpp index f9b05ab3c3..de5eac327c 100644 --- a/arangod/HashIndex/hash-index.cpp +++ b/arangod/HashIndex/hash-index.cpp @@ -60,9 +60,10 @@ static inline size_t KeyEntrySize (TRI_hash_index_t const* idx) { /// @brief fills the index search from hash index element //////////////////////////////////////////////////////////////////////////////// +template static int FillIndexSearchValueByHashIndexElement (TRI_hash_index_t* hashIndex, TRI_index_search_value_t* key, - TRI_hash_index_element_t* element) { + T* element) { key->_values = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, KeyEntrySize(hashIndex), false)); if (key->_values == nullptr) { @@ -85,8 +86,9 @@ static int FillIndexSearchValueByHashIndexElement (TRI_hash_index_t* hashIndex, /// @brief creates space for sub-objects in the hash index element //////////////////////////////////////////////////////////////////////////////// +template static int AllocateSubObjectsHashIndexElement (TRI_hash_index_t const* idx, - TRI_hash_index_element_t* element) { + T* element) { element->_subObjects = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, KeyEntrySize(idx), false)); @@ -101,7 +103,8 @@ static int AllocateSubObjectsHashIndexElement (TRI_hash_index_t const* idx, /// @brief frees space for sub-objects in the hash index element //////////////////////////////////////////////////////////////////////////////// -static void FreeSubObjectsHashIndexElement (TRI_hash_index_element_t* element) { +template +static void FreeSubObjectsHashIndexElement (T* element) { if (element->_subObjects != nullptr) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, element->_subObjects); } @@ -116,8 +119,9 @@ static void FreeSubObjectsHashIndexElement (TRI_hash_index_element_t* element) { /// hashed and the shape identifier of each part. //////////////////////////////////////////////////////////////////////////////// +template static int HashIndexHelper (TRI_hash_index_t const* hashIndex, - TRI_hash_index_element_t* hashElement, + T* hashElement, TRI_doc_mptr_t const* document) { TRI_shaper_t* shaper; // underlying shaper TRI_shaped_json_t shapedObject; // the sub-object @@ -185,22 +189,23 @@ static int HashIndexHelper (TRI_hash_index_t const* hashIndex, /// @brief index helper for hashing with allocation //////////////////////////////////////////////////////////////////////////////// +template static int HashIndexHelperAllocate (TRI_hash_index_t const* hashIndex, - TRI_hash_index_element_t* hashElement, + T* hashElement, TRI_doc_mptr_t const* document) { // ............................................................................. // Allocate storage to shaped json objects stored as a simple list. These // will be used for hashing. Fill the json field list from the document. // ............................................................................. - int res = AllocateSubObjectsHashIndexElement(hashIndex, hashElement); + int res = AllocateSubObjectsHashIndexElement(hashIndex, hashElement); if (res != TRI_ERROR_NO_ERROR) { // out of memory return res; } - res = HashIndexHelper(hashIndex, hashElement, document); + res = HashIndexHelper(hashIndex, hashElement, document); // ............................................................................. // It may happen that the document does not have the necessary attributes to @@ -213,7 +218,7 @@ static int HashIndexHelperAllocate (TRI_hash_index_t const* hashIndex, res = TRI_ERROR_NO_ERROR; } else if (res != TRI_ERROR_NO_ERROR) { - FreeSubObjectsHashIndexElement(hashElement); + FreeSubObjectsHashIndexElement(hashElement); } return res; @@ -235,16 +240,17 @@ static int HashIndexHelperAllocate (TRI_hash_index_t const* hashIndex, //////////////////////////////////////////////////////////////////////////////// static int HashIndex_insert (TRI_hash_index_t* hashIndex, - TRI_hash_index_element_t* element) { + TRI_hash_index_element_t* element, + bool isRollback) { TRI_index_search_value_t key; - int res = FillIndexSearchValueByHashIndexElement(hashIndex, &key, element); + int res = FillIndexSearchValueByHashIndexElement(hashIndex, &key, element); if (res != TRI_ERROR_NO_ERROR) { // out of memory return res; } - res = TRI_InsertKeyHashArray(&hashIndex->_hashArray, &key, element, false); + res = TRI_InsertKeyHashArray(&hashIndex->_hashArray, &key, element, false, isRollback); if (key._values != nullptr) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, key._values); @@ -323,8 +329,21 @@ static TRI_index_result_t HashIndex_find (TRI_hash_index_t* hashIndex, //////////////////////////////////////////////////////////////////////////////// static int MultiHashIndex_insert (TRI_hash_index_t* hashIndex, - TRI_hash_index_element_t* element) { - int res = TRI_InsertElementHashArrayMulti(&hashIndex->_hashArray, element, false); + TRI_hash_index_element_multi_t* element, + bool isRollback) { + TRI_index_search_value_t key; + int res = FillIndexSearchValueByHashIndexElement(hashIndex, &key, element); + + if (res != TRI_ERROR_NO_ERROR) { + // out of memory + return res; + } + + res = TRI_InsertElementHashArrayMulti(&hashIndex->_hashArrayMulti, &key, element, isRollback); + + if (key._values != nullptr) { + TRI_Free(TRI_UNKNOWN_MEM_ZONE, key._values); + } if (res == TRI_RESULT_ELEMENT_EXISTS) { return TRI_ERROR_INTERNAL; @@ -338,8 +357,20 @@ static int MultiHashIndex_insert (TRI_hash_index_t* hashIndex, //////////////////////////////////////////////////////////////////////////////// int MultiHashIndex_remove (TRI_hash_index_t* hashIndex, - TRI_hash_index_element_t* element) { - int res = TRI_RemoveElementHashArrayMulti(&hashIndex->_hashArray, element); + TRI_hash_index_element_multi_t* element) { + TRI_index_search_value_t key; + int res = FillIndexSearchValueByHashIndexElement(hashIndex, &key, element); + + if (res != TRI_ERROR_NO_ERROR) { + // out of memory + return res; + } + + res = TRI_RemoveElementHashArrayMulti(&hashIndex->_hashArrayMulti, &key, element); + + if (key._values != nullptr) { + TRI_Free(TRI_UNKNOWN_MEM_ZONE, key._values); + } if (res == TRI_RESULT_ELEMENT_NOT_FOUND) { return TRI_ERROR_INTERNAL; @@ -361,7 +392,7 @@ static TRI_index_result_t MultiHashIndex_find (TRI_hash_index_t* hashIndex, // we want more than one result returned! // ............................................................................. - TRI_vector_pointer_t result = TRI_LookupByKeyHashArrayMulti(&hashIndex->_hashArray, key); + TRI_vector_pointer_t result = TRI_LookupByKeyHashArrayMulti(&hashIndex->_hashArrayMulti, key); if (result._length == 0) { results._length = 0; @@ -369,7 +400,7 @@ static TRI_index_result_t MultiHashIndex_find (TRI_hash_index_t* hashIndex, } else { results._length = result._length; - results._documents = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, result._length* sizeof(TRI_doc_mptr_t*), false)); + results._documents = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, result._length * sizeof(TRI_doc_mptr_t*), false)); if (results._documents == nullptr) { // no memory. prevent worst case by re-setting results length to 0 @@ -380,7 +411,7 @@ static TRI_index_result_t MultiHashIndex_find (TRI_hash_index_t* hashIndex, } for (size_t j = 0; j < result._length; ++j) { - results._documents[j] = ((TRI_hash_index_element_t*)(result._buffer[j]))->_document; + results._documents[j] = ((TRI_hash_index_element_multi_t*)(result._buffer[j]))->_document; } } @@ -403,8 +434,14 @@ static TRI_index_result_t MultiHashIndex_find (TRI_hash_index_t* hashIndex, size_t MemoryHashIndex (TRI_index_t const* idx) { TRI_hash_index_t const* hashIndex = (TRI_hash_index_t const*) idx; - return static_cast(KeyEntrySize(hashIndex) * hashIndex->_hashArray._nrUsed + - TRI_MemoryUsageHashArray(&hashIndex->_hashArray)); + if (hashIndex->base._unique) { + return static_cast(KeyEntrySize(hashIndex) * hashIndex->_hashArray._nrUsed + + TRI_MemoryUsageHashArray(&hashIndex->_hashArray)); + } + else { + return static_cast(KeyEntrySize(hashIndex) * hashIndex->_hashArrayMulti._nrUsed + + TRI_MemoryUsageHashArrayMulti(&hashIndex->_hashArrayMulti)); + } } //////////////////////////////////////////////////////////////////////////////// @@ -460,22 +497,34 @@ static int InsertHashIndex (TRI_index_t* idx, bool isRollback) { TRI_hash_index_t* hashIndex = (TRI_hash_index_t*) idx; - TRI_hash_index_element_t hashElement; - int res = HashIndexHelperAllocate(hashIndex, &hashElement, document); - - if (res == TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING) { - return TRI_ERROR_NO_ERROR; - } - - if (res != TRI_ERROR_NO_ERROR) { - return res; - } - + int res; if (hashIndex->base._unique) { - res = HashIndex_insert(hashIndex, &hashElement); + TRI_hash_index_element_t hashElement; + res = HashIndexHelperAllocate(hashIndex, &hashElement, document); + + if (res == TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING) { + return TRI_ERROR_NO_ERROR; + } + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + res = HashIndex_insert(hashIndex, &hashElement, isRollback); } else { - res = MultiHashIndex_insert(hashIndex, &hashElement); + TRI_hash_index_element_multi_t hashElement; + res = HashIndexHelperAllocate(hashIndex, &hashElement, document); + + if (res == TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING) { + return TRI_ERROR_NO_ERROR; + } + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } + + res = MultiHashIndex_insert(hashIndex, &hashElement, isRollback); } return res; @@ -489,26 +538,37 @@ static int RemoveHashIndex (TRI_index_t* idx, TRI_doc_mptr_t const* document, bool isRollback) { TRI_hash_index_t* hashIndex = (TRI_hash_index_t*) idx; - - TRI_hash_index_element_t hashElement; - int res = HashIndexHelperAllocate(hashIndex, &hashElement, document); - - if (res == TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING) { - return TRI_ERROR_NO_ERROR; - } - - if (res != TRI_ERROR_NO_ERROR) { - return res; - } + int res; if (hashIndex->base._unique) { + TRI_hash_index_element_t hashElement; + res = HashIndexHelperAllocate(hashIndex, &hashElement, document); + + if (res == TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING) { + return TRI_ERROR_NO_ERROR; + } + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } res = HashIndex_remove(hashIndex, &hashElement); + FreeSubObjectsHashIndexElement(&hashElement); } else { + TRI_hash_index_element_multi_t hashElement; + res = HashIndexHelperAllocate(hashIndex, &hashElement, document); + + if (res == TRI_ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING) { + return TRI_ERROR_NO_ERROR; + } + + if (res != TRI_ERROR_NO_ERROR) { + return res; + } res = MultiHashIndex_remove(hashIndex, &hashElement); + FreeSubObjectsHashIndexElement(&hashElement); } - FreeSubObjectsHashIndexElement(&hashElement); return res; } @@ -519,7 +579,13 @@ static int RemoveHashIndex (TRI_index_t* idx, static int SizeHintHashIndex (TRI_index_t* idx, size_t size) { TRI_hash_index_t* hashIndex = (TRI_hash_index_t*) idx; - TRI_ResizeHashArray(&hashIndex->_hashArray, size); + + if (hashIndex->base._unique) { + TRI_ResizeHashArray(&hashIndex->_hashArray, size); + } + else { + TRI_ResizeHashArrayMulti(&hashIndex->_hashArrayMulti, size); + } return TRI_ERROR_NO_ERROR; } @@ -561,9 +627,15 @@ TRI_index_t* TRI_CreateHashIndex (TRI_document_collection_t* document, TRI_InitVectorString(&idx->_fields, TRI_CORE_MEM_ZONE); TRI_CopyDataFromVectorPointerVectorString(TRI_CORE_MEM_ZONE, &idx->_fields, fields); - // create a index preallocated for the current number of documents - int res = TRI_InitHashArray(&hashIndex->_hashArray, - hashIndex->_paths._length); + int res; + if (unique) { + res = TRI_InitHashArray(&hashIndex->_hashArray, + hashIndex->_paths._length); + } + else { + res = TRI_InitHashArrayMulti(&hashIndex->_hashArrayMulti, + hashIndex->_paths._length); + } // oops, out of memory? if (res != TRI_ERROR_NO_ERROR) { @@ -583,12 +655,16 @@ TRI_index_t* TRI_CreateHashIndex (TRI_document_collection_t* document, void TRI_DestroyHashIndex (TRI_index_t* idx) { TRI_hash_index_t* hashIndex; - TRI_DestroyVectorString(&idx->_fields); - hashIndex = (TRI_hash_index_t*) idx; + if (hashIndex->base._unique) { + TRI_DestroyHashArray(&hashIndex->_hashArray); + } + else { + TRI_DestroyHashArrayMulti(&hashIndex->_hashArrayMulti); + } + TRI_DestroyVectorString(&idx->_fields); TRI_DestroyVector(&hashIndex->_paths); - TRI_DestroyHashArray(&hashIndex->_hashArray); } //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/HashIndex/hash-index.h b/arangod/HashIndex/hash-index.h index 7c030a9d8e..e00a483745 100644 --- a/arangod/HashIndex/hash-index.h +++ b/arangod/HashIndex/hash-index.h @@ -34,6 +34,7 @@ #include "Basics/Common.h" #include "HashIndex/hash-array.h" +#include "HashIndex/hash-array-multi.h" #include "VocBase/index.h" // ----------------------------------------------------------------------------- @@ -63,6 +64,13 @@ typedef struct TRI_hash_index_element_s { } TRI_hash_index_element_t; +typedef struct TRI_hash_index_element_multi_s { + struct TRI_doc_mptr_t* _document; + struct TRI_shaped_sub_s* _subObjects; + struct TRI_hash_index_element_multi_s* _next; +} +TRI_hash_index_element_multi_t; + //////////////////////////////////////////////////////////////////////////////// /// @brief hash index //////////////////////////////////////////////////////////////////////////////// @@ -70,7 +78,10 @@ TRI_hash_index_element_t; typedef struct TRI_hash_index_s { TRI_index_t base; - TRI_hash_array_t _hashArray; // the hash array itself + union { + TRI_hash_array_t _hashArray; // the hash array itself, unique values + TRI_hash_array_multi_t _hashArrayMulti; // the hash array itself, non-unique values + }; TRI_vector_t _paths; // a list of shape pid which identifies the fields of the index } TRI_hash_index_t; diff --git a/arangod/Makefile.files b/arangod/Makefile.files index 38e3e910d2..fb5d586216 100644 --- a/arangod/Makefile.files +++ b/arangod/Makefile.files @@ -89,6 +89,7 @@ arangod_libarangod_a_SOURCES = \ arangod/GeoIndex/GeoIndex.cpp \ arangod/GeoIndex/geo-index.cpp \ arangod/HashIndex/hash-array.cpp \ + arangod/HashIndex/hash-array-multi.cpp \ arangod/HashIndex/hash-index.cpp \ arangod/IndexOperators/index-operator.cpp \ arangod/Replication/ContinuousSyncer.cpp \