Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

reduce element size in hash index

This commit is contained in:
Jan Steemann 2012-10-16 15:15:31 +02:00
parent b0f69deaf9
commit 0beeeb6c2a
7 changed files with 32 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
arangod/HashIndex/compare.h
View File

@ -39,16 +39,13 @@
#include <BasicsC/hashes.h>
#include <ShapedJson/json-shaper.h>
#include <ShapedJson/shaped-json.h>
#include <VocBase/primary-collection.h>
#define USE_STATIC_HASHARRAY_COMPARE 1
#define HASHARRAY_ELEMENT_TYPE(a,b) \
struct a { \
size_t numFields; \
TRI_shaped_json_t* fields; \
void* data; \
void* collection; \
} b
@ -90,7 +87,6 @@ static void IndexStaticClearElement(TRI_hasharray_t* array, void* element) {
LocalElement_t* hElement = (LocalElement_t*)(element);
if (element != NULL) {
hElement->numFields = 0;
hElement->fields = 0;
hElement->data = 0;
}
@ -113,15 +109,14 @@ static bool IndexStaticCopyElementElement (TRI_hasharray_t* array, void* left, v
return false;
}
leftElement->numFields = rightElement->numFields;
leftElement->data = rightElement->data;
leftElement->fields = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_shaped_json_t) * leftElement->numFields, false);
leftElement->fields = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_shaped_json_t) * array->_numFields, false);
if (leftElement->fields == NULL) {
return false;
}
memcpy(leftElement->fields, rightElement->fields, sizeof(TRI_shaped_json_t) * leftElement->numFields);
memcpy(leftElement->fields, rightElement->fields, sizeof(TRI_shaped_json_t) * array->_numFields);
return true;
}
@ -164,7 +159,7 @@ static uint64_t IndexStaticHashElement (TRI_hasharray_t* array, void* element) {
uint64_t hash = TRI_FnvHashBlockInitial();
size_t j;
for (j = 0; j < hElement->numFields; j++) {
for (j = 0; j < array->_numFields; j++) {
hash = IndexStaticHashShapedJson(hash, (j + hElement->fields) );
}
return hash;
@ -182,7 +177,7 @@ static uint64_t IndexStaticHashKey (TRI_hasharray_t* array, void* element) {
uint64_t hash = TRI_FnvHashBlockInitial();
size_t j;
for (j = 0; j < hElement->numFields; j++) {
for (j = 0; j < array->_numFields; j++) {
hash = IndexStaticHashShapedJson(hash, (j + hElement->fields) );
}
return hash;
@ -228,10 +223,6 @@ static bool IndexStaticIsEqualElementElement (TRI_hasharray_t* array, void* left
return false;
}
if (hLeftElement->numFields != hRightElement->numFields) {
return false; // should never happen
}
return (hLeftElement->data == hRightElement->data);
}
@ -269,11 +260,7 @@ static bool IndexStaticIsEqualKeyElement (TRI_hasharray_t* array, void* leftElem
return false;
}
if (hLeftElement->numFields != hRightElement->numFields) {
return false; // should never happen
}
for (j = 0; j < hLeftElement->numFields; j++) {
for (j = 0; j < array->_numFields; j++) {
if (!IndexStaticIsEqualShapedJsonShapedJson((j + hLeftElement->fields), (j + hRightElement->fields))) {
return false;
}
@ -297,11 +284,7 @@ static bool IndexStaticIsEqualKeyElementMulti (TRI_hasharray_t* array, void* lef
return false;
}
if (hLeftElement->numFields != hRightElement->numFields) {
return false; // should never happen
}
for (j = 0; j < hLeftElement->numFields; j++) {
for (j = 0; j < array->_numFields; j++) {
TRI_shaped_json_t* left = (j + hLeftElement->fields);
TRI_shaped_json_t* right = (j + hRightElement->fields);
if (!IndexStaticIsEqualShapedJsonShapedJson(left, right)) {

9
arangod/HashIndex/hasharray.c
View File

@ -67,6 +67,7 @@ static bool ResizeHashArrayMulti (TRI_hasharray_t*);
////////////////////////////////////////////////////////////////////////////////
bool TRI_InitHashArray (TRI_hasharray_t* array,
size_t numFields,
size_t elementSize,
uint64_t (*hashKey) (TRI_hasharray_t*, void*),
uint64_t (*hashElement) (TRI_hasharray_t*, void*),
@ -78,16 +79,20 @@ bool TRI_InitHashArray (TRI_hasharray_t* array,
// ...........................................................................
// Assign the callback functions
// ...........................................................................
assert(numFields > 0);
array->clearElement = clearElement;
array->isEmptyElement = isEmptyElement;
array->isEqualKeyElement = isEqualKeyElement;
array->isEqualElementElement = isEqualElementElement;
array->_numFields = numFields;
array->_elementSize = elementSize;
array->_table = NULL;
array->_nrAlloc = 10;
// set initial allocation size to 256 elements
array->_nrAlloc = 256;
// ...........................................................................

3
arangod/HashIndex/hasharray.h
View File

@ -59,7 +59,7 @@ typedef struct TRI_hasharray_s {
bool (*isEqualKeyElement) (struct TRI_hasharray_s*, void*, void*);
bool (*isEqualElementElement) (struct TRI_hasharray_s*, void*, void*);
size_t _numFields; // the number of fields indexes
uint64_t _elementSize;
uint64_t _nrAlloc; // the size of the table
uint64_t _nrUsed; // the number of used entries
@ -101,6 +101,7 @@ TRI_hasharray_t;
////////////////////////////////////////////////////////////////////////////////
bool TRI_InitHashArray (TRI_hasharray_t*,
size_t numFields,
size_t elementSize,
uint64_t (*hashKey) (TRI_hasharray_t*, void*),
uint64_t (*hashElement) (TRI_hasharray_t*, void*),

17
arangod/HashIndex/hashindex.c
View File

@ -100,19 +100,15 @@ void HashIndex_freeResult(TRI_hash_index_elements_t* const list) {
FreeResults(list);
}
// -----------------------------------------------------------------------------
// constructors public functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief Creates a new hash array used for storage of elements in the hash index
////////////////////////////////////////////////////////////////////////////////
HashIndex* HashIndex_new() {
HashIndex* HashIndex_new(size_t numFields) {
HashIndex* hashIndex;
hashIndex = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(HashIndex), false);
@ -124,18 +120,19 @@ HashIndex* HashIndex_new() {
hashIndex->hashArray = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_hasharray_t), false);
if (hashIndex->hashArray == NULL) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, hashIndex);
return NULL;
}
if (! TRI_InitHashArray(hashIndex->hashArray, sizeof(HashIndexElement), NULL, NULL, NULL, NULL, NULL, NULL) ) {
if (! TRI_InitHashArray(hashIndex->hashArray, numFields, sizeof(HashIndexElement), NULL, NULL, NULL, NULL, NULL, NULL) ) {
HashIndex_free(hashIndex);
return NULL;
}
return hashIndex;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief Assigns a static function call to a function pointer used by Query Engine
////////////////////////////////////////////////////////////////////////////////
@ -322,7 +319,7 @@ void MultiHashIndex_freeResult(TRI_hash_index_elements_t* const list) {
/// @brief Creates a new multi (non-unique) hash index
////////////////////////////////////////////////////////////////////////////////
HashIndex* MultiHashIndex_new() {
HashIndex* MultiHashIndex_new(size_t numFields) {
HashIndex* hashIndex;
hashIndex = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(HashIndex), false);
@ -334,11 +331,13 @@ HashIndex* MultiHashIndex_new() {
hashIndex->hashArray = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_hasharray_t), false);
if (hashIndex->hashArray == NULL) {
TRI_Free(TRI_UNKNOWN_MEM_ZONE, hashIndex);
return NULL;
}
if (! TRI_InitHashArray(hashIndex->hashArray, sizeof(HashIndexElement), NULL, NULL, NULL, NULL, NULL, NULL) ) {
if (! TRI_InitHashArray(hashIndex->hashArray, numFields, sizeof(HashIndexElement), NULL, NULL, NULL, NULL, NULL, NULL) ) {
HashIndex_free(hashIndex);
return NULL;
}

6
arangod/HashIndex/hashindex.h
View File

@ -60,10 +60,8 @@ typedef struct {
typedef struct {
size_t numFields; // the number of fields
TRI_shaped_json_t* fields; // list of shaped json objects the blob of data within will be hashed
void* data; // master document pointer
void* collection; // currently not used
} HashIndexElement;
typedef struct {
@ -91,7 +89,7 @@ int HashIndex_assignMethod (void*, TRI_index_method_assignment_type_e);
void HashIndex_destroy (HashIndex*);
HashIndex* HashIndex_new (void);
HashIndex* HashIndex_new (size_t);
void HashIndex_free (HashIndex*);
@ -128,7 +126,7 @@ void MultiHashIndex_free (HashIndex*);
void MultiHashIndex_freeResult(TRI_hash_index_elements_t* const);
HashIndex* MultiHashIndex_new (void);
HashIndex* MultiHashIndex_new (size_t);
int MultiHashIndex_add (HashIndex*, HashIndexElement*);

68
arangod/VocBase/index.c
View File

@ -1476,8 +1476,7 @@ static int InsertHashIndex (TRI_index_t* idx, TRI_doc_mptr_t const* doc) {
// These will be used for hashing.
// .............................................................................
hashElement.numFields = hashIndex->_paths._length;
hashElement.fields = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashElement.numFields, false);
hashElement.fields = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashIndex->_paths._length, false);
res = HashIndexHelper(hashIndex, &hashElement, doc, NULL);
@ -1553,8 +1552,7 @@ static int RemoveHashIndex (TRI_index_t* idx, TRI_doc_mptr_t const* doc) {
// Allocate some memory for the HashIndexElement structure
// .............................................................................
hashElement.numFields = hashIndex->_paths._length;
hashElement.fields = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashElement.numFields, false);
hashElement.fields = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashIndex->_paths._length, false);
// .............................................................................
// Fill the json field list from the document
@ -1648,8 +1646,7 @@ static int UpdateHashIndex (TRI_index_t* idx,
// Allocate some memory for the HashIndexElement structure
// .............................................................................
hashElement.numFields = hashIndex->_paths._length;
hashElement.fields = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashElement.numFields, false);
hashElement.fields = TRI_Allocate(TRI_CORE_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashIndex->_paths._length, false);
// .............................................................................
// Update for unique hash index
@ -1817,10 +1814,10 @@ TRI_index_t* TRI_CreateHashIndex (struct TRI_primary_collection_s* collection,
}
if (unique) {
hashIndex->_hashIndex = HashIndex_new();
hashIndex->_hashIndex = HashIndex_new(hashIndex->_paths._length);
}
else {
hashIndex->_hashIndex = MultiHashIndex_new();
hashIndex->_hashIndex = MultiHashIndex_new(hashIndex->_paths._length);
}
if (hashIndex->_hashIndex == NULL) { // oops out of memory?
@ -1906,56 +1903,6 @@ void TRI_FreeResultHashIndex (const TRI_index_t* const idx,
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief locates entries in the hash index given a JSON list
///
/// @warning who ever calls this function is responsible for destroying
/// TRI_hash_index_elements_t* results
////////////////////////////////////////////////////////////////////////////////
TRI_hash_index_elements_t* TRI_LookupJsonHashIndex (TRI_index_t* idx, TRI_json_t* values) {
TRI_hash_index_t* hashIndex;
TRI_hash_index_elements_t* result;
HashIndexElement element;
TRI_shaper_t* shaper;
size_t j;
element.numFields = values->_value._objects._length;
element.fields = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_shaped_json_t) * element.numFields, false);
if (element.fields == NULL) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
LOG_WARNING("out-of-memory in LookupJsonHashIndex");
return NULL;
}
hashIndex = (TRI_hash_index_t*) idx;
shaper = hashIndex->base._collection->_shaper;
for (j = 0; j < element.numFields; ++j) {
TRI_json_t* jsonObject = (TRI_json_t*) (TRI_AtVector(&(values->_value._objects), j));
TRI_shaped_json_t* shapedObject = TRI_ShapedJsonJson(shaper, jsonObject);
element.fields[j] = *shapedObject;
TRI_Free(TRI_UNKNOWN_MEM_ZONE, shapedObject);
}
if (hashIndex->base._unique) {
result = HashIndex_find(hashIndex->_hashIndex, &element);
}
else {
result = MultiHashIndex_find(hashIndex->_hashIndex, &element);
}
for (j = 0; j < element.numFields; ++j) {
TRI_DestroyShapedJson(shaper, element.fields + j);
}
TRI_Free(TRI_UNKNOWN_MEM_ZONE, element.fields);
return result;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief locates entries in the hash index given shaped json objects
////////////////////////////////////////////////////////////////////////////////
@ -1968,8 +1915,7 @@ TRI_hash_index_elements_t* TRI_LookupShapedJsonHashIndex (TRI_index_t* idx, TRI_
hashIndex = (TRI_hash_index_t*) idx;
element.numFields = hashIndex->_paths._length;
element.fields = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_shaped_json_t) * element.numFields, false);
element.fields = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_shaped_json_t) * hashIndex->_paths._length, false);
if (element.fields == NULL) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
@ -1977,7 +1923,7 @@ TRI_hash_index_elements_t* TRI_LookupShapedJsonHashIndex (TRI_index_t* idx, TRI_
return NULL;
}
for (j = 0; j < element.numFields; ++j) {
for (j = 0; j < hashIndex->_paths._length; ++j) {
element.fields[j] = *values[j];
}

9
arangod/VocBase/index.h
View File

@ -510,15 +510,6 @@ void TRI_FreeHashIndex (TRI_index_t* idx);
void TRI_FreeResultHashIndex (const TRI_index_t* const,
TRI_hash_index_elements_t* const);
////////////////////////////////////////////////////////////////////////////////
/// @brief locates entries in the hash index given a JSON list
///
/// @warning who ever calls this function is responsible for destroying
/// TRI_hash_index_elements_t* results
////////////////////////////////////////////////////////////////////////////////
TRI_hash_index_elements_t* TRI_LookupJsonHashIndex (TRI_index_t*, TRI_json_t*);
////////////////////////////////////////////////////////////////////////////////
/// @brief locates entries in the hash index given shaped json objects
///