//////////////////////////////////////////////////////////////////////////////// /// @brief json shaper used to compute the shape of an json object /// /// @file /// /// DISCLAIMER /// /// Copyright 2010-2011 triagens GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); /// you may not use this file except in compliance with the License. /// You may obtain a copy of the License at /// /// http://www.apache.org/licenses/LICENSE-2.0 /// /// Unless required by applicable law or agreed to in writing, software /// distributed under the License is distributed on an "AS IS" BASIS, /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. /// See the License for the specific language governing permissions and /// limitations under the License. /// /// Copyright holder is triAGENS GmbH, Cologne, Germany /// /// @author Dr. Frank Celler /// @author Martin Schoenert /// @author Copyright 2006-2011, triagens GmbH, Cologne, Germany //////////////////////////////////////////////////////////////////////////////// #include "voc-shaper.h" #include #include #include #include #include #include // ----------------------------------------------------------------------------- // --SECTION-- private constants // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// /// @addtogroup VocBase /// @{ //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// /// @brief datafile size //////////////////////////////////////////////////////////////////////////////// static TRI_voc_size_t const SHAPER_DATAFILE_SIZE = (2 * 1024 * 1204); //////////////////////////////////////////////////////////////////////////////// /// @} //////////////////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- // --SECTION-- private types // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// /// @addtogroup VocBase /// @{ //////////////////////////////////////////////////////////////////////////////// typedef struct attribute_weight_s { TRI_shape_aid_t _aid; int64_t _weight; char* _attribute; struct attribute_weight_s* _next; } attribute_weight_t; typedef struct attribute_weights_s { attribute_weight_t* _first; attribute_weight_t* _last; size_t _length; } attribute_weights_t; //////////////////////////////////////////////////////////////////////////////// /// @brief persistent, collection-bases shaper //////////////////////////////////////////////////////////////////////////////// typedef struct voc_shaper_s { TRI_shaper_t base; TRI_associative_synced_t _attributeNames; TRI_associative_synced_t _attributeIds; TRI_associative_synced_t _shapeDictionary; TRI_associative_synced_t _shapeIds; TRI_associative_pointer_t _accessors; TRI_vector_pointer_t _sortedAttributes; TRI_associative_pointer_t _weightedAttributes; attribute_weights_t _weights; TRI_shape_aid_t _nextAid; TRI_shape_sid_t _nextSid; TRI_shape_collection_t* _collection; TRI_mutex_t _shapeLock; TRI_mutex_t _attributeLock; TRI_mutex_t _accessorLock; } voc_shaper_t; //////////////////////////////////////////////////////////////////////////////// /// @} //////////////////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- // --SECTION-- private functions // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// /// @addtogroup VocBase /// @{ //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// /// @brief hashs the attribute name of a key //////////////////////////////////////////////////////////////////////////////// static uint64_t HashKeyAttributeName (TRI_associative_synced_t* array, void const* key) { char const* k = (char const*) key; return TRI_FnvHashString(k); } //////////////////////////////////////////////////////////////////////////////// /// @brief hashs the attribute name of an element //////////////////////////////////////////////////////////////////////////////// static uint64_t HashElementAttributeName (TRI_associative_synced_t* array, void const* element) { char const* e = (char const*) element; return TRI_FnvHashString(e + sizeof(TRI_df_attribute_marker_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief compares an attribute name and an attribute //////////////////////////////////////////////////////////////////////////////// static bool EqualKeyAttributeName (TRI_associative_synced_t* array, void const* key, void const* element) { char const* k = (char const*) key; char const* e = (char const*) element; return TRI_EqualString(k, e + sizeof(TRI_df_attribute_marker_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief compares two attribute strings stored in the attribute marker /// /// returns 0 if the strings are equal /// returns < 0 if the left string compares less than the right string /// returns > 0 if the left string compares more than the right string //////////////////////////////////////////////////////////////////////////////// static int attributeWeightCompareFunction(const void* leftItem, const void* rightItem) { const attribute_weight_t* l = (const attribute_weight_t*)(leftItem); const attribute_weight_t* r = (const attribute_weight_t*)(rightItem); assert(l); assert(r); #ifdef TRI_HAVE_ICU return TR_compare_utf8(l->_attribute, r->_attribute); #else return (strcmp(l->_attribute, r->_attribute)); #endif } static int attributeWeightCompareFunctionPointer(const void* leftItem, const void* rightItem) { const attribute_weight_t* l = *((const attribute_weight_t**)(leftItem)); const attribute_weight_t* r = *((const attribute_weight_t**)(rightItem)); assert(l); assert(r); #ifdef TRI_HAVE_ICU return TR_compare_utf8(l->_attribute, r->_attribute); #else return (strcmp(l->_attribute, r->_attribute)); #endif } //////////////////////////////////////////////////////////////////////////////// /// @brief performs a binary search on a list of attributes (attribute markers) /// and returns the position where a given attribute would be inserted //////////////////////////////////////////////////////////////////////////////// static int64_t sortedIndexOf(voc_shaper_t* shaper, attribute_weight_t* item) { int64_t leftPos; int64_t rightPos; int64_t midPos; int compareResult; leftPos = 0; rightPos = ((int64_t) shaper->_sortedAttributes._length) - 1; while (leftPos <= rightPos) { midPos = (leftPos + rightPos) / 2; compareResult = attributeWeightCompareFunction(TRI_AtVectorPointer(&(shaper->_sortedAttributes), midPos), (void*)(item)); if (compareResult < 0) { leftPos = midPos + 1; } else if (compareResult > 0) { rightPos = midPos - 1; } else { // should never happen since we do not allow duplicates here return -1; } } return leftPos; // insert it to the left of this position } //////////////////////////////////////////////////////////////////////////////// /// @brief helper method to store a list of attributes and their corresponding /// weights. //////////////////////////////////////////////////////////////////////////////// static void setAttributeWeight(voc_shaper_t* shaper, attribute_weight_t* item, int64_t* searchResult, bool* weighted) { uint64_t itemWeight; attribute_weight_t* leftItem; // nearest neighbour items attribute_weight_t* rightItem; // nearest neighbour items const int64_t resolution = 100; void* result; if (item == NULL && shaper == NULL) { // oops assert(false); return; } *searchResult = sortedIndexOf(shaper, item); if (*searchResult < 0 || *searchResult > shaper->_sortedAttributes._length) { // oops assert(false); return; } *weighted = false; switch ( (shaper->_sortedAttributes)._length) { case 0: { item->_weight = 0; *weighted = true; break; } case 1: { if (*searchResult == 0) { item->_weight = 0; rightItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), 0)); rightItem->_weight = resolution; } else { leftItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), 0)); leftItem->_weight = 0; item->_weight = resolution; } *weighted = true; break; } default: { if (*searchResult == 0) { rightItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), 0)); item->_weight = rightItem->_weight - resolution; *weighted = true; } else if (*searchResult == (shaper->_sortedAttributes)._length) { leftItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), (shaper->_sortedAttributes)._length - 1)); item->_weight = leftItem->_weight + resolution; *weighted = true; } else { leftItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), *searchResult - 1)); rightItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), *searchResult)); itemWeight = (rightItem->_weight + leftItem->_weight) / 2; if (leftItem->_weight != itemWeight && rightItem->_weight != itemWeight) { item->_weight = itemWeight; *weighted = true; } } break; } } // end of switch statement result = TRI_InsertKeyAssociativePointer(&(shaper->_weightedAttributes), &(item->_aid), item, false); if (result != NULL) { LOG_ERROR("attribute weight could not be inserted into associative array"); *searchResult = -1; return; } // ........................................................................... // Obtain the pointer of the weighted attribute structure which is stored // in the associative array. We need to pass this pointer to the Vector Pointer // ........................................................................... item = TRI_LookupByKeyAssociativePointer(&(shaper->_weightedAttributes), &(item->_aid)); if (item == NULL) { LOG_ERROR("attribute weight could not be located immediately after insert into associative array"); *searchResult = -1; return; } TRI_InsertVectorPointer(&(shaper->_sortedAttributes), item, *searchResult); } static void fullSetAttributeWeight(voc_shaper_t* shaper) { int64_t startWeight; attribute_weight_t* item; int j; startWeight = 0; for (j = 0; j < shaper->_sortedAttributes._length; ++j) { item = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), j)); item->_weight = startWeight; startWeight += 100; } } //////////////////////////////////////////////////////////////////////////////// /// @brief finds an attribute identifier by name //////////////////////////////////////////////////////////////////////////////// static TRI_shape_aid_t FindAttributeName (TRI_shaper_t* shaper, char const* name) { TRI_df_attribute_marker_t marker; TRI_df_marker_t* result; TRI_df_attribute_marker_t* markerResult; int res; size_t n; voc_shaper_t* s; void const* p; void* f; int64_t searchResult; bool weighted; attribute_weight_t* weightedAttribute; s = (voc_shaper_t*) shaper; p = TRI_LookupByKeyAssociativeSynced(&s->_attributeNames, name); if (p != NULL) { return ((TRI_df_attribute_marker_t const*) p)->_aid; } // create a new attribute name n = strlen(name) + 1; // lock the index and check that the element is still missing TRI_LockMutex(&s->_attributeLock); p = TRI_LookupByKeyAssociativeSynced(&s->_attributeNames, name); // if the element appeared, return the aid if (p != NULL) { TRI_UnlockMutex(&s->_attributeLock); return ((TRI_df_attribute_marker_t const*) p)->_aid; } // create new attribute identifier memset(&marker, 0, sizeof(TRI_df_attribute_marker_t)); marker.base._type = TRI_DF_MARKER_ATTRIBUTE; marker.base._size = sizeof(TRI_df_attribute_marker_t) + n; marker._aid = s->_nextAid++; marker._size = n; // write into the shape collection res = TRI_WriteShapeCollection(s->_collection, &marker.base, sizeof(TRI_df_attribute_marker_t), name, n, &result); if (res != TRI_ERROR_NO_ERROR) { TRI_UnlockMutex(&s->_attributeLock); return 0; } // enter into the dictionaries f = TRI_InsertKeyAssociativeSynced(&s->_attributeNames, name, result); assert(f == NULL); f = TRI_InsertKeyAssociativeSynced(&s->_attributeIds, &marker._aid, result); assert(f == NULL); // ........................................................................... // Each attribute has an associated integer as a weight. This // weight corresponds to the natural ordering of the attribute strings // ........................................................................... markerResult = (TRI_df_attribute_marker_t*)(result); weightedAttribute = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(attribute_weight_t), false); if (weightedAttribute != NULL) { weightedAttribute->_aid = markerResult->_aid; weightedAttribute->_weight = TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; weightedAttribute->_attribute = (char*)(markerResult) + sizeof(TRI_df_attribute_marker_t); weightedAttribute->_next = NULL; // .......................................................................... // Save the new attribute weight in the linked list. // .......................................................................... if ((s->_weights)._last == NULL) { (s->_weights)._first = weightedAttribute; } else { ((s->_weights)._last)->_next = weightedAttribute; } (s->_weights)._last = weightedAttribute; (s->_weights)._length += 1; setAttributeWeight(s, weightedAttribute, &searchResult, &weighted); assert(searchResult > -1); if (!weighted) { fullSetAttributeWeight(s); } } else { LOG_WARNING("FindAttributeName could not allocate memory, attribute is NOT weighted"); } // ........................................................................... // and release the lock // ........................................................................... TRI_UnlockMutex(&s->_attributeLock); return marker._aid; } //////////////////////////////////////////////////////////////////////////////// /// @brief hashs the attribute id //////////////////////////////////////////////////////////////////////////////// static uint64_t HashKeyAttributeId (TRI_associative_synced_t* array, void const* key) { TRI_shape_aid_t const* k = key; return TRI_FnvHashPointer(k, sizeof(TRI_shape_aid_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief hashs the attribute //////////////////////////////////////////////////////////////////////////////// static uint64_t HashElementAttributeId (TRI_associative_synced_t* array, void const* element) { TRI_df_attribute_marker_t const* e = element; return TRI_FnvHashPointer(&e->_aid, sizeof(TRI_shape_aid_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief compares an attribute name and an attribute //////////////////////////////////////////////////////////////////////////////// static bool EqualKeyAttributeId (TRI_associative_synced_t* array, void const* key, void const* element) { TRI_shape_aid_t const* k = key; TRI_df_attribute_marker_t const* e = element; return *k == e->_aid; } //////////////////////////////////////////////////////////////////////////////// /// @brief looks up an attribute name by identifier //////////////////////////////////////////////////////////////////////////////// static char const* LookupAttributeId (TRI_shaper_t* shaper, TRI_shape_aid_t aid) { voc_shaper_t* s = (voc_shaper_t*) shaper; void const* p; p = TRI_LookupByKeyAssociativeSynced(&s->_attributeIds, &aid); if (p == NULL) { return NULL; } else { char const* a; a = p; return a + sizeof(TRI_df_attribute_marker_t); } } //////////////////////////////////////////////////////////////////////////////// /// @brief looks up an attribute weight by identifier //////////////////////////////////////////////////////////////////////////////// static int64_t LookupAttributeWeight (TRI_shaper_t* shaper, TRI_shape_aid_t aid) { voc_shaper_t* s = (voc_shaper_t*) shaper; const attribute_weight_t* item; item = (const attribute_weight_t*)(TRI_LookupByKeyAssociativePointer(&s->_weightedAttributes, &aid)); if (item == NULL) { // ........................................................................ // return -9223372036854775807L 2^63-1 to indicate that the attribute is // weighted to be the lowest possible // ........................................................................ LOG_WARNING("LookupAttributeWeight returned NULL weight"); return TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; } if (item->_aid != aid) { // ........................................................................ // return -9223372036854775807L 2^63-1 to indicate that the attribute is // weighted to be the lowest possible // ........................................................................ LOG_WARNING("LookupAttributeWeight returned an UNDEFINED weight"); return TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; } return item->_weight; } //////////////////////////////////////////////////////////////////////////////// /// @brief hashes the shapes //////////////////////////////////////////////////////////////////////////////// static uint64_t HashElementShape (TRI_associative_synced_t* array, void const* element) { char const* e = element; TRI_shape_t const* ee = element; return TRI_FnvHashPointer(e + + sizeof(TRI_shape_sid_t), ee->_size - sizeof(TRI_shape_sid_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief compares shapes //////////////////////////////////////////////////////////////////////////////// static bool EqualElementShape (TRI_associative_synced_t* array, void const* left, void const* right) { char const* l = left; char const* r = right; TRI_shape_t const* ll = left; TRI_shape_t const* rr = right; return (ll->_size == rr->_size) && memcmp(l + sizeof(TRI_shape_sid_t), r + sizeof(TRI_shape_sid_t), ll->_size - sizeof(TRI_shape_sid_t)) == 0; } //////////////////////////////////////////////////////////////////////////////// /// @brief finds a shape //////////////////////////////////////////////////////////////////////////////// static TRI_shape_t const* FindShape (TRI_shaper_t* shaper, TRI_shape_t* shape) { TRI_df_marker_t* result; TRI_df_shape_marker_t marker; TRI_shape_t const* found; TRI_shape_t* l; int res; voc_shaper_t* s; void* f; s = (voc_shaper_t*) shaper; found = TRI_LookupByElementAssociativeSynced(&s->_shapeDictionary, shape); // shape found, free argument and return if (found != 0) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, shape); return found; } // lock the index and check the element is still missing TRI_LockMutex(&s->_shapeLock); found = TRI_LookupByElementAssociativeSynced(&s->_shapeDictionary, shape); if (found != 0) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, shape); TRI_UnlockMutex(&s->_shapeLock); return found; } // create a new shape marker memset(&marker, 0, sizeof(TRI_df_shape_marker_t)); marker.base._type = TRI_DF_MARKER_SHAPE; marker.base._size = sizeof(TRI_df_shape_marker_t) + shape->_size; shape->_sid = s->_nextSid++; // write into the shape collection res = TRI_WriteShapeCollection(s->_collection, &marker.base, sizeof(TRI_df_shape_marker_t), shape, shape->_size, &result); if (res != TRI_ERROR_NO_ERROR) { TRI_UnlockMutex(&s->_shapeLock); return NULL; } TRI_Free(TRI_UNKNOWN_MEM_ZONE, shape); // enter into the dictionaries l = (TRI_shape_t*) (((char*) result) + sizeof(TRI_df_shape_marker_t)); f = TRI_InsertElementAssociativeSynced(&s->_shapeDictionary, l); assert(f == NULL); f = TRI_InsertKeyAssociativeSynced(&s->_shapeIds, &l->_sid, l); assert(f == NULL); TRI_UnlockMutex(&s->_shapeLock); return l; } //////////////////////////////////////////////////////////////////////////////// /// @brief hashes the shape id //////////////////////////////////////////////////////////////////////////////// static uint64_t HashKeyShapeId (TRI_associative_synced_t* array, void const* key) { TRI_shape_sid_t const* k = key; return TRI_FnvHashPointer(k, sizeof(TRI_shape_sid_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief hashes the shape //////////////////////////////////////////////////////////////////////////////// static uint64_t HashElementShapeId (TRI_associative_synced_t* array, void const* element) { TRI_shape_t const* e = element; return TRI_FnvHashPointer(&e->_sid, sizeof(TRI_shape_sid_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief compares a shape id and a shape //////////////////////////////////////////////////////////////////////////////// static bool EqualKeyShapeId (TRI_associative_synced_t* array, void const* key, void const* element) { TRI_shape_sid_t const* k = key; TRI_shape_t const* e = element; return *k == e->_sid; } //////////////////////////////////////////////////////////////////////////////// /// @brief looks up a shape by identifier //////////////////////////////////////////////////////////////////////////////// static TRI_shape_t const* LookupShapeId (TRI_shaper_t* shaper, TRI_shape_sid_t sid) { voc_shaper_t* s = (voc_shaper_t*) shaper; return TRI_LookupByKeyAssociativeSynced(&s->_shapeIds, &sid); } //////////////////////////////////////////////////////////////////////////////// /// @brief iterator for open //////////////////////////////////////////////////////////////////////////////// static bool OpenIterator (TRI_df_marker_t const* marker, void* data, TRI_datafile_t* datafile, bool journal) { voc_shaper_t* shaper = data; void* f; attribute_weight_t* weightedAttribute; attribute_weight_t* weightedItem; attribute_weight_t* result; if (marker->_type == TRI_DF_MARKER_SHAPE) { char* p = ((char*) marker) + sizeof(TRI_df_shape_marker_t); TRI_shape_t* l = (TRI_shape_t*) p; LOG_TRACE("found shape %lu", (unsigned long) l->_sid); f = TRI_InsertElementAssociativeSynced(&shaper->_shapeDictionary, l); assert(f == NULL); f = TRI_InsertKeyAssociativeSynced(&shaper->_shapeIds, &l->_sid, l); assert(f == NULL); if (shaper->_nextSid <= l->_sid) { shaper->_nextSid = l->_sid + 1; } } else if (marker->_type == TRI_DF_MARKER_ATTRIBUTE) { TRI_df_attribute_marker_t* m = (TRI_df_attribute_marker_t*) marker; char* p = ((char*) m) + sizeof(TRI_df_attribute_marker_t); LOG_TRACE("found attribute %lu / '%s'", (unsigned long) m->_aid, p); f = TRI_InsertKeyAssociativeSynced(&shaper->_attributeNames, p, m); assert(f == NULL); f = TRI_InsertKeyAssociativeSynced(&shaper->_attributeIds, &m->_aid, m); assert(f == NULL); if (shaper->_nextAid <= m->_aid) { shaper->_nextAid = m->_aid + 1; } // ......................................................................... // Add the attributes to the 'sorted' vector. These are in random order // at the moment, however they will be sorted once all the attributes // have been loaded into memory. // ......................................................................... weightedAttribute = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(attribute_weight_t), false); if (weightedAttribute != NULL) { weightedAttribute->_aid = m->_aid; weightedAttribute->_weight = TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; weightedAttribute->_attribute = (char*)(m) + sizeof(TRI_df_attribute_marker_t); weightedAttribute->_next = NULL; // .......................................................................... // Save the new attribute weight in the linked list. // .......................................................................... if ((shaper->_weights)._last == NULL) { (shaper->_weights)._first = weightedAttribute; } else { ((shaper->_weights)._last)->_next = weightedAttribute; } (shaper->_weights)._last = weightedAttribute; (shaper->_weights)._length += 1; result = (attribute_weight_t*)(TRI_InsertKeyAssociativePointer(&(shaper->_weightedAttributes), &(weightedAttribute->_aid), weightedAttribute, false)); if (result == NULL) { weightedItem = TRI_LookupByKeyAssociativePointer(&(shaper->_weightedAttributes), &(weightedAttribute->_aid)); if (weightedItem == NULL) { LOG_ERROR("attribute weight could not be located immediately after insert into associative array"); } else if (weightedItem->_aid != weightedAttribute->_aid) { LOG_ERROR("attribute weight could not be located immediately after insert into associative array"); } else { TRI_PushBackVectorPointer(&shaper->_sortedAttributes, weightedItem); } } else { LOG_WARNING("weighted attribute could not be inserted into associative array"); } } else { LOG_WARNING("OpenIterator could not allocate memory, attribute is NOT weighted"); } } else { LOG_TRACE("skipping marker %lu", (unsigned long) marker->_type); } return true; } //////////////////////////////////////////////////////////////////////////////// /// @brief hashes the accessor //////////////////////////////////////////////////////////////////////////////// static uint64_t HashElementAccessor (TRI_associative_pointer_t* array, void const* element) { TRI_shape_access_t const* ee = element; uint64_t v[2]; v[0] = ee->_sid; v[1] = ee->_pid; return TRI_FnvHashPointer(v, sizeof(v)); } //////////////////////////////////////////////////////////////////////////////// /// @brief compares an accessor //////////////////////////////////////////////////////////////////////////////// static bool EqualElementAccessor (TRI_associative_pointer_t* array, void const* left, void const* right) { TRI_shape_access_t const* ll = left; TRI_shape_access_t const* rr = right; return ll->_sid == rr->_sid && ll->_pid == rr->_pid; } //////////////////////////////////////////////////////////////////////////////// /// @brief Hashes a weighted attribute //////////////////////////////////////////////////////////////////////////////// static bool EqualKeyElementWeightedAttribute (TRI_associative_pointer_t* array, const void* key, const void* element) { TRI_shape_aid_t* aid = (TRI_shape_aid_t*)(key); attribute_weight_t* item = (attribute_weight_t*)(element); return (*aid == item->_aid); } static uint64_t HashKeyWeightedAttribute (TRI_associative_pointer_t* array, const void* key) { TRI_shape_aid_t* aid = (TRI_shape_aid_t*)(key); return TRI_FnvHashBlock(TRI_FnvHashBlockInitial(), (char*)(aid), sizeof(TRI_shape_aid_t)); } static uint64_t HashElementWeightedAttribute (TRI_associative_pointer_t* array, const void* element) { attribute_weight_t* item = (attribute_weight_t*)(element); TRI_shape_aid_t* aid = &(item->_aid); return TRI_FnvHashBlock(TRI_FnvHashBlockInitial(), (char*)(aid), sizeof(TRI_shape_aid_t)); } //////////////////////////////////////////////////////////////////////////////// /// @brief initialises a persistent shaper //////////////////////////////////////////////////////////////////////////////// static void InitVocShaper (voc_shaper_t* shaper, TRI_shape_collection_t* collection) { shaper->base.findAttributeName = FindAttributeName; shaper->base.lookupAttributeId = LookupAttributeId; shaper->base.findShape = FindShape; shaper->base.lookupShapeId = LookupShapeId; TRI_InitAssociativeSynced(&shaper->_attributeNames, TRI_UNKNOWN_MEM_ZONE, HashKeyAttributeName, HashElementAttributeName, EqualKeyAttributeName, 0); TRI_InitAssociativeSynced(&shaper->_attributeIds, TRI_UNKNOWN_MEM_ZONE, HashKeyAttributeId, HashElementAttributeId, EqualKeyAttributeId, 0); TRI_InitAssociativeSynced(&shaper->_shapeDictionary, TRI_UNKNOWN_MEM_ZONE, 0, HashElementShape, 0, EqualElementShape); TRI_InitAssociativeSynced(&shaper->_shapeIds, TRI_UNKNOWN_MEM_ZONE, HashKeyShapeId, HashElementShapeId, EqualKeyShapeId, 0); TRI_InitAssociativePointer(&shaper->_accessors, TRI_UNKNOWN_MEM_ZONE, 0, HashElementAccessor, 0, EqualElementAccessor); TRI_InitMutex(&shaper->_shapeLock); TRI_InitMutex(&shaper->_attributeLock); TRI_InitMutex(&shaper->_accessorLock); shaper->_nextAid = 1; shaper->_nextSid = 1; shaper->_collection = collection; // .......................................................................... // Attribute weight // .......................................................................... shaper->base.lookupAttributeWeight = LookupAttributeWeight; TRI_InitVectorPointer(&shaper->_sortedAttributes, TRI_UNKNOWN_MEM_ZONE); TRI_InitAssociativePointer(&shaper->_weightedAttributes, TRI_UNKNOWN_MEM_ZONE, HashKeyWeightedAttribute, HashElementWeightedAttribute, EqualKeyElementWeightedAttribute, NULL); // .......................................................................... // We require a place to store the weights -- a linked list is as good as // anything for now. Later try and store in a smart 2D array. // .......................................................................... (shaper->_weights)._first = NULL; (shaper->_weights)._last = NULL; (shaper->_weights)._length = 0; } //////////////////////////////////////////////////////////////////////////////// /// @} //////////////////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// /// @addtogroup VocBase /// @{ //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// /// @brief creates persistent shaper //////////////////////////////////////////////////////////////////////////////// TRI_shaper_t* TRI_CreateVocShaper (TRI_vocbase_t* vocbase, char const* path, char const* name, const bool waitForSync) { voc_shaper_t* shaper; TRI_shape_collection_t* collection; TRI_col_parameter_t parameter; bool ok; TRI_InitParameterCollection(vocbase, ¶meter, name, TRI_COL_TYPE_SHAPE, SHAPER_DATAFILE_SIZE); // override wait for sync for shapes parameter._waitForSync = waitForSync; collection = TRI_CreateShapeCollection(vocbase, path, ¶meter); if (collection == NULL) { return NULL; } shaper = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(voc_shaper_t), false); // TODO: memory allocation might fail TRI_InitShaper(&shaper->base, TRI_UNKNOWN_MEM_ZONE); InitVocShaper(shaper, collection); // handle basics ok = TRI_InsertBasicTypesShaper(&shaper->base); if (! ok) { TRI_FreeVocShaper(&shaper->base); return NULL; } // and return return &shaper->base; } //////////////////////////////////////////////////////////////////////////////// /// @brief destroys a persistent shaper, but does not free the pointer //////////////////////////////////////////////////////////////////////////////// void TRI_DestroyVocShaper (TRI_shaper_t* s) { voc_shaper_t* shaper = (voc_shaper_t*) s; size_t i; attribute_weight_t* weightedAttribute; attribute_weight_t* nextWeightedAttribute; assert(shaper); assert(shaper->_collection); TRI_FreeShapeCollection(shaper->_collection); TRI_DestroyAssociativeSynced(&shaper->_attributeNames); TRI_DestroyAssociativeSynced(&shaper->_attributeIds); TRI_DestroyAssociativeSynced(&shaper->_shapeDictionary); TRI_DestroyAssociativeSynced(&shaper->_shapeIds); for (i = 0; i < shaper->_accessors._nrAlloc; ++i) { TRI_shape_access_t* accessor = (TRI_shape_access_t*) shaper->_accessors._table[i]; if (accessor != NULL) { TRI_FreeShapeAccessor(accessor); } } TRI_DestroyAssociativePointer(&shaper->_accessors); TRI_DestroyMutex(&shaper->_shapeLock); TRI_DestroyMutex(&shaper->_attributeLock); // .......................................................................... // Attribute weight // .......................................................................... TRI_DestroyVectorPointer(&shaper->_sortedAttributes); TRI_DestroyAssociativePointer(&shaper->_weightedAttributes); weightedAttribute = (shaper->_weights)._first; while (weightedAttribute != NULL) { nextWeightedAttribute = weightedAttribute->_next; TRI_Free(TRI_UNKNOWN_MEM_ZONE, weightedAttribute); weightedAttribute = nextWeightedAttribute; } TRI_DestroyShaper(s); } //////////////////////////////////////////////////////////////////////////////// /// @brief destroys a persistent shaper and frees the pointer //////////////////////////////////////////////////////////////////////////////// void TRI_FreeVocShaper (TRI_shaper_t* shaper) { TRI_DestroyVocShaper(shaper); TRI_Free(TRI_UNKNOWN_MEM_ZONE, shaper); } //////////////////////////////////////////////////////////////////////////////// /// @} //////////////////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- // --SECTION-- public functions // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// /// @addtogroup VocBase /// @{ //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// /// @brief returns the underlying collection //////////////////////////////////////////////////////////////////////////////// TRI_shape_collection_t* TRI_CollectionVocShaper (TRI_shaper_t* shaper) { return ((voc_shaper_t*) shaper)->_collection; } //////////////////////////////////////////////////////////////////////////////// /// @brief opens a persistent shaper //////////////////////////////////////////////////////////////////////////////// TRI_shaper_t* TRI_OpenVocShaper (TRI_vocbase_t* vocbase, char const* filename) { voc_shaper_t* shaper; TRI_shape_collection_t* collection; bool ok; collection = TRI_OpenShapeCollection(vocbase, filename); if (collection == NULL) { return NULL; } shaper = TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(voc_shaper_t), false); if (shaper == NULL) { return NULL; } TRI_InitShaper(&shaper->base, TRI_UNKNOWN_MEM_ZONE); InitVocShaper(shaper, collection); // read all shapes and attributes TRI_IterateCollection(&collection->base, OpenIterator, shaper); // ............................................................................. // Sort all the attributes using the attribute string // ............................................................................. qsort(shaper->_sortedAttributes._buffer, shaper->_sortedAttributes._length, sizeof(attribute_weight_t*), attributeWeightCompareFunctionPointer); // ............................................................................. // re-weigh all of the attributes // ............................................................................. fullSetAttributeWeight(shaper); // handle basics ok = TRI_InsertBasicTypesShaper(&shaper->base); if (! ok) { TRI_FreeVocShaper(&shaper->base); return NULL; } return &shaper->base; } //////////////////////////////////////////////////////////////////////////////// /// @brief closes a persistent shaper //////////////////////////////////////////////////////////////////////////////// int TRI_CloseVocShaper (TRI_shaper_t* s) { voc_shaper_t* shaper = (voc_shaper_t*) s; int err; err = TRI_CloseShapeCollection(shaper->_collection); if (err != TRI_ERROR_NO_ERROR) { LOG_ERROR("cannot close shape collection of shaper, error %lu", (unsigned long) err); } // TODO free the accessors return err; } //////////////////////////////////////////////////////////////////////////////// /// @brief finds an accessor for a persistent shaper //////////////////////////////////////////////////////////////////////////////// TRI_shape_access_t const* TRI_FindAccessorVocShaper (TRI_shaper_t* s, TRI_shape_sid_t sid, TRI_shape_pid_t pid) { voc_shaper_t* shaper = (voc_shaper_t*) s; TRI_shape_access_t search; TRI_shape_access_t* accessor; TRI_shape_access_t const* found; TRI_LockMutex(&shaper->_accessorLock); search._sid = sid; search._pid = pid; found = TRI_LookupByElementAssociativePointer(&shaper->_accessors, &search); if (found == NULL) { found = accessor = TRI_ShapeAccessor(&shaper->base, sid, pid); TRI_InsertElementAssociativePointer(&shaper->_accessors, accessor, true); } TRI_UnlockMutex(&shaper->_accessorLock); return found; } //////////////////////////////////////////////////////////////////////////////// /// @brief extracts a sub-shape //////////////////////////////////////////////////////////////////////////////// bool TRI_ExtractShapedJsonVocShaper (TRI_shaper_t* shaper, TRI_shaped_json_t const* document, TRI_shape_sid_t sid, TRI_shape_pid_t pid, TRI_shaped_json_t* result, TRI_shape_t const** shape) { TRI_shape_access_t const* accessor; bool ok; accessor = TRI_FindAccessorVocShaper(shaper, document->_sid, pid); if (accessor == NULL) { LOG_TRACE("failed to get accessor for sid %lu and path %lu", (unsigned long) document->_sid, (unsigned long) pid); return false; } *shape = accessor->_shape; if (accessor->_shape == NULL) { LOG_TRACE("expecting any object for path %lu, got nothing", (unsigned long) pid); return sid == 0; } if (sid != 0 && sid != accessor->_shape->_sid) { LOG_TRACE("expecting sid %lu for path %lu, got sid %lu", (unsigned long) sid, (unsigned long) pid, (unsigned long) accessor->_shape->_sid); return false; } ok = TRI_ExecuteShapeAccessor(accessor, document, result); if (! ok) { LOG_TRACE("failed to get accessor for sid %lu and path %lu", (unsigned long) document->_sid, (unsigned long) pid); return false; } return true; } //////////////////////////////////////////////////////////////////////////////// /// @} //////////////////////////////////////////////////////////////////////////////// // Local Variables: // mode: outline-minor // outline-regexp: "^\\(/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|// --SECTION--\\|/// @\\}\\)" // End: