diff --git a/CHANGELOG b/CHANGELOG index 8eac7d180d..83732de206 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,15 @@ +v2.2.0 (XXXX-XX-XX) +------------------- + +* removed sorting of attribute names for use in a collection's shaper + + sorting attribute names was done on document insert to keep attributes + of a collection in sorted order for faster comparisons. The sort order + of attributes was only used in one particular and unlikey case, so it + was removed. Collections with many different attribute names should + benefit from this change by faster inserts and slightly less memory usage. + + v2.1.0 (XXXX-XX-XX) ------------------- diff --git a/arangod/SkipLists/skiplistIndex.cpp b/arangod/SkipLists/skiplistIndex.cpp index f09c92c034..9cf08e124a 100644 --- a/arangod/SkipLists/skiplistIndex.cpp +++ b/arangod/SkipLists/skiplistIndex.cpp @@ -536,8 +536,8 @@ SkiplistIndex* SkiplistIndex_new (TRI_primary_collection_t* primary, skiplistIndex->_numFields = numFields; skiplistIndex->unique = unique; skiplistIndex->sparse = sparse; - skiplistIndex->skiplist = TRI_InitSkipList(CmpElmElm,CmpKeyElm,skiplistIndex, - FreeElm,unique); + skiplistIndex->skiplist = TRI_InitSkipList(CmpElmElm, CmpKeyElm, skiplistIndex, + FreeElm, unique); if (skiplistIndex->skiplist == NULL) { TRI_Free(TRI_CORE_MEM_ZONE, skiplistIndex); return NULL; diff --git a/arangod/VocBase/voc-shaper.cpp b/arangod/VocBase/voc-shaper.cpp index d25169675e..edf2b43de7 100644 --- a/arangod/VocBase/voc-shaper.cpp +++ b/arangod/VocBase/voc-shaper.cpp @@ -40,45 +40,6 @@ // --SECTION-- private types // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - -//////////////////////////////////////////////////////////////////////////////// -/// @brief attribute weight -//////////////////////////////////////////////////////////////////////////////// - -typedef struct attribute_weight_s { - TRI_shape_aid_t _aid; - int64_t _weight; - char* _attribute; - struct attribute_weight_s* _next; -} -attribute_weight_t; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief attribute weights -//////////////////////////////////////////////////////////////////////////////// - -typedef struct attribute_weights_s { - attribute_weight_t* _first; - attribute_weight_t* _last; - size_t _length; -} -attribute_weights_t; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief weighted attribute -//////////////////////////////////////////////////////////////////////////////// - -typedef struct weighted_attribute_s { - TRI_shape_aid_t _aid; - int64_t _weight; - TRI_shaped_json_t _value; -} -weighted_attribute_t; - //////////////////////////////////////////////////////////////////////////////// /// @brief collection-based shaper //////////////////////////////////////////////////////////////////////////////// @@ -93,10 +54,6 @@ typedef struct voc_shaper_s { TRI_associative_pointer_t _accessors; - TRI_vector_pointer_t _sortedAttributes; - TRI_associative_pointer_t _weightedAttributes; - attribute_weights_t _weights; - TRI_shape_aid_t _nextAid; TRI_shape_sid_t _nextSid; @@ -108,19 +65,10 @@ typedef struct voc_shaper_s { } voc_shaper_t; -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- private functions // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief hashs the attribute name of a key //////////////////////////////////////////////////////////////////////////////// @@ -152,191 +100,6 @@ static bool EqualKeyAttributeName (TRI_associative_synced_t* array, void const* return TRI_EqualString(k, e + sizeof(TRI_df_attribute_marker_t)); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief compares two attribute strings stored in the attribute marker -/// -/// returns 0 if the strings are equal -/// returns < 0 if the left string compares less than the right string -/// returns > 0 if the left string compares more than the right string -//////////////////////////////////////////////////////////////////////////////// - -static int CompareNameAttributeWeight (const void* leftItem, - const void* rightItem) { - const attribute_weight_t* l = (const attribute_weight_t*)(leftItem); - const attribute_weight_t* r = (const attribute_weight_t*)(rightItem); - - assert(l); - assert(r); - - return TRI_compare_utf8(l->_attribute, r->_attribute); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief compares two attribute strings stored in the attribute marker -//////////////////////////////////////////////////////////////////////////////// - -static int CompareNameAttributeWeightPointer (const void* leftItem, - const void* rightItem) { - const attribute_weight_t* l = *((const attribute_weight_t**)(leftItem)); - const attribute_weight_t* r = *((const attribute_weight_t**)(rightItem)); - - assert(l); - assert(r); - - return TRI_compare_utf8(l->_attribute, r->_attribute); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief search for attribute -/// -/// Performs a binary search on a list of attributes (attribute markers) and -/// returns the position where a given attribute would be inserted -//////////////////////////////////////////////////////////////////////////////// - -static int64_t SortedIndexOf (voc_shaper_t* shaper, - attribute_weight_t* item) { - int64_t leftPos; - int64_t rightPos; - int64_t midPos; - - leftPos = 0; - rightPos = ((int64_t) shaper->_sortedAttributes._length) - 1; - - while (leftPos <= rightPos) { - midPos = (leftPos + rightPos) / 2; - - int compareResult = CompareNameAttributeWeight(TRI_AtVectorPointer(&(shaper->_sortedAttributes), midPos), (void*) item); - if (compareResult < 0) { - leftPos = midPos + 1; - } - else if (compareResult > 0) { - rightPos = midPos - 1; - } - else { - // should never happen since we do not allow duplicates here - return -1; - } - } - return leftPos; // insert it to the left of this position -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief sets the attribute weight -/// -/// helper method to store a list of attributes and their corresponding weights. -//////////////////////////////////////////////////////////////////////////////// - -static void SetAttributeWeight (voc_shaper_t* shaper, - attribute_weight_t* item, - int64_t* searchResult, - bool* weighted) { - int64_t itemWeight; - attribute_weight_t* leftItem; // nearest neighbour items - attribute_weight_t* rightItem; // nearest neighbour items - const int64_t resolution = 100; - void* result; - - *weighted = false; - - assert(item != NULL); - assert(shaper != NULL); - - *searchResult = SortedIndexOf(shaper, item); - - if (*searchResult < 0 || - *searchResult > (int64_t) shaper->_sortedAttributes._length) { // oops - assert(false); - return; - } - - switch ( (shaper->_sortedAttributes)._length) { - case 0: { - item->_weight = 0; - *weighted = true; - break; - } - - case 1: { - if (*searchResult == 0) { - item->_weight = 0; - rightItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), 0)); - rightItem->_weight = resolution; - } - else { - leftItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), 0)); - leftItem->_weight = 0; - item->_weight = resolution; - } - *weighted = true; - break; - } - - default: { - if (*searchResult == 0) { - rightItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), 0)); - item->_weight = rightItem->_weight - resolution; - *weighted = true; - } - else if (*searchResult == (int64_t) (shaper->_sortedAttributes)._length) { - leftItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), (shaper->_sortedAttributes)._length - 1)); - item->_weight = leftItem->_weight + resolution; - *weighted = true; - } - else { - leftItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), *searchResult - 1)); - rightItem = (attribute_weight_t*)(TRI_AtVectorPointer(&(shaper->_sortedAttributes), *searchResult)); - itemWeight = (rightItem->_weight + leftItem->_weight) / 2; - - if (leftItem->_weight != itemWeight && - rightItem->_weight != itemWeight) { - item->_weight = itemWeight; - *weighted = true; - } - } - break; - } - } // end of switch statement - - result = TRI_InsertKeyAssociativePointer(&(shaper->_weightedAttributes), &(item->_aid), item, false); - - if (result != NULL) { - LOG_ERROR("attribute weight could not be inserted into associative array"); - *searchResult = -1; - return; - } - - // ........................................................................... - // Obtain the pointer of the weighted attribute structure which is stored - // in the associative array. We need to pass this pointer to the Vector Pointer - // ........................................................................... - - item = static_cast(TRI_LookupByKeyAssociativePointer(&(shaper->_weightedAttributes), &(item->_aid))); - - if (item == NULL) { - LOG_ERROR("attribute weight could not be located immediately after insert into associative array"); - *searchResult = -1; - return; - } - - TRI_InsertVectorPointer(&(shaper->_sortedAttributes), item, (size_t) (*searchResult)); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief sets the attribute weight -//////////////////////////////////////////////////////////////////////////////// - -static void FullSetAttributeWeight (voc_shaper_t* shaper) { - int64_t startWeight; - - startWeight = 0; - - for (size_t j = 0; j < shaper->_sortedAttributes._length; ++j) { - attribute_weight_t* item = (attribute_weight_t*) TRI_AtVectorPointer(&(shaper->_sortedAttributes), j); - item->_weight = startWeight; - startWeight += 100; - } -} - //////////////////////////////////////////////////////////////////////////////// /// @brief finds an attribute identifier by name //////////////////////////////////////////////////////////////////////////////// @@ -368,7 +131,6 @@ static TRI_shape_aid_t FindOrCreateAttributeByName (TRI_shaper_t* shaper, char* mem; TRI_df_attribute_marker_t* marker; TRI_df_marker_t* result; - TRI_df_attribute_marker_t* markerResult; TRI_doc_datafile_info_t* dfi; TRI_voc_fid_t fid; int res; @@ -378,8 +140,6 @@ static TRI_shape_aid_t FindOrCreateAttributeByName (TRI_shaper_t* shaper, TRI_voc_size_t totalSize; void const* p; void* f; - int64_t searchResult; - bool weighted; assert(name != NULL); @@ -469,47 +229,6 @@ static TRI_shape_aid_t FindOrCreateAttributeByName (TRI_shaper_t* shaper, f = TRI_InsertKeyAssociativeSynced(&s->_attributeNames, name, result, false); assert(f == NULL); - // ........................................................................... - // Each attribute has an associated integer as a weight. This - // weight corresponds to the natural ordering of the attribute strings - // ........................................................................... - - markerResult = (TRI_df_attribute_marker_t*) result; - - attribute_weight_t* weightedAttribute = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(attribute_weight_t), false)); - - if (weightedAttribute != NULL) { - weightedAttribute->_aid = markerResult->_aid; - weightedAttribute->_weight = TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; - weightedAttribute->_attribute = TRI_DuplicateStringZ(TRI_UNKNOWN_MEM_ZONE, name); - weightedAttribute->_next = NULL; - - // .......................................................................... - // Save the new attribute weight in the linked list. - // .......................................................................... - - if ((s->_weights)._last == NULL) { - (s->_weights)._first = weightedAttribute; - } - else { - ((s->_weights)._last)->_next = weightedAttribute; - } - - (s->_weights)._last = weightedAttribute; - (s->_weights)._length += 1; - - SetAttributeWeight(s, weightedAttribute, &searchResult, &weighted); - assert(searchResult > -1); - - if (! weighted) { - FullSetAttributeWeight(s); - } - } - - else { - LOG_WARNING("FindAttributeByName could not allocate memory, attribute is NOT weighted"); - } - // ........................................................................... // and release the lock // ........................................................................... @@ -567,38 +286,6 @@ static char const* LookupAttributeId (TRI_shaper_t* shaper, return static_cast(p) + sizeof(TRI_df_attribute_marker_t); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief looks up an attribute weight by identifier -//////////////////////////////////////////////////////////////////////////////// - -static int64_t LookupAttributeWeight (TRI_shaper_t* shaper, - TRI_shape_aid_t aid) { - voc_shaper_t* s = (voc_shaper_t*) shaper; - const attribute_weight_t* item; - - item = (const attribute_weight_t*)(TRI_LookupByKeyAssociativePointer(&s->_weightedAttributes, &aid)); - - if (item == NULL) { - // ........................................................................ - // return -9223372036854775807L 2^63-1 to indicate that the attribute is - // weighted to be the lowest possible - // ........................................................................ - LOG_WARNING("LookupAttributeWeight returned NULL weight"); - return TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; - } - - if (item->_aid != aid) { - // ........................................................................ - // return -9223372036854775807L 2^63-1 to indicate that the attribute is - // weighted to be the lowest possible - // ........................................................................ - LOG_WARNING("LookupAttributeWeight returned an UNDEFINED weight"); - return TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; - } - - return item->_weight; -} - //////////////////////////////////////////////////////////////////////////////// /// @brief hashes the shapes //////////////////////////////////////////////////////////////////////////////// @@ -761,131 +448,6 @@ static TRI_shape_t const* FindShape (TRI_shaper_t* shaper, return l; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief compares to weighted attributes -//////////////////////////////////////////////////////////////////////////////// - -static int AttributeWeightCompareFunction (const void* leftItem, const void* rightItem) { - const weighted_attribute_t* l = (const weighted_attribute_t*) leftItem; - const weighted_attribute_t* r = (const weighted_attribute_t*) rightItem; - - if (l->_weight < r->_weight) { - return -1; - } - if (l->_weight > r->_weight) { - return 1; - } - return 0; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief free weighted attribute -/// -/// Helper method to deal with freeing memory associated with the weight -/// comparisions. -//////////////////////////////////////////////////////////////////////////////// - -static void FreeShapeTypeJsonArrayHelper (weighted_attribute_t** leftWeightedList, - weighted_attribute_t** rightWeightedList) { - if (*leftWeightedList != NULL) { - TRI_Free(TRI_UNKNOWN_MEM_ZONE, *leftWeightedList); - *leftWeightedList = NULL; - } - - if (*rightWeightedList != NULL) { - TRI_Free(TRI_UNKNOWN_MEM_ZONE, *rightWeightedList); - *rightWeightedList = NULL; - } -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief returns the number of entries -//////////////////////////////////////////////////////////////////////////////// - -static int CompareShapeTypeJsonArrayHelper (const TRI_shape_t* shape, - const TRI_shaper_t* shaper, - const TRI_shaped_json_t* shapedJson, - weighted_attribute_t** attributeArray) { - char* charShape = (char*)(shape); - TRI_shape_size_t fixedEntries; // the number of entries in the JSON array whose value is of a fixed size - TRI_shape_size_t variableEntries; // the number of entries in the JSON array whose value is not of a known fixed size - TRI_shape_size_t j; - const TRI_shape_aid_t* aids; - const TRI_shape_sid_t* sids; - const TRI_shape_size_t* offsets; - - // ............................................................................. - // Ensure we return an empty array - in case of funny business below - // ............................................................................. - - *attributeArray = NULL; - - // ............................................................................. - // Determine the number of fixed sized values - // ............................................................................. - - charShape = charShape + sizeof(TRI_shape_t); - fixedEntries = *((TRI_shape_size_t*)(charShape)); - - // ............................................................................. - // Determine the number of variable sized values - // ............................................................................. - - charShape = charShape + sizeof(TRI_shape_size_t); - variableEntries = *((TRI_shape_size_t*)(charShape)); - - // ............................................................................. - // It may happen that the shaped_json_array is 'empty {}' - // ............................................................................. - - if ((fixedEntries + variableEntries) == 0) { - return 0; - } - - // ............................................................................. - // Allocate memory to hold the attribute information required for comparison - // ............................................................................. - - *attributeArray = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, (sizeof(weighted_attribute_t) * (fixedEntries + variableEntries)), false)); - - if (*attributeArray == NULL) { - return -1; - } - - // ............................................................................. - // Determine the list of shape identifiers - // ............................................................................. - - charShape = charShape + sizeof(TRI_shape_size_t); - sids = (const TRI_shape_sid_t*)(charShape); - - charShape = charShape + (sizeof(TRI_shape_sid_t) * (fixedEntries + variableEntries)); - aids = (const TRI_shape_aid_t*)(charShape); - - charShape = charShape + (sizeof(TRI_shape_aid_t) * (fixedEntries + variableEntries)); - offsets = (const TRI_shape_size_t*)(charShape); - - for (j = 0; j < fixedEntries; ++j) { - (*attributeArray)[j]._aid = aids[j]; - (*attributeArray)[j]._weight = shaper->lookupAttributeWeight((TRI_shaper_t*)(shaper),aids[j]); - (*attributeArray)[j]._value._sid = sids[j]; - (*attributeArray)[j]._value._data.data = shapedJson->_data.data + offsets[j]; - (*attributeArray)[j]._value._data.length = (uint32_t) (offsets[j + 1] - offsets[j]); - } - - offsets = (const TRI_shape_size_t*)(shapedJson->_data.data); - for (j = 0; j < variableEntries; ++j) { - TRI_shape_size_t jj = j + fixedEntries; - (*attributeArray)[jj]._aid = aids[jj]; - (*attributeArray)[jj]._weight = shaper->lookupAttributeWeight((TRI_shaper_t*)(shaper),aids[jj]); - (*attributeArray)[jj]._value._sid = sids[jj]; - (*attributeArray)[jj]._value._data.data = shapedJson->_data.data + offsets[j]; - (*attributeArray)[jj]._value._data.length = (uint32_t) (offsets[j + 1] - offsets[j]); - } - - return (int) (fixedEntries + variableEntries); -} - //////////////////////////////////////////////////////////////////////////////// /// @brief hashes the shape id //////////////////////////////////////////////////////////////////////////////// @@ -962,27 +524,6 @@ static bool EqualElementAccessor (TRI_associative_pointer_t* array, void const* return ll->_sid == rr->_sid && ll->_pid == rr->_pid; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief Hashes a weighted attribute -//////////////////////////////////////////////////////////////////////////////// - -static bool EqualKeyElementWeightedAttribute (TRI_associative_pointer_t* array, const void* key, const void* element) { - TRI_shape_aid_t* aid = (TRI_shape_aid_t*)(key); - attribute_weight_t* item = (attribute_weight_t*)(element); - return (*aid == item->_aid); -} - -static uint64_t HashKeyWeightedAttribute (TRI_associative_pointer_t* array, const void* key) { - TRI_shape_aid_t* aid = (TRI_shape_aid_t*)(key); - return TRI_FnvHashBlock(TRI_FnvHashBlockInitial(), (char*) aid, sizeof(TRI_shape_aid_t)); -} - -static uint64_t HashElementWeightedAttribute (TRI_associative_pointer_t* array, const void* element) { - attribute_weight_t* item = (attribute_weight_t*)(element); - TRI_shape_aid_t* aid = &(item->_aid); - return TRI_FnvHashBlock(TRI_FnvHashBlockInitial(), (char*) aid, sizeof(TRI_shape_aid_t)); -} - //////////////////////////////////////////////////////////////////////////////// /// @brief initialises a shaper //////////////////////////////////////////////////////////////////////////////// @@ -1065,13 +606,6 @@ static int InitStep1VocShaper (voc_shaper_t* shaper) { return res; } - res = TRI_InitAssociativePointer(&shaper->_weightedAttributes, - TRI_UNKNOWN_MEM_ZONE, - HashKeyWeightedAttribute, - HashElementWeightedAttribute, - EqualKeyElementWeightedAttribute, - NULL); - if (res != TRI_ERROR_NO_ERROR) { TRI_DestroyAssociativePointer(&shaper->_accessors); TRI_DestroyAssociativeSynced(&shaper->_shapeIds); @@ -1094,66 +628,16 @@ static int InitStep2VocShaper (voc_shaper_t* shaper) { TRI_InitMutex(&shaper->_attributeLock); TRI_InitMutex(&shaper->_accessorLock); - shaper->_nextAid = 1; - shaper->_nextSid = 7; // TODO!!!!!!!!!!!!!!!!! - - // .......................................................................... - // Attribute weight - // .......................................................................... - - shaper->base.lookupAttributeWeight = LookupAttributeWeight; - - TRI_InitVectorPointer(&shaper->_sortedAttributes, TRI_UNKNOWN_MEM_ZONE); - - // .......................................................................... - // We require a place to store the weights -- a linked list is as good as - // anything for now. Later try and store in a smart 2D array. - // .......................................................................... - - (shaper->_weights)._first = NULL; - (shaper->_weights)._last = NULL; - (shaper->_weights)._length = 0; + shaper->_nextAid = 1; // id of next attribute to hand out + shaper->_nextSid = TRI_FirstCustomShapeIdShaper(); // id of next shape to hand out return TRI_ERROR_NO_ERROR; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief initialises a shaper -//////////////////////////////////////////////////////////////////////////////// - -static int InitStep3VocShaper (voc_shaper_t* shaper) { - // ............................................................................. - // Sort all the attributes using the attribute string - // ............................................................................. - - qsort(shaper->_sortedAttributes._buffer, - shaper->_sortedAttributes._length, - sizeof(attribute_weight_t*), - CompareNameAttributeWeightPointer); - - - // ............................................................................. - // re-weigh all of the attributes - // ............................................................................. - - FullSetAttributeWeight(shaper); - - return TRI_ERROR_NO_ERROR; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief creates a shaper //////////////////////////////////////////////////////////////////////////////// @@ -1203,9 +687,6 @@ TRI_shaper_t* TRI_CreateVocShaper (TRI_vocbase_t* vocbase, void TRI_DestroyVocShaper (TRI_shaper_t* s) { voc_shaper_t* shaper = (voc_shaper_t*) s; - attribute_weight_t* weightedAttribute; - attribute_weight_t* nextWeightedAttribute; - size_t i; assert(shaper != NULL); @@ -1214,8 +695,9 @@ void TRI_DestroyVocShaper (TRI_shaper_t* s) { TRI_DestroyAssociativeSynced(&shaper->_shapeDictionary); TRI_DestroyAssociativeSynced(&shaper->_shapeIds); - for (i = 0; i < shaper->_accessors._nrAlloc; ++i) { + for (size_t i = 0; i < shaper->_accessors._nrAlloc; ++i) { TRI_shape_access_t* accessor = (TRI_shape_access_t*) shaper->_accessors._table[i]; + if (accessor != NULL) { TRI_FreeShapeAccessor(accessor); } @@ -1225,26 +707,6 @@ void TRI_DestroyVocShaper (TRI_shaper_t* s) { TRI_DestroyMutex(&shaper->_shapeLock); TRI_DestroyMutex(&shaper->_attributeLock); - // .......................................................................... - // Attribute weight - // .......................................................................... - - TRI_DestroyVectorPointer(&shaper->_sortedAttributes); - TRI_DestroyAssociativePointer(&shaper->_weightedAttributes); - - weightedAttribute = (shaper->_weights)._first; - while (weightedAttribute != NULL) { - nextWeightedAttribute = weightedAttribute->_next; - - // free attribute name - if (weightedAttribute->_attribute != NULL) { - TRI_Free(TRI_UNKNOWN_MEM_ZONE, weightedAttribute->_attribute); - } - - TRI_Free(TRI_UNKNOWN_MEM_ZONE, weightedAttribute); - weightedAttribute = nextWeightedAttribute; - } - TRI_DestroyShaper(s); } @@ -1257,27 +719,17 @@ void TRI_FreeVocShaper (TRI_shaper_t* shaper) { TRI_Free(TRI_UNKNOWN_MEM_ZONE, shaper); } -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- public functions // ----------------------------------------------------------------------------- //////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - -//////////////////////////////////////////////////////////////////////////////// -/// @brief destroys a shaper, but does not free the pointer +/// @brief initialise the shaper //////////////////////////////////////////////////////////////////////////////// int TRI_InitVocShaper (TRI_shaper_t* s) { - voc_shaper_t* shaper = (voc_shaper_t*) s; - - return InitStep3VocShaper(shaper); + // this is a no-op now as there are no attribute weights anymore + return TRI_ERROR_NO_ERROR; } //////////////////////////////////////////////////////////////////////////////// @@ -1399,60 +851,7 @@ int TRI_InsertAttributeVocShaper (TRI_shaper_t* s, shaper->_nextAid = m->_aid + 1; } - - // ......................................................................... - // Add the attributes to the 'sorted' vector. These are in random order - // at the moment, however they will be sorted once all the attributes - // have been loaded into memory. - // ......................................................................... - - attribute_weight_t* weightedAttribute = static_cast(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(attribute_weight_t), false)); - - if (weightedAttribute != NULL) { - attribute_weight_t* result; - - weightedAttribute->_aid = m->_aid; - weightedAttribute->_weight = TRI_VOC_UNDEFINED_ATTRIBUTE_WEIGHT; - weightedAttribute->_attribute = TRI_DuplicateStringZ(TRI_UNKNOWN_MEM_ZONE, (char*) m + sizeof(TRI_df_attribute_marker_t)); - weightedAttribute->_next = NULL; - - // .......................................................................... - // Save the new attribute weight in the linked list. - // .......................................................................... - - if ((shaper->_weights)._last == NULL) { - (shaper->_weights)._first = weightedAttribute; - } - else { - ((shaper->_weights)._last)->_next = weightedAttribute; - } - - (shaper->_weights)._last = weightedAttribute; - (shaper->_weights)._length += 1; - - result = (attribute_weight_t*) TRI_InsertKeyAssociativePointer(&(shaper->_weightedAttributes), &(weightedAttribute->_aid), weightedAttribute, false); - - if (result == NULL) { - attribute_weight_t* weightedItem = static_cast(TRI_LookupByKeyAssociativePointer(&(shaper->_weightedAttributes), &(weightedAttribute->_aid))); - - if (weightedItem == NULL || - weightedItem->_aid != weightedAttribute->_aid) { - LOG_ERROR("attribute weight could not be located immediately after insert into associative array"); - } - else { - TRI_PushBackVectorPointer(&shaper->_sortedAttributes, weightedItem); - } - - return TRI_ERROR_NO_ERROR; - } - - LOG_WARNING("weighted attribute could not be inserted into associative array"); - } - else { - LOG_WARNING("OpenIterator could not allocate memory, attribute is NOT weighted"); - } - - return TRI_ERROR_OUT_OF_MEMORY; + return TRI_ERROR_NO_ERROR; } //////////////////////////////////////////////////////////////////////////////// @@ -1541,7 +940,154 @@ bool TRI_ExtractShapedJsonVocShaper (TRI_shaper_t* shaper, } //////////////////////////////////////////////////////////////////////////////// -/// @brief helper method for recursion for comparison +/// @brief temporary structure for attributes +//////////////////////////////////////////////////////////////////////////////// + +typedef struct attribute_entry_s { + char* _attribute; + TRI_shaped_json_t _value; +} +attribute_entry_t; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief sort attribure names +//////////////////////////////////////////////////////////////////////////////// + +static int AttributeNameComparator (void const* lhs, + void const* rhs) { + attribute_entry_t const* l = static_cast(lhs); + attribute_entry_t const* r = static_cast(rhs); + + if (l->_attribute == NULL || r->_attribute == NULL) { + // error ! + return -1; + } + + return TRI_compare_utf8(l->_attribute, r->_attribute); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief create a sorted vector of attributes +//////////////////////////////////////////////////////////////////////////////// + +static int FillAttributesVector (TRI_vector_t* vector, + TRI_shaped_json_t const* shapedJson, + TRI_shape_t const* shape, + TRI_shaper_t const* shaper) { + + TRI_InitVector(vector, TRI_UNKNOWN_MEM_ZONE, sizeof(attribute_entry_t)); + + // ............................................................................. + // Determine the number of fixed sized values + // ............................................................................. + + char const* charShape = (char const*) shape; + + charShape = charShape + sizeof(TRI_shape_t); + TRI_shape_size_t fixedEntries = *((TRI_shape_size_t*)(charShape)); + + // ............................................................................. + // Determine the number of variable sized values + // ............................................................................. + + charShape = charShape + sizeof(TRI_shape_size_t); + TRI_shape_size_t variableEntries = *((TRI_shape_size_t*)(charShape)); + + // ............................................................................. + // It may happen that the shaped_json_array is 'empty {}' + // ............................................................................. + + if (fixedEntries + variableEntries == 0) { + return TRI_ERROR_NO_ERROR; + } + + // ............................................................................. + // Determine the list of shape identifiers + // ............................................................................. + + charShape = charShape + sizeof(TRI_shape_size_t); + TRI_shape_sid_t const* sids = (TRI_shape_sid_t const*) charShape; + + charShape = charShape + (sizeof(TRI_shape_sid_t) * (fixedEntries + variableEntries)); + TRI_shape_aid_t const* aids = (TRI_shape_aid_t const*) charShape; + + charShape = charShape + (sizeof(TRI_shape_aid_t) * (fixedEntries + variableEntries)); + TRI_shape_size_t const* offsets = (TRI_shape_size_t const*) charShape; + + for (TRI_shape_size_t i = 0; i < fixedEntries; ++i) { + attribute_entry_t attribute; + + char const* a = shaper->lookupAttributeId((TRI_shaper_t*) shaper, aids[i]); + + if (a == NULL) { + return TRI_ERROR_INTERNAL; + } + + char* copy = TRI_DuplicateStringZ(TRI_UNKNOWN_MEM_ZONE, a); + + if (copy == NULL) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + attribute._attribute = copy; + attribute._value._sid = sids[i]; + attribute._value._data.data = shapedJson->_data.data + offsets[i]; + attribute._value._data.length = (uint32_t) (offsets[i + 1] - offsets[i]); + + TRI_PushBackVector(vector, &attribute); + } + + offsets = (TRI_shape_size_t const*) shapedJson->_data.data; + + for (TRI_shape_size_t i = 0; i < variableEntries; ++i) { + attribute_entry_t attribute; + + char const* a = shaper->lookupAttributeId((TRI_shaper_t*) shaper, aids[i + fixedEntries]); + + if (a == NULL) { + return TRI_ERROR_INTERNAL; + } + + char* copy = TRI_DuplicateStringZ(TRI_UNKNOWN_MEM_ZONE, a); + + if (copy == NULL) { + return TRI_ERROR_OUT_OF_MEMORY; + } + + attribute._attribute = copy; + attribute._value._sid = sids[i + fixedEntries]; + attribute._value._data.data = shapedJson->_data.data + offsets[i]; + attribute._value._data.length = (uint32_t) (offsets[i + 1] - offsets[i]); + + TRI_PushBackVector(vector, &attribute); + } + + // sort the attributes by attribute name + qsort(vector->_buffer, TRI_LengthVector(vector), sizeof(attribute_entry_t), AttributeNameComparator); + + return TRI_ERROR_NO_ERROR; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief destroy a vector of attributes +//////////////////////////////////////////////////////////////////////////////// + +static void DestroyAttributesVector (TRI_vector_t* vector) { + size_t const n = TRI_LengthVector(vector); + + for (size_t i = 0; i < n; ++i) { + attribute_entry_t* entry = static_cast(TRI_AtVector(vector, i)); + + if (entry->_attribute != NULL) { + TRI_Free(TRI_UNKNOWN_MEM_ZONE, entry->_attribute); + } + } + + TRI_DestroyVector(vector); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief compares two shapes /// /// You must either supply (leftDocument, leftObject) or leftShaped. /// You must either supply (rightDocument, rightObject) or rightShaped. @@ -1566,9 +1112,6 @@ int TRI_CompareShapeTypes (TRI_doc_mptr_t* leftDocument, TRI_shaped_json_t rightElement; char const* ptr; int result; - size_t listLength; - weighted_attribute_t* leftWeightedList; - weighted_attribute_t* rightWeightedList; // left is either a shaped json or a shaped sub object if (leftDocument != NULL) { @@ -1799,6 +1342,7 @@ int TRI_CompareShapeTypes (TRI_doc_mptr_t* leftDocument, // unfortunately recursion: check the types of all the entries size_t leftListLength = *((TRI_shape_length_list_t*) left._data.data); size_t rightListLength = *((TRI_shape_length_list_t*) right._data.data); + size_t listLength; // determine the smallest list if (leftListLength > rightListLength) { @@ -1899,137 +1443,54 @@ int TRI_CompareShapeTypes (TRI_doc_mptr_t* leftDocument, } case TRI_SHAPE_ARRAY: { - // ............................................................................ // We are comparing a left JSON array with another JSON array on the right - // The comparison works as follows: - // - // Suppose that leftShape has m key/value pairs and that the - // rightShape has n key/value pairs - // - // Extract the m key aids (attribute identifiers) from the leftShape - // Extract the n key aids (attribute identifiers) from the rightShape - // - // Sort the key aids for both the left and right shape - // according to the weight of the key (attribute) - // - // Let lw_j denote the weight of the jth key from the sorted leftShape key list - // and rw_j the corresponding rightShape. - // - // If lw_j < rw_j return -1 - // If lw_j > rw_j return 1 - // If lw_j == rw_j, then we extract the values and compare the values - // using recursion. - // - // If lv_j < rv_j return -1 - // If lv_j > rv_j return 1 - // If lv_j == rv_j, then repeat the process with j+1. // ............................................................................ // ............................................................................ - // generate the left and right lists. + // generate the left and right lists sorted by attribute names // ............................................................................ - int leftNumWeightedList = CompareShapeTypeJsonArrayHelper(leftShape, leftShaper, &left, &leftWeightedList); - int rightNumWeightedList = CompareShapeTypeJsonArrayHelper(rightShape, rightShaper, &right, &rightWeightedList); + TRI_vector_t leftSorted; + TRI_vector_t rightSorted; - // ............................................................................ - // If the left and right both resulted in errors, we return equality for want - // of something better. - // ............................................................................ - - if ( (leftNumWeightedList < 0) && (rightNumWeightedList < 0) ) { // probably out of memory error - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); - return 0; - } - - // ............................................................................ - // If the left had an error, we rank the left as the smallest item in the order - // ............................................................................ - - if (leftNumWeightedList < 0) { // probably out of memory error - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); - return -1; // attempt to compare as low as possible - } - - // ............................................................................ - // If the right had an error, we rank the right as the largest item in the order - // ............................................................................ - - if (rightNumWeightedList < 0) { - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); - return 1; + bool error = false; + if (FillAttributesVector(&leftSorted, &left, leftShape, leftShaper) != TRI_ERROR_NO_ERROR) { + error = true; } - // ............................................................................ - // Are we comparing two empty shaped_json_arrays? - // ............................................................................ - - if ( (leftNumWeightedList == 0) && (rightNumWeightedList == 0) ) { - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); - return 0; + if (FillAttributesVector(&rightSorted, &right, rightShape, rightShaper) != TRI_ERROR_NO_ERROR) { + error = true; } - // ............................................................................ - // If the left is empty, then it is smaller than the right, right? - // ............................................................................ - - if (leftNumWeightedList == 0) { - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); - return -1; - } - - // ............................................................................ - // ...and the opposite of the above. - // ............................................................................ - - if (rightNumWeightedList == 0) { - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); - return 1; - } - - // .............................................................................. - // We now have to sort the left and right weighted list according to attribute weight - // .............................................................................. - - qsort(leftWeightedList, leftNumWeightedList, sizeof(weighted_attribute_t), AttributeWeightCompareFunction); - qsort(rightWeightedList, rightNumWeightedList, sizeof(weighted_attribute_t), AttributeWeightCompareFunction); - - // .............................................................................. - // check the weight and if equal check the values. Notice that numWeightedList - // below MUST be greater or equal to 1. - // .............................................................................. - - int numWeightedList = (leftNumWeightedList < rightNumWeightedList ? leftNumWeightedList : rightNumWeightedList); + size_t const leftLength = TRI_LengthVector(&leftSorted); + size_t const rightLength = TRI_LengthVector(&rightSorted); + size_t const numElements = (leftLength < rightLength ? leftLength : rightLength); result = 0; - for (int i = 0; i < numWeightedList; ++i) { - if (leftWeightedList[i]._weight != rightWeightedList[i]._weight) { - result = (leftWeightedList[i]._weight < rightWeightedList[i]._weight ? -1: 1); + for (size_t i = 0; i < numElements; ++i) { + attribute_entry_t const* l = static_cast(TRI_AtVector(&leftSorted, i)); + attribute_entry_t const* r = static_cast(TRI_AtVector(&rightSorted, i)); + + result = TRI_compare_utf8(l->_attribute, r->_attribute); + + if (result != 0) { break; } result = TRI_CompareShapeTypes(NULL, NULL, - &(leftWeightedList[i]._value), + &l->_value, NULL, NULL, - &(rightWeightedList[i]._value), + &r->_value, leftShaper, rightShaper); if (result != 0) { break; } - - // the attributes are equal now check for the values - /* start oreste debug - const char* name = leftShaper->lookupAttributeId(leftShaper,leftWeightedList[i]._aid); - printf("%s:%u:w=%ld:%s\n",__FILE__,__LINE__,leftWeightedList[i]._weight,name); - const char* name = rightShaper->lookupAttributeId(rightShaper,rightWeightedList[i]._aid); - printf("%s:%u:w=%ld:%s\n",__FILE__,__LINE__,rightWeightedList[i]._weight,name); - end oreste debug */ } if (result == 0) { @@ -2038,20 +1499,22 @@ int TRI_CompareShapeTypes (TRI_doc_mptr_t* leftDocument, // however one more check to determine if the number of elements in the arrays // are equal. // ............................................................................ - if (leftNumWeightedList < rightNumWeightedList) { + if (leftLength < rightLength) { result = -1; } - else if (leftNumWeightedList > rightNumWeightedList) { + else if (leftLength > rightLength) { result = 1; } } - - - // .............................................................................. - // Deallocate any memory for the comparisions and return the result - // .............................................................................. - - FreeShapeTypeJsonArrayHelper(&leftWeightedList, &rightWeightedList); + + // clean up + DestroyAttributesVector(&leftSorted); + DestroyAttributesVector(&rightSorted); + + if (error) { + return -1; + } + return result; } } // end of switch (rightType) @@ -2062,10 +1525,6 @@ int TRI_CompareShapeTypes (TRI_doc_mptr_t* leftDocument, return 0; //shut the vc++ up } -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // ----------------------------------------------------------------------------- diff --git a/arangod/VocBase/voc-shaper.h b/arangod/VocBase/voc-shaper.h index af01d9f3c4..98d8a550ac 100644 --- a/arangod/VocBase/voc-shaper.h +++ b/arangod/VocBase/voc-shaper.h @@ -47,11 +47,6 @@ struct TRI_document_collection_s; // --SECTION-- public types // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief datafile attribute marker //////////////////////////////////////////////////////////////////////////////// @@ -75,19 +70,10 @@ typedef struct TRI_df_shape_marker_s { } TRI_df_shape_marker_t; -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- constructors and destructors // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief creates a shaper //////////////////////////////////////////////////////////////////////////////// @@ -107,19 +93,10 @@ void TRI_DestroyVocShaper (TRI_shaper_t*); void TRI_FreeVocShaper (TRI_shaper_t*); -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - // ----------------------------------------------------------------------------- // --SECTION-- public functions // ----------------------------------------------------------------------------- -//////////////////////////////////////////////////////////////////////////////// -/// @addtogroup VocBase -/// @{ -//////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// /// @brief destroys a shaper, but does not free the pointer //////////////////////////////////////////////////////////////////////////////// @@ -182,10 +159,6 @@ int TRI_CompareShapeTypes (TRI_doc_mptr_t* leftDocument, TRI_shaper_t* leftShaper, TRI_shaper_t* rightShaper); -//////////////////////////////////////////////////////////////////////////////// -/// @} -//////////////////////////////////////////////////////////////////////////////// - #ifdef __cplusplus } #endif diff --git a/arangosh/Benchmark/arangob.cpp b/arangosh/Benchmark/arangob.cpp index 862f44d0c2..1ea3a57753 100644 --- a/arangosh/Benchmark/arangob.cpp +++ b/arangosh/Benchmark/arangob.cpp @@ -204,7 +204,7 @@ static void ParseProgramOptions (int argc, char* argv[]) { ("batch-size", &BatchSize, "number of operations in one batch (0 disables batching)") ("keep-alive", &KeepAlive, "use HTTP keep-alive") ("collection", &Collection, "collection name to use in tests") - ("test-case", &TestCase, "test case to use") + ("test-case", &TestCase, "test case to use (possible values: version, document, collection, hash, skiplist, edge, shapes, shapes-append, random-shapes, crud, crud-append, counttrx, multitrx)") ("complexity", &Complexity, "complexity parameter for the test") ("delay", &Delay, "use a startup delay (necessary only when run in series)") ("progress", &Progress, "show progress") diff --git a/etc/relative/arangod.conf b/etc/relative/arangod.conf index 12dd1cf7d9..6cd7d011d7 100644 --- a/etc/relative/arangod.conf +++ b/etc/relative/arangod.conf @@ -4,8 +4,8 @@ # remove-on-drop = true [server] -disable-authentication = false -endpoint = tcp://0.0.0.0:8888 +disable-authentication = true +endpoint = tcp://localhost:8529 threads = 5 # reuse-address = false @@ -15,7 +15,7 @@ threads = 3 [javascript] startup-directory = ./js app-path = ./js/apps -frontend-development = true +frontend-development = false [ruby] action-directory = ./mr/actions/system @@ -29,8 +29,8 @@ severity = human username = root password = -disable-dispatcher-kickstarter = false -disable-dispatcher-frontend = false +disable-dispatcher-kickstarter = true +disable-dispatcher-frontend = true data-path = ./cluster/data log-path = ./cluster/log agent-path = ./bin/etcd-arango diff --git a/js/apps/system/aardvark/frontend/js/modules/org/arangodb/arango-collection-common.js b/js/apps/system/aardvark/frontend/js/modules/org/arangodb/arango-collection-common.js index e7c1a69131..6a7c6d5c1f 100644 --- a/js/apps/system/aardvark/frontend/js/modules/org/arangodb/arango-collection-common.js +++ b/js/apps/system/aardvark/frontend/js/modules/org/arangodb/arango-collection-common.js @@ -33,7 +33,7 @@ var ArangoCollection = require("org/arangodb/arango-collection").ArangoCollectio var arangodb = require("org/arangodb"); -var ArangoError = arangodb.ArrangoError; +var ArangoError = arangodb.ArangoError; var sprintf = arangodb.sprintf; var db = arangodb.db; @@ -528,29 +528,42 @@ ArangoCollection.prototype.closedRange = function (name, left, right) { //////////////////////////////////////////////////////////////////////////////// /// @brief constructs a geo index selection /// -/// @FUN{@FA{collection}.geo(@FA{location})} -//////////////////////////////////////////// +/// @FUN{@FA{collection}.geo(@FA{location-attribute})} +////////////////////////////////////////////////////// /// -/// The next @FN{near} or @FN{within} operator will use the specific geo-spatial -/// index. +/// Looks up a geo index defined on attribute @FA{location-attribute}. /// -/// @FUN{@FA{collection}.geo(@FA{location}, @LIT{true})} -//////////////////////////////////////////////////////// +/// Returns a geo index object if an index was found. The @FN{near} or +/// @FN{within} operators can then be used to execute a geo-spatial query on +/// this particular index. /// -/// The next @FN{near} or @FN{within} operator will use the specific geo-spatial -/// index. +/// This is useful for collections with multiple defined geo indexes. /// -/// @FUN{@FA{collection}.geo(@FA{latitude}, @FA{longitude})} -//////////////////////////////////////////////////////////// +/// @FUN{@FA{collection}.geo(@FA{location-attribute}, @LIT{true})} +////////////////////////////////////////////////////////////////// /// -/// The next @FN{near} or @FN{within} operator will use the specific geo-spatial -/// index. +/// Looks up a geo index on a compound attribute @FA{location-attribute}. +/// +/// Returns a geo index object if an index was found. The @FN{near} or +/// @FN{within} operators can then be used to execute a geo-spatial query on +/// this particular index. +/// +/// @FUN{@FA{collection}.geo(@FA{latitude-attribute}, @FA{longitude-attribute})} +//////////////////////////////////////////////////////////////////////////////// +/// +/// Looks up a geo index defined on the two attributes @FA{latitude-attribute} +/// and @FA{longitude-attribute}. +/// +/// Returns a geo index object if an index was found. The @FN{near} or +/// @FN{within} operators can then be used to execute a geo-spatial query on +/// this particular index. /// /// @EXAMPLES /// /// Assume you have a location stored as list in the attribute @LIT{home} -/// and a destination stored in the attribute @LIT{work}. Than you can use the -/// @FN{geo} operator to select, which coordinates to use in a near query. +/// and a destination stored in the attribute @LIT{work}. Then you can use the +/// @FN{geo} operator to select which geo-spatial attributes (and thus which +/// index) to use in a near query. /// /// @TINYEXAMPLE{simple-query-geo,use a specific index} //////////////////////////////////////////////////////////////////////////////// diff --git a/js/common/modules/org/arangodb/general-graph.js b/js/common/modules/org/arangodb/general-graph.js index d9c645e7f6..0aeeb34d60 100644 --- a/js/common/modules/org/arangodb/general-graph.js +++ b/js/common/modules/org/arangodb/general-graph.js @@ -30,8 +30,9 @@ var arangodb = require("org/arangodb"), - ArangoCollection = arangodb.ArangoCollection, - db = arangodb.db; + ArangoCollection = arangodb.ArangoCollection, + db = arangodb.db, + _ = require("underscore"); // ----------------------------------------------------------------------------- @@ -48,10 +49,10 @@ var arangodb = require("org/arangodb"), var stringToArray = function (x) { - if (typeof(x) === "string") { - return [x]; - } - return x; + if (typeof(x) === "string") { + return [x]; + } + return x; }; //////////////////////////////////////////////////////////////////////////////// @@ -61,16 +62,16 @@ var stringToArray = function (x) { var isValidCollectionsParameter = function (x) { - if (!x) { - return false; - } - if (Array.isArray(x) && x.length === 0) { - return false; - } - if (typeof(x) !== "string" && !Array.isArray(x)) { - return false; - } - return true; + if (!x) { + return false; + } + if (Array.isArray(x) && x.length === 0) { + return false; + } + if (typeof(x) !== "string" && !Array.isArray(x)) { + return false; + } + return true; }; //////////////////////////////////////////////////////////////////////////////// @@ -78,20 +79,20 @@ var isValidCollectionsParameter = function (x) { //////////////////////////////////////////////////////////////////////////////// var findOrCreateCollectionByName = function (name, type) { - var col = db._collection(name),res = false; - if (col === null) { - if (type === ArangoCollection.TYPE_DOCUMENT) { - col = db._create(name); - } else { - col = db._createEdgeCollection(name); - } - res = true; - } else if (!(col instanceof ArangoCollection) || col.type() !== type) { - throw "<" + name + "> must be a " + - (type === ArangoCollection.TYPE_DOCUMENT ? "document" : "edge") - + " collection"; - } - return res; + var col = db._collection(name),res = false; + if (col === null) { + if (type === ArangoCollection.TYPE_DOCUMENT) { + col = db._create(name); + } else { + col = db._createEdgeCollection(name); + } + res = true; + } else if (!(col instanceof ArangoCollection) || col.type() !== type) { + throw "<" + name + "> must be a " + + (type === ArangoCollection.TYPE_DOCUMENT ? "document" : "edge") + + " collection"; + } + return res; }; // ----------------------------------------------------------------------------- @@ -109,23 +110,23 @@ var findOrCreateCollectionByName = function (name, type) { var _undirectedRelationDefinition = function (relationName, vertexCollections) { - if (arguments.length < 2) { - throw "method _undirectedRelationDefinition expects 2 arguments"; - } + if (arguments.length < 2) { + throw "method _undirectedRelationDefinition expects 2 arguments"; + } - if (typeof relationName !== "string" || relationName === "") { - throw " must be a not empty string"; - } + if (typeof relationName !== "string" || relationName === "") { + throw " must be a not empty string"; + } - if (!isValidCollectionsParameter(vertexCollections)) { - throw " must be a not empty string or array"; - } + if (!isValidCollectionsParameter(vertexCollections)) { + throw " must be a not empty string or array"; + } - return { - collection: "relationName", - from: stringToArray(vertexCollections), - to: stringToArray(vertexCollections) - }; + return { + collection: relationName, + from: stringToArray(vertexCollections), + to: stringToArray(vertexCollections) + }; }; @@ -135,29 +136,29 @@ var _undirectedRelationDefinition = function (relationName, vertexCollections) { var _directedRelationDefinition = function ( - relationName, fromVertexCollections, toVertexCollections) { + relationName, fromVertexCollections, toVertexCollections) { - if (arguments.length < 3) { - throw "method _undirectedRelationDefinition expects 3 arguments"; - } + if (arguments.length < 3) { + throw "method _undirectedRelationDefinition expects 3 arguments"; + } - if (typeof relationName !== "string" || relationName === "") { - throw " must be a not empty string"; - } + if (typeof relationName !== "string" || relationName === "") { + throw " must be a not empty string"; + } - if (!isValidCollectionsParameter(fromVertexCollections)) { - throw " must be a not empty string or array"; - } + if (!isValidCollectionsParameter(fromVertexCollections)) { + throw " must be a not empty string or array"; + } - if (!isValidCollectionsParameter(toVertexCollections)) { - throw " must be a not empty string or array"; - } + if (!isValidCollectionsParameter(toVertexCollections)) { + throw " must be a not empty string or array"; + } - return { - collection: "relationName", - from: stringToArray(fromVertexCollections), - to: stringToArray(toVertexCollections) - }; + return { + collection: relationName, + from: stringToArray(fromVertexCollections), + to: stringToArray(toVertexCollections) + }; }; //////////////////////////////////////////////////////////////////////////////// @@ -167,12 +168,12 @@ var _directedRelationDefinition = function ( var edgeDefinitions = function () { - var res = [], args = arguments; - Object.keys(args).forEach(function (x) { - res.push(args[x]); - }); + var res = [], args = arguments; + Object.keys(args).forEach(function (x) { + res.push(args[x]); + }); - return res; + return res; }; @@ -183,54 +184,51 @@ var edgeDefinitions = function () { var _create = function (graphName, edgeDefinitions) { - var gdb = db._collection("_graphs"), - g, - graphAlreadyExists = true, - createdCollections = [] - ; + var gdb = db._collection("_graphs"), + g, + graphAlreadyExists = true; - if (gdb === null) { - throw "_graphs collection does not exist."; - } + if (gdb === null) { + throw "_graphs collection does not exist."; + } - if (!graphName) { - throw "a graph name is required to create a graph."; - } - if (!Array.isArray(edgeDefinitions) || edgeDefinitions.length === 0) { - throw "at least one edge definition is required to create a graph."; - } + if (!graphName) { + throw "a graph name is required to create a graph."; + } + if (!Array.isArray(edgeDefinitions) || edgeDefinitions.length === 0) { + throw "at least one edge definition is required to create a graph."; + } - try { - g = gdb.document(graphName); - } catch (e) { - if (e.errorNum !== 1202) { - throw e; - } - graphAlreadyExists = false; - } + try { + g = gdb.document(graphName); + } catch (e) { + if (e.errorNum !== 1202) { + throw e; + } + graphAlreadyExists = false; + } - if (graphAlreadyExists) { - throw "graph " + graphName + " already exists."; - } + if (graphAlreadyExists) { + throw "graph " + graphName + " already exists."; + } - edgeDefinitions.forEach(function (e) { - e.from.concat(e.to).forEach(function (v) { - if (findOrCreateCollectionByName(v, ArangoCollection.TYPE_DOCUMENT)) { - createdCollections.push(v); - } - }); - if (findOrCreateCollectionByName(e.collection, ArangoCollection.TYPE_EDGE)) { - createdCollections.push(e.collection); - } - }); + var vertexCollections = {}; + var edgeCollections = {}; + edgeDefinitions.forEach(function (e) { + e.from.concat(e.to).forEach(function (v) { + findOrCreateCollectionByName(v, ArangoCollection.TYPE_DOCUMENT); + vertexCollections[v] = db[v]; + }); + findOrCreateCollectionByName(e.collection, ArangoCollection.TYPE_EDGE); + edgeCollections[e.collection] = db[e.collection]; + }); - gdb.save({ - 'edgeDefinitions' : edgeDefinitions, - '_key' : graphName - }); - - return new Graph(graphName, edgeDefinitions); + gdb.save({ + 'edgeDefinitions' : edgeDefinitions, + '_key' : graphName + }); + return new Graph(graphName, edgeDefinitions, vertexCollections, edgeCollections); }; @@ -239,7 +237,20 @@ var _create = function (graphName, edgeDefinitions) { /// @brief load a graph. //////////////////////////////////////////////////////////////////////////////// -var Graph = function(graphName, edgeDefinitions) { +var Graph = function(graphName, edgeDefinitions, vertexCollections, edgeCollections) { + var self = this; + this.__vertexCollections = vertexCollections; + this.__edgeCollections = edgeCollections; + this.__edgeDefinitions = edgeDefinitions; + + + _.each(vertexCollections, function(obj, key) { + self[key] = obj; + }); + + _.each(edgeCollections, function(obj, key) { + self[key] = obj; + }); }; @@ -247,14 +258,20 @@ var _graph = function() { return new Graph(); }; -Graph.prototype.edgeCollections = function() { - //testdummy - return [require("internal").db.w3]; +//////////////////////////////////////////////////////////////////////////////// +/// @brief return all edge collections of the graph. +//////////////////////////////////////////////////////////////////////////////// + +Graph.prototype._edgeCollections = function() { + return _.values(this.__edgeCollections); }; -Graph.prototype.vertexCollections = function() { - //testdummy - return [require("internal").db.a, require("internal").db.b]; +//////////////////////////////////////////////////////////////////////////////// +/// @brief return all vertex collections of the graph. +//////////////////////////////////////////////////////////////////////////////// + +Graph.prototype._vertexCollections = function() { + return _.values(this.__vertexCollections); }; //////////////////////////////////////////////////////////////////////////////// @@ -262,7 +279,7 @@ Graph.prototype.vertexCollections = function() { //////////////////////////////////////////////////////////////////////////////// Graph.prototype.edges = function(vertexId) { - var edgeCollections = this.edgeCollections(); + var edgeCollections = this._edgeCollections(); var result = []; @@ -280,7 +297,7 @@ Graph.prototype.edges = function(vertexId) { //////////////////////////////////////////////////////////////////////////////// Graph.prototype.inEdges = function(vertexId) { - var edgeCollections = this.edgeCollections(); + var edgeCollections = this._edgeCollections(); var result = []; @@ -298,7 +315,7 @@ Graph.prototype.inEdges = function(vertexId) { //////////////////////////////////////////////////////////////////////////////// Graph.prototype.outEdges = function(vertexId) { - var edgeCollections = this.edgeCollections(); + var edgeCollections = this._edgeCollections(); var result = []; @@ -316,11 +333,11 @@ Graph.prototype.outEdges = function(vertexId) { //////////////////////////////////////////////////////////////////////////////// Graph.prototype.getInVertex = function(edgeId) { - var edgeCollection = this.getEdgeCollectionByName(edgeId.split("/")[0]); + var edgeCollection = this._getEdgeCollectionByName(edgeId.split("/")[0]); var document = edgeCollection.document(edgeId); if (document) { var vertexId = document._from; - var vertexCollection = this.getVertexCollectionByName(vertexId.split("/")[0]); + var vertexCollection = this._getVertexCollectionByName(vertexId.split("/")[0]); return vertexCollection.document(vertexId); } }; @@ -330,11 +347,11 @@ Graph.prototype.getInVertex = function(edgeId) { //////////////////////////////////////////////////////////////////////////////// Graph.prototype.getOutVertex = function(edgeId) { - var edgeCollection = this.getEdgeCollectionByName(edgeId.split("/")[0]); + var edgeCollection = this._getEdgeCollectionByName(edgeId.split("/")[0]); var document = edgeCollection.document(edgeId); if (document) { var vertexId = document._to; - var vertexCollection = this.getVertexCollectionByName(vertexId.split("/")[0]); + var vertexCollection = this._getVertexCollectionByName(vertexId.split("/")[0]); return vertexCollection.document(vertexId); } }; @@ -344,16 +361,9 @@ Graph.prototype.getOutVertex = function(edgeId) { /// @brief get edge collection by name. //////////////////////////////////////////////////////////////////////////////// -Graph.prototype.getEdgeCollectionByName = function(name) { - var edgeCollections = this.edgeCollections(); - var results = edgeCollections.filter( - function(edgeCollection) { - return edgeCollection.name() === name; - } - ); - - if (results.length === 1) { - return results[0]; +Graph.prototype._getEdgeCollectionByName = function(name) { + if (this.__edgeCollections[name]) { + return this.__edgeCollections[name]; } throw "Collection " + name + " does not exist in graph."; }; @@ -362,16 +372,9 @@ Graph.prototype.getEdgeCollectionByName = function(name) { /// @brief get vertex collection by name. //////////////////////////////////////////////////////////////////////////////// -Graph.prototype.getVertexCollectionByName = function(name) { - var vertexCollections = this.vertexCollections(); - var results = vertexCollections.filter( - function(vertexCollection) { - return vertexCollection.name() === name; - } - ); - - if (results.length === 1) { - return results[0]; +Graph.prototype._getVertexCollectionByName = function(name) { + if (this.__vertexCollections[name]) { + return this.__vertexCollections[name]; } throw "Collection " + name + " does not exist in graph."; }; diff --git a/js/common/tests/shell-general-graph.js b/js/common/tests/shell-general-graph.js index b41827f55e..5ea8943ebf 100644 --- a/js/common/tests/shell-general-graph.js +++ b/js/common/tests/shell-general-graph.js @@ -24,7 +24,7 @@ /// /// Copyright holder is triAGENS GmbH, Cologne, Germany /// -/// @author Florian Bartels +/// @author Florian Bartels, Michael Hackstein /// @author Copyright 2014, triAGENS GmbH, Cologne, Germany //////////////////////////////////////////////////////////////////////////////// @@ -33,7 +33,7 @@ var jsunity = require("jsunity"); var arangodb = require("org/arangodb"); var console = require("console"); -var graph = require("org/arangodb/general-graph") +var graph = require("org/arangodb/general-graph"); var print = arangodb.print; @@ -223,7 +223,6 @@ function GeneralGraphCreationSuite() { } - /*test_create : function () { @@ -242,11 +241,160 @@ function GeneralGraphCreationSuite() { }*/ - - } + }; } +// ----------------------------------------------------------------------------- +// --SECTION-- Simple Queries +// ----------------------------------------------------------------------------- + +function GeneralGraphSimpleQueriesSuite() { + + var dropInclExcl = function() { + var col = require("internal").db._collection("_graphs"); + try { + col.remove("graph"); + } catch (e) { + return; + } + }; + + var createInclExcl = function() { + dropInclExcl(); + var inc = graph._directedRelationDefinition("included", ["v1"], ["v1", "v2"]); + var exc = graph._directedRelationDefinition("excluded", ["v1"], ["v3"]); + var g = graph._create("graph", [inc, exc]); + return g; + }; + + return { + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test: restrict construct on edges +//////////////////////////////////////////////////////////////////////////////// + + test_restrictOnEdges: function() { + var g = createInclExcl(); + var incEdge1 = g.included.save( + "v1/1", + "v2/1", + { + included: true + } + ); + var incEdge2 = g.included.save( + "v1/2", + "v1/1", + { + included: true + } + ); + var excEdge = g.excluded.save( + "v1/1", + "v3/1", + { + included: false + } + ); + /* + var result = g.edges().restrict("v1"); + assertEqual(result.length, 2); + assertNotEqual(result.indexOf(incEdge1), -1); + assertNotEqual(result.indexOf(incEdge2), -1); + assertEqual(result.indexOf(excEdge), -1); + */ + dropInclExcl(); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test: restrict construct on inEdges +//////////////////////////////////////////////////////////////////////////////// + + test_restrictOnInEdges: function() { + var g = createInclExcl(); + var excEdge1 = g.included.save( + "v1/1", + "v2/1", + { + included: false + } + ); + var incEdge = g.included.save( + "v1/2", + "v1/1", + { + included: true + } + ); + var excEdge2 = g.excluded.save( + "v1/1", + "v3/1", + { + included: false + } + ); + /* + var result = g.edges().restrict("v1"); + assertEqual(result.length, 1); + assertEqual(result.indexOf(excEdge1), -1); + assertNotEqual(result.indexOf(incEdge), -1); + assertEqual(result.indexOf(excEdge2), -1); + */ + dropInclExcl(); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test: restrict construct on outEdges +//////////////////////////////////////////////////////////////////////////////// + + test_restrictOnOutEdges: function() { + var g = createInclExcl(); + var incEdge = g.included.save( + "v1/1", + "v2/1", + { + included: true + } + ); + var excEdge1 = g.included.save( + "v1/2", + "v1/1", + { + included: false + } + ); + var excEdge2 = g.excluded.save( + "v1/1", + "v3/1", + { + included: false + } + ); + /* + var result = g.edges().restrict("v1"); + assertEqual(result.length, 1); + assertNotEqual(result.indexOf(incEdge), -1); + assertEqual(result.indexOf(excEdge1), -1); + assertEqual(result.indexOf(excEdge2), -1); + */ + dropInclExcl(); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test: filter construct on Edges +//////////////////////////////////////////////////////////////////////////////// + + test_filterOnEdges: function() { + + } + + }; + + +} + + // ----------------------------------------------------------------------------- // --SECTION-- main @@ -257,6 +405,7 @@ function GeneralGraphCreationSuite() { //////////////////////////////////////////////////////////////////////////////// jsunity.run(GeneralGraphCreationSuite); +jsunity.run(GeneralGraphSimpleQueriesSuite); return jsunity.done(); diff --git a/lib/ShapedJson/json-shaper.cpp b/lib/ShapedJson/json-shaper.cpp index 66c4d0edf0..800bc081b9 100644 --- a/lib/ShapedJson/json-shaper.cpp +++ b/lib/ShapedJson/json-shaper.cpp @@ -505,6 +505,14 @@ TRI_shape_t* TRI_LookupBasicShapeShaper (TRI_shape_t const* shape) { return NULL; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief returns the first id for user-defined shapes +//////////////////////////////////////////////////////////////////////////////// + +TRI_shape_sid_t TRI_FirstCustomShapeIdShaper () { + return 7; +} + //////////////////////////////////////////////////////////////////////////////// /// @brief initialises global basic shape types //////////////////////////////////////////////////////////////////////////////// @@ -555,6 +563,8 @@ void TRI_InitialiseShaper () { shape->_type = TRI_SHAPE_LIST; shape->_dataSize = TRI_SHAPE_SIZE_VARIABLE; shape->_sid = BasicShapes._sidList = 6; + + assert(shape->_sid + 1 == TRI_FirstCustomShapeIdShaper()); } //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/ShapedJson/json-shaper.h b/lib/ShapedJson/json-shaper.h index affa6f47f0..e95d98bd91 100644 --- a/lib/ShapedJson/json-shaper.h +++ b/lib/ShapedJson/json-shaper.h @@ -165,6 +165,12 @@ TRI_shape_t* TRI_LookupSidBasicShapeShaper (TRI_shape_sid_t); TRI_shape_t* TRI_LookupBasicShapeShaper (TRI_shape_t const*); +//////////////////////////////////////////////////////////////////////////////// +/// @brief returns the first id for user-defined shapes +//////////////////////////////////////////////////////////////////////////////// + +TRI_shape_sid_t TRI_FirstCustomShapeIdShaper (void); + //////////////////////////////////////////////////////////////////////////////// /// @brief initialises global basic shape types //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/ShapedJson/shape-accessor.cpp b/lib/ShapedJson/shape-accessor.cpp index 5d36d71997..9a7adf47c6 100644 --- a/lib/ShapedJson/shape-accessor.cpp +++ b/lib/ShapedJson/shape-accessor.cpp @@ -75,25 +75,8 @@ static bool BytecodeShapeAccessor (TRI_shaper_t* shaper, TRI_shape_access_t* acc paids = (TRI_shape_aid_t*) (((char const*) path) + sizeof(TRI_shape_path_t)); // collect the bytecode - // we need at least 4 entries in the vector to store an accessor - TRI_InitVectorPointer2(&ops, shaper->_memoryZone, 4); - - // start with the shape - res = TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_SHAPE_PTR); - - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("out of memory"); - TRI_DestroyVectorPointer(&ops); - return false; - } - - res = TRI_PushBackVectorPointer(&ops, CONST_CAST(shape)); - - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("out of memory"); - TRI_DestroyVectorPointer(&ops); - return false; - } + // we need at least 2 entries in the vector to store an accessor + TRI_InitVectorPointer2(&ops, shaper->_memoryZone, 2); // and follow it for (i = 0; i < path->_aidLength; ++i, ++paids) { @@ -179,22 +162,6 @@ static bool BytecodeShapeAccessor (TRI_shaper_t* shaper, TRI_shape_access_t* acc return false; } - res = TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_SHAPE_PTR); - - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("out of memory"); - TRI_DestroyVectorPointer(&ops); - return false; - } - - res = TRI_PushBackVectorPointer(&ops, CONST_CAST(shape)); - - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("out of memory"); - TRI_DestroyVectorPointer(&ops); - return false; - } - break; } } @@ -240,22 +207,6 @@ static bool BytecodeShapeAccessor (TRI_shaper_t* shaper, TRI_shape_access_t* acc return false; } - res = TRI_PushBackVectorPointer(&ops, (void*) TRI_SHAPE_AC_SHAPE_PTR); - - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("out of memory"); - TRI_DestroyVectorPointer(&ops); - return false; - } - - res = TRI_PushBackVectorPointer(&ops, CONST_CAST(shape)); - - if (res != TRI_ERROR_NO_ERROR) { - LOG_ERROR("out of memory"); - TRI_DestroyVectorPointer(&ops); - return false; - } - break; } } @@ -333,10 +284,6 @@ static bool ExecuteBytecodeShapeAccessor (TRI_shape_access_t const* accessor, case TRI_SHAPE_AC_DONE: return true; - case TRI_SHAPE_AC_SHAPE_PTR: - ops++; - break; - case TRI_SHAPE_AC_OFFSET_FIX: b = (TRI_shape_size_t) (uintptr_t) *ops++; // offset is always smaller than 4 GByte e = (TRI_shape_size_t) (uintptr_t) *ops++; // offset is always smaller than 4 GByte @@ -478,13 +425,6 @@ void TRI_PrintShapeAccessor (TRI_shape_access_t* accessor) { case TRI_SHAPE_AC_DONE: return; - case TRI_SHAPE_AC_SHAPE_PTR: - shape = static_cast(*ops++); - - printf(" OP: shape %lu\n", - (unsigned long) shape->_sid); - break; - case TRI_SHAPE_AC_OFFSET_FIX: b = (TRI_shape_size_t) (uintptr_t) *ops++; // offset is always smaller than 4 GByte e = (TRI_shape_size_t) (uintptr_t) *ops++; // offset is always smaller than 4 GByte diff --git a/lib/ShapedJson/shape-accessor.h b/lib/ShapedJson/shape-accessor.h index 542c5539fb..ab1a93bda4 100644 --- a/lib/ShapedJson/shape-accessor.h +++ b/lib/ShapedJson/shape-accessor.h @@ -62,14 +62,13 @@ typedef struct TRI_shape_access_s { TRI_shape_access_t; //////////////////////////////////////////////////////////////////////////////// -/// @brief size of short strings +/// @brief shape accessor bytecode operations //////////////////////////////////////////////////////////////////////////////// typedef enum { - TRI_SHAPE_AC_DONE = 1, - TRI_SHAPE_AC_SHAPE_PTR = 2, - TRI_SHAPE_AC_OFFSET_FIX = 3, - TRI_SHAPE_AC_OFFSET_VAR = 4 + TRI_SHAPE_AC_DONE = 1, + TRI_SHAPE_AC_OFFSET_FIX = 2, + TRI_SHAPE_AC_OFFSET_VAR = 3 } TRI_shape_ac_bc_e;