From f89af3f6f4e18039c9d28d2e19469d8c30f428cd Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 4 Aug 2015 14:02:04 +0200 Subject: [PATCH] adjusted tests for indexBuckets --- CHANGELOG | 16 ++++- arangod/Replication/ContinuousSyncer.cpp | 14 ++-- arangod/Replication/Syncer.cpp | 11 ++-- arangod/VocBase/collection.cpp | 4 +- arangod/VocBase/collection.h | 6 ++ js/server/tests/replication-data.js | 81 ++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 14 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 5556a63e3f..453fc59112 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,20 @@ v2.7.0 (XXXX-XX-XX) ------------------- +* increased default value collection-specific `indexBuckets` value from 1 to 16 + + Collections created from 2.7 on will use the new default if not overriden on + collection creation or later using `collection.properties({ indexBuckets: ... })`. + + The `indexBuckets` value determines the number of buckets to use when creating + an edges index in the collection, and it may be used in future versions of ArangoDB + for other index types, too. + +* allow parallel processing of multiple buckets when loading edge indexes + + Buckets will be built in parallel if the collection has an `indexBuckets` value + greater than 1 and the collection contains a significant amount of documents/edges. + * changed HTTP client to use poll instead of select on Linux and MacOS This affects the ArangoShell and user-defined JavaScript code running inside @@ -17,7 +31,7 @@ v2.7.0 (XXXX-XX-XX) FOR doc IN collection RETURN DISTINCT doc.status -* removed `createNamedQueue()` and `addJob()` from org/arangodb/tasks +* removed `createNamedQueue()` and `addJob()` functions from org/arangodb/tasks * moved dispatcher to atomic implementation diff --git a/arangod/Replication/ContinuousSyncer.cpp b/arangod/Replication/ContinuousSyncer.cpp index ddd5a17e7e..58ec5394ea 100644 --- a/arangod/Replication/ContinuousSyncer.cpp +++ b/arangod/Replication/ContinuousSyncer.cpp @@ -623,9 +623,10 @@ int ContinuousSyncer::renameCollection (TRI_json_t const* json) { int ContinuousSyncer::changeCollection (TRI_json_t const* json) { TRI_json_t const* collectionJson = TRI_LookupObjectJson(json, "collection"); - bool waitForSync = JsonHelper::getBooleanValue(collectionJson, "waitForSync", false); - bool doCompact = JsonHelper::getBooleanValue(collectionJson, "doCompact", true); - int maximalSize = JsonHelper::getNumericValue(collectionJson, "maximalSize", TRI_JOURNAL_DEFAULT_MAXIMAL_SIZE); + bool waitForSync = JsonHelper::getBooleanValue(collectionJson, "waitForSync", false); + bool doCompact = JsonHelper::getBooleanValue(collectionJson, "doCompact", true); + int maximalSize = JsonHelper::getNumericValue(collectionJson, "maximalSize", TRI_JOURNAL_DEFAULT_MAXIMAL_SIZE); + uint32_t indexBuckets = JsonHelper::getNumericValue(collectionJson, "indexBuckets", TRI_DEFAULT_INDEX_BUCKETS); TRI_voc_cid_t cid = getCid(json); TRI_vocbase_col_t* col = TRI_LookupCollectionByIdVocBase(_vocbase, cid); @@ -640,9 +641,10 @@ int ContinuousSyncer::changeCollection (TRI_json_t const* json) { TRI_col_info_t parameters; // only need to set these three properties as the others cannot be updated on the fly - parameters._doCompact = doCompact; - parameters._maximalSize = maximalSize; - parameters._waitForSync = waitForSync; + parameters._doCompact = doCompact; + parameters._maximalSize = maximalSize; + parameters._waitForSync = waitForSync; + parameters._indexBuckets = indexBuckets; bool doSync = _vocbase->_settings.forceSyncProperties; return TRI_UpdateCollectionInfo(_vocbase, guard.collection()->_collection, ¶meters, doSync); diff --git a/arangod/Replication/Syncer.cpp b/arangod/Replication/Syncer.cpp index 80e57e8e32..337af3ddb7 100644 --- a/arangod/Replication/Syncer.cpp +++ b/arangod/Replication/Syncer.cpp @@ -393,11 +393,12 @@ int Syncer::createCollection (TRI_json_t const* json, TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, keyOptions); } - params._doCompact = JsonHelper::getBooleanValue(json, "doCompact", true); - params._waitForSync = JsonHelper::getBooleanValue(json, "waitForSync", _vocbase->_settings.defaultWaitForSync); - params._isVolatile = JsonHelper::getBooleanValue(json, "isVolatile", false); - params._isSystem = (name[0] == '_'); - params._planId = 0; + params._doCompact = JsonHelper::getBooleanValue(json, "doCompact", true); + params._waitForSync = JsonHelper::getBooleanValue(json, "waitForSync", _vocbase->_settings.defaultWaitForSync); + params._isVolatile = JsonHelper::getBooleanValue(json, "isVolatile", false); + params._isSystem = (name[0] == '_'); + params._planId = 0; + params._indexBuckets = JsonHelper::getNumericValue(json, "indexBuckets", (uint32_t) TRI_DEFAULT_INDEX_BUCKETS); TRI_voc_cid_t planId = JsonHelper::stringUInt64(json, "planId"); if (planId > 0) { diff --git a/arangod/VocBase/collection.cpp b/arangod/VocBase/collection.cpp index 4a9e51c3be..1d870fca26 100644 --- a/arangod/VocBase/collection.cpp +++ b/arangod/VocBase/collection.cpp @@ -804,7 +804,7 @@ static void FillParametersFromJson (TRI_col_info_t* parameters, // init with defaults memset(parameters, 0, sizeof(TRI_col_info_t)); parameters->_initialCount = -1; - parameters->_indexBuckets = 1; + parameters->_indexBuckets = TRI_DEFAULT_INDEX_BUCKETS; // convert json size_t const n = TRI_LengthVector(&json->_value._objects); @@ -904,7 +904,7 @@ void TRI_InitCollectionInfo (TRI_vocbase_t* vocbase, parameters->_maximalSize = static_cast(PageSize); } parameters->_initialCount = -1; - parameters->_indexBuckets = 1; + parameters->_indexBuckets = TRI_DEFAULT_INDEX_BUCKETS; // fill name with 0 bytes memset(parameters->_name, 0, sizeof(parameters->_name)); diff --git a/arangod/VocBase/collection.h b/arangod/VocBase/collection.h index 2075738219..5713cb66ba 100644 --- a/arangod/VocBase/collection.h +++ b/arangod/VocBase/collection.h @@ -116,6 +116,12 @@ struct TRI_vocbase_col_s; #define TRI_COL_NAME_STATISTICS "_statistics" +//////////////////////////////////////////////////////////////////////////////// +/// @brief default number of index buckets +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_DEFAULT_INDEX_BUCKETS 8 + // ----------------------------------------------------------------------------- // --SECTION-- public types // ----------------------------------------------------------------------------- diff --git a/js/server/tests/replication-data.js b/js/server/tests/replication-data.js index 4c37a66cf7..212978ed09 100644 --- a/js/server/tests/replication-data.js +++ b/js/server/tests/replication-data.js @@ -1064,6 +1064,7 @@ function ReplicationSuite () { assertTrue(properties.waitForSync); assertTrue(properties.doCompact); assertEqual(2097152, properties.journalSize); + assertTrue(properties.hasOwnProperty("indexBuckets")); state.cid = c._id; state.properties = c.properties(); @@ -1075,6 +1076,8 @@ function ReplicationSuite () { assertTrue(properties.waitForSync); assertTrue(properties.doCompact); assertEqual(2097152, properties.journalSize); + assertTrue(properties.hasOwnProperty("indexBuckets")); + assertEqual(properties.indexBuckets, properties.indexBuckets); } ); }, @@ -1101,6 +1104,7 @@ function ReplicationSuite () { assertFalse(properties.waitForSync); assertFalse(properties.doCompact); assertEqual(1048576, properties.journalSize); + assertTrue(properties.hasOwnProperty("indexBuckets")); state.cid = c._id; state.properties = c.properties(); @@ -1112,6 +1116,35 @@ function ReplicationSuite () { assertFalse(properties.waitForSync); assertFalse(properties.doCompact); assertEqual(1048576, properties.journalSize); + assertTrue(properties.hasOwnProperty("indexBuckets")); + assertEqual(properties.indexBuckets, properties.indexBuckets); + } + ); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test change collection +//////////////////////////////////////////////////////////////////////////////// + + testChangeCollectionIndexBuckets : function () { + compare( + function (state) { + var c = db._create(cn, { + indexBuckets: 4 + }); + + var properties = c.properties(); + assertEqual(4, properties.indexBuckets); + + properties = c.properties({ indexBuckets: 8 }); + assertEqual(8, properties.indexBuckets); + + state.cid = c._id; + state.properties = c.properties(); + }, + function (state) { + var properties = db._collection(cn).properties(); + assertEqual(8, properties.indexBuckets); } ); }, @@ -1144,6 +1177,7 @@ function ReplicationSuite () { assertEqual(1048576, properties.journalSize); assertTrue(properties.keyOptions.allowUserKeys); assertEqual("traditional", properties.keyOptions.type); + assertTrue(properties.hasOwnProperty("indexBuckets")); } ); }, @@ -1180,6 +1214,53 @@ function ReplicationSuite () { assertEqual(2097152, properties.journalSize); assertFalse(properties.keyOptions.allowUserKeys); assertEqual("autoincrement", properties.keyOptions.type); + assertTrue(properties.hasOwnProperty("indexBuckets")); + } + ); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test create collection +//////////////////////////////////////////////////////////////////////////////// + + testCreateCollectionIndexBuckets1 : function () { + compare( + function (state) { + var c = db._create(cn, { + indexBuckets: 16 + }); + + state.cid = c._id; + state.properties = c.properties(); + }, + function (state) { + var properties = db._collection(cn).properties(); + assertEqual(state.cid, db._collection(cn)._id); + assertEqual(cn, db._collection(cn).name()); + assertEqual(16, properties.indexBuckets); + } + ); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test create collection +//////////////////////////////////////////////////////////////////////////////// + + testCreateCollectionIndexBuckets2 : function () { + compare( + function (state) { + var c = db._create(cn, { + indexBuckets: 8 + }); + + state.cid = c._id; + state.properties = c.properties(); + }, + function (state) { + var properties = db._collection(cn).properties(); + assertEqual(state.cid, db._collection(cn)._id); + assertEqual(cn, db._collection(cn).name()); + assertEqual(8, properties.indexBuckets); } ); },