Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

CHANGELOG

@@ -1,6 +1,20 @@
 v2.7.0 (XXXX-XX-XX)
 -------------------
 
+* increased default value collection-specific `indexBuckets` value from 1 to 16
+
+  Collections created from 2.7 on will use the new default if not overriden on
+  collection creation or later using `collection.properties({ indexBuckets: ... })`.
+
+  The `indexBuckets` value determines the number of buckets to use when creating
+  an edges index in the collection, and it may be used in future versions of ArangoDB
+  for other index types, too.
+
+* allow parallel processing of multiple buckets when loading edge indexes
+
+  Buckets will be built in parallel if the collection has an `indexBuckets` value
+  greater than 1 and the collection contains a significant amount of documents/edges.
+
 * changed HTTP client to use poll instead of select on Linux and MacOS
 
   This affects the ArangoShell and user-defined JavaScript code running inside 
@@ -17,7 +31,7 @@ v2.7.0 (XXXX-XX-XX)
       FOR doc IN collection
         RETURN DISTINCT doc.status 
 
-* removed `createNamedQueue()` and `addJob()` from org/arangodb/tasks
+* removed `createNamedQueue()` and `addJob()` functions from org/arangodb/tasks
 
 * moved dispatcher to atomic implementation
 

arangod/Replication/ContinuousSyncer.cpp

@@ -626,6 +626,7 @@ int ContinuousSyncer::changeCollection (TRI_json_t const* json) {
   bool waitForSync      = JsonHelper::getBooleanValue(collectionJson, "waitForSync", false);
   bool doCompact        = JsonHelper::getBooleanValue(collectionJson, "doCompact", true);
   int maximalSize       = JsonHelper::getNumericValue<int>(collectionJson, "maximalSize", TRI_JOURNAL_DEFAULT_MAXIMAL_SIZE);
+  uint32_t indexBuckets = JsonHelper::getNumericValue<uint32_t>(collectionJson, "indexBuckets", TRI_DEFAULT_INDEX_BUCKETS);
 
   TRI_voc_cid_t cid = getCid(json);
   TRI_vocbase_col_t* col = TRI_LookupCollectionByIdVocBase(_vocbase, cid);
@@ -643,6 +644,7 @@ int ContinuousSyncer::changeCollection (TRI_json_t const* json) {
     parameters._doCompact    = doCompact;
     parameters._maximalSize  = maximalSize;
     parameters._waitForSync  = waitForSync;
+    parameters._indexBuckets = indexBuckets;
 
     bool doSync = _vocbase->_settings.forceSyncProperties;
     return TRI_UpdateCollectionInfo(_vocbase, guard.collection()->_collection, &parameters, doSync);

arangod/Replication/Syncer.cpp

@@ -398,6 +398,7 @@ int Syncer::createCollection (TRI_json_t const* json,
   params._isVolatile   = JsonHelper::getBooleanValue(json, "isVolatile", false);
   params._isSystem     = (name[0] == '_');
   params._planId       = 0;
+  params._indexBuckets = JsonHelper::getNumericValue<uint32_t>(json, "indexBuckets", (uint32_t) TRI_DEFAULT_INDEX_BUCKETS);
 
   TRI_voc_cid_t planId = JsonHelper::stringUInt64(json, "planId");
   if (planId > 0) {

arangod/VocBase/collection.cpp

@@ -804,7 +804,7 @@ static void FillParametersFromJson (TRI_col_info_t* parameters,
   // init with defaults
   memset(parameters, 0, sizeof(TRI_col_info_t));
   parameters->_initialCount = -1;
-  parameters->_indexBuckets = 1;
+  parameters->_indexBuckets = TRI_DEFAULT_INDEX_BUCKETS;
 
   // convert json
   size_t const n = TRI_LengthVector(&json->_value._objects);
@@ -904,7 +904,7 @@ void TRI_InitCollectionInfo (TRI_vocbase_t* vocbase,
     parameters->_maximalSize = static_cast<TRI_voc_size_t>(PageSize);
   }
   parameters->_initialCount  = -1;
-  parameters->_indexBuckets  = 1;
+  parameters->_indexBuckets  = TRI_DEFAULT_INDEX_BUCKETS;
 
   // fill name with 0 bytes
   memset(parameters->_name, 0, sizeof(parameters->_name));

arangod/VocBase/collection.h

@@ -116,6 +116,12 @@ struct TRI_vocbase_col_s;
 
 #define TRI_COL_NAME_STATISTICS  "_statistics"
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief default number of index buckets
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_DEFAULT_INDEX_BUCKETS 8
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                      public types
 // -----------------------------------------------------------------------------

js/server/tests/replication-data.js

@@ -1064,6 +1064,7 @@ function ReplicationSuite () {
           assertTrue(properties.waitForSync);
           assertTrue(properties.doCompact);
           assertEqual(2097152, properties.journalSize);
+          assertTrue(properties.hasOwnProperty("indexBuckets"));
 
           state.cid = c._id;
           state.properties = c.properties();
@@ -1075,6 +1076,8 @@ function ReplicationSuite () {
           assertTrue(properties.waitForSync);
           assertTrue(properties.doCompact);
           assertEqual(2097152, properties.journalSize);
+          assertTrue(properties.hasOwnProperty("indexBuckets"));
+          assertEqual(properties.indexBuckets, properties.indexBuckets);
         }
       );
     },
@@ -1101,6 +1104,7 @@ function ReplicationSuite () {
           assertFalse(properties.waitForSync);
           assertFalse(properties.doCompact);
           assertEqual(1048576, properties.journalSize);
+          assertTrue(properties.hasOwnProperty("indexBuckets"));
 
           state.cid = c._id;
           state.properties = c.properties();
@@ -1112,6 +1116,35 @@ function ReplicationSuite () {
           assertFalse(properties.waitForSync);
           assertFalse(properties.doCompact);
           assertEqual(1048576, properties.journalSize);
+          assertTrue(properties.hasOwnProperty("indexBuckets"));
+          assertEqual(properties.indexBuckets, properties.indexBuckets);
+        }
+      );
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test change collection
+////////////////////////////////////////////////////////////////////////////////
+
+    testChangeCollectionIndexBuckets : function () {
+      compare(
+        function (state) {
+          var c = db._create(cn, {
+            indexBuckets: 4
+          });
+
+          var properties = c.properties();
+          assertEqual(4, properties.indexBuckets);
+
+          properties = c.properties({ indexBuckets: 8 });
+          assertEqual(8, properties.indexBuckets);
+
+          state.cid = c._id;
+          state.properties = c.properties();
+        },
+        function (state) {
+          var properties = db._collection(cn).properties();
+          assertEqual(8, properties.indexBuckets);
         }
       );
     },
@@ -1144,6 +1177,7 @@ function ReplicationSuite () {
           assertEqual(1048576, properties.journalSize);
           assertTrue(properties.keyOptions.allowUserKeys);
           assertEqual("traditional", properties.keyOptions.type);
+          assertTrue(properties.hasOwnProperty("indexBuckets"));
         }
       );
     },
@@ -1180,6 +1214,53 @@ function ReplicationSuite () {
           assertEqual(2097152, properties.journalSize);
           assertFalse(properties.keyOptions.allowUserKeys);
           assertEqual("autoincrement", properties.keyOptions.type);
+          assertTrue(properties.hasOwnProperty("indexBuckets"));
+        }
+      );
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test create collection
+////////////////////////////////////////////////////////////////////////////////
+
+    testCreateCollectionIndexBuckets1 : function () {
+      compare(
+        function (state) {
+          var c = db._create(cn, {
+            indexBuckets: 16
+          });
+
+          state.cid = c._id;
+          state.properties = c.properties();
+        },
+        function (state) {
+          var properties = db._collection(cn).properties();
+          assertEqual(state.cid, db._collection(cn)._id);
+          assertEqual(cn, db._collection(cn).name());
+          assertEqual(16, properties.indexBuckets);
+        }
+      );
+    },
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test create collection
+////////////////////////////////////////////////////////////////////////////////
+
+    testCreateCollectionIndexBuckets2 : function () {
+      compare(
+        function (state) {
+          var c = db._create(cn, {
+            indexBuckets: 8
+          });
+
+          state.cid = c._id;
+          state.properties = c.properties();
+        },
+        function (state) {
+          var properties = db._collection(cn).properties();
+          assertEqual(state.cid, db._collection(cn)._id);
+          assertEqual(cn, db._collection(cn).name());
+          assertEqual(8, properties.indexBuckets);
         }
       );
     },