Merge branch 'devel' of github.com:arangodb/arangodb into devel

Documentation/Books/Makefile

@@ -406,7 +406,7 @@ build-books:
 	done
 
 	set -e; for book in $(ALLBOOKS); do \
-		make build-books-keep-md NAME=$${book}; NODE_MODULES_DIR=${NODE_MODULES_DIR}\
+		make build-books-keep-md NAME=$${book} NODE_MODULES_DIR=${NODE_MODULES_DIR}; \
 	done
 
 	set -e; for book in $(ALLBOOKS); do \

Documentation/Books/Manual/Administration/Configuration/RocksDB.md

@@ -12,7 +12,7 @@ underlying RocksDB instance, and we change very few of their default settings.
 `--rocksdb.write-buffer-size`
 
 The amount of data to build up in each in-memory buffer (backed by a log file)
-before closing the buffer and queueing it to be flushed into standard storage.
+before closing the buffer and queuing it to be flushed into standard storage.
 Default: 64MiB. Larger values may improve performance, especially for bulk
 loads.
 
@@ -44,16 +44,40 @@ The number of levels that do not use compression. The default value is 2.
 Levels above this number will use Snappy compression to reduce the disk
 space requirements for storing data in these levels.
 
-`--rocksdb.max-bytes-for-level-base` (Hidden)
+`--rocksdb.dynamic-level-bytes`
 
-The maximum total data size in bytes in level-1 of the LSM tree. Default:
+If true, the amount of data in each level of the LSM tree is determined
-256MiB.
+dynamically so as to minimize the space amplification; otherwise, the level
+sizes are fixed. The dynamic sizing allows RocksDB to maintain a well-structured
+LSM tree regardless of total data size. Default: true.
+
+`--rocksdb.max-bytes-for-level-base`
+
+The maximum total data size in bytes in level-1 of the LSM tree. Only effective
+if `--rocksdb.dynamic-level-bytes` is false. Default: 256MiB.
 
 `--rocksdb.max-bytes-for-level-multiplier`
 
 The maximum total data size in bytes for level L of the LSM tree can be
 calculated as `max-bytes-for-level-base * (max-bytes-for-level-multiplier ^
-(L-1))`. Default: 10.
+(L-1))`. Only effective if `--rocksdb.dynamic-level-bytes` is false. Default:
+10.
+
+`--rocksdb.level0-compaction-trigger`
+
+Compaction of level-0 to level-1 is triggered when this many files exist in
+level-0. Setting this to a higher number may help bulk writes at the expense of
+slowing down reads. Default: 2.
+
+`--rocksdb.level0-slowdown-trigger`
+
+When this many files accumulate in level-0, writes will be slowed down to
+`--rocksdb.delayed-write-rate` to allow compaction to catch up. Default: 20.
+
+`--rocksdb.level0-stop-trigger`
+
+When this many files accumulate in level-0, writes will be stopped to allow
+compaction to catch up. Default: 36.
 
 ### File I/O
 
@@ -119,7 +143,7 @@ The number of bits used to shard the block cache to allow concurrent operations.
 To keep individual shards at a reasonable size (i.e. at least 512KB), keep this
 value to at most `block-cache-shard-bits / 512KB`. Default: `block-cache-size /
 2^19`.
-  
+
 `--rocksdb.table-block-size`
 
 Approximate size of user data (in bytes) packed per block for uncompressed data.
@@ -153,15 +177,15 @@ If true, skip corrupted records in WAL recovery. Default: false.
 
 Timeout after which unused WAL files are deleted (in seconds). Default: 10.0s.
 
-Data of ongoing transactions is stored in RAM. Transactions that get too big 
+Data of ongoing transactions is stored in RAM. Transactions that get too big
 (in terms of number of operations involved or the total size of data created or
-modified by the transaction) will be committed automatically. Effectively this 
+modified by the transaction) will be committed automatically. Effectively this
-means that big user transactions are split into multiple smaller RocksDB 
+means that big user transactions are split into multiple smaller RocksDB
-transactions that are committed individually. The entire user transaction will 
+transactions that are committed individually. The entire user transaction will
 not necessarily have ACID properties in this case.
- 
+
-The following options can be used to control the RAM usage and automatic 
+The following options can be used to control the RAM usage and automatic
-intermediate commits for the RocksDB engine: 
+intermediate commits for the RocksDB engine:
 
 `--rocksdb.max-transaction-size`
 
@@ -174,10 +198,11 @@ limit exceeded").
 
 `--rocksdb.intermediate-commit-size`
 
-If the size of all operations in a transaction reaches this threshold, the transaction 
+If the size of all operations in a transaction reaches this threshold, the
-is committed automatically and a new transaction is started. The value is specified in bytes.
+transaction is committed automatically and a new transaction is started. The
-  
+value is specified in bytes.
+
 `--rocksdb.intermediate-commit-count`
 
-If the number of operations in a transaction reaches this value, the transaction is 
+If the number of operations in a transaction reaches this value, the transaction
-committed automatically and a new transaction is started.
+is committed automatically and a new transaction is started.

Installation/Jenkins/build.sh

@@ -147,6 +147,7 @@ CONFIGURE_OPTIONS+=("$CMAKE_OPENSSL")
 INSTALL_PREFIX="/"
 MAINTAINER_MODE="-DUSE_MAINTAINER_MODE=off"
 
+RETRY_N_TIMES=1
 TAR_SUFFIX=""
 TARGET_DIR=""
 CLANG36=0
@@ -389,6 +390,11 @@ while [ $# -gt 0 ];  do
             shift
             CONFIGURE_OPTIONS+=(-DUSE_ENTERPRISE=On)
             ;;
+        --retryPackages)
+            shift
+            RETRY_N_TIMES=$1
+            shift
+            ;;
         *)
             echo "Unknown option: $1"
             exit 1
@@ -700,7 +706,16 @@ set -e
 
 if [ -n "${CPACK}" ] && [ -n "${TARGET_DIR}" ];  then
     ${PACKAGE_MAKE} clean_packages || exit 1
-    ${PACKAGE_MAKE} packages || exit 1
+    while test "${RETRY_N_TIMES}" -gt 0; do
+          ${PACKAGE_MAKE} packages || break
+          RETRY_N_TIMES=$((RETRY_N_TIMES - 1))
+          echo "failed to build packages - waiting 5 mins maybe the situation gets better?"
+          sleep 600
+    done
+    if test "${RETRY_N_TIMES}" -eq 0; then
+        echo "building packages failed terminally"
+        exit 1
+    fi
 fi
 # and install
 

arangod/Aql/EnumerateCollectionBlock.cpp

@@ -185,28 +185,12 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t,  // atLeast,
     // only copy 1st row of registers inherited from previous frame(s)
     inheritRegisters(cur, res.get(), _pos);
 
-    auto col = _collection->getCollection();
+    IndexIterator::DocumentCallback cb;
-    LogicalCollection* c = col.get();
-    std::function<void(DocumentIdentifierToken const& tkn)> cb;
     if (_mustStoreResult) {
-      cb = [&](DocumentIdentifierToken const& tkn) {
+      cb = [&](ManagedDocumentResult const& mdr) {
-        if (c->readDocument(_trx, tkn, *_mmdr)) {
+        res->setValue(send,
-          // The result is in the first variable of this depth,
+                      static_cast<arangodb::aql::RegisterId>(curRegs),
-          // we do not need to do a lookup in
+                      mdr.createAqlValue());
-          // getPlanNode()->_registerPlan->varInfo,
-          // but can just take cur->getNrRegs() as registerId:
-          uint8_t const* vpack = _mmdr->vpack();
-          if (_mmdr->canUseInExternal()) {
-            res->setValue(send, static_cast<arangodb::aql::RegisterId>(curRegs),
-                          AqlValue(vpack, AqlValueFromManagedDocument()));
-          } else {
-            AqlValue a(_mmdr->createAqlValue());
-            AqlValueGuard guard(a, true);
-            res->setValue(send, static_cast<arangodb::aql::RegisterId>(curRegs), a);
-            guard.steal();
-          }
-        }
-
         if (send > 0) {
           // re-use already copied AQLValues
           res->copyValuesFromFirstRow(send, static_cast<RegisterId>(curRegs));
@@ -214,7 +198,7 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t,  // atLeast,
         ++send;
       };
     } else {
-      cb = [&](DocumentIdentifierToken const& tkn) {
+      cb = [&](ManagedDocumentResult const& mdr) {
         if (send > 0) {
           // re-use already copied AQLValues
           res->copyValuesFromFirstRow(send, static_cast<RegisterId>(curRegs));
@@ -229,7 +213,7 @@ AqlItemBlock* EnumerateCollectionBlock::getSome(size_t,  // atLeast,
       THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
     }
 
-    bool tmp = _cursor->next(cb, atMost);
+    bool tmp = _cursor->nextDocument(cb, atMost);
     if (!tmp) {
       TRI_ASSERT(!_cursor->hasMore());
     }

arangod/Cluster/ClusterComm.cpp

@@ -1161,6 +1161,7 @@ void ClusterCommThread::run() {
       abortRequestsToFailedServers();
       _cc->communicator()->work_once();
       _cc->communicator()->wait();
+      LOG_TOPIC(TRACE, Logger::CLUSTER) << "done waiting in ClusterCommThread";
     } catch (std::exception const& ex) {
       LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "caught exception in ClusterCommThread: " << ex.what();
     } catch (...) {

arangod/MMFiles/MMFilesCollection.cpp

@@ -2778,8 +2778,6 @@ int MMFilesCollection::insert(transaction::Methods* trx, VPackSlice const slice,
     bool const useDeadlockDetector =
       (lock && !trx->isSingleOperationTransaction() && !trx->state()->hasHint(transaction::Hints::Hint::NO_DLD));
     try {
-      // TODO Do we use the CollectionLocker on LogicalCollections
-      // or do we use it on the SE specific one?
       arangodb::MMFilesCollectionWriteLocker collectionLocker(
           this, useDeadlockDetector, lock);
 

arangod/MMFiles/MMFilesTransactionState.cpp

@@ -282,7 +282,9 @@ int MMFilesTransactionState::addOperation(TRI_voc_rid_t revisionId,
     }
     if (localWaitForSync) {
       // also sync RocksDB WAL
-      MMFilesPersistentIndexFeature::syncWal();
+      if (collection->getPhysical()->hasIndexOfType(arangodb::Index::TRI_IDX_TYPE_PERSISTENT_INDEX)) {
+        MMFilesPersistentIndexFeature::syncWal();
+      }
     }
     operation.setTick(slotInfo.tick);
     fid = slotInfo.logfileId;

arangod/RocksDBEngine/RocksDBEngine.cpp

@@ -199,6 +199,7 @@ void RocksDBEngine::start() {
   _options.min_write_buffer_number_to_merge =
       static_cast<int>(opts->_minWriteBufferNumberToMerge);
   _options.num_levels = static_cast<int>(opts->_numLevels);
+  _options.level_compaction_dynamic_level_bytes = opts->_dynamicLevelBytes;
   _options.max_bytes_for_level_base = opts->_maxBytesForLevelBase;
   _options.max_bytes_for_level_multiplier =
       static_cast<int>(opts->_maxBytesForLevelMultiplier);
@@ -217,6 +218,8 @@ void RocksDBEngine::start() {
       static_cast<int>(opts->_baseBackgroundCompactions);
   _options.max_background_compactions =
       static_cast<int>(opts->_maxBackgroundCompactions);
+  _options.max_subcompactions =
+      static_cast<int>(opts->_maxSubcompactions);
   _options.max_background_flushes = static_cast<int>(opts->_maxFlushes);
   _options.use_fsync = opts->_useFSync;
 
@@ -228,20 +231,20 @@ void RocksDBEngine::start() {
                                                  : rocksdb::kNoCompression);
   }
 
-  // TODO: try out the effects of these options 
+  // TODO: try out the effects of these options
   // Number of files to trigger level-0 compaction. A value <0 means that
   // level-0 compaction will not be triggered by number of files at all.
   //
   // Default: 4
-  // _options.level0_file_num_compaction_trigger = -1;
+  _options.level0_file_num_compaction_trigger = opts->_level0CompactionTrigger;
 
   // Soft limit on number of level-0 files. We start slowing down writes at this
   // point. A value <0 means that no writing slow down will be triggered by
   // number of files in level-0.
-  // _options.level0_slowdown_writes_trigger = -1;
+  _options.level0_slowdown_writes_trigger = opts->_level0SlowdownTrigger;
 
   // Maximum number of level-0 files.  We stop writes at this point.
-  // _options.level0_stop_writes_trigger = 256;
+  _options.level0_stop_writes_trigger = opts->_level0StopTrigger;
 
   _options.recycle_log_file_num = static_cast<size_t>(opts->_recycleLogFileNum);
   _options.compaction_readahead_size =
@@ -309,7 +312,7 @@ void RocksDBEngine::start() {
   std::vector<rocksdb::ColumnFamilyHandle*> cfHandles;
   size_t const numberOfColumnFamilies = RocksDBColumnFamily::numberOfColumnFamilies;
   {
-    rocksdb::Options testOptions; 
+    rocksdb::Options testOptions;
     testOptions.create_if_missing = false;
     testOptions.create_missing_column_families = false;
     std::vector<std::string> existingColumnFamilies;
@@ -335,14 +338,14 @@ void RocksDBEngine::start() {
         }
         names.append(it);
       }
-      
+
       LOG_TOPIC(DEBUG, arangodb::Logger::STARTUP) << "found existing column families: " << names;
 
       if (existingColumnFamilies.size() < numberOfColumnFamilies) {
-        LOG_TOPIC(FATAL, arangodb::Logger::STARTUP) 
+        LOG_TOPIC(FATAL, arangodb::Logger::STARTUP)
-            << "unexpected number of column families found in database (" << cfHandles.size() << "). " 
+            << "unexpected number of column families found in database (" << cfHandles.size() << "). "
             << "expecting at least " << numberOfColumnFamilies
-            << ". if you are upgrading from an alpha version of ArangoDB 3.2, " 
+            << ". if you are upgrading from an alpha version of ArangoDB 3.2, "
             << "it is required to restart with a new database directory and re-import data";
         FATAL_ERROR_EXIT();
       }
@@ -364,8 +367,8 @@ void RocksDBEngine::start() {
   }
 
   if (cfHandles.size() < numberOfColumnFamilies) {
-    LOG_TOPIC(FATAL, arangodb::Logger::STARTUP) 
+    LOG_TOPIC(FATAL, arangodb::Logger::STARTUP)
-        << "unexpected number of column families found in database. " 
+        << "unexpected number of column families found in database. "
         << "got " << cfHandles.size() << ", expecting at least " << numberOfColumnFamilies;
     FATAL_ERROR_EXIT();
   }

arangod/StorageEngine/PhysicalCollection.cpp

@@ -66,6 +66,16 @@ void PhysicalCollection::drop() {
   }
 }
 
+bool PhysicalCollection::hasIndexOfType(arangodb::Index::IndexType type) const {
+  READ_LOCKER(guard, _indexesLock);
+  for (auto const& idx : _indexes) {
+    if (idx->type() == type) {
+      return true;
+    }
+  }
+  return false;
+}
+
 std::shared_ptr<Index> PhysicalCollection::lookupIndex(
     TRI_idx_iid_t idxId) const {
   READ_LOCKER(guard, _indexesLock);

arangod/StorageEngine/PhysicalCollection.h

@@ -26,6 +26,7 @@
 
 #include "Basics/Common.h"
 #include "Basics/ReadWriteLock.h"
+#include "Indexes/Index.h"
 #include "VocBase/voc-types.h"
 
 #include <velocypack/Builder.h>
@@ -94,6 +95,8 @@ class PhysicalCollection {
   ///////////////////////////////////
 
   virtual void prepareIndexes(arangodb::velocypack::Slice indexesSlice) = 0;
+  
+  bool hasIndexOfType(arangodb::Index::IndexType type) const;
 
   /// @brief Find index by definition
   virtual std::shared_ptr<Index> lookupIndex(

js/apps/system/_admin/aardvark/APP/frontend/build/index-min.html

@@ -1724,7 +1724,10 @@ if (list.length > 0) {
               </th>
             </tr> <% } %> </table>
 
-      </div> <% if (figuresData.figures.alive && figuresData.figures.dead) { %> <table class="figures1 arango-table">
+      </div> <% if (figuresData.figures.alive && figuresData.figures.dead) { %> <% if (window.frontendConfig.isCluster) { %> <div class="infoBox">
+              <h4>Figures</h4>
+              <p>The following information does not contain data stored in the write-ahead log. It can be inaccurate, until the write-ahead log has been completely flushed.</p>
+            </div> <% } %> <table class="figures1 arango-table">
             <tr class="figuresHeader">
               <th>Type</th>
               <th>Count</th>
@@ -1733,9 +1736,7 @@ if (list.length > 0) {
               <th>Info</th>
             </tr>
             <tr>
-              <th class="modal-text">Alive</th>
+              <th class="modal-text">Alive</th> <% if (figuresData.walMessage) { %> <th class="modal-text"><%=figuresData.walMessage%></th> <% } else { %> <th class="modal-text"><%=numeral(figuresData.figures.alive.count).format('0,0')%></th> <% } %> <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=prettyBytes(figuresData.figures.alive.size)%> <% } %> </th>
-              <th class="modal-text"><%=numeral(figuresData.figures.alive.count).format('0,0')%></th>
-              <th class="modal-text"> <%=prettyBytes(figuresData.figures.alive.size)%> </th>
               <th class="modal-text"> -</th>
               <th class="tooltipInfoTh">
                 <span class="modalTooltips arangoicon icon_arangodb_info" title="Total number and size of all living documents.">
@@ -1744,9 +1745,9 @@ if (list.length > 0) {
             </tr>
             <tr>
               <th class="modal-text">Dead</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.dead.count).format('0,0')%></th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=numeral(figuresData.figures.dead.count).format('0,0')%> <% } %> </th>
-              <th class="modal-text"> <%=prettyBytes(figuresData.figures.dead.size)%> </th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=prettyBytes(figuresData.figures.dead.size)%> <% } %> </th>
-              <th class="modal-text"><%=figuresData.figures.dead.deletion%></th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=figuresData.figures.dead.deletion%> <% } %> </th>
 
               <th class="tooltipInfoTh">
                 <div>
@@ -1767,8 +1768,8 @@ if (list.length > 0) {
             </tr>
             <tr>
               <th class="modal-text">Datafiles</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.datafiles.count).format('0,0')%></th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=numeral(figuresData.figures.datafiles.count).format('0,0')%> <% } %> </th>
-              <th class="modal-text"> <%= prettyBytes(figuresData.figures.datafiles.fileSize) %> </th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%= prettyBytes(figuresData.figures.datafiles.fileSize) %> <% } %> </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
                 <div>
@@ -1779,8 +1780,8 @@ if (list.length > 0) {
             </tr>
             <tr>
               <th class="modal-text">Journals</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.journals.count).format('0,0')%></th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=numeral(figuresData.figures.journals.count).format('0,0')%> <% } %> </th>
-              <th class="modal-text"> <%=prettyBytes(figuresData.figures.journals.fileSize)%> </th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=prettyBytes(figuresData.figures.journals.fileSize)%> <% } %> </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
                 <span class="modalTooltips arangoicon icon_arangodb_info" title="Number and total size of journal files.">
@@ -1789,8 +1790,8 @@ if (list.length > 0) {
             </tr>
             <tr>
               <th class="modal-text">Compactors</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.compactors.count).format('0,0')%></th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=numeral(figuresData.figures.compactors.count).format('0,0')%> <% } %> </th>
-              <th class="modal-text"> <%=prettyBytes(figuresData.figures.compactors.fileSize)%> </th>
+              <th class="modal-text"> <% if (figuresData.walMessage) { %> <%=figuresData.walMessage%> <% } else { %> <%=prettyBytes(figuresData.figures.compactors.fileSize)%> <% } %> </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
                 <span class="modalTooltips arangoicon icon_arangodb_info" title="Number and total size of compactor files."></span>
@@ -1798,7 +1799,7 @@ if (list.length > 0) {
             </tr>
             <tr>
               <th class="modal-text">Indexes</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.indexes.count).format('0,0')%></th>
+              <th class="modal-text"> <%=numeral(figuresData.figures.indexes.count).format('0,0')%> </th>
               <th class="modal-text"> <%=prettyBytes(figuresData.figures.indexes.size)%> </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
@@ -2779,4 +2780,4 @@ var cutByResolution = function (str) {
   </div>
 
   <div id="workMonitorContent" class="innerContent">
-  </div></script></head><body><nav class="navbar" style="display: none"><div class="primary"><div class="navlogo"><a class="logo big" href="#"><img id="ArangoDBLogo" class="arangodbLogo" src="img/arangodb-edition-optimized.svg"></a><a class="logo small" href="#"><img class="arangodbLogo" src="img/arangodb_logo_small.png"></a><a class="version"><span id="currentVersion"></span></a></div><div class="statmenu" id="statisticBar"></div><div class="navmenu" id="navigationBar"></div></div></nav><div id="modalPlaceholder"></div><div class="bodyWrapper" style="display: none"><div class="centralRow"><div id="navbar2" class="navbarWrapper secondary"><div class="subnavmenu" id="subNavigationBar"></div></div><div class="resizecontainer contentWrapper"><div id="loadingScreen" class="loadingScreen" style="display: none"><i class="fa fa-circle-o-notch fa-spin fa-3x fa-fw margin-bottom"></i> <span class="sr-only">Loading...</span></div><div id="content" class="centralContent"></div><footer class="footer"><div id="footerBar"></div></footer></div></div></div><div id="progressPlaceholder" style="display:none"></div><div id="spotlightPlaceholder" style="display:none"></div><div id="graphSettingsContent" style="display: none"></div><div id="filterSelectDiv" style="display:none"></div><div id="offlinePlaceholder" style="display:none"><div class="offline-div"><div class="pure-u"><div class="pure-u-1-4"></div><div class="pure-u-1-2 offline-window"><div class="offline-header"><h3>You have been disconnected from the server</h3></div><div class="offline-body"><p>The connection to the server has been lost. The server may be under heavy load.</p><p>Trying to reconnect in <span id="offlineSeconds">10</span> seconds.</p><p class="animation_state"><span><button class="button-success">Reconnect now</button></span></p></div></div><div class="pure-u-1-4"></div></div></div></div><div class="arangoFrame" style=""><div class="outerDiv"><div class="innerDiv"></div></div></div><script src="libs.js?version=1494757959829"></script><script src="app.js?version=1494757959829"></script></body></html>
+  </div></script></head><body><nav class="navbar" style="display: none"><div class="primary"><div class="navlogo"><a class="logo big" href="#"><img id="ArangoDBLogo" class="arangodbLogo" src="img/arangodb-edition-optimized.svg"></a><a class="logo small" href="#"><img class="arangodbLogo" src="img/arangodb_logo_small.png"></a><a class="version"><span id="currentVersion"></span></a></div><div class="statmenu" id="statisticBar"></div><div class="navmenu" id="navigationBar"></div></div></nav><div id="modalPlaceholder"></div><div class="bodyWrapper" style="display: none"><div class="centralRow"><div id="navbar2" class="navbarWrapper secondary"><div class="subnavmenu" id="subNavigationBar"></div></div><div class="resizecontainer contentWrapper"><div id="loadingScreen" class="loadingScreen" style="display: none"><i class="fa fa-circle-o-notch fa-spin fa-3x fa-fw margin-bottom"></i> <span class="sr-only">Loading...</span></div><div id="content" class="centralContent"></div><footer class="footer"><div id="footerBar"></div></footer></div></div></div><div id="progressPlaceholder" style="display:none"></div><div id="spotlightPlaceholder" style="display:none"></div><div id="graphSettingsContent" style="display: none"></div><div id="filterSelectDiv" style="display:none"></div><div id="offlinePlaceholder" style="display:none"><div class="offline-div"><div class="pure-u"><div class="pure-u-1-4"></div><div class="pure-u-1-2 offline-window"><div class="offline-header"><h3>You have been disconnected from the server</h3></div><div class="offline-body"><p>The connection to the server has been lost. The server may be under heavy load.</p><p>Trying to reconnect in <span id="offlineSeconds">10</span> seconds.</p><p class="animation_state"><span><button class="button-success">Reconnect now</button></span></p></div></div><div class="pure-u-1-4"></div></div></div></div><div class="arangoFrame" style=""><div class="outerDiv"><div class="innerDiv"></div></div></div><script src="libs.js?version=1496756709895"></script><script src="app.js?version=1496756709895"></script></body></html>

js/apps/system/_admin/aardvark/APP/frontend/js/templates/modalCollectionInfo.ejs

@@ -145,6 +145,12 @@
       </div>
 
         <% if (figuresData.figures.alive && figuresData.figures.dead) { %>
+          <% if (window.frontendConfig.isCluster) { %>
+            <div class="infoBox">
+              <h4>Figures</h4>
+              <p>The following information does not contain data stored in the write-ahead log. It can be inaccurate, until the write-ahead log has been completely flushed.</p>
+            </div>
+          <% } %>
           <table class="figures1 arango-table">
             <tr class="figuresHeader">
               <th>Type</th>
@@ -155,9 +161,17 @@
             </tr>
             <tr>
               <th class="modal-text">Alive</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.alive.count).format('0,0')%></th>
+              <% if (figuresData.walMessage) { %> 
+                <th class="modal-text"><%=figuresData.walMessage%></th>
+              <% } else { %>
+                <th class="modal-text"><%=numeral(figuresData.figures.alive.count).format('0,0')%></th>
+              <% } %>
               <th class="modal-text">
-              <%=prettyBytes(figuresData.figures.alive.size)%>
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=prettyBytes(figuresData.figures.alive.size)%>
+                <% } %>
               </th>
               <th class="modal-text"> -</th>
               <th class="tooltipInfoTh">
@@ -167,11 +181,27 @@
             </tr>
             <tr>
               <th class="modal-text">Dead</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.dead.count).format('0,0')%></th>
               <th class="modal-text">
-              <%=prettyBytes(figuresData.figures.dead.size)%>
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=numeral(figuresData.figures.dead.count).format('0,0')%>
+                <% } %>
+              </th>
+              <th class="modal-text">
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=prettyBytes(figuresData.figures.dead.size)%>
+                <% } %>
+              </th>
+              <th class="modal-text">
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=figuresData.figures.dead.deletion%>
+                <% } %>
               </th>
-              <th class="modal-text"><%=figuresData.figures.dead.deletion%></th>
 
               <th class="tooltipInfoTh">
                 <div>
@@ -196,9 +226,19 @@
             </tr>
             <tr>
               <th class="modal-text">Datafiles</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.datafiles.count).format('0,0')%></th>
               <th class="modal-text">
-                <%= prettyBytes(figuresData.figures.datafiles.fileSize) %>
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=numeral(figuresData.figures.datafiles.count).format('0,0')%>
+                <% } %>
+              </th>
+              <th class="modal-text">
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%= prettyBytes(figuresData.figures.datafiles.fileSize) %>
+                <% } %>
               </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
@@ -210,9 +250,19 @@
             </tr>
             <tr>
               <th class="modal-text">Journals</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.journals.count).format('0,0')%></th>
               <th class="modal-text">
-              <%=prettyBytes(figuresData.figures.journals.fileSize)%>
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=numeral(figuresData.figures.journals.count).format('0,0')%>
+                <% } %>
+              </th>
+              <th class="modal-text">
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=prettyBytes(figuresData.figures.journals.fileSize)%>
+                <% } %>
               </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
@@ -222,9 +272,19 @@
             </tr>
             <tr>
               <th class="modal-text">Compactors</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.compactors.count).format('0,0')%></th>
               <th class="modal-text">
-              <%=prettyBytes(figuresData.figures.compactors.fileSize)%>
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=numeral(figuresData.figures.compactors.count).format('0,0')%>
+                <% } %>
+              </th>
+              <th class="modal-text">
+                <% if (figuresData.walMessage) { %>
+                  <%=figuresData.walMessage%>
+                <% } else { %>
+                  <%=prettyBytes(figuresData.figures.compactors.fileSize)%>
+                <% } %>
               </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">
@@ -233,9 +293,11 @@
             </tr>
             <tr>
               <th class="modal-text">Indexes</th>
-              <th class="modal-text"><%=numeral(figuresData.figures.indexes.count).format('0,0')%></th>
               <th class="modal-text">
-              <%=prettyBytes(figuresData.figures.indexes.size)%>
+                <%=numeral(figuresData.figures.indexes.count).format('0,0')%>
+              </th>
+              <th class="modal-text">
+                <%=prettyBytes(figuresData.figures.indexes.size)%>
               </th>
               <th class="modal-text">&nbsp;</th>
               <th class="tooltipInfoTh">

js/apps/system/_admin/aardvark/APP/frontend/js/views/infoView.js

@@ -1,6 +1,6 @@
 /* jshint browser: true */
 /* jshint unused: false */
-/* global arangoHelper, Backbone, window, $ */
+/* global arangoHelper, Backbone, window, $, frontendConfig */
 
 (function () {
   'use strict';
@@ -38,6 +38,22 @@
           arangoHelper.arangoError('Figures', 'Could not get revision.');
         } else {
           var buttons = [];
+          // analyse figures in cluster
+          if (frontendConfig.isCluster) {
+            console.log(figures);
+            if (figures.figures.alive.size === 0 &&
+              figures.figures.alive.count === 0 &&
+              figures.figures.datafiles.count === 0 &&
+              figures.figures.datafiles.fileSize === 0 &&
+              figures.figures.journals.count === 0 &&
+              figures.figures.journals.fileSize === 0 &&
+              figures.figures.compactors.count === 0 &&
+              figures.figures.compactors.fileSize === 0 &&
+              figures.figures.dead.size === 0 &&
+              figures.figures.dead.count === 0) {
+              figures.walMessage = ' - not ready yet - ';
+            }
+          }
           var tableContent = {
             figures: figures,
             revision: revision,

js/apps/system/_admin/aardvark/APP/frontend/js/views/queryView.js

@@ -187,6 +187,11 @@
       });
     },
 
+    removeInputEditors: function () {
+      this.closeAceResults(null, $('#aqlEditor'));
+      this.closeAceResults(null, $('#bindParamAceEditor'));
+    },
+
     getCustomQueryParameterByName: function (qName) {
       return this.collection.findWhere({name: qName}).get('parameter');
     },
@@ -641,7 +646,11 @@
 
     closeAceResults: function (counter, target) {
       var self = this;
-      ace.edit('outputEditor' + counter).destroy();
+      if (counter) {
+        ace.edit('outputEditor' + counter).destroy();
+      } else {
+        ace.edit($(target).attr('id')).destroy();
+      }
       $('#outputEditorWrapper' + this.outputCounter).hide();
 
       var cleanup = function (target) {

js/apps/system/_admin/aardvark/APP/frontend/scss/_modals.scss

@@ -545,3 +545,16 @@
   margin-left: -320px;
   width: 640px;
 }
+
+.infoBox {
+  border: 1px solid $c-content-border;
+  border-left-color: $c-info-blue;
+  border-left-width: 5px;
+  border-radius: 3px;
+  margin: 20px;
+  padding: 20px;
+
+  h4 {
+    color: $c-info-blue;
+  }
+}

js/server/modules/@arangodb/database-version.js

@@ -128,7 +128,7 @@ exports.databaseVersion = function () {
   }
 
   // path to the VERSION file
-  var versionFile = db._path() + '/VERSION';
+  let versionFile = db._versionFilename();
   var lastVersion = null;
 
   // VERSION file exists, read its contents

lib/ApplicationFeatures/RocksDBOptionFeature.cpp

@@ -60,6 +60,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(
           rocksDBDefaults.max_bytes_for_level_multiplier),
       _baseBackgroundCompactions(rocksDBDefaults.base_background_compactions),
       _maxBackgroundCompactions(rocksDBDefaults.max_background_compactions),
+      _maxSubcompactions(rocksDBDefaults.max_subcompactions),
       _maxFlushes(rocksDBDefaults.max_background_flushes),
       _numThreadsHigh(0),
       _numThreadsLow(0),
@@ -70,13 +71,17 @@ RocksDBOptionFeature::RocksDBOptionFeature(
       _tableBlockSize(std::max(rocksDBTableOptionsDefaults.block_size, static_cast<decltype(rocksDBTableOptionsDefaults.block_size)>(16 * 1024))),
       _recycleLogFileNum(rocksDBDefaults.recycle_log_file_num),
       _compactionReadaheadSize(rocksDBDefaults.compaction_readahead_size),
+      _level0CompactionTrigger(2),
+      _level0SlowdownTrigger(rocksDBDefaults.level0_slowdown_writes_trigger),
+      _level0StopTrigger(rocksDBDefaults.level0_stop_writes_trigger),
       _verifyChecksumsInCompaction(
           rocksDBDefaults.verify_checksums_in_compaction),
       _optimizeFiltersForHits(rocksDBDefaults.optimize_filters_for_hits),
       _useDirectReads(rocksDBDefaults.use_direct_reads),
       _useDirectWrites(rocksDBDefaults.use_direct_writes),
       _useFSync(rocksDBDefaults.use_fsync),
-      _skipCorrupted(false) {
+      _skipCorrupted(false),
+      _dynamicLevelBytes(true) {
   uint64_t testSize = _blockCacheSize >> 19;
   while (testSize > 0) {
     _blockCacheShardBits++;
@@ -111,8 +116,9 @@ void RocksDBOptionFeature::collectOptions(
   options->addHiddenOption(
       "--rocksdb.delayed_write_rate",
       "limited write rate to DB (in bytes per second) if we are writing to the "
-      "last "
+      "last mem-table allowed and we allow more than 3 mem-tables, or if we "
-      "mem table allowed and we allow more than 3 mem tables",
+      "have surpassed a certain number of level-0 files and need to slowdown "
+      "writes",
       new UInt64Parameter(&_delayedWriteRate));
 
   options->addOption("--rocksdb.min-write-buffer-number-to-merge",
@@ -129,13 +135,20 @@ void RocksDBOptionFeature::collectOptions(
                      "number of uncompressed levels for the database",
                      new UInt64Parameter(&_numUncompressedLevels));
 
-  options->addHiddenOption("--rocksdb.max-bytes-for-level-base",
+  options->addOption("--rocksdb.dynamic-level-bytes",
-                           "control maximum total data size for level-1",
+                     "if true, determine the number of bytes for each level "
-                           new UInt64Parameter(&_maxBytesForLevelBase));
+                     "dynamically to minimize space amplification",
+                     new BooleanParameter(&_dynamicLevelBytes));
+
+  options->addOption("--rocksdb.max-bytes-for-level-base",
+                     "if not using dynamic level sizes, this controls the "
+                     "maximum total data size for level-1",
+                     new UInt64Parameter(&_maxBytesForLevelBase));
 
   options->addOption("--rocksdb.max-bytes-for-level-multiplier",
-                     "maximum number of bytes for level L can be calculated as "
+                     "if not using dynamic level sizes, the maximum number of "
-                     "max-bytes-for-level-base * "
+                     "bytes for level L can be calculated as "
+                     " max-bytes-for-level-base * "
                      "(max-bytes-for-level-multiplier ^ (L-1))",
                      new DoubleParameter(&_maxBytesForLevelMultiplier));
 
@@ -149,11 +162,9 @@ void RocksDBOptionFeature::collectOptions(
       "--rocksdb.optimize-filters-for-hits",
       "this flag specifies that the implementation should optimize the filters "
       "mainly for cases where keys are found rather than also optimize for "
-      "keys "
+      "keys missed. This would be used in cases where the application knows "
-      "missed. This would be used in cases where the application knows that "
+      "that there are very few misses or the performance in the case of "
-      "there are very few misses or the performance in the case of misses is "
+      "misses is not important",
-      "not "
-      "important",
       new BooleanParameter(&_optimizeFiltersForHits));
 
 #ifdef __linux__
@@ -180,10 +191,27 @@ void RocksDBOptionFeature::collectOptions(
                      "maximum number of concurrent background compaction jobs",
                      new UInt64Parameter(&_maxBackgroundCompactions));
 
+  options->addOption("--rocksdb.max-subcompactions",
+                     "maximum number of concurrent subjobs for a background "
+                     "compaction",
+                     new UInt64Parameter(&_maxSubcompactions));
+
   options->addOption("--rocksdb.max-background-flushes",
                      "maximum number of concurrent flush operations",
                      new UInt64Parameter(&_maxFlushes));
 
+  options->addOption("--rocksdb.level0-compaction-trigger",
+                     "number of level-0 files that triggers a compaction",
+                     new UInt64Parameter(&_level0CompactionTrigger));
+
+  options->addOption("--rocksdb.level0-slowdown-trigger",
+                     "number of level-0 files that triggers a write slowdown",
+                     new UInt64Parameter(&_level0SlowdownTrigger));
+
+  options->addOption("--rocksdb.level0-stop-trigger",
+                     "number of level-0 files that triggers a full write stall",
+                     new UInt64Parameter(&_level0StopTrigger));
+
   options->addOption(
       "--rocksdb.num-threads-priority-high",
       "number of threads for high priority operations (e.g. flush)",
@@ -201,7 +229,7 @@ void RocksDBOptionFeature::collectOptions(
   options->addOption("--rocksdb.block-cache-shard-bits",
                      "number of shard bits to use for block cache",
                      new UInt64Parameter(&_blockCacheShardBits));
-  
+
   options->addOption("--rocksdb.table-block-size",
                      "approximate size (in bytes) of user data packed per block",
                      new UInt64Parameter(&_tableBlockSize));
@@ -263,6 +291,9 @@ void RocksDBOptionFeature::validateOptions(
         << "invalid value for '--rocksdb.num-threads-priority-low'";
     FATAL_ERROR_EXIT();
   }
+  if (_maxSubcompactions > _numThreadsLow) {
+    _maxSubcompactions = _numThreadsLow;
+  }
   if (_blockCacheShardBits > 32) {
     LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
         << "invalid value for '--rocksdb.block-cache-shard-bits'";

lib/ApplicationFeatures/RocksDBOptionFeature.h

@@ -57,6 +57,7 @@ class RocksDBOptionFeature final
   double _maxBytesForLevelMultiplier;
   uint64_t _baseBackgroundCompactions;
   uint64_t _maxBackgroundCompactions;
+  uint64_t _maxSubcompactions;
   uint64_t _maxFlushes;
   uint64_t _numThreadsHigh;
   uint64_t _numThreadsLow;
@@ -65,12 +66,16 @@ class RocksDBOptionFeature final
   uint64_t _tableBlockSize;
   uint64_t _recycleLogFileNum;
   uint64_t _compactionReadaheadSize;
+  uint64_t _level0CompactionTrigger;
+  uint64_t _level0SlowdownTrigger;
+  uint64_t _level0StopTrigger;
   bool _verifyChecksumsInCompaction;
   bool _optimizeFiltersForHits;
   bool _useDirectReads;
   bool _useDirectWrites;
   bool _useFSync;
   bool _skipCorrupted;
+  bool _dynamicLevelBytes;
 };
 
 }  // namespace arangodb

lib/Basics/files.cpp

@@ -2078,8 +2078,6 @@ static std::string TRI_ApplicationName = "arangodb";
 
 void TRI_SetApplicationName(char const* name) {
   TRI_ASSERT(name != nullptr);
-  TRI_ASSERT(strlen(name) <= 13);
-
   TRI_ApplicationName = name;
 }
 

scripts/generateDocumentation.sh

@@ -42,7 +42,7 @@ test_tools(){
     if test "${ARANGODB_VERSION_REVISION}" == "devel"; then
         export NODE_MODULES_DIR="/tmp/devel/node_modules"
     else
-        export NODE_MODULES_DIR="/tmp/${ARANGODB_VERSION_MAJOR}.${ARANGODB_VERSION_MINOR}"
+        export NODE_MODULES_DIR="/tmp/${ARANGODB_VERSION_MAJOR}.${ARANGODB_VERSION_MINOR}/node_modules"
     fi
 
     if test ! -d ${NODE_MODULES_DIR}; then