Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

CHANGELOG

@@ -5,6 +5,18 @@ devel
 
 * changed index filling to make it more parallel, dispatch tasks to boost::asio
 
+* more detailed stacktraces in Foxx apps
+
+
+v3.1.10 (2017-XX-XX)
+--------------------
+
+* updated versions of bundled node modules:
+  - joi: from 8.4.2 to 9.2.0
+  - joi-to-json-schema: from 2.2.0 to 2.3.0
+  - sinon: from 1.17.4 to 1.17.6
+  - lodash: from 4.13.1 to 4.16.6
+
 * added shortcut for AQL ternary operator
   instead of `condition ? true-part : false-part` it is now possible to also use a
   shortcut variant `condition ? : false-part`, e.g.
@@ -74,6 +86,13 @@ edge attribute `label`.
 
 * added option -D to define a configuration file environment key=value
 
+* changed encoding behavior for URLs encoded in the C++ code of ArangoDB:
+  previously the special characters `-`, `_`, `~` and `.` were returned as-is 
+  after URL-encoding, now `.` will be encoded to be `%2e`. 
+  This also changes the behavior of how incoming URIs are processed: previously
+  occurrences of `..` in incoming request URIs were collapsed (e.g. `a/../b/` was 
+  collapsed to a plain `b/`). Now `..` in incoming request URIs are not collapsed.
+
 * Foxx request URL suffix is no longer unescaped
 
 * @arangodb/request option json now defaults to `true` if the response body is not empty and encoding is not explicitly set to `null` (binary).

Documentation/DocuBlocks/schedulerBackend.md

@@ -1,10 +0,0 @@
-
-
-@brief scheduler backend
-`--scheduler.backend arg`
-
-The I/O method used by the event handler. The default (if this option is
-not specified) is to try all recommended backends. This is platform
-specific. See libev for further details and the meaning of select, poll
-and epoll.
-

Installation/systemd/arangodb3.service.in

@@ -30,7 +30,8 @@ ExecStartPre=@CHOWN_EXECUTABLE@ -R arangodb:arangodb /var/lib/arangodb3-apps
 ExecStartPre=@CHMOD_EXECUTABLE@ 700 /var/lib/arangodb3-apps
 ExecStartPre=/usr/sbin/arangod --uid arangodb --gid arangodb --pid-file /var/run/arangodb3/arangod.pid --server.rest-server false --database.auto-upgrade true
 ExecStart=/usr/sbin/arangod    --uid arangodb --gid arangodb --pid-file /var/run/arangodb3/arangod.pid --temp.path /var/tmp/arangodb3 --supervisor --log.foreground-tty false
-TimeoutStopSec=120
+TimeoutStopSec=3600
+TimeoutSec=3600
 Restart=on-failure
 
 [Install]

LICENSES-OTHER-COMPONENTS.md

@@ -51,11 +51,6 @@
 * Project Home: http://site.icu-project.org/
 * License: [ICU License](http://source.icu-project.org/repos/icu/icu/trunk/license.html)
 
-### libev 4.11
-
-* Project Home: http://software.schmorp.de/pkg/libev.html
-* License: Dual-License [BSD-style 2-Clause License](http://cvs.schmorp.de/libev/LICENSE?revision=1.11&view=markup)
-
 ### linenoise-ng
 
 * GitHub: https://github.com/arangodb/linenoise-ng

arangod/Agency/AgencyComm.cpp

@@ -612,7 +612,7 @@ std::string AgencyCommManager::redirect(
     << specification << ", url = " << rest;
   
   if (endpoint == specification) {
-    LOG_TOPIC(WARN, Logger::AGENCYCOMM)
+    LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
       << "got an agency redirect back to the old agency '" << endpoint << "'";
     failedNonLocking(std::move(connection), endpoint);
     return "";
@@ -632,7 +632,7 @@ std::string AgencyCommManager::redirect(
     std::remove(_endpoints.begin(), _endpoints.end(), specification),
     _endpoints.end());
   
-  LOG_TOPIC(WARN, Logger::AGENCYCOMM)
+  LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
     << "Got an agency redirect from '" << endpoint
     << "' to '" << specification << "'";
   
@@ -1360,15 +1360,9 @@ AgencyCommResult AgencyComm::sendWithFailover(
       return result;
     }
 
-    if (1 < tries) {
+    double elapsed = 1.e-2 * (
-      LOG_TOPIC(WARN, Logger::AGENCYCOMM)
+      std::round(1.e+2 * std::chrono::duration<double>(
-        << "Retrying agency communication at '" << endpoint
+                   std::chrono::steady_clock::now() - started).count()));
-        << "', tries: " << tries << " ("
-        << 1.e-2 * (
-          std::round(
-            1.e+2 * std::chrono::duration<double>(
-              std::chrono::steady_clock::now() - started).count())) << "s)";
-    }
 
     // try to send; if we fail completely, do not retry
     try {
@@ -1470,6 +1464,26 @@ AgencyCommResult AgencyComm::sendWithFailover(
       break;
     }
 
+    if (tries%50 == 0) {
+      LOG_TOPIC(WARN, Logger::AGENCYCOMM)
+        << "Bad agency communiction! Unsuccessful consecutive tries:"
+        << tries << " (" << elapsed << "s). Network checks needed!";
+    } else if (tries%15 == 0) {
+      LOG_TOPIC(INFO, Logger::AGENCYCOMM)
+        << "Flaky agency communication. Unsuccessful consecutive tries: "
+        << tries << " (" << elapsed << "s). Network checks advised.";
+    } 
+    
+    if (1 < tries) {
+      LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
+        << "Retrying agency communication at '" << endpoint
+        << "', tries: " << tries << " ("
+        << 1.e-2 * (
+          std::round(
+            1.e+2 * std::chrono::duration<double>(
+              std::chrono::steady_clock::now() - started).count())) << "s)";
+    }
+    
     // here we have failed and want to try next endpoint
     AgencyCommManager::MANAGER->failed(std::move(connection), endpoint);
     endpoint.clear();

arangod/Aql/AqlItemBlock.cpp

@@ -22,6 +22,8 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "AqlItemBlock.h"
+#include "Aql/BlockCollector.h"
+#include "Aql/ExecutionBlock.h"
 #include "Aql/ExecutionNode.h"
 #include "Basics/VelocyPackHelper.h"
 
@@ -430,6 +432,37 @@ AqlItemBlock* AqlItemBlock::steal(std::vector<size_t> const& chosen, size_t from
   return res.release();
 }
 
+/// @brief concatenate multiple blocks
+AqlItemBlock* AqlItemBlock::concatenate(ResourceMonitor* resourceMonitor,
+    BlockCollector* collector) {
+  
+  size_t totalSize = collector->totalSize();
+  RegisterId nrRegs = collector->nrRegs();
+
+  TRI_ASSERT(totalSize > 0);
+  TRI_ASSERT(nrRegs > 0);
+
+  auto res = std::make_unique<AqlItemBlock>(resourceMonitor, totalSize, nrRegs);
+
+  size_t pos = 0;
+  for (auto& it : collector->_blocks) {
+    size_t const n = it->size();
+    for (size_t row = 0; row < n; ++row) {
+      for (RegisterId col = 0; col < nrRegs; ++col) {
+        // copy over value
+        AqlValue const& a = it->getValueReference(row, col);
+        if (!a.isEmpty()) {
+          res->setValue(pos + row, col, a);
+        }
+      }
+    }
+    it->eraseAll();
+    pos += n;
+  }
+
+  return res.release();
+}
+
 /// @brief concatenate multiple blocks, note that the new block now owns all
 /// AqlValue pointers in the old blocks, therefore, the latter are all
 /// set to nullptr, just to be sure.
@@ -455,7 +488,6 @@ AqlItemBlock* AqlItemBlock::concatenate(ResourceMonitor* resourceMonitor,
 
   size_t pos = 0;
   for (auto& it : blocks) {
-    TRI_ASSERT(it != res.get());
     size_t const n = it->size();
     for (size_t row = 0; row < n; ++row) {
       for (RegisterId col = 0; col < nrRegs; ++col) {
@@ -467,7 +499,7 @@ AqlItemBlock* AqlItemBlock::concatenate(ResourceMonitor* resourceMonitor,
       }
     }
     it->eraseAll();
-    pos += it->size();
+    pos += n;
   }
 
   return res.release();

arangod/Aql/AqlItemBlock.h

@@ -32,6 +32,7 @@
 
 namespace arangodb {
 namespace aql {
+class BlockCollector;
 
 // an <AqlItemBlock> is a <nrItems>x<nrRegs> vector of <AqlValue>s (not
 // pointers). The size of an <AqlItemBlock> is the number of items.
@@ -255,6 +256,9 @@ class AqlItemBlock {
   /// to which our AqlValues point will vanish.
   AqlItemBlock* steal(std::vector<size_t> const& chosen, size_t from, size_t to);
   
+  /// @brief concatenate multiple blocks from a collector
+  static AqlItemBlock* concatenate(ResourceMonitor*, BlockCollector* collector);
+
   /// @brief concatenate multiple blocks, note that the new block now owns all
   /// AqlValue pointers in the old blocks, therefore, the latter are all
   /// set to nullptr, just to be sure.

arangod/Aql/Ast.cpp

@@ -2749,6 +2749,12 @@ AstNode* Ast::optimizeFunctionCall(AstNode* node) {
         return createNodeFunctionCall("COLLECTION_COUNT", countArgs);
       }
     }
+  } else if (func->externalName == "IS_NULL") {
+    auto args = node->getMember(0);
+    if (args->numMembers() == 1) {
+      // replace IS_NULL(x) function call with `x == null`
+      return createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, args->getMemberUnchecked(0), createNodeValueNull()); 
+    }
   }
 
   if (!func->isDeterministic) {

arangod/Aql/BasicBlocks.cpp

@@ -22,12 +22,18 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "BasicBlocks.h"
+#include "Aql/AqlItemBlock.h"
 #include "Aql/ExecutionEngine.h"
 #include "Basics/Exceptions.h"
 #include "VocBase/vocbase.h"
 
 using namespace arangodb::aql;
   
+void SingletonBlock::deleteInputVariables() {
+  delete _inputRegisterValues;
+  _inputRegisterValues = nullptr;
+}
+
 void SingletonBlock::buildWhitelist() {
   if (!_whitelistBuilt) {
     auto en = static_cast<SingletonNode const*>(getPlanNode());
@@ -155,6 +161,11 @@ FilterBlock::FilterBlock(ExecutionEngine* engine, FilterNode const* en)
 
 FilterBlock::~FilterBlock() {}
   
+/// @brief internal function to actually decide if the document should be used
+bool FilterBlock::takeItem(AqlItemBlock* items, size_t index) const {
+  return items->getValueReference(index, _inReg).toBoolean();
+}
+
 /// @brief internal function to get another block
 bool FilterBlock::getBlock(size_t atLeast, size_t atMost) {
   DEBUG_BEGIN_BLOCK();  

arangod/Aql/BasicBlocks.h

@@ -59,10 +59,7 @@ class SingletonBlock : public ExecutionBlock {
   int64_t remaining() override final { return _done ? 0 : 1; }
 
  private:
-  void deleteInputVariables() {
+  void deleteInputVariables();
-    delete _inputRegisterValues;
-    _inputRegisterValues = nullptr;
-  }
 
   void buildWhitelist();
 
@@ -85,9 +82,7 @@ class FilterBlock : public ExecutionBlock {
 
  private:
   /// @brief internal function to actually decide if the document should be used
-  inline bool takeItem(AqlItemBlock* items, size_t index) const {
+  bool takeItem(AqlItemBlock* items, size_t index) const;
-    return items->getValueReference(index, _inReg).toBoolean();
-  }
 
   /// @brief internal function to get another block
   bool getBlock(size_t atLeast, size_t atMost);

arangod/Aql/BlockCollector.cpp

@@ -0,0 +1,84 @@
+////////////////////////////////////////////////////////////////////////////////
+/// @brief Infrastructure for ExecutionPlans
+///
+/// DISCLAIMER
+///
+/// Copyright 2010-2014 triagens GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is triAGENS GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+/// @author Copyright 2014, triagens GmbH, Cologne, Germany
+////////////////////////////////////////////////////////////////////////////////
+
+#include "BlockCollector.h"
+#include "Aql/AqlItemBlock.h"
+#include "Aql/ResourceUsage.h"
+
+using namespace arangodb::aql;
+
+BlockCollector::BlockCollector() : _totalSize(0) {}
+
+BlockCollector::~BlockCollector() { clear(); }
+
+size_t BlockCollector::totalSize() const { return _totalSize; }
+
+size_t BlockCollector::nrRegs() const {
+  TRI_ASSERT(_totalSize > 0);
+  TRI_ASSERT(!_blocks.empty());
+  return _blocks[0]->getNrRegs();
+}
+
+void BlockCollector::clear() {
+  for (auto& it : _blocks) {
+    it->eraseAll();
+    delete it;
+  }
+  _blocks.clear();
+  _totalSize = 0;
+}
+  
+void BlockCollector::add(std::unique_ptr<AqlItemBlock> block) {
+  TRI_ASSERT(block->size() > 0);
+
+  _blocks.push_back(block.get());
+  _totalSize += block->size();
+  block.release();
+}
+
+AqlItemBlock* BlockCollector::steal(ResourceMonitor* resourceMonitor) {
+  if (_blocks.empty()) {
+    return nullptr;
+  }
+
+  TRI_ASSERT(_totalSize > 0);
+  AqlItemBlock* result = nullptr;
+
+  if (_blocks.size() == 1) {
+    // only got a single result. return it as it is
+    result = _blocks[0];
+  } else {
+    result = AqlItemBlock::concatenate(resourceMonitor, this);
+    for (auto& it : _blocks) {
+      delete it;
+    }
+  }
+
+  // ownership is now passed to result
+  _totalSize = 0;
+  _blocks.clear();
+  return result;
+}
+

arangod/Aql/BlockCollector.h

@@ -0,0 +1,61 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGOD_AQL_BLOCK_COLLECTOR_H
+#define ARANGOD_AQL_BLOCK_COLLECTOR_H 1
+
+#include "Basics/Common.h"
+
+namespace arangodb {
+namespace aql {
+class AqlItemBlock;
+class ResourceMonitor;
+
+class BlockCollector {
+  friend class AqlItemBlock;
+
+ public:
+  BlockCollector(BlockCollector const&) = delete;
+  BlockCollector& operator=(BlockCollector const&) = delete;
+
+  BlockCollector();
+  ~BlockCollector();
+
+  size_t totalSize() const;
+  size_t nrRegs() const;
+
+  void clear();
+  
+  void add(std::unique_ptr<AqlItemBlock> block);
+
+  AqlItemBlock* steal(ResourceMonitor*);
+
+ private:
+  std::vector<AqlItemBlock*> _blocks;
+  size_t _totalSize;
+};
+
+}  // namespace arangodb::aql
+}  // namespace arangodb
+
+#endif

arangod/Aql/ClusterBlocks.cpp

@@ -29,6 +29,7 @@
 #include <velocypack/Slice.h>
 #include <velocypack/velocypack-aliases.h>
 
+#include "Aql/AqlItemBlock.h"
 #include "Aql/AqlValue.h"
 #include "Aql/ExecutionEngine.h"
 #include "Basics/Exceptions.h"

arangod/Aql/ExecutionBlock.cpp

@@ -23,8 +23,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "ExecutionBlock.h"
-#include "Aql/ExecutionEngine.h"
+#include "Aql/AqlItemBlock.h"
 #include "Aql/Ast.h"
+#include "Aql/ExecutionEngine.h"
 
 using namespace arangodb::aql;
   

arangod/Aql/ExecutionBlock.h

@@ -24,7 +24,6 @@
 #ifndef ARANGOD_AQL_EXECUTION_BLOCK_H
 #define ARANGOD_AQL_EXECUTION_BLOCK_H 1
 
-#include "AqlItemBlock.h"
 #include "Aql/ExecutionNode.h"
 #include "Aql/Variable.h"
 
@@ -61,6 +60,7 @@ namespace arangodb {
 class Transaction;
 
 namespace aql {
+class AqlItemBlock;
 
 /// @brief sort element for block, consisting of register, sort direction, 
 /// and a possible attribute path to dig into the document

arangod/Aql/ExecutionEngine.cpp

@@ -1042,7 +1042,7 @@ struct CoordinatorInstanciator : public WalkerWorker<ExecutionNode> {
           id = TRI_NewTickServer();
 
           try {
-            queryRegistry->insert(id, engine->getQuery(), 3600.0);
+            queryRegistry->insert(id, engine->getQuery(), 600.0);
           } catch (...) {
             delete engine->getQuery();
             // This deletes the new query as well as the engine

arangod/Aql/IndexBlock.cpp

@@ -470,6 +470,7 @@ bool IndexBlock::readIndex(size_t atMost) {
 
 int IndexBlock::initializeCursor(AqlItemBlock* items, size_t pos) {
   DEBUG_BEGIN_BLOCK();
+  _collector.clear();
   int res = ExecutionBlock::initializeCursor(items, pos);
 
   if (res != TRI_ERROR_NO_ERROR) {
@@ -492,7 +493,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
   traceGetSomeBegin();
   if (_done) {
     traceGetSomeEnd(nullptr);
-    return nullptr;
+    return _collector.steal(_engine->getQuery()->resourceMonitor());
   }
  
   std::unique_ptr<AqlItemBlock> res;
@@ -508,7 +509,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
       if (!ExecutionBlock::getBlock(toFetch, toFetch) || (!initIndexes())) {
         _done = true;
         traceGetSomeEnd(nullptr);
-        return nullptr;
+        return _collector.steal(_engine->getQuery()->resourceMonitor());
       }
       _pos = 0;  // this is in the first block
 
@@ -530,7 +531,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
           if (!ExecutionBlock::getBlock(DefaultBatchSize(), DefaultBatchSize())) {
             _done = true;
             traceGetSomeEnd(nullptr);
-            return nullptr;
+            return _collector.steal(_engine->getQuery()->resourceMonitor());
           }
           _pos = 0;  // this is in the first block
         }
@@ -538,7 +539,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
         if (!initIndexes()) {
           _done = true;
           traceGetSomeEnd(nullptr);
-          return nullptr;
+          return _collector.steal(_engine->getQuery()->resourceMonitor());
         }
         readIndex(atMost);
       }
@@ -581,6 +582,13 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
           res->copyValuesFromFirstRow(j, static_cast<RegisterId>(curRegs));
         }
       }
+
+      _collector.add(std::move(res));
+      TRI_ASSERT(res.get() == nullptr);
+
+      if (_collector.totalSize() >= atMost) {
+        res.reset(_collector.steal(_engine->getQuery()->resourceMonitor()));
+      }
     }
 
   } while (res.get() == nullptr);
@@ -588,6 +596,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
   // Clear out registers no longer needed later:
   clearRegisters(res.get());
   traceGetSomeEnd(res.get());
+  
   return res.release();
 
   // cppcheck-suppress style

arangod/Aql/IndexBlock.h

@@ -25,6 +25,7 @@
 #ifndef ARANGOD_AQL_INDEX_BLOCK_H
 #define ARANGOD_AQL_INDEX_BLOCK_H 1
 
+#include "Aql/BlockCollector.h"
 #include "Aql/ExecutionBlock.h"
 #include "Aql/ExecutionNode.h"
 #include "Aql/IndexNode.h"
@@ -155,6 +156,8 @@ class IndexBlock : public ExecutionBlock {
   bool _hasV8Expression;
   
   std::unique_ptr<ManagedDocumentResult> _mmdr;
+
+  BlockCollector _collector;
 };
 
 }  // namespace arangodb::aql

arangod/Aql/QueryRegistry.h

@@ -45,7 +45,7 @@ class QueryRegistry {
   /// a query for this <vocbase> and <id> combination and an exception will
   /// be thrown in that case. The time to live <ttl> is in seconds and the
   /// query will be deleted if it is not opened for that amount of time.
-  void insert(QueryId id, Query* query, double ttl = 3600.0);
+  void insert(QueryId id, Query* query, double ttl = 600.0);
 
   /// @brief open, find a query in the registry, if none is found, a nullptr
   /// is returned, otherwise, ownership of the query is transferred to the

arangod/Aql/RestAqlHandler.cpp

@@ -105,7 +105,7 @@ void RestAqlHandler::createQueryFromVelocyPack() {
   }
 
   // Now the query is ready to go, store it in the registry and return:
-  double ttl = 3600.0;
+  double ttl = 600.0;
   bool found;
   std::string const& ttlstring = _request->header("ttl", found);
 
@@ -317,7 +317,7 @@ void RestAqlHandler::createQueryFromString() {
   }
 
   // Now the query is ready to go, store it in the registry and return:
-  double ttl = 3600.0;
+  double ttl = 600.0;
   bool found;
   std::string const& ttlstring = _request->header("ttl", found);
 

arangod/CMakeLists.txt

@@ -115,6 +115,7 @@ SET(ARANGOD_SOURCES
   Aql/BaseExpressionContext.cpp
   Aql/BasicBlocks.cpp
   Aql/BindParameters.cpp
+  Aql/BlockCollector.cpp
   Aql/CalculationBlock.cpp
   Aql/ClusterBlocks.cpp
   Aql/ClusterNodes.cpp

arangod/Cluster/HeartbeatThread.cpp

@@ -87,6 +87,60 @@ HeartbeatThread::HeartbeatThread(AgencyCallbackRegistry* agencyCallbackRegistry,
 
 HeartbeatThread::~HeartbeatThread() { shutdown(); }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief running of heartbeat background jobs (in JavaScript), we run
+/// these by instantiating an object in class HeartbeatBackgroundJob,
+/// which is a std::function<void()> and holds a shared_ptr to the
+/// HeartbeatThread singleton itself. This instance is then posted to
+/// the io_service for execution in the thread pool. Should the heartbeat
+/// thread itself terminate during shutdown, then the HeartbeatThread
+/// singleton itself is still kept alive by the shared_ptr in the instance
+/// of HeartbeatBackgroundJob. The operator() method simply calls the
+/// runBackgroundJob() method of the heartbeat thread. Should this have
+/// to schedule another background job, then it can simply create a new
+/// HeartbeatBackgroundJob instance, again using shared_from_this() to
+/// create a new shared_ptr keeping the HeartbeatThread object alive.
+////////////////////////////////////////////////////////////////////////////////
+
+class HeartbeatBackgroundJob {
+  std::shared_ptr<HeartbeatThread> _heartbeatThread;
+ public:
+  explicit HeartbeatBackgroundJob(std::shared_ptr<HeartbeatThread> hbt)
+    : _heartbeatThread(hbt) {}
+
+  void operator()() {
+    _heartbeatThread->runBackgroundJob();
+  }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief method runBackgroundJob()
+////////////////////////////////////////////////////////////////////////////////
+
+void HeartbeatThread::runBackgroundJob() {
+  uint64_t jobNr = ++_backgroundJobsLaunched;
+  LOG_TOPIC(INFO, Logger::HEARTBEAT) << "sync callback started " << jobNr;
+  {
+    DBServerAgencySync job(this);
+    job.work();
+  }
+  LOG_TOPIC(INFO, Logger::HEARTBEAT) << "sync callback ended " << jobNr;
+
+  {
+    MUTEX_LOCKER(mutexLocker, *_statusLock);
+    TRI_ASSERT(_backgroundJobScheduledOrRunning);
+    if (_launchAnotherBackgroundJob) {
+      jobNr = ++_backgroundJobsPosted;
+      LOG_TOPIC(INFO, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr;
+      _launchAnotherBackgroundJob = false;
+      _ioService->post(HeartbeatBackgroundJob(shared_from_this()));
+    } else {
+      _backgroundJobScheduledOrRunning = false;
+      _launchAnotherBackgroundJob = false;
+    }
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief heartbeat main loop
 /// the heartbeat thread constantly reports the current server status to the
@@ -104,31 +158,6 @@ void HeartbeatThread::run() {
   if (ServerState::instance()->isCoordinator()) {
     runCoordinator();
   } else {
-    // Set the member variable that holds a closure to run background
-    // jobs in JS:
-    auto self = shared_from_this();
-    _backgroundJob = [self, this]() {
-      uint64_t jobNr = ++_backgroundJobsLaunched;
-      LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback started " << jobNr;
-      {
-        DBServerAgencySync job(this);
-        job.work();
-      }
-      LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback ended " << jobNr;
-
-      {
-        MUTEX_LOCKER(mutexLocker, *_statusLock);
-        if (_launchAnotherBackgroundJob) {
-          LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail "
-            << ++_backgroundJobsPosted;
-          _launchAnotherBackgroundJob = false;
-          _ioService->post(_backgroundJob);
-        } else {
-          _backgroundJobScheduledOrRunning = false;
-          _launchAnotherBackgroundJob = false;
-        }
-      }
-    };
     runDBServer();
   }
 }
@@ -340,16 +369,6 @@ void HeartbeatThread::runDBServer() {
   }
 
   _agencyCallbackRegistry->unregisterCallback(planAgencyCallback);
-  int count = 0;
-  while (++count < 3000) {
-    {
-      MUTEX_LOCKER(mutexLocker, *_statusLock);
-      if (!_backgroundJobScheduledOrRunning) {
-        break;
-      }
-    }
-    usleep(100000);
-  }
   LOG_TOPIC(TRACE, Logger::HEARTBEAT)
       << "stopped heartbeat thread (DBServer version)";
 }
@@ -593,7 +612,7 @@ bool HeartbeatThread::init() {
 ////////////////////////////////////////////////////////////////////////////////
 
 void HeartbeatThread::dispatchedJobResult(DBServerAgencySyncResult result) {
-  LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Dispatched job returned!";
+  LOG_TOPIC(INFO, Logger::HEARTBEAT) << "Dispatched job returned!";
   bool doSleep = false;
   {
     MUTEX_LOCKER(mutexLocker, *_statusLock);
@@ -777,10 +796,10 @@ void HeartbeatThread::syncDBServerStatusQuo() {
   }
 
   // schedule a job for the change:
-  LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync "
+  uint64_t jobNr = ++_backgroundJobsPosted;
-    << ++_backgroundJobsPosted;
+  LOG_TOPIC(INFO, Logger::HEARTBEAT) << "dispatching sync " << jobNr;
   _backgroundJobScheduledOrRunning = true;
-  _ioService->post(_backgroundJob);
+  _ioService->post(HeartbeatBackgroundJob(shared_from_this()));
 }
 
 ////////////////////////////////////////////////////////////////////////////////

arangod/Cluster/HeartbeatThread.h

@@ -74,6 +74,8 @@ class HeartbeatThread : public Thread,
 
   void setReady() { _ready.store(true); }
 
+  void runBackgroundJob();
+
   void dispatchedJobResult(DBServerAgencySyncResult);
 
   //////////////////////////////////////////////////////////////////////////////
@@ -253,12 +255,6 @@ class HeartbeatThread : public Thread,
   //////////////////////////////////////////////////////////////////////////////
   
   bool _launchAnotherBackgroundJob;
-
-  //////////////////////////////////////////////////////////////////////////////
-  /// @brief _backgroundJob, the closure that does the work
-  //////////////////////////////////////////////////////////////////////////////
-  
-  std::function<void()> _backgroundJob;
 };
 }
 

arangod/Indexes/PathBasedIndex.cpp

@@ -124,7 +124,7 @@ int PathBasedIndex::fillElement(std::vector<T*>& elements,
     auto slices = buildIndexValue(doc);
 
     if (slices.size() == n) {
-      // if shapes.size() != n, then the value is not inserted into the index
+      // if slices.size() != n, then the value is not inserted into the index
       // because of index sparsity!
       T* element = static_cast<T*>(_allocator->allocate());
       TRI_ASSERT(element != nullptr);

arangod/Wal/LogfileManager.cpp

@@ -876,7 +876,7 @@ int LogfileManager::flush(bool waitForSync, bool waitForCollector,
       res = this->waitForCollector(lastOpenLogfileId, maxWaitTime);
 
       if (res == TRI_ERROR_LOCK_TIMEOUT) {
-        LOG(ERR) << "got lock timeout when waiting for WAL flush. lastOpenLogfileId: " << lastOpenLogfileId;
+        LOG(DEBUG) << "got lock timeout when waiting for WAL flush. lastOpenLogfileId: " << lastOpenLogfileId;
       }
     } else if (res == TRI_ERROR_ARANGO_DATAFILE_EMPTY) {
       // current logfile is empty and cannot be collected
@@ -887,7 +887,7 @@ int LogfileManager::flush(bool waitForSync, bool waitForCollector,
         res = this->waitForCollector(lastSealedLogfileId, maxWaitTime);
       
         if (res == TRI_ERROR_LOCK_TIMEOUT) {
-          LOG(ERR) << "got lock timeout when waiting for WAL flush. lastSealedLogfileId: " << lastSealedLogfileId;
+          LOG(DEBUG) << "got lock timeout when waiting for WAL flush. lastSealedLogfileId: " << lastSealedLogfileId;
         }
       }
     }
@@ -1731,8 +1731,7 @@ int LogfileManager::waitForCollector(Logfile::IdType logfileId,
     // try again
   }
 
-  // TODO: remove debug info here
+  LOG(DEBUG) << "going into lock timeout. having waited for logfile: " << logfileId << ", maxWaitTime: " << maxWaitTime;
-  LOG(ERR) << "going into lock timeout. having waited for logfile: " << logfileId << ", maxWaitTime: " << maxWaitTime;
   logStatus();
 
   // waited for too long
@@ -1740,11 +1739,10 @@ int LogfileManager::waitForCollector(Logfile::IdType logfileId,
 }
   
 void LogfileManager::logStatus() {
-  // TODO: remove debug info here
+  LOG(DEBUG) << "logfile manager status report: lastCollectedId: " << _lastCollectedId.load() << ", lastSealedId: " << _lastSealedId.load();
-  LOG(ERR) << "logfile manager status report: lastCollectedId: " << _lastCollectedId.load() << ", lastSealedId: " << _lastSealedId.load();
   READ_LOCKER(locker, _logfilesLock);
   for (auto logfile : _logfiles) {
-    LOG(ERR) << "- logfile " << logfile.second->id() << ", filename '" << logfile.second->filename()
+    LOG(DEBUG) << "- logfile " << logfile.second->id() << ", filename '" << logfile.second->filename()
                << "', status " << logfile.second->statusText();
   }
 }

etc/testing/arangod-common.conf

@@ -2,6 +2,7 @@
 force-direct = true
 level = info
 level = replication=warn
+level = threads=debug
 
 [database]
 force-sync-properties = false

js/server/modules/@arangodb/cluster.js

@@ -745,7 +745,7 @@ function executePlanForCollections(plannedCollections) {
               let collection;
               if (!localCollections.hasOwnProperty(shardName)) {
                 // must create this shard
-                console.info("creating local shard '%s/%s' for central '%s/%s'",
+                console.debug("creating local shard '%s/%s' for central '%s/%s'",
                   database,
                   shardName,
                   database,
@@ -813,7 +813,7 @@ function executePlanForCollections(plannedCollections) {
                 }, {});
 
                 if (Object.keys(properties).length > 0) {
-                  console.info("updating properties for local shard '%s/%s'",
+                  console.debug("updating properties for local shard '%s/%s'",
                     database,
                     shardName);
 
@@ -831,17 +831,17 @@ function executePlanForCollections(plannedCollections) {
 
               // Now check whether the status is OK:
               if (collectionStatus !== collectionInfo.status) {
-                console.info("detected status change for local shard '%s/%s'",
+                console.debug("detected status change for local shard '%s/%s'",
                   database,
                   shardName);
 
                 if (collectionInfo.status === ArangoCollection.STATUS_UNLOADED) {
-                  console.info("unloading local shard '%s/%s'",
+                  console.debug("unloading local shard '%s/%s'",
                     database,
                     shardName);
                   collection.unload();
                 } else if (collectionInfo.status === ArangoCollection.STATUS_LOADED) {
-                  console.info("loading local shard '%s/%s'",
+                  console.debug("loading local shard '%s/%s'",
                     database,
                     shardName);
                   collection.load();
@@ -1264,7 +1264,7 @@ function executePlanForDatabases(plannedDatabases) {
     if (!plannedDatabases.hasOwnProperty(name) && name.substr(0, 1) !== '_') {
       // must drop database
 
-      console.info("dropping local database '%s'", name);
+      console.debug("dropping local database '%s'", name);
 
       // Do we have to stop a replication applier first?
       if (ArangoServerState.role() === 'SECONDARY') {
@@ -1273,7 +1273,7 @@ function executePlanForDatabases(plannedDatabases) {
           var rep = require('@arangodb/replication');
           var state = rep.applier.state();
           if (state.state.running === true) {
-            console.info('stopping replication applier first');
+            console.debug('stopping replication applier first');
             rep.applier.stop();
           }
         }

js/server/tests/aql/aql-optimizer-geoindex.js

@@ -53,23 +53,8 @@ function optimizerRuleTestSuite() {
     sorted : true
   };
 
-  var ruleName = "use-geoindex";
+  var ruleName = "geoindex";
-  var secondRuleName = "use-geoindexes";
-  var removeCalculationNodes = "remove-unnecessary-calculations-2";
   var colName = "UnitTestsAqlOptimizer" + ruleName.replace(/-/g, "_");
-  var colNameOther = colName + "_XX";
-
-  // various choices to control the optimizer: 
-  var paramNone = { optimizer: { rules: [ "-all" ] } };
-  var paramIndexFromSort  = { optimizer: { rules: [ "-all", "+" + ruleName ] } };
-  var paramIndexRange   = { optimizer: { rules: [ "-all", "+" + secondRuleName ] } };
-  var paramIndexFromSort_IndexRange = { optimizer: { rules: [ "-all", "+" + ruleName, "+" + secondRuleName ] } };
-  var paramIndexFromSort_IndexRange_RemoveCalculations = {
-    optimizer: { rules: [ "-all", "+" + ruleName, "+" + secondRuleName, "+" + removeCalculationNodes ] }
-  };
-  var paramIndexFromSort_RemoveCalculations = {
-    optimizer: { rules: [ "-all", "+" + ruleName, "+" + removeCalculationNodes ] }
-  };
 
   var geocol;
   var sortArray = function (l, r) {
@@ -113,19 +98,6 @@ function optimizerRuleTestSuite() {
   };
 
   var geodistance = function(latitude1, longitude1, latitude2, longitude2) {
-    //if (TYPEWEIGHT(latitude1) !== TYPEWEIGHT_NUMBER ||
-    //  TYPEWEIGHT(longitude1) !== TYPEWEIGHT_NUMBER ||
-    //  TYPEWEIGHT(latitude2) !== TYPEWEIGHT_NUMBER ||
-    //  TYPEWEIGHT(longitude2) !== TYPEWEIGHT_NUMBER) {
-    //  WARN('DISTANCE', INTERNAL.errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH);
-    //  return null;
-    //}
-
-    //var p1 = AQL_TO_NUMBER(latitude1) * (Math.PI / 180.0);
-    //var p2 = AQL_TO_NUMBER(latitude2) * (Math.PI / 180.0);
-    //var d1 = AQL_TO_NUMBER(latitude2 - latitude1) * (Math.PI / 180.0);
-    //var d2 = AQL_TO_NUMBER(longitude2 - longitude1) * (Math.PI / 180.0);
-
     var p1 = (latitude1) * (Math.PI / 180.0);
     var p2 = (latitude2) * (Math.PI / 180.0);
     var d1 = (latitude2 - latitude1) * (Math.PI / 180.0);
@@ -165,7 +137,6 @@ function optimizerRuleTestSuite() {
 
     tearDown : function () {
       internal.db._drop(colName);
-      internal.db._drop(colNameOther);
       geocol = null;
     },
 
@@ -215,14 +186,6 @@ function optimizerRuleTestSuite() {
         queries.forEach(function(query) {
           var result = AQL_EXPLAIN(query.string);
 
-          // //optimized on cluster
-          // if (query[1]) {
-          //   assertNotEqual(-1, removeAlwaysOnClusterRules(result.plan.rules).indexOf(ruleName), query[0]);
-          // }
-          // else {
-          //   assertEqual(-1, removeAlwaysOnClusterRules(result.plan.rules).indexOf(ruleName), query[0]);
-          // }
-
           //sort nodes
           if (query.sort) {
             hasSortNode(result,query);
@@ -268,7 +231,6 @@ function optimizerRuleTestSuite() {
           var pairs = result.json.map(function(res){
               return [res.lat,res.lon];
           });
-          //internal.print(pairs)
           assertEqual(expected[qindex].sort(),pairs.sort());
           //expect(expected[qindex].sort()).to.be.equal(result.json.sort())
         });

js/server/tests/resilience/moving-shards-cluster.js

@@ -163,7 +163,7 @@ function MovingShardsSuite () {
       }
 
       if (!ok) {
-        console.info(
+        console.error(
           "Failed: Server " + id + " was not cleaned out. List of cleaned servers: ["
             + obj.cleanedServers + "]");
       }

lib/Basics/ArangoGlobalContext.cpp

@@ -289,6 +289,23 @@ void ArangoGlobalContext::runStartupChecks() {
              "necessary to set the value in '"
           << filename << "' to 2";
     }
+    std::string const proc_cpuinfo_filename("/proc/cpuinfo");
+    try {
+      std::string const cpuInfo =
+          arangodb::basics::FileUtils::slurp(proc_cpuinfo_filename);
+      auto start = cpuInfo.find("ARMv6");
+
+      if (start != std::string::npos) {
+          LOG(FATAL)
+              << "possibly incompatible ARMv6 CPU detected.";
+          FATAL_ERROR_EXIT();
+        }
+    } catch (...) {
+      // ignore that we cannot detect the alignment
+      LOG(TRACE)
+          << "unable to detect CPU type '"
+          << filename << "'";
+    }
   }
 #endif
 }

lib/Logger/Logger.cpp

@@ -84,7 +84,7 @@ void Logger::setLogLevel(std::string const& levelName) {
 
   if (l == "fatal") {
     level = LogLevel::FATAL;
-  } else if (l == "error") {
+  } else if (l == "error" || l == "err") {
     level = LogLevel::ERR;
   } else if (l == "warning" || l == "warn") {
     level = LogLevel::WARN;