Merge branch 'devel' of github.com:arangodb/arangodb into devel

CHANGELOG

@@ -48,6 +48,8 @@ devel
 v3.0.6 (XXXX-XX-XX)
 -------------------
 
+* fixed issue #2026
+
 * slightly better error diagnostics for AQL query compilation and replication
 
 * fixed issue #2018

CMakeLists.txt

@@ -37,7 +37,7 @@ endif ()
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # where to find CMAKE modules
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
+set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
 
 # be verbose about flags used
 option(VERBOSE OFF)
@@ -708,7 +708,7 @@ if (NOT USE_BOOST_UNITTESTS)
   message(STATUS "BOOST unit-tests are disabled")
 endif ()
 
-include_directories(${Boost_INCLUDE_DIR})
+include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 add_definitions(-DARANGODB_BOOST_VERSION=\"${Boost_VERSION}\")
 
 ################################################################################

Documentation/Books/Manual/GettingStarted/Installing/Linux.mdpp

@@ -39,8 +39,8 @@ For unattended installation, you can set the password using the
 [debconf helpers](http://www.microhowto.info/howto/perform_an_unattended_installation_of_a_debian_package.html).
 
 ```
-echo arangodb3 arangodb/password password NEWPASSWORD | debconf-set-selections
+echo arangodb3 arangodb3/password password NEWPASSWORD | debconf-set-selections
-echo arangodb3 arangodb/password_again password NEWPASSWORD | debconf-set-selections
+echo arangodb3 arangodb3/password_again password NEWPASSWORD | debconf-set-selections
 ```
 
 The commands should be executed prior to the installation.

Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md

@@ -94,8 +94,19 @@ and the hash value is used to determine the target shard.
 **Note**: Values of shard key attributes cannot be changed once set.
   This option is meaningless in a single server setup.
 
+@RESTBODYPARAM{replicationFactor,integer,optional,int64} 
+(The default is *1*): in a cluster, this attribute determines how many copies 
+of each shard are kept on different DBServers. The value 1 means that only one 
+copy (no synchronous replication) is kept. A value of k means that k-1 replicas 
+are kept. Any two copies reside on different DBServers. Replication between them is 
+synchronous, that is, every write operation to the "leader" copy will be replicated 
+to all "follower" replicas, before the write operation is reported successful.
+
+If a server fails, this is detected automatically and one of the servers holding 
+copies take over, usually without an error being reported.
+
 @RESTDESCRIPTION
-Creates an new collection with a given name. The request must contain an
+Creates a new collection with a given name. The request must contain an
 object with the following attributes.
 
 

Installation/Jenkins/build.sh

@@ -101,16 +101,6 @@ if test -f last_compiled_version.sha;  then
 fi
 
 COMPILE_MATTERS="3rdParty"
-CLEAN_IT=1
-
-if test -n "$LASTREV"; then
-    lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l`
-
-    if test $lines -eq 0; then
-        echo "no relevant changes, no need for full recompile"
-        CLEAN_IT=0
-    fi
-fi
 
 # setup make options
 if test -z "${CXX}"; then
@@ -193,6 +183,8 @@ case "$1" in
         ;;
 esac
 
+CLEAN_IT=0
+
 
 while [ $# -gt 0 ];  do
     case "$1" in
@@ -304,6 +296,10 @@ while [ $# -gt 0 ];  do
             shift
             ;;
         
+        --checkCleanBuild)
+            CLEAN_IT=1
+            shift
+            ;;
         *)
             echo "Unknown option: $1"
             exit 1
@@ -311,6 +307,18 @@ while [ $# -gt 0 ];  do
     esac
 done
 
+
+if test -n "$LASTREV"; then
+    lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l`
+
+    if test $lines -eq 0; then
+        echo "no relevant changes, no need for full recompile"
+        CLEAN_IT=0
+    fi
+fi
+
+
+
 if [ "$GCC5" == 1 ]; then
     CC=/usr/bin/gcc-5
     CXX=/usr/bin/g++-5

Installation/Windows/Templates/NSIS.template.in

@@ -946,7 +946,7 @@ Function default_installation_directory
    Return
 FunctionEnd
 
-Function assign_proper_access_rigths
+Function assign_proper_access_rights
     StrCpy $0 "0"
     AccessControl::GrantOnFile \
      "$INSTDIR" "(BU)" "GenericRead + GenericWrite + GenericExecute"
@@ -963,7 +963,7 @@ Function is_writable
 ; is does not matter if we do some errors here
    ${If} $TRI_INSTALL_ALL_USERS == '1'
      CreateDirectory $INSTDIR
-     Call assign_proper_access_rigths
+     Call assign_proper_access_rights
     ${EndIf}
 FunctionEnd
 

Installation/Windows/client/Templates/NSIS.template.in

@@ -1,4 +1,5 @@
 ; CPack install script designed for a nmake build
+; TODO !addplugindir  '@CPACK_PLUGIN_PATH@/AccessControl/Plugins'
 
 ;--------------------------------
 ; Include LogicLib for more readable code
@@ -484,14 +485,10 @@ FunctionEnd
 
 ;--------------------------------
 ;Pages
-  !define MUI_PAGE_CUSTOMFUNCTION_PRE skip_page
   !insertmacro MUI_PAGE_WELCOME
 
-  !define MUI_PAGE_CUSTOMFUNCTION_PRE skip_page
   !insertmacro MUI_PAGE_LICENSE "@CPACK_RESOURCE_FILE_LICENSE@"
 
-  Page custom InstallOptionsPage skip_page
-
   !define MUI_PAGE_CUSTOMFUNCTION_PRE default_installation_directory
   !define MUI_PAGE_CUSTOMFUNCTION_LEAVE check_installation_directory
   !insertmacro MUI_PAGE_DIRECTORY
@@ -709,7 +706,6 @@ displayAgain:
   StrCmp $PASSWORD $PASSWORD_AGAIN +3 0
     MessageBox MB_OK|MB_ICONSTOP "Passwords don't match, try again"
     Goto displayAgain
-  done:
     Pop ${TEMP1}
 
     Return  
@@ -736,9 +732,6 @@ Function default_installation_directory
    ${If} $R4 == '1'
      StrCpy $TRI_INSTALL_TYPE 'SingleUser'
    ${EndIf}
-     Call read_options
-   ${EndIf}
-
 
   ${Switch} $TRI_INSTALL_TYPE
     ${Case} 'SingleUser'
@@ -750,24 +743,24 @@ Function default_installation_directory
    Return
 FunctionEnd
 
-Function assign_proper_access_rigths
+; TODO Function assign_proper_access_rights
-    StrCpy $0 "0"
+; TODO     StrCpy $0 "0"
-    AccessControl::GrantOnFile \
+; TODO     AccessControl::GrantOnFile \
-     "$INSTDIR" "(BU)" "GenericRead + GenericWrite + GenericExecute"
+; TODO      "$INSTDIR" "(BU)" "GenericRead + GenericWrite + GenericExecute"
-    Pop $R0
+; TODO     Pop $R0
-    ${If} $R0 == error
+; TODO     ${If} $R0 == error
-      Pop $R0
+; TODO       Pop $R0
-      StrCpy $0 "1"
+; TODO       StrCpy $0 "1"
-      DetailPrint `AccessControl error: $R0`
+; TODO       DetailPrint `AccessControl error: $R0`
-      ; MessageBox MB_OK "target directory  $INSTDIR can not get cannot get correct access rigths"
+; TODO       ; MessageBox MB_OK "target directory  $INSTDIR can not get cannot get correct access rigths"
-  ${EndIf}
+; TODO   ${EndIf}
-FunctionEnd
+; TODO FunctionEnd
 
 Function is_writable
 ; is does not matter if we do some errors here
    ${If} $TRI_INSTALL_ALL_USERS == '1'
      CreateDirectory $INSTDIR
-     Call assign_proper_access_rigths
+     ; TODO Call assign_proper_access_rights
      ${EndIf}
 FunctionEnd
 

Installation/debian/postinst

@@ -15,8 +15,9 @@ if [ "$1" = "configure" -a -z "$2" ]; then
          /usr/sbin/arango-init-database \
          --uid arangodb --gid arangodb || true
     fi
-
+    db_set arangodb3/password_again ""
     db_set arangodb3/password ""
+    db_go
 fi
 
 # check if we should upgrade the database directory

Installation/debian/preinst

@@ -2,7 +2,7 @@
 set -e
 
 getent group arangodb >/dev/null || groupadd -r arangodb 
-getent passwd arangodb >/dev/null || useradd -r -g arangodb -d /usr/share/arangodb -s /bin/false -c "ArangoDB Application User" arangodb 
+getent passwd arangodb >/dev/null || useradd -r -g arangodb -d /usr/share/arangodb3 -s /bin/false -c "ArangoDB Application User" arangodb 
 
 install -o arangodb -g arangodb -m 755 -d /var/lib/arangodb3
 install -o arangodb -g arangodb -m 755 -d /var/lib/arangodb3-apps

Installation/release.sh

@@ -50,6 +50,11 @@ fi
 
 VERSION="$1"
 
+if echo ${VERSION} | grep -q -- '-'; then
+    echo "${VERSION} mustn't contain minuses! "
+    exit 1
+fi
+
 if git tag | grep -q "^v$VERSION$";  then
   echo "$0: version $VERSION already defined"
   exit 1

arangod/Agency/AgentConfiguration.cpp

@@ -476,7 +476,7 @@ bool config_t::merge(VPackSlice const& conf) {
   ss << "Min RAFT interval: ";
   if (_minPing == 0) { // Command line beats persistence
     if (conf.hasKey(minPingStr)) {
-      _minPing = conf.get(minPingStr).getUInt();
+      _minPing = conf.get(minPingStr).getDouble();
       ss << _minPing << " (persisted)";
     } else {
       _minPing = 0.5;
@@ -491,7 +491,7 @@ bool config_t::merge(VPackSlice const& conf) {
   ss << "Max RAFT interval: ";
   if (_maxPing == 0) { // Command line beats persistence
     if (conf.hasKey(maxPingStr)) {
-      _maxPing = conf.get(maxPingStr).getUInt();
+      _maxPing = conf.get(maxPingStr).getDouble();
       ss << _maxPing << " (persisted)";
     } else {
       _maxPing = 2.5;

arangod/Aql/CollectNode.cpp

@@ -192,6 +192,7 @@ struct UserVarFinder final : public WalkerWorker<ExecutionNode> {
                en->getType() == ExecutionNode::INDEX ||
                en->getType() == ExecutionNode::ENUMERATE_LIST ||
                en->getType() == ExecutionNode::TRAVERSAL ||
+               en->getType() == ExecutionNode::SHORTEST_PATH ||
                en->getType() == ExecutionNode::COLLECT) {
       depth += 1;
     }

arangod/Aql/ExecutionNode.cpp

@@ -581,7 +581,7 @@ ExecutionNode const* ExecutionNode::getLoop() const {
     auto type = node->getType();
 
     if (type == ENUMERATE_COLLECTION || type == INDEX || type == TRAVERSAL ||
-        type == ENUMERATE_LIST) {
+        type == ENUMERATE_LIST || type == SHORTEST_PATH) {
       return node;
     }
   }
@@ -1160,7 +1160,7 @@ void ExecutionNode::RegisterPlan::after(ExecutionNode* en) {
 
         if (it2 == varInfo.end()) {
           // report an error here to prevent crashing
-          THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "missing variable #" + std::to_string(v->id) + " for node " + en->getTypeString() + " while planning registers"); 
+          THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "missing variable #" + std::to_string(v->id) + " (" + v->name + ") for node " + en->getTypeString() + " while planning registers"); 
         }
 
         // finally adjust the variable inside the IN calculation

arangod/Aql/ExecutionPlan.cpp

@@ -749,7 +749,7 @@ ExecutionNode* ExecutionPlan::fromNodeTraversal(ExecutionNode* previous,
   return addDependency(previous, en);
 }
 
-AstNode const* ExecutionPlan::parseTraversalVertexNode(ExecutionNode* previous,
+AstNode const* ExecutionPlan::parseTraversalVertexNode(ExecutionNode*& previous,
                                                        AstNode const* vertex) {
   if (vertex->type == NODE_TYPE_OBJECT && vertex->isConstant()) {
     size_t n = vertex->numMembers();
@@ -767,6 +767,8 @@ AstNode const* ExecutionPlan::parseTraversalVertexNode(ExecutionNode* previous,
     // operand is some misc expression
     auto calc = createTemporaryCalculation(vertex, previous);
     vertex = _ast->createNodeReference(getOutVariable(calc));
+    // update previous so the caller has an updated value
+    previous = calc;
   }
 
   return vertex;
@@ -1960,6 +1962,7 @@ bool ExecutionPlan::isDeadSimple() const {
         nodeType == ExecutionNode::ENUMERATE_COLLECTION ||
         nodeType == ExecutionNode::ENUMERATE_LIST ||
         nodeType == ExecutionNode::TRAVERSAL ||
+        nodeType == ExecutionNode::SHORTEST_PATH ||
         nodeType == ExecutionNode::INDEX) {
       // these node types are not simple
       return false;

arangod/Aql/ExecutionPlan.h

@@ -286,7 +286,7 @@ class ExecutionPlan {
   ExecutionNode* fromJson(arangodb::basics::Json const& Json);
 
   /// @brief create an vertex element for graph nodes
-  AstNode const* parseTraversalVertexNode(ExecutionNode*, AstNode const*);
+  AstNode const* parseTraversalVertexNode(ExecutionNode*&, AstNode const*);
 
  private:
   /// @brief map from node id to the actual node

arangod/Aql/OptimizerRules.cpp

@@ -32,6 +32,7 @@
 #include "Aql/Function.h"
 #include "Aql/IndexNode.h"
 #include "Aql/ModificationNodes.h"
+#include "Aql/ShortestPathNode.h"
 #include "Aql/SortCondition.h"
 #include "Aql/SortNode.h"
 #include "Aql/TraversalConditionFinder.h"
@@ -324,7 +325,8 @@ void arangodb::aql::removeRedundantSortsRule(Optimizer* opt,
           }
         } else if (current->getType() == EN::ENUMERATE_LIST ||
                    current->getType() == EN::ENUMERATE_COLLECTION ||
-                   current->getType() == EN::TRAVERSAL) {
+                   current->getType() == EN::TRAVERSAL ||
+                   current->getType() == EN::SHORTEST_PATH) {
           // ok, but we cannot remove two different sorts if one of these node
           // types is between them
           // example: in the following query, the one sort will be optimized
@@ -764,10 +766,10 @@ void arangodb::aql::removeSortRandRule(Optimizer* opt, ExecutionPlan* plan,
         case EN::SUBQUERY:
         case EN::ENUMERATE_LIST:
         case EN::TRAVERSAL:
+        case EN::SHORTEST_PATH:
         case EN::INDEX: {
-          // if we found another SortNode, an CollectNode, FilterNode, a
+          // if we found another SortNode, a CollectNode, FilterNode, a
-          // SubqueryNode,
+          // SubqueryNode, an EnumerateListNode, a TraversalNode or an IndexNode
-          // an EnumerateListNode, a TraversalNode or an IndexNode
           // this means we cannot apply our optimization
           collectionNode = nullptr;
           current = nullptr;
@@ -949,7 +951,9 @@ void arangodb::aql::moveCalculationsDownRule(Optimizer* opt,
       } else if (currentType == EN::INDEX ||
                  currentType == EN::ENUMERATE_COLLECTION ||
                  currentType == EN::ENUMERATE_LIST ||
-                 currentType == EN::TRAVERSAL || currentType == EN::COLLECT ||
+                 currentType == EN::TRAVERSAL || 
+                 currentType == EN::SHORTEST_PATH || 
+                 currentType == EN::COLLECT ||
                  currentType == EN::NORESULTS) {
         // we will not push further down than such nodes
         shouldMove = false;
@@ -1242,6 +1246,17 @@ class arangodb::aql::RedundantCalculationsReplacer final
       : _replacements(replacements) {
   }
   
+  template <typename T>
+  void replaceStartTargetVariables(ExecutionNode* en) {
+    auto node = static_cast<T*>(en);
+    if (node->_inStartVariable != nullptr) {
+      node->_inStartVariable = Variable::replace(node->_inStartVariable, _replacements);
+    }
+    if (node->_inTargetVariable != nullptr) {
+      node->_inTargetVariable = Variable::replace(node->_inTargetVariable, _replacements);
+    }
+  }
+
   template <typename T>
   void replaceInVariable(ExecutionNode* en) {
     auto node = static_cast<T*>(en);
@@ -1291,6 +1306,11 @@ class arangodb::aql::RedundantCalculationsReplacer final
         break;
       }
       
+      case EN::SHORTEST_PATH: {
+        replaceStartTargetVariables<ShortestPathNode>(en);
+        break;
+      }
+
       case EN::COLLECT: {
         auto node = static_cast<CollectNode*>(en);
         for (auto& variable : node->_groupVariables) {
@@ -3589,7 +3609,7 @@ void arangodb::aql::patchUpdateStatementsRule(Optimizer* opt,
         }
       }
 
-      if (type == EN::TRAVERSAL) {
+      if (type == EN::TRAVERSAL || type == EN::SHORTEST_PATH) {
         // unclear what will be read by the traversal
         modified = false;
         break;

arangod/Aql/ShortestPathNode.h

@@ -38,6 +38,7 @@ namespace aql {
 /// @brief class ShortestPathNode
 class ShortestPathNode : public ExecutionNode {
   friend class ExecutionBlock;
+  friend class RedundantCalculationsReplacer;
   friend class ShortestPathBlock;
 
   /// @brief constructor with a vocbase and a collection name

arangod/Cluster/ClusterMethods.cpp

@@ -123,6 +123,11 @@ static void mergeResults(
   resultBody->openArray();
   for (auto const& pair : reverseMapping) {
     VPackSlice arr = resultMap.find(pair.first)->second->slice();
+    if (arr.isObject() && arr.hasKey("error") && arr.get("error").isBoolean() && arr.get("error").getBoolean()) {
+      // an error occurred, now rethrow the error
+      int res = arr.get("errorNum").getNumericValue<int>();
+      THROW_ARANGO_EXCEPTION(res);
+    }
     resultBody->add(arr.at(pair.second));
   }
   resultBody->close();
@@ -736,6 +741,7 @@ int createDocumentOnCoordinator(
   bool useMultiple = slice.isArray();
 
   int res = TRI_ERROR_NO_ERROR;
+
   if (useMultiple) {
     VPackValueLength length = slice.length();
     for (VPackValueLength idx = 0; idx < length; ++idx) {
@@ -766,6 +772,7 @@ int createDocumentOnCoordinator(
   // Now prepare the requests:
   std::vector<ClusterCommRequest> requests;
   auto body = std::make_shared<std::string>();
+
   for (auto const& it : shardMap) {
     if (!useMultiple) {
       TRI_ASSERT(it.second.size() == 1);

arangod/GeneralServer/HttpCommTask.cpp

@@ -47,9 +47,9 @@ class HttpResponse;
 namespace rest {
 class GeneralServer;
 
-size_t const HttpCommTask::MaximalHeaderSize = 1 * 1024 * 1024;      //   1 MB
+size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024;       //    2 MB
-size_t const HttpCommTask::MaximalBodySize = 512 * 1024 * 1024;      // 512 MB
+size_t const HttpCommTask::MaximalBodySize = 1024 * 1024 * 1024;      // 1024 MB
-size_t const HttpCommTask::MaximalPipelineSize = 512 * 1024 * 1024;  // 512 MB
+size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024;  // 1024 MB
 size_t const HttpCommTask::RunCompactEvery = 500;
 
 HttpCommTask::HttpCommTask(GeneralServer* server, TRI_socket_t sock,

arangod/RestServer/BootstrapFeature.cpp

@@ -139,7 +139,7 @@ void BootstrapFeature::start() {
   auto vocbase = DatabaseFeature::DATABASE->vocbase();
   
   auto ss = ServerState::instance();
-  if (!ss->isRunningInCluster() && !ss->isAgent()) {
+  if (!ss->isRunningInCluster()) {
     LOG_TOPIC(DEBUG, Logger::STARTUP) << "Running server/server.js";
     V8DealerFeature::DEALER->loadJavascript(vocbase, "server/server.js");
   } else if (ss->isCoordinator()) {

arangod/VocBase/datafile.cpp

@@ -34,6 +34,7 @@
 #include "VocBase/server.h"
 
 #include <sstream>
+#include <iomanip>
 
 // #define DEBUG_DATAFILE 1
 
@@ -756,7 +757,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
         bool nextMarkerOk = false;
 
         if (size > 0) {
-          auto next = reinterpret_cast<char const*>(marker) + size;
+          auto next = reinterpret_cast<char const*>(marker) + DatafileHelper::AlignedSize<size_t>(size);
           auto p = next;
 
           if (p < end) {
@@ -810,10 +811,67 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
         datafile->_next = datafile->_data + datafile->_currentSize;
         datafile->_state = TRI_DF_STATE_OPEN_ERROR;
 
-        LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
+        LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' of size " 
+                  << datafile->_maximalSize << ", at position " << currentSize;
+
+        LOG(WARN) << "crc mismatch found inside marker of type '" << TRI_NameMarkerDatafile(marker) 
+                  << "' and size " << size
+                  << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
+ 
+        {
+          LOG(INFO) << "raw marker data following:";
+          char const* p = reinterpret_cast<char const*>(marker);
+          char const* e = reinterpret_cast<char const*>(marker) + DatafileHelper::AlignedSize<size_t>(size);
+          std::string line;
+          std::string raw;
+          size_t printed = 0;
+          while (p < e) {
+            // print offset
+            line.append("0x");
+            uintptr_t offset = static_cast<uintptr_t>(p - datafile->_data);
+            for (size_t i = 0; i < 8; ++i) {
+              uint8_t c = static_cast<uint8_t>((offset & (0xFFULL << 8 * (7 - i))) >> 8 * (7 - i));
+              uint8_t n1 = c >> 4;
+              uint8_t n2 = c & 0x0F;
+
+              line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10);
+              line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10);
+            }
+
+            // print data
+            line.append(": ");
+            for (size_t i = 0; i < 16; ++i) {
+              if (p >= e) {
+                line.append("   ");
+              } else {
+                uint8_t c = static_cast<uint8_t>(*p++);
+                uint8_t n1 = c >> 4;
+                uint8_t n2 = c & 0x0F;
+
+                line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10);
+                line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10);
+                line.push_back(' ');
+
+                raw.push_back((c < 32 || c >= 127) ? '.' : static_cast<unsigned char>(c));
+
+                ++printed;
+              }
+            }
+
+            LOG(INFO) << line << "  " << raw;
+            line.clear();
+            raw.clear();
+
+            if (printed >= 2048) {
+              LOG(INFO) << "(output truncated due to excessive length)";
+              break;
+            }
+          }
+        }
 
         if (nextMarkerOk) {
           LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it";
+          LOG(INFO) << "please restart the server with the parameter '--wal.ignore-logfile-errors true' to repair the marker";
         } else {
           LOG(WARN) << "data directly following this marker cannot be analyzed";
         }

arangosh/Import/ImportFeature.cpp

@@ -161,6 +161,16 @@ void ImportFeature::validateOptions(
                       StringUtils::join(positionals, ", ");
     FATAL_ERROR_EXIT();
   }
+
+  static unsigned const MaxBatchSize = 768 * 1024 * 1024;
+
+  if (_chunkSize > MaxBatchSize) {
+    // it's not sensible to raise the batch size beyond this value 
+    // because the server has a built-in limit for the batch size too 
+    // and will reject bigger HTTP request bodies
+    LOG(WARN) << "capping --batch-size value to " << MaxBatchSize;
+    _chunkSize = MaxBatchSize;
+  }
 }
 
 void ImportFeature::start() {

cmake/FindPythonInterp.cmake

@@ -0,0 +1,165 @@
+#.rst:
+# FindPythonInterp
+# ----------------
+#
+# Find python interpreter
+#
+# This module finds if Python interpreter is installed and determines
+# where the executables are.  This code sets the following variables:
+#
+# ::
+#
+#   PYTHONINTERP_FOUND         - Was the Python executable found
+#   PYTHON_EXECUTABLE          - path to the Python interpreter
+#
+#
+#
+# ::
+#
+#   PYTHON_VERSION_STRING      - Python version found e.g. 2.5.2
+#   PYTHON_VERSION_MAJOR       - Python major version found e.g. 2
+#   PYTHON_VERSION_MINOR       - Python minor version found e.g. 5
+#   PYTHON_VERSION_PATCH       - Python patch version found e.g. 2
+#
+#
+#
+# The Python_ADDITIONAL_VERSIONS variable can be used to specify a list
+# of version numbers that should be taken into account when searching
+# for Python.  You need to set this variable before calling
+# find_package(PythonInterp).
+#
+# If calling both ``find_package(PythonInterp)`` and
+# ``find_package(PythonLibs)``, call ``find_package(PythonInterp)`` first to
+# get the currently active Python version by default with a consistent version
+# of PYTHON_LIBRARIES.
+
+#=============================================================================
+# Copyright 2005-2010 Kitware, Inc.
+# Copyright 2011 Bjoern Ricks <bjoern.ricks@gmail.com>
+# Copyright 2012 Rolf Eike Beer <eike@sf-mail.de>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+unset(_Python_NAMES)
+
+set(_PYTHON1_VERSIONS 1.6 1.5)
+set(_PYTHON2_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0)
+set(_PYTHON3_VERSIONS 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
+
+if(PythonInterp_FIND_VERSION)
+    if(PythonInterp_FIND_VERSION_COUNT GREATER 1)
+        set(_PYTHON_FIND_MAJ_MIN "${PythonInterp_FIND_VERSION_MAJOR}.${PythonInterp_FIND_VERSION_MINOR}")
+        list(APPEND _Python_NAMES
+             python${_PYTHON_FIND_MAJ_MIN}
+             python${PythonInterp_FIND_VERSION_MAJOR})
+        unset(_PYTHON_FIND_OTHER_VERSIONS)
+        if(NOT PythonInterp_FIND_VERSION_EXACT)
+            foreach(_PYTHON_V ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS})
+                if(NOT _PYTHON_V VERSION_LESS _PYTHON_FIND_MAJ_MIN)
+                    list(APPEND _PYTHON_FIND_OTHER_VERSIONS ${_PYTHON_V})
+                endif()
+             endforeach()
+        endif()
+        unset(_PYTHON_FIND_MAJ_MIN)
+    else()
+        list(APPEND _Python_NAMES python${PythonInterp_FIND_VERSION_MAJOR})
+        set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS})
+    endif()
+else()
+    set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON3_VERSIONS} ${_PYTHON2_VERSIONS} ${_PYTHON1_VERSIONS})
+endif()
+find_program(PYTHON_EXECUTABLE NAMES ${_Python_NAMES})
+
+# Set up the versions we know about, in the order we will search. Always add
+# the user supplied additional versions to the front.
+set(_Python_VERSIONS ${Python_ADDITIONAL_VERSIONS})
+# If FindPythonInterp has already found the major and minor version,
+# insert that version next to get consistent versions of the interpreter and
+# library.
+if(DEFINED PYTHONLIBS_VERSION_STRING)
+  string(REPLACE "." ";" _PYTHONLIBS_VERSION "${PYTHONLIBS_VERSION_STRING}")
+  list(GET _PYTHONLIBS_VERSION 0 _PYTHONLIBS_VERSION_MAJOR)
+  list(GET _PYTHONLIBS_VERSION 1 _PYTHONLIBS_VERSION_MINOR)
+  list(APPEND _Python_VERSIONS ${_PYTHONLIBS_VERSION_MAJOR}.${_PYTHONLIBS_VERSION_MINOR})
+endif()
+# Search for the current active python version first
+list(APPEND _Python_VERSIONS ";")
+list(APPEND _Python_VERSIONS ${_PYTHON_FIND_OTHER_VERSIONS})
+
+unset(_PYTHON_FIND_OTHER_VERSIONS)
+unset(_PYTHON1_VERSIONS)
+unset(_PYTHON2_VERSIONS)
+unset(_PYTHON3_VERSIONS)
+
+# Search for newest python version if python executable isn't found
+if(NOT PYTHON_EXECUTABLE)
+    foreach(_CURRENT_VERSION IN LISTS _Python_VERSIONS)
+      set(_Python_NAMES python${_CURRENT_VERSION})
+      if(WIN32)
+        list(APPEND _Python_NAMES python)
+      endif()
+      find_program(PYTHON_EXECUTABLE
+        NAMES ${_Python_NAMES}
+        PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]
+        )
+    endforeach()
+endif()
+
+# determine python version string
+if(PYTHON_EXECUTABLE)
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c
+                            "import sys; sys.stdout.write(';'.join([str(x) for x in sys.version_info[:3]]))"
+                    OUTPUT_VARIABLE _VERSION
+                    RESULT_VARIABLE _PYTHON_VERSION_RESULT
+                    ERROR_QUIET)
+    if(NOT _PYTHON_VERSION_RESULT)
+        string(REPLACE ";" "." PYTHON_VERSION_STRING "${_VERSION}")
+        list(GET _VERSION 0 PYTHON_VERSION_MAJOR)
+        list(GET _VERSION 1 PYTHON_VERSION_MINOR)
+        list(GET _VERSION 2 PYTHON_VERSION_PATCH)
+        if(PYTHON_VERSION_PATCH EQUAL 0)
+            # it's called "Python 2.7", not "2.7.0"
+            string(REGEX REPLACE "\\.0$" "" PYTHON_VERSION_STRING "${PYTHON_VERSION_STRING}")
+        endif()
+    else()
+        # sys.version predates sys.version_info, so use that
+        execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c "import sys; sys.stdout.write(sys.version)"
+                        OUTPUT_VARIABLE _VERSION
+                        RESULT_VARIABLE _PYTHON_VERSION_RESULT
+                        ERROR_QUIET)
+        if(NOT _PYTHON_VERSION_RESULT)
+            string(REGEX REPLACE " .*" "" PYTHON_VERSION_STRING "${_VERSION}")
+            string(REGEX REPLACE "^([0-9]+)\\.[0-9]+.*" "\\1" PYTHON_VERSION_MAJOR "${PYTHON_VERSION_STRING}")
+            string(REGEX REPLACE "^[0-9]+\\.([0-9])+.*" "\\1" PYTHON_VERSION_MINOR "${PYTHON_VERSION_STRING}")
+            if(PYTHON_VERSION_STRING MATCHES "^[0-9]+\\.[0-9]+\\.([0-9]+)")
+                set(PYTHON_VERSION_PATCH "${CMAKE_MATCH_1}")
+            else()
+                set(PYTHON_VERSION_PATCH "0")
+            endif()
+        else()
+            # sys.version was first documented for Python 1.5, so assume
+            # this is older.
+            set(PYTHON_VERSION_STRING "1.4")
+            set(PYTHON_VERSION_MAJOR "1")
+            set(PYTHON_VERSION_MINOR "4")
+            set(PYTHON_VERSION_PATCH "0")
+        endif()
+    endif()
+    unset(_PYTHON_VERSION_RESULT)
+    unset(_VERSION)
+endif()
+
+# handle the QUIETLY and REQUIRED arguments and set PYTHONINTERP_FOUND to TRUE if
+# all listed variables are TRUE
+include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonInterp REQUIRED_VARS PYTHON_EXECUTABLE VERSION_VAR PYTHON_VERSION_STRING)
+
+mark_as_advanced(PYTHON_EXECUTABLE)

cmake/packages/client/deb.txt

@@ -80,11 +80,12 @@ if(CMAKE_TARGET_ARCHITECTURES MATCHES ".*x86_64.*")
 elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "aarch64")
   set(ARANGODB_PACKAGE_ARCHITECTURE "arm64")
 elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "armv7")
-  set(ARANGODB_PACKAGE_ARCHITECTURE "arm7")
+  set(ARANGODB_PACKAGE_ARCHITECTURE "armhf")
 else()
   set(ARANGODB_PACKAGE_ARCHITECTURE "i386")
 endif()
 
+set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE ${ARANGODB_PACKAGE_ARCHITECTURE})
 set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${ARANGODB_PACKAGE_REVISION}_${ARANGODB_PACKAGE_ARCHITECTURE}")
 
 ################################################################################
@@ -98,3 +99,12 @@ include(${ORIGINAL_SOURCE_DIR}/cmake/InstallArangoDBJSClient.cmake)
 include(${ORIGINAL_SOURCE_DIR}/arangosh/install.cmake)
 
 include(CPack)
+
+# TODO: missing usr/share/man
+# TODO: missing usr/share/man/man1
+# TODO: missing usr/share/man/man1/arangobench.1
+# TODO: missing usr/share/man/man1/arangodump.1
+# TODO: missing usr/share/man/man1/arangoimp.1
+# TODO: missing usr/share/man/man1/arangorestore.1
+# TODO: missing usr/share/man/man1/arangosh.1
+# usr/share/man/man8/foxx-manager.8

cmake/packages/deb.cmake

@@ -24,10 +24,11 @@ if(CMAKE_TARGET_ARCHITECTURES MATCHES ".*x86_64.*")
 elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "aarch64")
   set(ARANGODB_PACKAGE_ARCHITECTURE "arm64")
 elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "armv7")
-  set(ARANGODB_PACKAGE_ARCHITECTURE "arm7")
+  set(ARANGODB_PACKAGE_ARCHITECTURE "armhf")
 else()
   set(ARANGODB_PACKAGE_ARCHITECTURE "i386")
 endif()
+set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE ${ARANGODB_PACKAGE_ARCHITECTURE})
 set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${ARANGODB_PACKAGE_REVISION}_${ARANGODB_PACKAGE_ARCHITECTURE}")
 
 

js/client/modules/@arangodb/index.js

@@ -105,7 +105,7 @@ exports.plainServerVersion = function () {
       devel = version.match(/(.*)((alpha|beta|devel|rc)[0-9]*)$/);
 
       if (devel !== null) {
-        version = devel[1];
+        version = devel[1] + '0';
       }
     }
 

js/server/modules/@arangodb/foxx/queues/index.js

@@ -24,6 +24,7 @@
 // //////////////////////////////////////////////////////////////////////////////
 
 const isCluster = require("@arangodb/cluster").isCluster();
+const isAgent = global.ArangoAgent.enabled();
 
 var _ = require('lodash');
 var flatten = require('internal').flatten;
@@ -231,7 +232,7 @@ function asNumber (num) {
 }
 
 function updateQueueDelayClusterAware() {
-  if (isCluster) {
+  if (isCluster && !isAgent) {
     global.ArangoAgency.set('Current/FoxxmasterQueueupdate', true);
   }
   updateQueueDelay();

js/server/modules/@arangodb/index.js

@@ -86,7 +86,7 @@ exports.plainServerVersion = function () {
     devel = version.match(/(.*)((alpha|beta|devel|rc)[0-9]*)$/);
 
     if (devel !== null) {
-      version = devel[1];
+      version = devel[1] + '0';;
     }
   }
 

lib/SimpleHttpClient/SimpleHttpResult.cpp

@@ -162,7 +162,7 @@ void SimpleHttpResult::addHeaderField(char const* key, size_t keyLength,
 
           if (_returnCode == 204) {
             // HTTP 204 = No content. Assume we will have a content-length of 0.
-            // note that will value can be overridden later if the response has the content-length
+            // note that the value can be overridden later if the response has the content-length
             // header set to some other value
             setContentLength(0);
           }

scripts/build-deb.sh

@@ -14,5 +14,6 @@ cd ${DIR}/..
     --buildDir build-deb \
     --targetDir /var/tmp/ \
     --jemalloc \
+    $@
 
 cd ${DIR}/..

scripts/build-nsis.sh

@@ -11,6 +11,7 @@ cd ${DIR}/..
     --msvc \
     --buildDir /cygdrive/c/b/y/ \
     --package NSIS \
-    --targetDir /var/tmp/ 
+    --targetDir /var/tmp/ \
+    $@
 
 cd ${DIR}/..

scripts/build-rpm.sh

@@ -13,5 +13,6 @@ cd ${DIR}/..
     --buildDir build-rpm \
     --targetDir /var/tmp/ \
     --jemalloc \
+    $@
 
 cd ${DIR}/..

scripts/perfanalysis.cpp

@@ -1,17 +1,20 @@
 // Compile with
 //   g++ perfanalysis.cpp -o perfanalyis -std=c++11 -Wall -O3
 
-#include <iostream>
-#include <string>
-#include <vector>
-#include <cstdlib>
-#include <cstring>
-#include <unordered_map>
 #include <algorithm>
+#include <iostream>
+#include <regex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <memory>
 
 using namespace std;
 
 struct Event {
+  static const regex re;
   string threadName;
   int tid;
   string cpu;
@@ -22,45 +25,27 @@ struct Event {
   bool isRet;
 
   Event(string& line) : isRet(false) {
-    char* s = strdup(line.c_str());
+    std::smatch match_obj;
-    char* p = strtok(s, " ");
+    if(!std::regex_search(line, match_obj, re)){
-    char* q;
+      throw std::logic_error("could not parse line");
-    if (p != nullptr) {
-      threadName = p;
-      p = strtok(nullptr, " ");
-      tid = strtol(p, nullptr, 10);
-      p = strtok(nullptr, " ");
-      cpu = p;
-      p = strtok(nullptr, " ");
-      startTime = strtod(p, nullptr);
-      p = strtok(nullptr, " ");
-      q = strtok(nullptr, " ");
-      if (strcmp(q, "cs:") == 0) {
-        free(s);
-        return;
     }
-      name = p;
+
-      name.pop_back();
+    threadName = match_obj[1];
-      auto l = name.size();
+    tid = std::stoi(match_obj[2]);
-      if (l >= 3 && name[l-1] == 't' && name[l-2] == 'e' && 
+    cpu = match_obj[3];
-          name[l-3] == 'R') {
+    startTime = std::stod(match_obj[4]);
+    duration = 0;
+    name = match_obj[6];
+    inbrackets = match_obj[7];
+    if (match_obj[9].length() > 0) {
       isRet = true;
-        name.pop_back();
+      name.erase(name.end() - 3, name.end());  // remove Ret suffix form name
-        name.pop_back();
-        name.pop_back();
     }
-      inbrackets = q;
-    }
-    free(s);
   }
 
-  bool empty() {
+  bool empty() { return name.empty(); }
-    return name.empty();
-  }
 
-  string id() {
+  string id() { return to_string(tid) + name; }
-    return to_string(tid) + name;
-  }
 
   string pretty() {
     return to_string(duration) + " " + name + " " + to_string(startTime);
@@ -77,31 +62,46 @@ struct Event {
   }
 };
 
-int main(int argc, char* argv[]) {
+// sample output:
-  unordered_map<string, Event*> table;
+// arangod 32636 [005] 16678249.324973: probe_arangod:insertLocalRet: (14a7f60 <- 14a78d6)
-  vector<Event*> list;
+// process tid   core  timepoint        scope:name                          frame
+const regex Event::re(
+    R"_(\s*(\S+))_"                                  // name 1
+    R"_(\s+(\d+))_"                                  // tid  2
+    R"_(\s+\[(\d+)\])_"                              // cup  3
+    R"_(\s+(\d+\.\d+):)_"                            // time 4
+    R"_(\s+([^: ]+):([^: ]+):)_"                     // scope:func 5:6
+    R"_(\s+\(([0-9a-f]+)(\s+<-\s+([0-9a-f]+))?\))_"  // (start -> stop) 7 -> 9
+    ,
+    std::regex_constants::ECMAScript | std::regex_constants::optimize);
+
+int main(int /*argc*/, char* /*argv*/ []) {
+  unordered_map<string, unique_ptr<Event>> table;
+  vector<unique_ptr<Event>> list;
 
   string line;
   while (getline(cin, line)) {
-    Event* e = new Event(line);
+    auto event = std::make_unique<Event>(line);
-    if (!e->empty()) {
+    if (!event->empty()) {
-      string id = e->id();
+      string id = event->id();
-      if (!e->isRet) {
+      // insert to table if it is not a function return
+      if (!event->isRet) {
         auto it = table.find(id);
         if (it != table.end()) {
           cout << "Alarm, double event found:\n" << line << std::endl;
         } else {
-          table.insert(make_pair(id, e));
+          table.insert(make_pair(id, std::move(event)));
         }
+      // update duration in table
       } else {
         auto it = table.find(id);
         if (it == table.end()) {
           cout << "Return for unknown event found:\n" << line << std::endl;
         } else {
-          Event* s = it->second;
+          unique_ptr<Event> ev = std::move(it->second);
           table.erase(it);
-          s->duration = e->startTime - s->startTime;
+          ev->duration = event->startTime - ev->startTime;
-          list.push_back(s);
+          list.push_back(std::move(ev));
         }
       }
     }
@@ -109,13 +109,11 @@ int main(int argc, char* argv[]) {
   cout << "Unreturned events:\n";
   for (auto& p : table) {
     cout << p.second->pretty() << "\n";
-    delete p.second;
   }
-  sort(list.begin(), list.end(), [](Event* a, Event* b) -> bool {
+  sort(list.begin(), list.end(),
-                                   return *a < *b;
+       [](unique_ptr<Event>const& a, unique_ptr<Event>const& b) -> bool { return *a < *b; });
-                                 });
   cout << "Events sorted by name and time:\n";
-  for (auto* e : list) {
+  for (auto& e : list) {
     cout << e->pretty() << "\n";
   }
   return 0;

scripts/setupPerfEvents.sh

@@ -6,54 +6,60 @@
 #
 # This script sets up performance monitoring events to measure single
 # document operations. Run this script with sudo when the ArangoDB
-# process is already running. Then do
+# process is already running:
+#
+#   ./setupPerfEvents.sh
+#
+# Now you are able to recrod the event with:
+#
 #   sudo perf record -e "probe_arangod:*" -aR sleep 60
-# (to sample for 60 seconds). A file "perf.data" is written to the 
+#
-# current directory.
+# The above command will get sample data for 60 seconds. A file "perf.data" is
-# Dump the events in this file with
+# written to the current directory. Dump the events in this file with:
+#
 #   sudo perf script > perf.history
+#
 # This logs the times when individual threads hit the events.
 # Use the program perfanalyis.cpp in this directory in the following way:
+#
 #   sudo ./perfanalyis < perf.history > perf.statistics
-# This will group enter and exit events of functions together, compute
+#
-# the time spent and sort by function.
+# This will group enter and exit events of functions together, compute the time
-# Remove all events with
+# spent and sort by function. When finised remove all events with:
+#
 #   sudo perf probe -d "probe_arangod:*"
-# List events with
+#
+# List events with:
+#
 #   sudo perf probe -l
+#
+#
 
-ARANGOD_EXECUTABLE=build/bin/arangod
+main(){
+  local ARANGOD_EXECUTABLE=${1-build/bin/arangod}
+
+  #delete all existing events
   perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*"
 
-echo Adding events, this takes a few seconds...
+  echo "Adding events, this takes a few seconds..."
 
-addEvent() {
+  echo "Single document operations..."
-  x=$1
-  y=$2
-  if [ "x$y" == "x" ] ; then
-    y=$x
-  fi
-  echo $x
-  perf probe -x $ARANGOD_EXECUTABLE -a $x=$y 2> /dev/null
-  perf probe -x $ARANGOD_EXECUTABLE -a ${x}Ret=$y%return 2> /dev/null
-}
-echo Single document operations...
   addEvent insertLocal
   addEvent removeLocal
   addEvent modifyLocal
   addEvent documentLocal
 
-echo Single document operations on coordinator...
+  echo "Single document operations on coordinator..."
   addEvent insertCoordinator
   addEvent removeCoordinator
   addEvent updateCoordinator
   addEvent replaceCoordinator
   addEvent documentCoordinator
 
-echo work method in HttpServerJob
+  echo "work method in HttpServerJob"
   addEvent workHttpServerJob work@HttpServerJob.cpp
 
-echo work method in RestDocumentHandler
+  echo "work method in RestDocumentHandler"
   addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp
   addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp
   addEvent handleRequest handleRequest@HttpServer.cpp
@@ -63,3 +69,15 @@ addEvent tcp_sendmsg
   addEvent tcp_recvmsg
 
   echo Done.
+}
+
+addEvent() {
+  local name="$1"
+  local func="${2-"${name}"}"
+
+  echo "setting up $name for function: $func"
+  perf probe -x $ARANGOD_EXECUTABLE -a $name=$func 2> /dev/null             #enter function
+  perf probe -x $ARANGOD_EXECUTABLE -a ${name}Ret=$func%return 2> /dev/null #return form function
+}
+
+main "$@"

scripts/startLocalCluster.sh

@@ -96,6 +96,7 @@ start() {
     PORT=$2
     mkdir cluster/data$PORT
     echo Starting $TYPE on port $PORT
+    mkdir -p cluster/apps$PORT 
     build/bin/arangod -c none \
                 --database.directory cluster/data$PORT \
                 --cluster.agency-endpoint tcp://127.0.0.1:$BASE \