Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

CMakeLists.txt

@@ -708,7 +708,7 @@ if (NOT USE_BOOST_UNITTESTS)
   message(STATUS "BOOST unit-tests are disabled")
 endif ()
 
-include_directories(${Boost_INCLUDE_DIR})
+include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 add_definitions(-DARANGODB_BOOST_VERSION=\"${Boost_VERSION}\")
 
 ################################################################################

Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md

@@ -94,8 +94,19 @@ and the hash value is used to determine the target shard.
 **Note**: Values of shard key attributes cannot be changed once set.
   This option is meaningless in a single server setup.
 
+@RESTBODYPARAM{replicationFactor,integer,optional,int64} 
+(The default is *1*): in a cluster, this attribute determines how many copies 
+of each shard are kept on different DBServers. The value 1 means that only one 
+copy (no synchronous replication) is kept. A value of k means that k-1 replicas 
+are kept. Any two copies reside on different DBServers. Replication between them is 
+synchronous, that is, every write operation to the "leader" copy will be replicated 
+to all "follower" replicas, before the write operation is reported successful.
+
+If a server fails, this is detected automatically and one of the servers holding 
+copies take over, usually without an error being reported.
+
 @RESTDESCRIPTION
-Creates an new collection with a given name. The request must contain an
+Creates a new collection with a given name. The request must contain an
 object with the following attributes.
 
 

arangod/GeneralServer/HttpCommTask.cpp

@@ -47,9 +47,9 @@ class HttpResponse;
 namespace rest {
 class GeneralServer;
 
-size_t const HttpCommTask::MaximalHeaderSize = 1 * 1024 * 1024;      //   1 MB
+size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024;      //   1 MB
-size_t const HttpCommTask::MaximalBodySize = 512 * 1024 * 1024;      // 512 MB
+size_t const HttpCommTask::MaximalBodySize = 1024 * 1024 * 1024;      // 1024 MB
-size_t const HttpCommTask::MaximalPipelineSize = 512 * 1024 * 1024;  // 512 MB
+size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024;  // 512 MB
 size_t const HttpCommTask::RunCompactEvery = 500;
 
 HttpCommTask::HttpCommTask(GeneralServer* server, TRI_socket_t sock,

arangosh/Import/ImportFeature.cpp

@@ -161,6 +161,16 @@ void ImportFeature::validateOptions(
                       StringUtils::join(positionals, ", ");
     FATAL_ERROR_EXIT();
   }
+
+  static unsigned const MaxBatchSize = 768 * 1024 * 1024;
+
+  if (_chunkSize > MaxBatchSize) {
+    // it's not sensible to raise the batch size beyond this value 
+    // because the server has a built-in limit for the batch size too 
+    // and will reject bigger HTTP request bodies
+    LOG(WARN) << "capping --batch-size value to " << MaxBatchSize;
+    _chunkSize = MaxBatchSize;
+  }
 }
 
 void ImportFeature::start() {

scripts/perfanalysis.cpp

@@ -1,17 +1,20 @@
 // Compile with
 //   g++ perfanalysis.cpp -o perfanalyis -std=c++11 -Wall -O3
 
-#include <iostream>
-#include <string>
-#include <vector>
-#include <cstdlib>
-#include <cstring>
-#include <unordered_map>
 #include <algorithm>
+#include <iostream>
+#include <regex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <memory>
 
 using namespace std;
 
 struct Event {
+  static const regex re;
   string threadName;
   int tid;
   string cpu;
@@ -22,45 +25,27 @@ struct Event {
   bool isRet;
 
   Event(string& line) : isRet(false) {
-    char* s = strdup(line.c_str());
+    std::smatch match_obj;
-    char* p = strtok(s, " ");
+    if(!std::regex_search(line, match_obj, re)){
-    char* q;
+      throw std::logic_error("could not parse line");
-    if (p != nullptr) {
-      threadName = p;
-      p = strtok(nullptr, " ");
-      tid = strtol(p, nullptr, 10);
-      p = strtok(nullptr, " ");
-      cpu = p;
-      p = strtok(nullptr, " ");
-      startTime = strtod(p, nullptr);
-      p = strtok(nullptr, " ");
-      q = strtok(nullptr, " ");
-      if (strcmp(q, "cs:") == 0) {
-        free(s);
-        return;
     }
-      name = p;
+
-      name.pop_back();
+    threadName = match_obj[1];
-      auto l = name.size();
+    tid = std::stoi(match_obj[2]);
-      if (l >= 3 && name[l-1] == 't' && name[l-2] == 'e' && 
+    cpu = match_obj[3];
-          name[l-3] == 'R') {
+    startTime = std::stod(match_obj[4]);
+    duration = 0;
+    name = match_obj[6];
+    inbrackets = match_obj[7];
+    if (match_obj[9].length() > 0) {
       isRet = true;
-        name.pop_back();
+      name.erase(name.end() - 3, name.end());  // remove Ret suffix form name
-        name.pop_back();
-        name.pop_back();
     }
-      inbrackets = q;
-    }
-    free(s);
   }
 
-  bool empty() {
+  bool empty() { return name.empty(); }
-    return name.empty();
-  }
 
-  string id() {
+  string id() { return to_string(tid) + name; }
-    return to_string(tid) + name;
-  }
 
   string pretty() {
     return to_string(duration) + " " + name + " " + to_string(startTime);
@@ -77,31 +62,46 @@ struct Event {
   }
 };
 
-int main(int argc, char* argv[]) {
+// sample output:
-  unordered_map<string, Event*> table;
+// arangod 32636 [005] 16678249.324973: probe_arangod:insertLocalRet: (14a7f60 <- 14a78d6)
-  vector<Event*> list;
+// process tid   core  timepoint        scope:name                          frame
+const regex Event::re(
+    R"_(\s*(\S+))_"                                  // name 1
+    R"_(\s+(\d+))_"                                  // tid  2
+    R"_(\s+\[(\d+)\])_"                              // cup  3
+    R"_(\s+(\d+\.\d+):)_"                            // time 4
+    R"_(\s+([^: ]+):([^: ]+):)_"                     // scope:func 5:6
+    R"_(\s+\(([0-9a-f]+)(\s+<-\s+([0-9a-f]+))?\))_"  // (start -> stop) 7 -> 9
+    ,
+    std::regex_constants::ECMAScript | std::regex_constants::optimize);
+
+int main(int /*argc*/, char* /*argv*/ []) {
+  unordered_map<string, unique_ptr<Event>> table;
+  vector<unique_ptr<Event>> list;
 
   string line;
   while (getline(cin, line)) {
-    Event* e = new Event(line);
+    auto event = std::make_unique<Event>(line);
-    if (!e->empty()) {
+    if (!event->empty()) {
-      string id = e->id();
+      string id = event->id();
-      if (!e->isRet) {
+      // insert to table if it is not a function return
+      if (!event->isRet) {
         auto it = table.find(id);
         if (it != table.end()) {
           cout << "Alarm, double event found:\n" << line << std::endl;
         } else {
-          table.insert(make_pair(id, e));
+          table.insert(make_pair(id, std::move(event)));
         }
+      // update duration in table
       } else {
         auto it = table.find(id);
         if (it == table.end()) {
           cout << "Return for unknown event found:\n" << line << std::endl;
         } else {
-          Event* s = it->second;
+          unique_ptr<Event> ev = std::move(it->second);
           table.erase(it);
-          s->duration = e->startTime - s->startTime;
+          ev->duration = event->startTime - ev->startTime;
-          list.push_back(s);
+          list.push_back(std::move(ev));
         }
       }
     }
@@ -109,13 +109,11 @@ int main(int argc, char* argv[]) {
   cout << "Unreturned events:\n";
   for (auto& p : table) {
     cout << p.second->pretty() << "\n";
-    delete p.second;
   }
-  sort(list.begin(), list.end(), [](Event* a, Event* b) -> bool {
+  sort(list.begin(), list.end(),
-                                   return *a < *b;
+       [](unique_ptr<Event>const& a, unique_ptr<Event>const& b) -> bool { return *a < *b; });
-                                 });
   cout << "Events sorted by name and time:\n";
-  for (auto* e : list) {
+  for (auto& e : list) {
     cout << e->pretty() << "\n";
   }
   return 0;

scripts/setupPerfEvents.sh

@@ -6,60 +6,78 @@
 #
 # This script sets up performance monitoring events to measure single
 # document operations. Run this script with sudo when the ArangoDB
-# process is already running. Then do
+# process is already running:
+#
+#   ./setupPerfEvents.sh
+#
+# Now you are able to recrod the event with:
+#
 #   sudo perf record -e "probe_arangod:*" -aR sleep 60
-# (to sample for 60 seconds). A file "perf.data" is written to the 
+#
-# current directory.
+# The above command will get sample data for 60 seconds. A file "perf.data" is
-# Dump the events in this file with
+# written to the current directory. Dump the events in this file with:
+#
 #   sudo perf script > perf.history
+#
 # This logs the times when individual threads hit the events.
 # Use the program perfanalyis.cpp in this directory in the following way:
+#
 #   sudo ./perfanalyis < perf.history > perf.statistics
-# This will group enter and exit events of functions together, compute
+#
-# the time spent and sort by function.
+# This will group enter and exit events of functions together, compute the time
-# Remove all events with
+# spent and sort by function. When finised remove all events with:
+#
 #   sudo perf probe -d "probe_arangod:*"
-# List events with
+#
+# List events with:
+#
 #   sudo perf probe -l
+#
+#
 
-ARANGOD_EXECUTABLE=build/bin/arangod
+main(){
-perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*"
+  local ARANGOD_EXECUTABLE=${1-build/bin/arangod}
 
-echo Adding events, this takes a few seconds...
+  #delete all existing events
+  perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*"
+
+  echo "Adding events, this takes a few seconds..."
+
+  echo "Single document operations..."
+  addEvent insertLocal
+  addEvent removeLocal
+  addEvent modifyLocal
+  addEvent documentLocal
+
+  echo "Single document operations on coordinator..."
+  addEvent insertCoordinator
+  addEvent removeCoordinator
+  addEvent updateCoordinator
+  addEvent replaceCoordinator
+  addEvent documentCoordinator
+
+  echo "work method in HttpServerJob"
+  addEvent workHttpServerJob work@HttpServerJob.cpp
+
+  echo "work method in RestDocumentHandler"
+  addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp
+  addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp
+  addEvent handleRequest handleRequest@HttpServer.cpp
+  addEvent handleWrite handleWrite@SocketTask.cpp
+
+  addEvent tcp_sendmsg
+  addEvent tcp_recvmsg
+
+  echo Done.
+}
 
 addEvent() {
-  x=$1
+  local name="$1"
-  y=$2
+  local func="${2-"${name}"}"
-  if [ "x$y" == "x" ] ; then
+
-    y=$x
+  echo "setting up $name for function: $func"
-  fi
+  perf probe -x $ARANGOD_EXECUTABLE -a $name=$func 2> /dev/null             #enter function
-  echo $x
+  perf probe -x $ARANGOD_EXECUTABLE -a ${name}Ret=$func%return 2> /dev/null #return form function
-  perf probe -x $ARANGOD_EXECUTABLE -a $x=$y 2> /dev/null
-  perf probe -x $ARANGOD_EXECUTABLE -a ${x}Ret=$y%return 2> /dev/null
 }
-echo Single document operations...
-addEvent insertLocal
-addEvent removeLocal
-addEvent modifyLocal
-addEvent documentLocal
 
-echo Single document operations on coordinator...
+main "$@"
-addEvent insertCoordinator
-addEvent removeCoordinator
-addEvent updateCoordinator
-addEvent replaceCoordinator
-addEvent documentCoordinator
-
-echo work method in HttpServerJob
-addEvent workHttpServerJob work@HttpServerJob.cpp
-
-echo work method in RestDocumentHandler
-addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp
-addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp
-addEvent handleRequest handleRequest@HttpServer.cpp
-addEvent handleWrite handleWrite@SocketTask.cpp
-
-addEvent tcp_sendmsg
-addEvent tcp_recvmsg
-
-echo Done.