Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

CHANGELOG

@@ -163,6 +163,8 @@ v3.1.16 (2017-XX-XX)
 
 * fixed issue #2392
 
+* try to raise file descriptors to at least 8192, warn otherwise
+
 * ui - aql editor improvements + updated ace editor version (memory leak)
 
 * fixed lost HTTP requests

Documentation/Books/Makefile

@@ -292,7 +292,7 @@ build-book:
 	make ppbook-check-directory-link
 	make book-check-images-referenced
 
-	if test -n $(NODE_MODLUES_DIR); then \
+	if test -n "$(NODE_MODLUES_DIR)"; then \
 		cp -a $(NODE_MODLUES_DIR) ppbooks/$(NAME); \
 	else \
 		cd ppbooks/$(NAME); gitbook install -g ; \

arangod/Replication/InitialSyncer.cpp

@@ -183,10 +183,12 @@ int InitialSyncer::run(std::string& errorMsg, bool incremental) {
       return res;
     }
 
-    if (_masterInfo._majorVersion == 1 ||
+    if (incremental) {
-        (_masterInfo._majorVersion == 2 && _masterInfo._minorVersion <= 6)) {
+      if (_masterInfo._majorVersion == 1 ||
-      LOG_TOPIC(WARN, Logger::REPLICATION) << "incremental replication is not supported with a master < ArangoDB 2.7";
+          (_masterInfo._majorVersion == 2 && _masterInfo._minorVersion <= 6)) {
-      incremental = false;
+        LOG_TOPIC(WARN, Logger::REPLICATION) << "incremental replication is not supported with a master < ArangoDB 2.7";
+        incremental = false;
+      }
     }
       
     if (incremental) {
@@ -605,7 +607,7 @@ int InitialSyncer::handleCollectionDump(
     appendix = "&flush=false";
   } else {
     // only flush WAL once
-    appendix = "&flush=true&flushWait=5";
+    appendix = "&flush=true&flushWait=15";
     _hasFlushed = true;
   }
 

arangod/RestServer/FileDescriptorsFeature.cpp

@@ -33,9 +33,11 @@ using namespace arangodb::application_features;
 using namespace arangodb::basics;
 using namespace arangodb::options;
 
+uint64_t const FileDescriptorsFeature::RECOMMENDED = 8192;
+
 FileDescriptorsFeature::FileDescriptorsFeature(
     application_features::ApplicationServer* server)
-    : ApplicationFeature(server, "FileDescriptors"), _descriptorsMinimum(1024) {
+    : ApplicationFeature(server, "FileDescriptors"), _descriptorsMinimum(0) {
   setOptional(false);
   requiresElevatedPrivileges(false);
   startsAfter("Logger");
@@ -76,76 +78,84 @@ void FileDescriptorsFeature::start() {
         << StringifyLimitValue(rlim.rlim_max) << ", soft limit is "
         << StringifyLimitValue(rlim.rlim_cur);
   }
+
+  if (rlim.rlim_cur < RECOMMENDED) {
+    LOG_TOPIC(WARN, arangodb::Logger::SYSCALL)
+        << "file-descriptors limit is too low, currently "
+        << StringifyLimitValue(rlim.rlim_cur) << ", raise to at least "
+        << RECOMMENDED;
+  }
 #endif
 }
 
 void FileDescriptorsFeature::adjustFileDescriptors() {
 #ifdef TRI_HAVE_GETRLIMIT
-  if (0 < _descriptorsMinimum) {
+  struct rlimit rlim;
-    struct rlimit rlim;
+  int res = getrlimit(RLIMIT_NOFILE, &rlim);
-    int res = getrlimit(RLIMIT_NOFILE, &rlim);
 
-    if (res != 0) {
+  if (res != 0) {
-      LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
+    LOG_TOPIC(FATAL, arangodb::Logger::SYSCALL)
-          << "cannot get the file descriptor limit: " << strerror(errno);
+        << "cannot get the file descriptor limit: " << strerror(errno);
+    FATAL_ERROR_EXIT();
+  }
+
+  LOG_TOPIC(DEBUG, arangodb::Logger::SYSCALL)
+      << "file-descriptors (nofiles) hard limit is "
+      << StringifyLimitValue(rlim.rlim_max) << ", soft limit is "
+      << StringifyLimitValue(rlim.rlim_cur);
+
+  uint64_t recommended = RECOMMENDED;
+  uint64_t minimum = _descriptorsMinimum;
+
+  if (recommended < minimum) {
+    recommended = minimum;
+  }
+
+  if (rlim.rlim_max < recommended) {
+    LOG_TOPIC(DEBUG, arangodb::Logger::SYSCALL)
+        << "hard limit " << rlim.rlim_max << " is too small, trying to raise";
+
+    rlim.rlim_max = recommended;
+    rlim.rlim_cur = recommended;
+
+    res = setrlimit(RLIMIT_NOFILE, &rlim);
+
+    if (0 < minimum && minimum < recommended && res < 0) {
+      rlim.rlim_max = minimum;
+      rlim.rlim_cur = minimum;
+
+      res = setrlimit(RLIMIT_NOFILE, &rlim);
+    }
+
+    if (0 < minimum && res < 0) {
+      LOG_TOPIC(FATAL, arangodb::Logger::SYSCALL)
+          << "cannot raise the file descriptor limit to " << minimum << ": "
+          << strerror(errno);
       FATAL_ERROR_EXIT();
     }
+  } else if (rlim.rlim_cur < recommended) {
+    LOG_TOPIC(DEBUG, arangodb::Logger::SYSCALL)
+        << "soft limit " << rlim.rlim_cur << " is too small, trying to raise";
 
-    LOG_TOPIC(DEBUG, arangodb::Logger::FIXME)
+    if (recommended < rlim.rlim_max) {
-        << "file-descriptors (nofiles) hard limit is "
+      recommended = rlim.rlim_max;
-        << StringifyLimitValue(rlim.rlim_max) << ", soft limit is "
-        << StringifyLimitValue(rlim.rlim_cur);
-
-    bool changed = false;
-
-    if (rlim.rlim_max < _descriptorsMinimum) {
-      LOG_TOPIC(DEBUG, arangodb::Logger::FIXME)
-          << "hard limit " << rlim.rlim_max << " is too small, trying to raise";
-
-      rlim.rlim_max = _descriptorsMinimum;
-      rlim.rlim_cur = _descriptorsMinimum;
-
-      res = setrlimit(RLIMIT_NOFILE, &rlim);
-
-      if (res < 0) {
-        LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
-            << "cannot raise the file descriptor limit to "
-            << _descriptorsMinimum << ": " << strerror(errno);
-        FATAL_ERROR_EXIT();
-      }
-
-      changed = true;
-    } else if (rlim.rlim_cur < _descriptorsMinimum) {
-      LOG_TOPIC(DEBUG, arangodb::Logger::FIXME)
-          << "soft limit " << rlim.rlim_cur << " is too small, trying to raise";
-
-      rlim.rlim_cur = _descriptorsMinimum;
-
-      res = setrlimit(RLIMIT_NOFILE, &rlim);
-
-      if (res < 0) {
-        LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
-            << "cannot raise the file descriptor limit to "
-            << _descriptorsMinimum << ": " << strerror(errno);
-        FATAL_ERROR_EXIT();
-      }
-
-      changed = true;
     }
 
-    if (changed) {
+    rlim.rlim_cur = recommended;
-      res = getrlimit(RLIMIT_NOFILE, &rlim);
 
-      if (res != 0) {
+    res = setrlimit(RLIMIT_NOFILE, &rlim);
-        LOG_TOPIC(FATAL, arangodb::Logger::SYSCALL)
-            << "cannot get the file descriptor limit: " << strerror(errno);
-        FATAL_ERROR_EXIT();
-      }
 
-      LOG_TOPIC(INFO, arangodb::Logger::SYSCALL)
+    if (0 < minimum && minimum < recommended && res < 0) {
-          << "file-descriptors (nofiles) new hard limit is "
+      rlim.rlim_cur = minimum;
-          << StringifyLimitValue(rlim.rlim_max) << ", new soft limit is "
+
-          << StringifyLimitValue(rlim.rlim_cur);
+      res = setrlimit(RLIMIT_NOFILE, &rlim);
+    }
+
+    if (0 < minimum && res < 0) {
+      LOG_TOPIC(FATAL, arangodb::Logger::SYSCALL)
+          << "cannot raise the file descriptor limit to " << minimum << ": "
+          << strerror(errno);
+      FATAL_ERROR_EXIT();
     }
   }
 #endif

arangod/RestServer/FileDescriptorsFeature.h

@@ -28,6 +28,9 @@
 
 namespace arangodb {
 class FileDescriptorsFeature : public application_features::ApplicationFeature {
+public:
+  static uint64_t const RECOMMENDED;
+
  public:
   explicit FileDescriptorsFeature(application_features::ApplicationServer*);
 

lib/Logger/LogTopic.cpp

@@ -61,7 +61,7 @@ LogTopic Logger::REQUESTS("requests", LogLevel::FATAL);  // suppress
 LogTopic Logger::SSL("ssl", LogLevel::WARN);
 LogTopic Logger::STARTUP("startup", LogLevel::INFO);
 LogTopic Logger::SUPERVISION("supervision", LogLevel::INFO);
-LogTopic Logger::SYSCALL("syscall", LogLevel::WARN);
+LogTopic Logger::SYSCALL("syscall", LogLevel::INFO);
 LogTopic Logger::THREADS("threads", LogLevel::WARN);
 LogTopic Logger::TRANSACTIONS("trx", LogLevel::WARN);
 LogTopic Logger::V8("v8", LogLevel::WARN);

scripts/unittest

@@ -10,7 +10,7 @@ else
     PS='/'
 fi;
 
-ulimit -n 2048
+ulimit -n 8192
 
 export PORT=`expr 1024 + $RANDOM`