Feature/make rspec timeouteable (#9354)

* make rspec timeouteable (on windows for now)

* add timeout state, fix windows implementation

* abort testrun if a timeout in rspec occured

* lint

* fix scope of procdump stopper

* fix include scope

* fix invokation

* use proper variable to wait for

js/client/modules/@arangodb/process-utils.js

@@ -542,7 +542,7 @@ function stopProcdump (options, instanceInfo, force = false) {
       print(Date() + " sending TERM to procdump to make it exit");
       instanceInfo.monitor.status = killExternal(instanceInfo.monitor.pid, termSignal);
     } else {
-      print(Date() + " wating for procdump to exit");
+      print(Date() + " waiting for procdump to exit");
       statusExternal(instanceInfo.monitor.pid, true);
     }
     instanceInfo.monitor.pid = null;
@@ -561,7 +561,7 @@ function killWithCoreDump (options, instanceInfo) {
 // / @brief executes a command and waits for result
 // //////////////////////////////////////////////////////////////////////////////
 
-function executeAndWait (cmd, args, options, valgrindTest, rootDir, circumventCores, coreCheck = false) {
+function executeAndWait (cmd, args, options, valgrindTest, rootDir, circumventCores, coreCheck = false, timeout = 0) {
   if (valgrindTest && options.valgrind) {
     let valgrindOpts = {};
 
@@ -620,7 +620,19 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir, circumventCo
     instanceInfo.exitStatus = res;
     if (runProcdump(options, instanceInfo, rootDir, res.pid)) {
       Object.assign(instanceInfo.exitStatus, 
-                    statusExternal(res.pid, true));
+                    statusExternal(res.pid, true, timeout * 1000));
+      if (instanceInfo.exitStatus.status === 'TIMEOUT') {
+        print('Timeout while running ' + cmd + ' - will kill it now! ' + JSON.stringify(args));
+        killExternal(res.pid);
+        stopProcdump(options, instanceInfo);
+        instanceInfo.exitStatus.status = 'ABORTED';
+        const deltaTime = time() - startTime;
+        return {
+          status: false,
+          message: 'irregular termination by TIMEOUT',
+          duration: deltaTime
+        };
+      }
       stopProcdump(options, instanceInfo);
     } else {
       print('Killing ' + cmd + ' - ' + JSON.stringify(args));
@@ -629,7 +641,7 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir, circumventCo
       instanceInfo.exitStatus = res;
     }
   } else {
-    res = executeExternalAndWait(cmd, args);
+    res = executeExternalAndWait(cmd, args, false, timeout);
     instanceInfo.pid = res.pid;
     instanceInfo.exitStatus = res;
   }

js/client/modules/@arangodb/test-utils.js

@@ -615,7 +615,7 @@ function runThere (options, instanceInfo, file) {
 
     let httpOptions = pu.makeAuthorizationHeaders(options);
     httpOptions.method = 'POST';
-    httpOptions.timeout = 2700;
+    httpOptions.timeout = options.oneTestTimeout;
 
     if (options.valgrind) {
       httpOptions.timeout *= 2;
@@ -886,7 +886,17 @@ function runInRSpec (options, instanceInfo, file, addArgs) {
     args = [rspec].concat(args);
   }
 
-  const res = pu.executeAndWait(command, args, options, 'arangosh', instanceInfo.rootDir, false, false);
+  const res = pu.executeAndWait(command, args, options, 'arangosh', instanceInfo.rootDir, false, false, options.oneTestTimeout);
+
+  if (!res.status) {
+    return {
+      total: 0,
+      failed: 1,
+      status: false,
+      forceTerminate: true,
+      message: res.message
+    };
+  }
 
   let result = {
     total: 0,

js/client/modules/@arangodb/testing.js

@@ -74,6 +74,7 @@ let optionsDocumentation = [
   '   - `agency`: if set to true agency tests are done',
   '   - `agencySize`: number of agents in agency',
   '   - `agencySupervision`: run supervision in agency',
+  '   - `oneTestTimeout`: how long a single testsuite (.js, .rb)  should run',
   '   - `test`: path to single test to execute for "single" test target',
   '   - `cleanup`: if set to true (the default), the cluster data files',
   '     and logs are removed after termination of the test.',
@@ -166,6 +167,7 @@ const optionsDefaults = {
   'skipNondeterministic': false,
   'skipGrey': false,
   'onlyGrey': false,
+  'oneTestTimeout': 2700,
   'skipTimeCritical': false,
   'storageEngine': 'rocksdb',
   'test': undefined,

lib/ApplicationFeatures/DaemonFeature.cpp

@@ -28,6 +28,7 @@
 #include <sys/types.h>
 #include <fcntl.h>
 
+#include "Basics/process-utils.h"
 #include "Basics/FileUtils.h"
 #include "Basics/StringUtils.h"
 #include "Logger/LogAppender.h"

lib/ApplicationFeatures/DaemonFeature.h

@@ -25,8 +25,6 @@
 
 #include "ApplicationFeatures/ApplicationFeature.h"
 
-#include "Basics/process-utils.h"
-
 namespace arangodb {
 
 class DaemonFeature final : public application_features::ApplicationFeature {

lib/ApplicationFeatures/SupervisorFeature.cpp

@@ -22,6 +22,7 @@
 
 #include "SupervisorFeature.h"
 
+#include "Basics/process-utils.h"
 #include "ApplicationFeatures/DaemonFeature.h"
 #include "Basics/ArangoGlobalContext.h"
 #include "Logger/LogAppender.h"

lib/Basics/process-utils.cpp

@@ -960,7 +960,7 @@ void TRI_CreateExternalProcess(char const* executable,
 /// @brief returns the status of an external process
 ////////////////////////////////////////////////////////////////////////////////
 
-ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait) {
+ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait, uint32_t timeout) {
   ExternalProcessStatus status;
   status._status = TRI_EXT_NOT_FOUND;
   status._exitStatus = 0;
@@ -1062,7 +1062,11 @@ ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait) {
       bool wantGetExitCode = wait;
       if (wait) {
         DWORD result;
-        result = WaitForSingleObject(external->_process, INFINITE);
+        DWORD waitFor = INFINITE;
+        if (timeout != 0) {
+          waitFor = timeout;
+        }
+        result = WaitForSingleObject(external->_process, waitFor);
         if (result == WAIT_FAILED) {
           FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0,
                         windowsErrorBuf, sizeof(windowsErrorBuf), NULL);
@@ -1074,6 +1078,10 @@ ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait) {
               arangodb::basics::StringUtils::itoa(static_cast<int64_t>(external->_pid)) +
               windowsErrorBuf;
           status._exitStatus = GetLastError();
+        } else if ((result == WAIT_TIMEOUT)  && (timeout != 0)) {
+          wantGetExitCode = false;
+          external->_status = TRI_EXT_TIMEOUT;
+          external->_exitStatus = -1;
         }
       } else {
         DWORD result;
@@ -1134,7 +1142,7 @@ ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait) {
             external->_exitStatus = exitCode;
           }
         }
-      } else {
+      } else if (timeout == 0) {
         external->_status = TRI_EXT_RUNNING;
       }
     }
@@ -1153,7 +1161,7 @@ ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait) {
   status._exitStatus = external->_exitStatus;
 
   // Do we have to free our data?
-  if (external->_status != TRI_EXT_RUNNING && external->_status != TRI_EXT_STOPPED) {
+  if (external->_status != TRI_EXT_RUNNING && external->_status != TRI_EXT_STOPPED && external->_status != TRI_EXT_TIMEOUT) {
     MUTEX_LOCKER(mutexLocker, ExternalProcessesLock);
 
     for (auto it = ExternalProcesses.begin(); it != ExternalProcesses.end(); ++it) {
@@ -1417,7 +1425,7 @@ ExternalProcessStatus TRI_KillExternalProcess(ExternalId pid, int signal, bool i
     // if the process wasn't spawned by us, no waiting required.
     int count = 0;
     while (true) {
-      ExternalProcessStatus status = TRI_CheckExternalProcess(pid, false);
+      ExternalProcessStatus status = TRI_CheckExternalProcess(pid, false, 0);
       if (!isTerminal) {
         // we just sent a signal, don't care whether
         // the process is gone by now.
@@ -1451,7 +1459,7 @@ ExternalProcessStatus TRI_KillExternalProcess(ExternalId pid, int signal, bool i
       count++;
     }
   }
-  return TRI_CheckExternalProcess(pid, false);
+  return TRI_CheckExternalProcess(pid, false, 0);
 }
 
 #ifdef _WIN32

lib/Basics/process-utils.h

@@ -73,6 +73,7 @@ typedef enum {
   TRI_EXT_TERMINATED = 5,   // process has terminated normally
   TRI_EXT_ABORTED = 6,      // process has terminated abnormally
   TRI_EXT_STOPPED = 7,      // process has been stopped
+  TRI_EXT_TIMEOUT = 9      // waiting for the process timed out
 } TRI_external_status_e;
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -175,7 +176,7 @@ void TRI_CreateExternalProcess(char const* executable,
 /// @brief returns the status of an external process
 ////////////////////////////////////////////////////////////////////////////////
 
-ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait);
+ExternalProcessStatus TRI_CheckExternalProcess(ExternalId pid, bool wait, uint32_t timeout);
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief whether a signal is expected to be terminal

lib/V8/v8-utils.cpp

@@ -4047,6 +4047,9 @@ static char const* convertProcessStatusToString(TRI_external_status_e processSta
     case TRI_EXT_STOPPED:
       status = "STOPPED";
       break;
+    case TRI_EXT_TIMEOUT:
+      status = "TIMEOUT";
+      break;
   }
   return status;
 }
@@ -4273,9 +4276,9 @@ static void JS_StatusExternal(v8::FunctionCallbackInfo<v8::Value> const& args) {
   v8::HandleScope scope(isolate);
 
   // extract the arguments
-  if (args.Length() < 1 || args.Length() > 2) {
+  if (args.Length() < 1 || args.Length() > 3) {
     TRI_V8_THROW_EXCEPTION_USAGE(
-        "statusExternal(<external-identifier>[, <wait>])");
+        "statusExternal(<external-identifier>[, <wait>[, <timeoutms>]])");
   }
 
   V8SecurityFeature* v8security =
@@ -4297,11 +4300,15 @@ static void JS_StatusExternal(v8::FunctionCallbackInfo<v8::Value> const& args) {
   pid._pid = static_cast<DWORD>(TRI_ObjectToUInt64(isolate, args[0], true));
 #endif
   bool wait = false;
-  if (args.Length() == 2) {
+  if (args.Length() >= 2) {
     wait = TRI_ObjectToBoolean(isolate, args[1]);
   }
+  uint32_t timeoutms = 0;
+  if (args.Length() >= 2) {
+    timeoutms = static_cast<uint32_t>(TRI_ObjectToUInt64(isolate, args[2], true));
+  }
 
-  ExternalProcessStatus external = TRI_CheckExternalProcess(pid, wait);
+  ExternalProcessStatus external = TRI_CheckExternalProcess(pid, wait, timeoutms);
 
   v8::Handle<v8::Object> result = v8::Object::New(isolate);
 
@@ -4332,9 +4339,9 @@ static void JS_ExecuteAndWaitExternal(v8::FunctionCallbackInfo<v8::Value> const&
   v8::HandleScope scope(isolate);
 
   // extract the arguments
-  if (3 < args.Length() || args.Length() < 1) {
+  if (4 < args.Length() || args.Length() < 1) {
     TRI_V8_THROW_EXCEPTION_USAGE(
-        "executeAndWaitExternal(<filename>[, <arguments> [,<usePipes>] ])");
+        "executeAndWaitExternal(<filename>[, <arguments> [,<usePipes>, [, <timoutms>]] ])");
   }
 
   TRI_Utf8ValueNFC name(isolate, args[0]);
@@ -4389,9 +4396,13 @@ static void JS_ExecuteAndWaitExternal(v8::FunctionCallbackInfo<v8::Value> const&
     }
   }
   bool usePipes = false;
-  if (3 <= args.Length()) {
+  if (args.Length() >= 3) {
     usePipes = TRI_ObjectToBoolean(isolate, args[2]);
   }
+  uint32_t timeoutms = 0;
+  if (args.Length() >= 4) {
+    timeoutms = static_cast<uint32_t>(TRI_ObjectToUInt64(isolate, args[3], true));
+  }
 
   ExternalId external;
   TRI_CreateExternalProcess(*name, arguments, usePipes, &external);
@@ -4404,7 +4415,7 @@ static void JS_ExecuteAndWaitExternal(v8::FunctionCallbackInfo<v8::Value> const&
   ExternalId pid;
   pid._pid = external._pid;
 
-  auto external_status = TRI_CheckExternalProcess(pid, true);
+  auto external_status = TRI_CheckExternalProcess(pid, true, timeoutms);
 
   convertStatusToV8(args, result, external_status, external);