Feature/cleanup rspec integration (#5517)

README_maintainers.md

@@ -155,6 +155,22 @@ Dependencies
   `cd UnitTests/HttpInterface; bundler`
 * catch (compile time, shipped in the 3rdParty directory)
 
+Invoking
+--------
+Since several testing technoligies are utilized, and different arangodb startup options may be required 
+(even different compilation options may be required) the framework is split into testsuites. 
+Get a list of the available testsuites and options by invoking: 
+
+    ./scripts/unittest
+
+To locate the suite(s) associated with a specific test file use: 
+
+    ./scripts/unittest find --test js/common/tests/shell/shell-aqlfunctions.js
+
+or to run all of them: 
+
+    ./scripts/unittest auto --test js/common/tests/shell/shell-aqlfunctions.js
+
 
 Filename conventions
 ====================

UnitTests/HttpInterface/README

@@ -1,12 +0,0 @@
-Conventions for testing framework:
-==================================
-
-All files in this directory whose name starts with "api-"
-and ends with ".js" automatically take part in http
-or https server tests, with the exception of 
-"api-replication-spec.js".
-
-If the filename contains the string "-cluster-", then it is
-only executed when testing in cluster mode. If the filename
-contains the string "-noncluster-", then it is only executed
-when testing in single instance mode.

UnitTests/HttpInterface/run-tests

@@ -1,4 +0,0 @@
-#!/bin/sh
-test -d logs || mkdir logs
-
-rspec -I . --color --format d `find . -name "api-*.rb" -a \! -name "*-cluster-*"`

UnitTests/HttpInterface/run-tests-replication

@@ -1,4 +0,0 @@
-#!/bin/sh
-test -d logs || mkdir logs
-
-rspec -I . --color --format d `find . -name "api-replication-*.rb" -a \! -name "*-cluster-*"`

UnitTests/HttpInterface/stress-test.rb

@@ -1,253 +0,0 @@
-#! ruby -rubygems
-# coding: utf-8
-
-require 'arangodb'
-
-# configuration
-$number_reader = 30
-$number_writer = 20
-$number_deleter = 10
-$maximal_size = 1000
-$verbose = false
-
-
-################################################################################
-## list of all active documents
-################################################################################
-
-$documents = []
-$doc_mutex = Mutex.new
-$doc_cv = ConditionVariable.new
-
-################################################################################
-## print version number of the server
-################################################################################
-
-doc = ArangoDB.get("/_admin/version")
-
-puts "starting stress test, ArangoDB #{doc.parsed_response['version']}"
-
-################################################################################
-## create a collection for testing
-################################################################################
-
-cn = "StressTest#{Time.now.to_i}"
-
-ArangoDB.delete("/_api/collection/#{cn}")
-
-body = "{ \"name\" : \"#{cn}\" }"
-doc = ArangoDB.post("/_api/collection", :body => body)
-$cid = doc.parsed_response['id']
-
-puts "create collection \"#{cn}\": #{$cid}"
-
-################################################################################
-## function to a read a document
-################################################################################
-
-def read_document (i, pos)
-  did = nil
-
-  $doc_mutex.synchronize {
-    if $documents.length == 0
-      return pos
-    end
-
-    pos = (pos * 13 + 1) % $documents.length
-
-    did = $documents[pos]
-
-    if pos + 1 == $documents.length
-      $documents.pop()
-    else
-      $documents[pos] = $documents.pop()
-    end
-  }
-
-  res = ArangoDB.get("/_api/document/#{did}")
-
-  if res.code == 200
-    if $verbose
-      puts "read document #{pos} in thread #{i}: #{did}"
-    end
-  else
-    puts "FAILED READ:"
-    puts "thread: #{i}"
-    puts "document identifier: #{did}"
-    puts JSON.pretty_generate(res.parsed_response)
-    puts "--------------"
-  end
-
-  $doc_mutex.synchronize {
-    $documents << did
-  }
-
-  return pos
-end
-
-################################################################################
-## function to a create a document
-################################################################################
-
-def create_document (i)
-  len = 0
-  body = "{ \"Hallo\" : #{i} }"
-  res = ArangoDB.post("/_api/document?collection=#{$cid}", :body => body)
-
-  if res.code == 201 or res.code == 202
-    did = res.parsed_response['_id']
-
-    $doc_mutex.synchronize {
-      $documents << did
-      len = $documents.length
-    }
-
-    if $verbose
-      puts "created document #{len} in thread #{i}: #{did}"
-    end
-  else
-    puts "FAILED CREATE:"
-    puts JSON.pretty_generate(res.parsed_response)
-    puts "--------------"
-  end
-end
-
-################################################################################
-## function to a delete a document
-################################################################################
-
-def delete_document (i, pos)
-  did = nil
-
-  $doc_mutex.synchronize {
-    if $documents.length == 0
-      return pos
-    end
-
-    pos = (pos * 13 + 1) % $documents.length
-
-    did = $documents[pos]
-
-    if pos + 1 == $documents.length
-      $documents.pop()
-    else
-      $documents[pos] = $documents.pop()
-    end
-  }
-
-  res = ArangoDB.delete("/_api/document/#{did}")
-
-  if res.code == 200
-    if $verbose
-      puts "deleted document #{pos} in thread #{i}: #{did}"
-    end
-  else
-    puts "FAILED DELETE:"
-    puts "thread: #{i}"
-    puts "document identifier: #{did}"
-    puts JSON.pretty_generate(res.parsed_response)
-    puts "--------------"
-  end
-
-  return pos
-end
-
-################################################################################
-## reader
-################################################################################
-
-def reader (i) 
-  pos = i % $maximal_size
-
-  while true
-    $doc_mutex.synchronize {
-      while $documents.length <= 10
-  $doc_cv.wait($doc_mutex)
-      end
-    }
-
-    pos = read_document(i, pos)
-
-    $doc_mutex.synchronize {
-      $doc_cv.broadcast
-    }
-  end
-end
-
-################################################################################
-## writer
-################################################################################
-
-def writer (i) 
-  while true
-    $doc_mutex.synchronize {
-      while $maximal_size <= $documents.length 
-  $doc_cv.wait($doc_mutex)
-      end
-    }
-
-    create_document(i)
-
-    $doc_mutex.synchronize {
-      $doc_cv.broadcast
-    }
-  end
-end
-
-################################################################################
-## deleter
-################################################################################
-
-def deleter (i) 
-  pos = i % $maximal_size
-
-  while true
-    $doc_mutex.synchronize {
-      while $documents.length < $maximal_size
-  $doc_cv.wait($doc_mutex)
-      end
-    }
-
-    pos = delete_document(i, pos)
-
-    $doc_mutex.synchronize {
-      $doc_cv.broadcast
-    }
-  end
-end
-
-################################################################################
-## main
-################################################################################
-
-threads = []
-
-$number_reader.times { |i|
-  threads << Thread.new(i) {
-    reader(i)
-  }
-}
-
-$number_writer.times { |i|
-  threads << Thread.new(i) {
-    writer(i)
-  }
-}
-
-$number_deleter.times { |i|
-  threads << Thread.new(i) {
-    deleter(i)
-  }
-}
-
-if $verbose
-  sleep
-else
-  while true
-    $doc_mutex.synchronize {
-      puts "number of documents: #{$documents.length}"
-    }
-
-    sleep(10)
-  end
-end

js/client/modules/@arangodb/process-utils.js

@@ -1249,7 +1249,8 @@ function startInstance (protocol, options, addArgs, testname, tmpDir) {
 
   let instanceInfo = {
     rootDir,
-    arangods: []
+    arangods: [],
+    protocol: protocol
   };
 
   const startTime = time();

js/client/modules/@arangodb/test-utils.js

@@ -117,7 +117,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
     _.defaults(env, customInstanceInfos.preStart.env);
   }
 
-  let instanceInfo = pu.startInstance('tcp', options, serverOptions, testname);
+  let instanceInfo = pu.startInstance(options.protocol, options, serverOptions, testname);
 
   if (instanceInfo === false) {
     if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('startFailed')) {
@@ -223,7 +223,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
         let delta = diffArray(collectionsBefore, collectionsAfter).filter(function(name) {
           return (name[0] !== '_'); // exclude system collections from the comparison
         });
-	print(delta);
+
         if (delta.length !== 0) {
           results[te] = {
             status: false,
@@ -345,7 +345,6 @@ function filterTestcaseByOptions (testname, options, whichFilter) {
 
   if (options.replication) {
     whichFilter.filter = 'replication';
-
     if (options.hasOwnProperty('test') && (typeof (options.test) !== 'undefined')) {
       whichFilter.filter = 'testcase';
       return ((testname.search(options.test) >= 0) &&
@@ -354,6 +353,7 @@ function filterTestcaseByOptions (testname, options, whichFilter) {
       return testname.indexOf('replication') !== -1;
     }
   } else if (testname.indexOf('replication') !== -1) {
+
     whichFilter.filter = 'replication';
     return false;
   }
@@ -571,15 +571,16 @@ function runInArangosh (options, instanceInfo, file, addArgs) {
     args = Object.assign(args, addArgs);
   }
   require('internal').env.INSTANCEINFO = JSON.stringify(instanceInfo);
+  const jsonFN = fs.join(instanceInfo.rootDir, 'testresult.json');
   let rc = pu.executeAndWait(pu.ARANGOSH_BIN, toArgv(args), options, 'arangosh', instanceInfo.rootDir, options.coreCheck);
   let result;
   try {
-    result = JSON.parse(fs.read(instanceInfo.rootDir + '/testresult.json'));
+    result = JSON.parse(fs.read(jsonFN));
 
-    fs.remove(instanceInfo.rootDir + '/testresult.json');
+    fs.remove(jsonFN);
   } catch (x) {
     if (options.extremeVerbosity) {
-      print('failed to read ' + instanceInfo.rootDir + '/testresult.json');
+      print('failed to read ' + jsonFN);
     }
     return rc;
   }
@@ -594,6 +595,138 @@ function runInArangosh (options, instanceInfo, file, addArgs) {
 }
 runInArangosh.info = 'arangosh';
 
+
+
+// //////////////////////////////////////////////////////////////////////////////
+// / @brief runs a unittest file using rspec
+// //////////////////////////////////////////////////////////////////////////////
+function camelize (str) {
+  return str.replace(/(?:^\w|[A-Z]|\b\w)/g, function (letter, index) {
+    return index === 0 ? letter.toLowerCase() : letter.toUpperCase();
+  }).replace(/\s+/g, '');
+}
+
+const parseRspecJson = function (testCase, res, totalDuration) {
+  let tName = camelize(testCase.description);
+  let status = (testCase.status === 'passed');
+
+  res[tName] = {
+    status: status,
+    message: testCase.full_description,
+    duration: totalDuration // RSpec doesn't offer per testcase time...
+  };
+
+  res.total++;
+
+  if (!status) {
+    const msg = yaml.safeDump(testCase)
+          .replace(/.*rspec\/core.*\n/gm, '')
+          .replace(/.*rspec\\core.*\n/gm, '')
+          .replace(/.*lib\/ruby.*\n/, '')
+          .replace(/.*- >-.*\n/gm, '')
+          .replace(/\n *`/gm, ' `');
+    print('RSpec test case falied: \n' + msg);
+    res[tName].message += '\n' + msg;
+  }
+  return status ? 0 : 1;
+};
+function runInRSpec (options, instanceInfo, file, addArgs) {
+  const tmpname = fs.join(instanceInfo.rootDir, 'testconfig.rb');
+  const jsonFN = fs.join(instanceInfo.rootDir, 'testresult.json');
+
+  let command;
+  let args;
+  let rspec;
+  let ssl = "0";
+  if (instanceInfo.protocol === 'ssl') {
+    ssl = "1";
+  }
+  let rspecConfig = 'RSpec.configure do |c|\n' +
+                    '  c.add_setting :ARANGO_SERVER\n' +
+      '  c.ARANGO_SERVER = "' +
+      instanceInfo.endpoint.substr(6) + '"\n' +
+      '  c.add_setting :ARANGO_SSL\n' +
+      '  c.ARANGO_SSL = "' + ssl + '"\n' +
+      '  c.add_setting :ARANGO_USER\n' +
+      '  c.ARANGO_USER = "' + options.username + '"\n' +
+      '  c.add_setting :ARANGO_PASSWORD\n' +
+      '  c.ARANGO_PASSWORD = "' + options.password + '"\n' +
+      '  c.add_setting :SKIP_TIMECRITICAL\n' +
+      '  c.SKIP_TIMECRITICAL = ' + JSON.stringify(options.skipTimeCritical) + '\n' +
+      'end\n';
+
+  fs.write(tmpname, rspecConfig);
+
+  if (options.extremeVerbosity === true) {
+    print('rspecConfig: \n' + rspecConfig);
+  }
+
+  try {
+    fs.makeDirectory(pu.LOGS_DIR);
+  } catch (err) {}
+
+  if (options.ruby === '') {
+    command = options.rspec;
+    rspec = undefined;
+  } else {
+    command = options.ruby;
+    rspec = options.rspec;
+  }
+
+  args = ['--color',
+          '-I', fs.join('UnitTests', 'arangodbRspecLib'),
+          '--format', 'd',
+          '--format', 'j',
+          '--out', jsonFN,
+          '--require', tmpname,
+          file
+         ];
+
+  if (rspec !== undefined) {
+    args = [rspec].concat(args);
+  }
+
+  const res = pu.executeAndWait(command, args, options, 'arangosh', instanceInfo.rootDir);
+
+  let result = {
+    total: 0,
+    failed: 0,
+    status: res.status
+  };
+
+  try {
+    const jsonResult = JSON.parse(fs.read(jsonFN));
+
+    if (options.extremeVerbosity) {
+      print(yaml.safeDump(jsonResult));
+    }
+
+    for (let j = 0; j < jsonResult.examples.length; ++j) {
+      result.failed += parseRspecJson(
+        jsonResult.examples[j], result,
+        jsonResult.summary.duration);
+    }
+
+    result.duration = jsonResult.summary.duration;
+  } catch (x) {
+    result.failed = 1;
+    result.message = 'Failed to parse rspec results for: ' + file;
+
+    if (res.status === false) {
+      options.cleanup = false;
+    }
+  }
+
+  fs.remove(jsonFN);
+  fs.remove(tmpname);
+  return result;
+}
+runInRSpec.info = 'runInRSpec';
+
+// //////////////////////////////////////////////////////////////////////////////
+// / @brief
+// //////////////////////////////////////////////////////////////////////////////
+
 function makeResults (testname, instanceInfo) {
   const startTime = time();
 
@@ -640,6 +773,7 @@ function makeResults (testname, instanceInfo) {
 
 exports.runThere = runThere;
 exports.runInArangosh = runInArangosh;
+exports.runInRSpec = runInRSpec;
 
 exports.makePathUnix = makePathUnix;
 exports.makePathGeneric = makePathGeneric;

js/client/modules/@arangodb/testing.js

@@ -74,6 +74,7 @@ let optionsDocumentation = [
   '   - `cleanup`: if set to true (the default), the cluster data files',
   '     and logs are removed after termination of the test.',
   '',
+  '   - `protocol`: the protocol to talk to the server - [tcp (default), ssl, unix]',
   '   - `build`: the directory containing the binaries',
   '   - `buildType`: Windows build type (Debug, Release), leave empty on linux',
   '   - `configDir`: the directory containing the config files, defaults to',
@@ -135,6 +136,7 @@ const optionsDefaults = {
   'mochaGrep': undefined,
   'onlyNightly': false,
   'password': '',
+  'protocol': 'tcp',
   'replication': false,
   'rr': false,
   'sanitizer': false,

js/client/modules/@arangodb/testsuites/rspec.js

@@ -53,7 +53,7 @@ const RESET = require('internal').COLORS.COLOR_RESET;
 // const YELLOW = require('internal').COLORS.COLOR_YELLOW;
 
 const testPaths = {
-  'http_replication': [fs.join('UnitTests', 'HttpInterface')],
+  'http_replication': [fs.join('UnitTests', 'HttpReplication')],
   'http_server': [fs.join('UnitTests', 'HttpInterface')],
   'server_http': ['js/common/tests/http'],
   'ssl_server': [fs.join('UnitTests', 'HttpInterface')]
@@ -70,243 +70,6 @@ function serverHttp (options) {
   return tu.performTests(options, testCases, 'server_http', tu.runThere);
 }
 
-// //////////////////////////////////////////////////////////////////////////////
-// / @brief runs ruby tests using RSPEC
-// //////////////////////////////////////////////////////////////////////////////
-
-function camelize (str) {
-  return str.replace(/(?:^\w|[A-Z]|\b\w)/g, function (letter, index) {
-    return index === 0 ? letter.toLowerCase() : letter.toUpperCase();
-  }).replace(/\s+/g, '');
-}
-
-function rubyTests (options, ssl) {
-  let instanceInfo;
-
-  if (ssl) {
-    instanceInfo = pu.startInstance('ssl', options, {}, 'ssl_server');
-  } else {
-    instanceInfo = pu.startInstance('tcp', options, {}, 'http_server');
-  }
-
-  if (instanceInfo === false) {
-    return {
-      status: false,
-      message: 'failed to start server!'
-    };
-  }
-
-  const tmpname = fs.getTempFile() + '.rb';
-
-  let rspecConfig = 'RSpec.configure do |c|\n' +
-                    '  c.add_setting :ARANGO_SERVER\n' +
-                    '  c.ARANGO_SERVER = "' +
-                    instanceInfo.endpoint.substr(6) + '"\n' +
-                    '  c.add_setting :ARANGO_SSL\n' +
-                    '  c.ARANGO_SSL = "' + (ssl ? '1' : '0') + '"\n' +
-                    '  c.add_setting :ARANGO_USER\n' +
-                    '  c.ARANGO_USER = "' + options.username + '"\n' +
-                    '  c.add_setting :ARANGO_PASSWORD\n' +
-                    '  c.ARANGO_PASSWORD = "' + options.password + '"\n' +
-                    '  c.add_setting :SKIP_TIMECRITICAL\n' +
-                    '  c.SKIP_TIMECRITICAL = ' + JSON.stringify(options.skipTimeCritical) + '\n' +
-                    'end\n';
-
-  fs.write(tmpname, rspecConfig);
-
-  if (options.extremeVerbosity === true) {
-    print('rspecConfig: \n' + rspecConfig);
-  }
-
-  try {
-    fs.makeDirectory(pu.LOGS_DIR);
-  } catch (err) {}
-
-  let files = [];
-  if (ssl) {
-    files = tu.scanTestPaths(testPaths.ssl_server);
-  } else {
-    files = tu.scanTestPaths(testPaths.http_server);
-  }
-  let continueTesting = true;
-  let filtered = {};
-  let results = {};
-
-  let args;
-  let command;
-  let rspec;
-
-  if (options.ruby === '') {
-    command = options.rspec;
-    rspec = undefined;
-  } else {
-    command = options.ruby;
-    rspec = options.rspec;
-  }
-
-  const parseRspecJson = function (testCase, res, totalDuration) {
-    let tName = camelize(testCase.description);
-    let status = (testCase.status === 'passed');
-
-    res[tName] = {
-      status: status,
-      message: testCase.full_description,
-      duration: totalDuration // RSpec doesn't offer per testcase time...
-    };
-
-    res.total++;
-
-    if (!status) {
-      const msg = yaml.safeDump(testCase)
-            .replace(/.*rspec\/core.*\n/gm, '')
-            .replace(/.*rspec\\core.*\n/gm, '')
-            .replace(/.*lib\/ruby.*\n/, '')
-            .replace(/.*- >-.*\n/gm, '')
-            .replace(/\n *`/gm, ' `');
-      print('RSpec test case falied: \n' + msg);
-      res[tName].message += '\n' + msg;
-    }
-    return status ? 0 : 1;
-  };
-
-  let count = 0;
-  let graphCount = 0;
-  files = tu.splitBuckets(options, files);
-
-  for (let i = 0; i < files.length; i++) {
-    const te = files[i];
-
-    if ((te.search('api-') !== -1) && te.substr(-3) === '.rb') {
-      let tfn = te;
-      if (tu.filterTestcaseByOptions(tfn, options, filtered)) {
-        count += 1;
-        if (!continueTesting) {
-          print('Skipping ' + te + ' server is gone.');
-
-          results[te] = {
-            status: false,
-            message: instanceInfo.exitStatus
-          };
-
-          instanceInfo.exitStatus = 'server is gone.';
-          break;
-        }
-
-        let collectionsBefore = [];
-        db._collections().forEach(collection => {
-          collectionsBefore.push(collection._name);
-        });
-
-        const subFolder = ssl ? 'ssl_server' : 'http_server';
-        const resultfn = fs.join(options.testOutputDirectory, subFolder, te + '.json');
-
-        args = ['--color',
-                '-I', fs.join('UnitTests', 'HttpInterface'),
-                '--format', 'd',
-                '--format', 'j',
-                '--out', resultfn,
-                '--require', tmpname,
-                tfn
-               ];
-
-        if (rspec !== undefined) {
-          args = [rspec].concat(args);
-        }
-
-        print('\n' + Date() + ' rspec trying', tfn, '...');
-        const res = pu.executeAndWait(command, args, options, 'arangosh', instanceInfo.rootDir);
-
-        results[te] = {
-          total: 0,
-          failed: 0,
-          status: res.status
-        };
-
-        try {
-          const jsonResult = JSON.parse(fs.read(resultfn));
-
-          if (options.extremeVerbosity) {
-            print(yaml.safeDump(jsonResult));
-          }
-
-          for (let j = 0; j < jsonResult.examples.length; ++j) {
-            results[te].failed += parseRspecJson(
-              jsonResult.examples[j], results[te],
-              jsonResult.summary.duration);
-          }
-
-          results[te].duration = jsonResult.summary.duration;
-        } catch (x) {
-          print('Failed to parse rspec result: ' + x);
-          results[te]['complete_' + te] = res;
-
-          if (res.status === false) {
-            options.cleanup = false;
-
-            if (!options.force) {
-              break;
-            }
-          }
-        }
-
-        continueTesting = pu.arangod.check.instanceAlive(instanceInfo, options);
-        if (continueTesting) {
-          // Check whether some collections were left behind, and if mark test as failed.
-          let collectionsAfter = [];
-          db._collections().forEach(collection => {
-            collectionsAfter.push(collection._name);
-          });
-          let delta = tu.diffArray(collectionsBefore, collectionsAfter, _.isEqual).filter(function (name) {
-            return (name[0] !== '_'); // exclude system collections from the comparison
-          });
-          if (delta.length !== 0) {
-            results[te] = {
-              status: false,
-              message: 'Cleanup missing - test left over collections! [' + delta + '] - Original test status: ' + JSON.stringify(results[te])
-            };
-            collectionsBefore = [];
-            db._collections().forEach(collection => {
-              collectionsBefore.push(collection._name);
-            });
-          }
-
-          if (db._graphs.count() !== graphCount) {
-            results[te] = {
-              status: false,
-              message: 'Cleanup of graphs missing - found graph definitions: [ ' +
-                JSON.stringify(db._graphs.toArray()) +
-                ' ] - Original test status: ' +
-                JSON.stringify(results[te])
-            };
-            graphCount = db._graphs.count();
-          }
-        }
-      } else {
-        if (options.extremeVerbosity) {
-          print('Skipped ' + te + ' because of ' + filtered.filter);
-        }
-      }
-    }
-  }
-
-  print('Shutting down...');
-
-  if (count === 0) {
-    results['ALLTESTS'] = {
-      status: false,
-      skipped: true
-    };
-    results.status = false;
-    print(RED + 'No testcase matched the filter.' + RESET);
-  }
-
-  fs.remove(tmpname);
-  pu.shutdownInstance(instanceInfo, options);
-  print('done.');
-
-  return results;
-}
-
 // //////////////////////////////////////////////////////////////////////////////
 // / @brief TEST: http_replication
 // //////////////////////////////////////////////////////////////////////////////
@@ -317,7 +80,9 @@ function httpReplication (options) {
   };
   _.defaults(opts, options);
   
-  return rubyTests(opts, false);
+  let testCases = tu.scanTestPaths(testPaths.http_replication);
+
+  return tu.performTests(opts, testCases, 'http_replication', tu.runInRSpec);
 }
 
 // //////////////////////////////////////////////////////////////////////////////
@@ -329,7 +94,10 @@ function httpServer (options) {
     'httpTrustedOrigin': 'http://was-erlauben-strunz.it'
   };
   _.defaults(opts, options);
-  return rubyTests(opts, false);
+
+  let testCases = tu.scanTestPaths(testPaths.http_server);
+
+  return tu.performTests(opts, testCases, 'http_server', tu.runInRSpec);
 }
 
 // //////////////////////////////////////////////////////////////////////////////
@@ -346,10 +114,14 @@ function sslServer (options) {
     };
   }
   var opts = {
-    'httpTrustedOrigin': 'http://was-erlauben-strunz.it'
+    'httpTrustedOrigin': 'http://was-erlauben-strunz.it',
+    'protocol': 'ssl'
   };
   _.defaults(opts, options);
-  return rubyTests(opts, true);
+
+  let testCases = tu.scanTestPaths(testPaths.ssl_server);
+
+  return tu.performTests(opts, testCases, 'ssl_server', tu.runInRSpec);
 }
 
 exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {