Work on documentation & usage:

- add unittest readme
  - unify the commandline help output of testing.js
  - utilise the commandline output of testing.js in unittest.js

README_unittests.md

@@ -0,0 +1,187 @@
+ArangoDB Unittesting Framework
+------------------------------
+------------------------------
+------------------------------
+
+
+
+Filename conventions
+====================
+Special patterns in filenames are used to select tests to be executed or skipped depending no parameters:
+
+-cluster
+--------
+These tests will only run if clustering is tested. (option 'cluster' needs to be true).
+
+-noncluster
+-----------
+These tests will only run if no cluster is used. (option 'cluster' needs to be false)
+
+-timecritical
+-------------
+These tests are critical to execution time - and thus may fail if arangod is to slow. This may happen i.e. if you run the tests in valgrind, so you want to avoid them. To skip them, set the option 'skipTimeCritical' to true.
+
+-disabled
+---------
+These tests are disabled. You may however want to run them by hand.
+
+replication
+-----------
+(only applies to ruby tests)
+These tests aren't run automaticaly since they require a manual set up environment.
+
+-spec
+-----
+These tests are ran using the jasmine framework instead of jsunity.
+
+Test frameworks used
+====================
+There are several major places where unittests live: 
+ - UnitTests/HttpInterface
+ - UnitTests/Basics
+ - UnitTests/Geo
+ - js/server/tests
+ - js/common/tests
+ - js/common/test-data
+ - /js/apps/system/aardvark/test
+
+
+HttpInterface - RSpec Client Tests
+---------------------------------
+TODO: which tests are these?
+
+
+jsUnity on arangod
+------------------
+
+jsUnity via arangosh
+--------------------
+
+
+
+
+jasmine tests
+-------------
+
+Jasmine tests cover two important usecase:
+  - testing the UI components of aardvark
+-spec
+
+aardvark
+
+
+
+
+
+UnitTests/Makefile.unittests
+
+
+Invocation methods
+==================
+
+
+
+Make-targets
+------------
+(used in travis CI integration)
+Most of the tests can be invoked via the main Makefile:
+ - unittests
+  - unittests-brief
+  - unittests-verbose
+ - unittests-recovery
+ - unittests-config
+ - unittests-boost
+ - unittests-single
+ - unittests-shell-server
+ - unittests-shell-server-only
+ - unittests-shell-server-ahuacatl
+ - unittests-shell-client-readonly
+ - unittests-shell-client
+ - unittests-http-server
+ - unittests-ssl-server
+ - unittests-import
+ - unittests-replication
+  - unittests-replication-server
+  - unittests-replication-http
+  - unittests-replication-data
+ - unittests-upgrade
+ - unittests-dfdb
+ - unittests-foxx-manager
+ - unittests-dump
+ - unittests-arangob
+ - unittests-authentication
+ - unittests-authentication-parameters
+
+
+Javascript framework
+--------------------
+(used in Jenkins integration)
+Invoked like that:
+scripts/run scripts/unittest.js all
+
+Choosing facility
+_________________
+
+The first parameter chooses the facility to execute.
+Available choices include:
+ - all: (calls multiple) This target is utilized by most of the jenkins builds invoking unit tests.
+ - single_client: (see Running a single unittestsuite)
+ 
+ - single_server: (see Running a single unittestsuite)
+ - many more - call without arguments for more details.
+     
+
+Passing Options
+_______________
+
+Options are passed in as one json; Please note that formating blanks may cause problems.
+
+so a commandline for running a single test using valgrind could look like this: 
+
+ scripts/run scripts/unittest.js single_server \
+    '{"test":"js/server/tests/aql-escaping.js",'\
+    '"extraargs":["--server.threads=1",'\
+        '"--scheduler.threads=1",'\
+        '"--javascript.gc-frequency","1000000",'\
+        '"--javascript.gc-interval","65536"],'\
+    '"valgrind":"/usr/bin/valgrind",'\
+    '"valgrindargs":["--log-file=/tmp/valgrindlog.%p"]}'
+
+ - we specify the test to execute
+ - we specify some arangod arguments which increase the server performance
+ - we specify to run using valgrind
+ - we specify some valgrind commandline arguments
+
+Running a single unittestsuite
+------------------------------
+Testing a single test with the frame work directly on a server:
+scripts/run scripts/unittest.js single_server '{"test":"js/server/tests/aql-escaping.js"}'
+
+Testing a single test with the frame work via arangosh:
+scripts/run scripts/unittest.js single_client '{"test":"js/server/tests/aql-escaping.js"}'
+
+unittest.js is mostly only a wrapper; The backend functionality lives in:
+js/server/modules/org/arangodb/testing.js
+
+
+
+arangod Emergency console
+-------------------------
+
+require("jsunity").runTest("js/server/tests/aql-escaping.js");
+
+arangosh client
+---------------
+
+require("jsunity").runTest("js/server/tests/aql-escaping.js");
+
+
+arangod commandline arguments
+-----------------------------
+
+bin/arangod /tmp/dataUT --javascript.unit-tests="js/server/tests/aql-escaping.js" --no-server
+
+
+make unittest
+
+js/common/modules/loadtestrunner.js

js/server/modules/org/arangodb/testing.js

@@ -35,57 +35,51 @@
 /// This function gets one or two arguments, the first describes which tests
 /// to perform and the second is an options object. For `which` the following
 /// values are allowed:
-///
+///  Empty will give you a complete list.
-///   - "all": do all tests
+var functionDoku = {
-///   - "config"
+  'all'               : " do all tests (marked with [x])",
-///   - "boost"
+  "shell_server_perf" : "bulk tests intended to get an overview of executiontime needed.",
-///   - "shell_server"
+  "single_client"     : "run one test suite isolated via the arangosh; options required\n" +
-///   - "shell_server_ahuacatl"
+    "            Run without to get more detail",
-///   - "shell_server_aql"
+  "single_server"     : "run one test suite on the server; options required\n" + 
-///   - "shell_server_perf": bulk tests intended to get an overview of executiontime needed.
+    "            Run without to get more detail"
-///   - "http_server"
+};
-///   - "ssl_server"
+
-///   - "shell_client"
+var optiondoku = [
-///   - "single_client" : run one test suite isolated via the arangosh
+  '',
-///   - "single_server" : run one test suite on the server
+  ' The following properties of `options` are defined:',
-///   - "dump"
+  '',
-///   - "arangob"
+  '   - `jsonReply`: if set a json is returned which the caller has to ',
-///   - "import"
+  '        present the user',
-///   - "upgrade"
+  '   - `force`: if set to true the tests are continued even if one fails',
-///   - "dfdb"
+  '   - `skipBoost`: if set to true the boost unittests are skipped',
-///   - "foxx-manager"
+  '   - `skipGeo`: if set to true the geo index tests are skipped',
-///   - "authentication"
+  '   - `skipAhuacatl`: if set to true the ahuacatl tests are skipped',
-///   - "authentication-parameters
+  '   - `skipAql`: if set to true the AQL tests are skipped',
-///   - "single": convenience to execute a single test file
+  '   - `skipRanges`: if set to true the ranges tests are skipped',
-///
+  '   - `skipTimeCritical`: if set to true, time critical tests will be skipped.',
-/// The following properties of `options` are defined:
+  '',
-///
+  '   - `cluster`: if set to true the tests are run with the coordinator',
-///   - `jsonReply`: if set a json is returned which the caller has to 
+  '     of a small local cluster',
-///        present the user
+  '   - `test`: path to single test to execute for "single" test target',
-///   - `force`: if set to true the tests are continued even if one fails
+  '   - `cleanup`: if set to true (the default), the cluster data files',
-///   - `skipBoost`: if set to true the boost unittests are skipped
+  '     and logs are removed after termination of the test.',
-///   - `skipGeo`: if set to true the geo index tests are skipped
+  '   - `jasmineReportFormat`: this option is passed on to the `format`',
-///   - `skipAhuacatl`: if set to true the ahuacatl tests are skipped
+  '     option of the Jasmin options object, only for Jasmin tests.',
-///   - `skipAql`: if set to true the AQL tests are skipped
+  '',
-///   - `skipRanges`: if set to true the ranges tests are skipped
+  '   - `valgrind`: if set to true the arangods are run with the valgrind',
-///   - `skipTimeCritical`: if set to true, time critical tests will be skipped.
+  '     memory checker',
-///   - `valgrind`: if set to true the arangods are run with the valgrind
+  '   - `valgrindXmlFileBase`: string to prepend to the xml report name',
-///     memory checker
+  '   - `valgrindargs`: list of commandline parameters to add to valgrind',
-///   - `valgrindXmlFileBase`: string to prepend to the xml report name
+  '',
-///   - `valgrindargs`: commandline parameters to add to valgrind
+  '   - `extraargs`: list of extra commandline arguments to add to arangod',
-///   - `extraargs`: extra commandline arguments to add to arangod
+  ''
-///   - `cluster`: if set to true the tests are run with the coordinator
+];
-///     of a small local cluster
-///   - `test`: path to single test to execute for "single" test target
-///   - `cleanup`: if set to true (the default), the cluster data files
-///     and logs are removed after termination of the test.
-///   - `jasmineReportFormat`: this option is passed on to the `format`
-///     option of the Jasmin options object, only for Jasmin tests.
 ////////////////////////////////////////////////////////////////////////////////
 
 var _ = require("underscore");
 
-var testFuncs = {};
+var testFuncs = {'all': function(){}};
 var print = require("internal").print;
 var time = require("internal").time;
 var fs = require("fs");
@@ -117,6 +111,57 @@ var optionsDefaults = { "cluster": false,
                         "test": undefined,
                         "cleanup": true,
                         "jsonReply": false};
+var allTests =
+  [
+    "config",
+    "boost",
+    "shell_server",
+    "shell_server_ahuacatl",
+    "shell_server_aql",
+    "http_server",
+    "ssl_server",
+    "shell_client",
+    "dump",
+    "arangob",
+    "importing",
+    "upgrade",
+    "foxx_manager",
+    "authentication",
+    "authentication_parameters"
+  ];
+
+
+function printUsage () {
+  print();
+  print("Usage: UnitTest( which, options )");
+  print();
+  print('       where "which" is one of:\n');
+  var i;
+  var checkAll;
+  var oneFunctionDoku;
+  for (i in testFuncs) {
+    if (testFuncs.hasOwnProperty(i)) {
+      if (functionDoku.hasOwnProperty(i)) {
+        oneFunctionDoku = ' - ' + functionDoku[i];
+      }
+      else {
+        oneFunctionDoku = '';
+      }
+      if (allTests.indexOf(i) !== -1) {
+        checkAll = '[x]';
+      }
+      else {
+        checkAll = '   ';
+      }
+      print('    ' + checkAll + ' '+i+' ' + oneFunctionDoku);
+    }
+  }
+  for (i in optiondoku) {
+    if (optiondoku.hasOwnProperty(i)) {
+      print(optiondoku[i]);
+    }
+  }
+}
 
 function findTopDir () {
   var topDir = fs.normalize(fs.makeAbsolute("."));
@@ -520,6 +565,72 @@ function performTests(options, testList, testname) {
   return results;
 }
 
+function single_usage(testsuite) {
+  print("single_" + testsuite + ": No test specified!\n Available tests:");
+  var filelist = "";
+  var list = fs.list(makePath("js/server/tests"));
+  for (var fileNo in list) {
+    if (/\.js$/.test(list[fileNo])) {
+      filelist += " js/server/tests/"+list[fileNo];
+    }
+  }
+  print(filelist);
+  print("usage: single_" + testsuite + " '{\"test\":\"<testfilename>\"}'");
+  print(" where <testfilename> is one from the list above.");
+  return { status: false, message: "No test specified!"};
+}
+
+
+testFuncs.single_server = function (options) {
+  var instanceInfo = startInstance("tcp", options, [], "single");
+  var result = { };
+  if (options.test !== undefined) {
+    var te = options.test;
+    print("\nTrying",te,"on server...");
+    result = {};
+    result[te] = runThere(options, instanceInfo, makePath(te));
+    print("Shutting down...");
+    shutdownInstance(instanceInfo,options);
+    print("done.");
+    return result;
+  }
+  else {
+    return single_usage("server");
+  }
+};
+
+testFuncs.single_client = function (options) {
+  var instanceInfo = startInstance("tcp", options, [], "single");
+  var result = { };
+  if (options.test !== undefined) {
+    var te = options.test;
+    var topDir = findTopDir();
+    var args = makeTestingArgsClient(options);
+    args.push("--server.endpoint");
+    args.push(instanceInfo.endpoint);
+    args.push("--javascript.unit-tests");
+    args.push(fs.join(topDir,te));
+    print("\nTrying",te,"on client...");
+    var arangosh = fs.join("bin","arangosh");
+    result = {};
+    result[te] = executeAndWait(arangosh, args);
+    print("Shutting down...");
+    shutdownInstance(instanceInfo,options);
+    print("done.");
+    return result;
+  }
+  else {
+    return single_usage("client");
+  }
+};
+
+testFuncs.shell_server_perf = function(options) {
+  findTests();
+  return performTests(options,
+                      tests_shell_server_aql_performance,
+                      'tests_shell_server_aql_performance');
+};
+
 testFuncs.shell_server = function (options) {
   findTests();
   return performTests(options, tests_shell_server, 'tests_shell_server');
@@ -564,13 +675,6 @@ testFuncs.shell_server_aql = function(options) {
   return "skipped";
 };
 
-testFuncs.shell_server_perf = function(options) {
-  findTests();
-  return performTests(options,
-                      tests_shell_server_aql_performance,
-                      'tests_shell_server_aql_performance');
-};
-
 testFuncs.shell_client = function(options) {
   findTests();
   var instanceInfo = startInstance("tcp", options, [], "shell_client");
@@ -651,66 +755,6 @@ testFuncs.boost = function (options) {
   return results;
 };
 
-
-function single_usage(testsuite) {
-  print("single_" + testsuite + ": No test specified!\n Available tests:");
-  var filelist = "";
-  var list = fs.list(makePath("js/server/tests"));
-  for (var fileNo in list) {
-    if (/\.js$/.test(list[fileNo])) {
-      filelist += " js/server/tests/"+list[fileNo];
-    }
-  }
-  print(filelist);
-  print("usage: single_" + testsuite + " '{\"test\":\"<testfilename>\"}'");
-  print(" where <testfilename> is one from the list above.");
-  return { status: false, message: "No test specified!"};
-}
-
-
-testFuncs.single_server = function (options) {
-  var instanceInfo = startInstance("tcp", options, [], "single");
-  var result = { };
-  if (options.test !== undefined) {
-    var te = options.test;
-    print("\nTrying",te,"on server...");
-    result = {};
-    result[te] = runThere(options, instanceInfo, makePath(te));
-    print("Shutting down...");
-    shutdownInstance(instanceInfo,options);
-    print("done.");
-    return result;
-  }
-  else {
-    return single_usage("server");
-  }
-};
-
-testFuncs.single_client = function (options) {
-  var instanceInfo = startInstance("tcp", options, [], "single");
-  var result = { };
-  if (options.test !== undefined) {
-    var te = options.test;
-    var topDir = findTopDir();
-    var args = makeTestingArgsClient(options);
-    args.push("--server.endpoint");
-    args.push(instanceInfo.endpoint);
-    args.push("--javascript.unit-tests");
-    args.push(fs.join(topDir,te));
-    print("\nTrying",te,"on client...");
-    var arangosh = fs.join("bin","arangosh");
-    result = {};
-    result[te] = executeAndWait(arangosh, args);
-    print("Shutting down...");
-    shutdownInstance(instanceInfo,options);
-    print("done.");
-    return result;
-  }
-  else {
-    return single_usage("client");
-  }
-};
-
 function rubyTests (options, ssl) {
   var instanceInfo;
   if (ssl) {
@@ -751,6 +795,7 @@ function rubyTests (options, ssl) {
       if ((n.indexOf("-cluster") === -1 || options.cluster) &&
           (n.indexOf("-noncluster") === -1 || options.cluster === false) &&
           (n.indexOf("-timecritical") === -1 || options.skipTimeCritical === false) &&
+          (n.indexOf("-disabled") === -1) && 
           n.indexOf("replication") === -1) {
         args = ["--color", "-I", fs.join("UnitTests","HttpInterface"),
                 "--format", "d", "--require", tmpname,
@@ -1255,50 +1300,6 @@ function unitTestPrettyPrintResults(r) {
   }
 }
 
-
-var allTests =
-  [
-    "config",
-    "boost",
-    "shell_server",
-    "shell_server_ahuacatl",
-    "shell_server_aql",
-    "http_server",
-    "ssl_server",
-    "shell_client",
-    "dump",
-    "arangob",
-    "importing",
-    "upgrade",
-    "foxx_manager",
-    "authentication",
-    "authentication_parameters"
-  ];
-
-function printUsage () {
-  print("Usage: UnitTest( which, options )");
-  print('       where "which" is one of:');
-  print('         "all": do all tests');
-  var i;
-  for (i in testFuncs) {
-    if (testFuncs.hasOwnProperty(i)) {
-      print('         "'+i+'"');
-    }
-  }
-  print('       and options can contain the following boolean properties:');
-  print('         "force": continue despite a failed test');
-  print('         "skipBoost": skip the boost unittests');
-  print('         "skipGeo": skip the geo index tests');
-  print('         "skipAhuacatl": skip the ahuacatl tests');
-  print('         "skipAql": skip the AQL tests');
-  print('         "skipRanges": skip the ranges tests');
-  print('         "valgrind": arangods are run with valgrind');
-  print('         "cluster": tests are run on a small local cluster');
-  print('         "test": name of test to run for "single" test');
-  print('         "skipClient": for "single" test');
-  print('         "skipServer": for "single" test');
-}
-
 function UnitTest (which, options) {
   var allok = true;
   var results = {};

scripts/unittest.js

@@ -89,10 +89,6 @@ function resultsToXml(results, baseName) {
 
 
 function main (argv) {
-  if (argv.length < 2) {
-    print("Usage: unittest TESTNAME [OPTIONS]");
-    return;
-  }
   var test = argv[1];
   var options = {};
   var r;