fixed generateExample

Documentation/CMakeLists.txt

@@ -9,6 +9,11 @@ add_custom_target (swagger
     ${PROJECT_SOURCE_DIR}/Documentation/DocuBlocks/Rest/
     > ${PROJECT_SOURCE_DIR}/js/apps/system/_admin/aardvark/APP/api-docs.json)
 
+# swagger
+add_custom_target(examples
+  COMMAND ${PROJECT_SOURCE_DIR}/utils/generateExamples
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
+
 # manual pages
 if (USE_MAINTAINER_MODE)
   set(MAN_NAMES

utils/generateExamples

@@ -19,39 +19,27 @@ mkdir -p ${DBDIR}
 echo Database has its data in ${DBDIR}
 echo Logfile is in ${LOGFILE}
 
-if [ -z "${ARANGOD}" ];  then
+if [ -z "${ARANGOSH}" ];  then
-  if [ -x build/bin/arangod ];  then
+  if [ -x build/bin/arangosh ];  then
-    ARANGOD=build/bin/arangod
+    ARANGOSH=build/bin/arangosh
   elif [ -x bin/arangosh ];  then
-    ARANGOD=bin/arangod
+    ARANGOSH=bin/arangosh
   else
-    echo "$0: cannot locate arangod"
+    echo "$0: cannot locate arangosh"
   fi
 fi
 
-${ARANGOD} \
+${ARANGOSH} \
     --configuration none \
-    --cluster.agent-path bin${PS}etcd-arango${EXT} \
+    --server.endpoint none \
-    --cluster.arangod-path bin${PS}arangod \
-    --cluster.coordinator-config etc${PS}relative${PS}arangod-coordinator.conf \
-    --cluster.dbserver-config etc${PS}relative${PS}arangod-dbserver.conf \
-    --cluster.disable-dispatcher-frontend false \
-    --cluster.disable-dispatcher-kickstarter false \
-    --cluster.data-path cluster \
-    --cluster.log-path cluster \
-    --database.directory ${DBDIR} \
     --log.file ${LOGFILE} \
-    --server.endpoint tcp://127.0.0.1:$PORT \
     --javascript.startup-directory js \
-    --javascript.app-path js${PS}apps \
+    --javascript.execute $SCRIPT \
-    --javascript.script $SCRIPT \
+    "${ARGS[@]}"
-    --no-server \
-    --temp-path ${PS}var${PS}tmp \
-    "${ARGS[@]}" \
 
 if test $? -eq 0; then
   echo "removing ${LOGFILE} ${DBDIR}"
-  rm -rf ${LOGFILE} ${DBDIR}
+  rm -rf ${LOGFILE} ${DBDIR} arangosh.examples.js
 else
   echo "failed - don't remove ${LOGFILE} ${DBDIR} - here's the logfile:"
   cat ${LOGFILE}

utils/generateExamples.js

@@ -1,58 +1,93 @@
 /*jshint globalstrict:false, unused:false */
 /*global start_pretty_print */
+'use strict';
 
-var fs = require("fs");
+const fs = require("fs");
-var internal = require("internal");
+const internal = require("internal");
-var executeExternal = require("internal").executeExternal;
+const executeExternal = internal.executeExternal;
-var executeExternalAndWait = internal.executeExternalAndWait;
+const executeExternalAndWait = internal.executeExternalAndWait;
-var download = require("internal").download;
+const download = internal.download;
-var print = internal.print;
+const print = internal.print;
-var wait = require("internal").wait;
+const wait = internal.wait;
-var killExternal = require("internal").killExternal;
+const killExternal = internal.killExternal;
-var toArgv = require("internal").toArgv;
+const toArgv = internal.toArgv;
-var statusExternal = require("internal").statusExternal;
+const statusExternal = internal.statusExternal;
+const testPort = internal.testPort;
 
-var yaml = require("js-yaml");
+const yaml = require("js-yaml");
-var endpointToURL = require("@arangodb/cluster/planner").endpointToURL;
-var PortFinder = require("@arangodb/cluster").PortFinder;
 
-var documentationSourceDirs = [
+const documentationSourceDirs = [
   fs.join(fs.makeAbsolute(''), "Documentation/Examples/setup-arangosh.js"),
-  fs.join(fs.makeAbsolute(''), "Documentation/Books/Users"),
+  fs.join(fs.makeAbsolute(''), "Documentation/DocuBlocks"),
-  fs.join(fs.makeAbsolute(''), "js/actions"),
+  fs.join(fs.makeAbsolute(''), "Documentation/Books/Users")
-  fs.join(fs.makeAbsolute(''), "js/client"),
+];
-  fs.join(fs.makeAbsolute(''), "js/common"),
-  fs.join(fs.makeAbsolute(''), "js/server"),
-  fs.join(fs.makeAbsolute(''), "js/apps/system/_api/gharial/APP")];
 
-var theScript = 'Documentation/Scripts/generateExamples.py';
+const theScript = 'utils/generateExamples.py';
 
-var scriptArguments = {
+const scriptArguments = {
   'outputDir': fs.join(fs.makeAbsolute(''), "Documentation/Examples"),
-  'outputFile': '/tmp/arangosh.examples.js'
+  'outputFile': fs.join(fs.makeAbsolute(''), "arangosh.examples.js")
 };
 
+let ARANGOD;
+let ARANGOSH;
+
+if (fs.exists("bin")) {
+  ARANGOD = fs.join(fs.join(fs.makeAbsolute('')), "bin/arangod");
+  ARANGOSH = fs.join(fs.join(fs.makeAbsolute('')), "bin/arangosh");
+}
+else {
+  ARANGOD = fs.join(fs.join(fs.makeAbsolute('')), "build/bin/arangod");
+  ARANGOSH = fs.join(fs.join(fs.makeAbsolute('')), "build/bin/arangosh");
+}
+
+function endpointToURL(endpoint) {
+  if (endpoint.substr(0, 6) === "ssl://") {
+    return "https://" + endpoint.substr(6);
+  }
+
+  const pos = endpoint.indexOf("://");
+
+  if (pos === -1) {
+    return "http://" + endpoint;
+  }
+
+  return "http" + endpoint.substr(pos);
+}
+
+function findFreePort() {
+  while (true) {
+    const port = Math.floor(Math.random() * (65536 - 1024)) + 1024;
+    const free = testPort("tcp://0.0.0.0:" + port);
+
+    if (free) {
+      return port;
+    }
+  }
+
+  return 8529;
+}
+
 function main(argv) {
-  "use strict";
+  let thePython = 'python';
-  var thePython = 'python';
+  let options = {};
-  var test = argv[1];
+  let serverEndpoint = '';
-  var options = {};
+  let startServer = true;
-  var serverEndpoint = '';
+  let instanceInfo = {};
-  var startServer = true;
+  let serverCrashed = false;
-  var instanceInfo = {};
+  let protocol = 'tcp';
-  var serverCrashed = false;
+  let tmpDataDir = fs.getTempFile();
-  var protocol = 'tcp';
+  let count = 0;
-  var tmpDataDir = fs.getTempFile();
-  var count = 0;
 
   try {
-    options = internal.parseArgv(argv, 1);
+    options = internal.parseArgv(argv, 0);
-  }
+  } catch (x) {
-  catch (x) {
     print("failed to parse the options: " + x.message);
     return -1;
   }
 
+  print(options);
+
   if (options.hasOwnProperty('withPython')) {
     thePython = options.withPython;
   }
@@ -65,13 +100,13 @@ function main (argv) {
     startServer = false;
     serverEndpoint = options['server.endpoint'];
   }
-  var args = [theScript].concat(internal.toArgv(scriptArguments));
+
+  let args = [theScript].concat(internal.toArgv(scriptArguments));
   args = args.concat(['--arangoshSetup']);
   args = args.concat(documentationSourceDirs);
 
-  // internal.print(JSON.stringify(args));
+  let res = executeExternalAndWait(thePython, args);
 
-  var res = executeExternalAndWait(thePython, args);
   if (res.exit !== 0) {
     print("parsing the examples failed - aborting!");
     print(res);
@@ -79,36 +114,46 @@ function main (argv) {
   }
 
   if (startServer) {
-    // We use the PortFinder to find a free port for our subinstance,
+    let port = findFreePort();
-    // to this end, we have to fake a dummy dispatcher:
-    var dispatcher = {endpoint: "tcp://127.0.0.1:", avoidPorts: {}, id: "me"};
-    var pf = new PortFinder([8529],dispatcher);
-    var port = pf.next();
     instanceInfo.port = port;
     serverEndpoint = protocol + "://127.0.0.1:" + port;
 
     instanceInfo.url = endpointToURL(serverEndpoint);
 
-    var serverArgs = {};
-    serverArgs["server.endpoint"] = serverEndpoint;
-    serverArgs["database.directory"] = fs.join(tmpDataDir,"data");
     fs.makeDirectoryRecursive(fs.join(tmpDataDir, "data"));
-    args["log.file"] = fs.join(tmpDataDir,"log");
+
-    instanceInfo.pid = executeExternal(fs.join("bin","arangod"), toArgv(serverArgs));
+    let serverArgs = {};
+
+    serverArgs["configuration"] = "none";
+    serverArgs["database.directory"] = fs.join(tmpDataDir, "data");
+    serverArgs["javascript.app-path"] = fs.join(tmpDataDir, "apps");
+    serverArgs["javascript.startup-directory"] = "js";
+    serverArgs["log.file"] = fs.join(tmpDataDir, "log");
+    serverArgs["server.disable-authentication"] = "true";
+    serverArgs["server.endpoint"] = serverEndpoint;
+    serverArgs["server.threads"] = "3";
+
+    print("================================================================================");
+    print(toArgv(serverArgs));
+    instanceInfo.pid = executeExternal(ARANGOD, toArgv(serverArgs));
+
     // Wait until the server is up:
     count = 0;
     instanceInfo.endpoint = serverEndpoint;
 
     while (true) {
       wait(0.5, false);
-      var r = download(instanceInfo.url + "/_api/version", "");
+      let r = download(instanceInfo.url + "/_api/version", "");
 
       if (!r.error && r.code === 200) {
         break;
       }
+
       count++;
+
       if (count % 60 === 0) {
         res = statusExternal(instanceInfo.pid, false);
+
         if (res.status !== "RUNNING") {
           print("start failed - process is gone: " + yaml.safeDump(res));
           return 1;
@@ -116,14 +161,15 @@ function main (argv) {
       }
     }
   }
-  var arangoshArgs = {
+
+  let arangoshArgs = {
     'configuration': fs.join(fs.makeAbsolute(''), 'etc', 'relative', 'arangosh.conf'),
     'server.password': "",
     'server.endpoint': serverEndpoint,
     'javascript.execute': scriptArguments.outputFile
   };
 
-  res = executeExternalAndWait('bin/arangosh', internal.toArgv(arangoshArgs));
+  res = executeExternalAndWait(ARANGOSH, internal.toArgv(arangoshArgs));
 
   if (startServer) {
     if (typeof(instanceInfo.exitStatus) === 'undefined') {
@@ -131,9 +177,11 @@ function main (argv) {
 
       print("Waiting for server shut down");
       count = 0;
-      var bar = "[";
+      let bar = "[";
+
       while (1) {
         instanceInfo.exitStatus = statusExternal(instanceInfo.pid, false);
+
         if (instanceInfo.exitStatus.status === "RUNNING") {
           count++;
           if (typeof(options.valgrind) === 'string') {
@@ -149,12 +197,10 @@ function main (argv) {
             serverCrashed = true;
             killExternal(instanceInfo.pid);
             break;
-          }
+          } else {
-          else {
             wait(1);
           }
-        }
+        } else if (instanceInfo.exitStatus.status !== "TERMINATED") {
-        else if (instanceInfo.exitStatus.status !== "TERMINATED") {
           if (instanceInfo.exitStatus.hasOwnProperty('signal')) {
             print("Server shut down with : " +
               yaml.safeDump(instanceInfo.exitStatus) +
@@ -162,21 +208,24 @@ function main (argv) {
             serverCrashed = true;
             break;
           }
-          if (require("internal").platform.substr(0,3) === 'win') {
+          if (internal.platform.substr(0, 3) === 'win') {
             // Windows: wait for procdump to do its job...
             statusExternal(instanceInfo.monitor, true);
           }
-        }
+        } else {
-        else {
           print("Server shutdown: Success.");
           break; // Success.
         }
       }
+
       if (count > 10) {
         print("long Server shutdown: " + bar + ']');
       }
 
     }
   }
+
   return 0;
 }
+
+main(ARGUMENTS);

utils/generateExamples.py

@@ -1,8 +1,4 @@
 ################################################################################
-### @brief creates examples from documentation files
-###
-### @file
-###
 ### DISCLAIMER
 ###
 ### Copyright by triAGENS GmbH - All rights reserved.
@@ -128,14 +124,13 @@ OPTION_OUTPUT_DIR = 2
 OPTION_FILTER = 3
 OPTION_OUTPUT_FILE = 4
 
-fstate = OPTION_NORMAL
-
 escapeBS = re.compile("\\\\")
 doubleBS = "\\\\\\\\"
 
 ################################################################################
 ### @brief generate arangosh example headers with functions etc. needed later
 ################################################################################
+
 def generateArangoshHeader():
     headerF = open("./Documentation/Scripts/exampleHeader.js", "r")
     print headerF.read()
@@ -144,6 +139,7 @@ def generateArangoshHeader():
 ################################################################################
 ### @brief Try to match the start of a command section
 ################################################################################
+
 regularStartLine = re.compile(r'^(/// )? *@EXAMPLE_ARANGOSH_OUTPUT{([^}]*)}')
 runLine = re.compile(r'^(/// )? *@EXAMPLE_ARANGOSH_RUN{([^}]*)}')
     
@@ -160,6 +156,7 @@ def matchStartLine(line, filename):
         if name in ArangoshFiles:
             print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
             sys.exit(1)
+
         # if we match for filters, only output these!
         if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
             filterTestList.append(name)
@@ -184,6 +181,7 @@ def matchStartLine(line, filename):
 
         ArangoshFiles[name] = True
         return (name, STATE_ARANGOSH_RUN)
+
     # Not found, remain in STATE_BEGIN
     return ("", STATE_BEGIN)
 
@@ -194,9 +192,11 @@ TESTLINES="testlines"
 TYPE="type"
 LINE_NO="lineNo"
 STRING="string"
+
 ################################################################################
 ### @brief loop over the lines of one input file
 ################################################################################
+
 def analyzeFile(f, filename): 
     global RunTests, TESTLINES, TYPE, LINE_NO, STRING
     strip = None
@@ -294,7 +294,6 @@ def generateSetupFunction():
     print "(function () {\n%s}());" % ArangoshSetup
     print
 
-
 ################################################################################
 ### @brief generate arangosh example
 ################################################################################
@@ -302,6 +301,7 @@ def generateSetupFunction():
 loopDetectRE = re.compile(r'^[ \n]*(while|if|var|throw|for) ')
 expectErrorRE = re.compile(r'.*// *xpError\((.*)\).*')
 #expectErrorRE = re.compile(r'.*//\s*xpError\(([^)]*)\)/')
+
 def generateArangoshOutput(testName):
     value = RunTests[testName]
     #print value
@@ -370,7 +370,6 @@ def generateArangoshOutput(testName):
 }());
 '''
 
-
 ################################################################################
 ### @brief generate arangosh run
 ################################################################################
@@ -429,6 +428,7 @@ def generateArangoshRun(testName):
 ################################################################################
 ### @brief generate arangosh run
 ################################################################################
+
 def generateArangoshShutdown():
     print '''
 if (allErrors.length > 0) {
@@ -437,29 +437,35 @@ if (allErrors.length > 0) {
 }
 '''
 
-
 ################################################################################
 ### @brief get file names
 ################################################################################
+
 def loopDirectories():
     global ArangoshSetup, OutputDir, FilterForTestcase
+
     argv = sys.argv
     argv.pop(0)
     filenames = []
+    fstate = OPTION_NORMAL
     
     for filename in argv:
         if filename == "--arangoshSetup":
             fstate = OPTION_ARANGOSH_SETUP
             continue
+
         if filename == "--onlyThisOne": 
             fstate = OPTION_FILTER
             continue
+
         if filename == "--outputDir":
             fstate = OPTION_OUTPUT_DIR
             continue
+
         if filename == "--outputFile":
             fstate = OPTION_OUTPUT_FILE
             continue
+
         if fstate == OPTION_NORMAL:
             if os.path.isdir(filename):
                 for root, dirs, files in os.walk(filename):
@@ -468,6 +474,7 @@ def loopDirectories():
                             filenames.append(os.path.join(root, file))
             else:
                 filenames.append(filename)
+
         elif fstate == OPTION_FILTER:
             fstate = OPTION_NORMAL
             if (len(filename) > 0): 
@@ -486,11 +493,11 @@ def loopDirectories():
         elif fstate == OPTION_OUTPUT_DIR:
             fstate = OPTION_NORMAL
             OutputDir = filename
+
         elif fstate == OPTION_OUTPUT_FILE:
             fstate = OPTION_NORMAL
             sys.stdout = open(filename, 'w')
 
-    print >> sys.stderr, repr(filenames)
     for filename in filenames:
         if (filename.find("#") < 0):
             f = open(filename, "r")
@@ -512,10 +519,10 @@ def generateTestCases():
         elif RunTests[thisTest][TYPE] == STATE_ARANGOSH_RUN:
             generateArangoshRun(thisTest)
 
-
 ################################################################################
 ### @brief main
 ################################################################################
+
 loopDirectories()
 print >> sys.stderr, "filtering test cases %s" %(filterTestList)