Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fixed generateExample

This commit is contained in:
Frank Celler 2016-03-17 12:00:14 +01:00
parent 13603e876e
commit e0bb505dda
4 changed files with 156 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Documentation/CMakeLists.txt
View File

@ -1,7 +1,7 @@
# -*- mode: CMAKE; -*-
# swagger
add_custom_target (swagger
add_custom_target(swagger
COMMAND ${PYTHON_EXECUTABLE}
${PROJECT_SOURCE_DIR}/Documentation/Scripts/generateSwagger.py
${PROJECT_SOURCE_DIR}
@ -9,6 +9,11 @@ add_custom_target (swagger
${PROJECT_SOURCE_DIR}/Documentation/DocuBlocks/Rest/
> ${PROJECT_SOURCE_DIR}/js/apps/system/_admin/aardvark/APP/api-docs.json)
# swagger
add_custom_target(examples
COMMAND ${PROJECT_SOURCE_DIR}/utils/generateExamples
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
# manual pages
if (USE_MAINTAINER_MODE)
set(MAN_NAMES

32
utils/generateExamples
View File

@ -19,39 +19,27 @@ mkdir -p ${DBDIR}
echo Database has its data in ${DBDIR}
echo Logfile is in ${LOGFILE}
if [ -z "${ARANGOD}" ]; then
if [ -x build/bin/arangod ]; then
ARANGOD=build/bin/arangod
if [ -z "${ARANGOSH}" ]; then
if [ -x build/bin/arangosh ]; then
ARANGOSH=build/bin/arangosh
elif [ -x bin/arangosh ]; then
ARANGOD=bin/arangod
ARANGOSH=bin/arangosh
else
echo "$0: cannot locate arangod"
echo "$0: cannot locate arangosh"
fi
fi
${ARANGOD} \
${ARANGOSH} \
--configuration none \
--cluster.agent-path bin${PS}etcd-arango${EXT} \
--cluster.arangod-path bin${PS}arangod \
--cluster.coordinator-config etc${PS}relative${PS}arangod-coordinator.conf \
--cluster.dbserver-config etc${PS}relative${PS}arangod-dbserver.conf \
--cluster.disable-dispatcher-frontend false \
--cluster.disable-dispatcher-kickstarter false \
--cluster.data-path cluster \
--cluster.log-path cluster \
--database.directory ${DBDIR} \
--server.endpoint none \
--log.file ${LOGFILE} \
--server.endpoint tcp://127.0.0.1:$PORT \
--javascript.startup-directory js \
--javascript.app-path js${PS}apps \
--javascript.script $SCRIPT \
--no-server \
--temp-path ${PS}var${PS}tmp \
"${ARGS[@]}" \
--javascript.execute $SCRIPT \
"${ARGS[@]}"
if test $? -eq 0; then
echo "removing ${LOGFILE} ${DBDIR}"
rm -rf ${LOGFILE} ${DBDIR}
rm -rf ${LOGFILE} ${DBDIR} arangosh.examples.js
else
echo "failed - don't remove ${LOGFILE} ${DBDIR} - here's the logfile:"
cat ${LOGFILE}

193
utils/generateExamples.js
View File

@ -1,58 +1,93 @@
/*jshint globalstrict:false, unused:false */
/*global start_pretty_print */
'use strict';
var fs = require("fs");
var internal = require("internal");
var executeExternal = require("internal").executeExternal;
var executeExternalAndWait = internal.executeExternalAndWait;
var download = require("internal").download;
var print = internal.print;
var wait = require("internal").wait;
var killExternal = require("internal").killExternal;
var toArgv = require("internal").toArgv;
var statusExternal = require("internal").statusExternal;
const fs = require("fs");
const internal = require("internal");
const executeExternal = internal.executeExternal;
const executeExternalAndWait = internal.executeExternalAndWait;
const download = internal.download;
const print = internal.print;
const wait = internal.wait;
const killExternal = internal.killExternal;
const toArgv = internal.toArgv;
const statusExternal = internal.statusExternal;
const testPort = internal.testPort;
var yaml = require("js-yaml");
var endpointToURL = require("@arangodb/cluster/planner").endpointToURL;
var PortFinder = require("@arangodb/cluster").PortFinder;
const yaml = require("js-yaml");
var documentationSourceDirs = [
const documentationSourceDirs = [
fs.join(fs.makeAbsolute(''), "Documentation/Examples/setup-arangosh.js"),
fs.join(fs.makeAbsolute(''), "Documentation/Books/Users"),
fs.join(fs.makeAbsolute(''), "js/actions"),
fs.join(fs.makeAbsolute(''), "js/client"),
fs.join(fs.makeAbsolute(''), "js/common"),
fs.join(fs.makeAbsolute(''), "js/server"),
fs.join(fs.makeAbsolute(''), "js/apps/system/_api/gharial/APP")];
fs.join(fs.makeAbsolute(''), "Documentation/DocuBlocks"),
fs.join(fs.makeAbsolute(''), "Documentation/Books/Users")
];
var theScript = 'Documentation/Scripts/generateExamples.py';
const theScript = 'utils/generateExamples.py';
var scriptArguments = {
const scriptArguments = {
'outputDir': fs.join(fs.makeAbsolute(''), "Documentation/Examples"),
'outputFile': '/tmp/arangosh.examples.js'
'outputFile': fs.join(fs.makeAbsolute(''), "arangosh.examples.js")
};
function main (argv) {
"use strict";
var thePython = 'python';
var test = argv[1];
var options = {};
var serverEndpoint = '';
var startServer = true;
var instanceInfo = {};
var serverCrashed = false;
var protocol = 'tcp';
var tmpDataDir = fs.getTempFile();
var count = 0;
let ARANGOD;
let ARANGOSH;
if (fs.exists("bin")) {
ARANGOD = fs.join(fs.join(fs.makeAbsolute('')), "bin/arangod");
ARANGOSH = fs.join(fs.join(fs.makeAbsolute('')), "bin/arangosh");
}
else {
ARANGOD = fs.join(fs.join(fs.makeAbsolute('')), "build/bin/arangod");
ARANGOSH = fs.join(fs.join(fs.makeAbsolute('')), "build/bin/arangosh");
}
function endpointToURL(endpoint) {
if (endpoint.substr(0, 6) === "ssl://") {
return "https://" + endpoint.substr(6);
}
const pos = endpoint.indexOf("://");
if (pos === -1) {
return "http://" + endpoint;
}
return "http" + endpoint.substr(pos);
}
function findFreePort() {
while (true) {
const port = Math.floor(Math.random() * (65536 - 1024)) + 1024;
const free = testPort("tcp://0.0.0.0:" + port);
if (free) {
return port;
}
}
return 8529;
}
function main(argv) {
let thePython = 'python';
let options = {};
let serverEndpoint = '';
let startServer = true;
let instanceInfo = {};
let serverCrashed = false;
let protocol = 'tcp';
let tmpDataDir = fs.getTempFile();
let count = 0;
try {
options = internal.parseArgv(argv, 1);
}
catch (x) {
options = internal.parseArgv(argv, 0);
} catch (x) {
print("failed to parse the options: " + x.message);
return -1;
}
print(options);
if (options.hasOwnProperty('withPython')) {
thePython = options.withPython;
}
@ -65,13 +100,13 @@ function main (argv) {
startServer = false;
serverEndpoint = options['server.endpoint'];
}
var args = [theScript].concat(internal.toArgv(scriptArguments));
let args = [theScript].concat(internal.toArgv(scriptArguments));
args = args.concat(['--arangoshSetup']);
args = args.concat(documentationSourceDirs);
// internal.print(JSON.stringify(args));
let res = executeExternalAndWait(thePython, args);
var res = executeExternalAndWait(thePython, args);
if (res.exit !== 0) {
print("parsing the examples failed - aborting!");
print(res);
@ -79,36 +114,46 @@ function main (argv) {
}
if (startServer) {
// We use the PortFinder to find a free port for our subinstance,
// to this end, we have to fake a dummy dispatcher:
var dispatcher = {endpoint: "tcp://127.0.0.1:", avoidPorts: {}, id: "me"};
var pf = new PortFinder([8529],dispatcher);
var port = pf.next();
let port = findFreePort();
instanceInfo.port = port;
serverEndpoint = protocol+"://127.0.0.1:"+port;
serverEndpoint = protocol + "://127.0.0.1:" + port;
instanceInfo.url = endpointToURL(serverEndpoint);
var serverArgs = {};
fs.makeDirectoryRecursive(fs.join(tmpDataDir, "data"));
let serverArgs = {};
serverArgs["configuration"] = "none";
serverArgs["database.directory"] = fs.join(tmpDataDir, "data");
serverArgs["javascript.app-path"] = fs.join(tmpDataDir, "apps");
serverArgs["javascript.startup-directory"] = "js";
serverArgs["log.file"] = fs.join(tmpDataDir, "log");
serverArgs["server.disable-authentication"] = "true";
serverArgs["server.endpoint"] = serverEndpoint;
serverArgs["database.directory"] = fs.join(tmpDataDir,"data");
fs.makeDirectoryRecursive(fs.join(tmpDataDir,"data"));
args["log.file"] = fs.join(tmpDataDir,"log");
instanceInfo.pid = executeExternal(fs.join("bin","arangod"), toArgv(serverArgs));
serverArgs["server.threads"] = "3";
print("================================================================================");
print(toArgv(serverArgs));
instanceInfo.pid = executeExternal(ARANGOD, toArgv(serverArgs));
// Wait until the server is up:
count = 0;
instanceInfo.endpoint = serverEndpoint;
while (true) {
wait(0.5, false);
var r = download(instanceInfo.url + "/_api/version", "");
let r = download(instanceInfo.url + "/_api/version", "");
if (! r.error && r.code === 200) {
if (!r.error && r.code === 200) {
break;
}
count ++;
count++;
if (count % 60 === 0) {
res = statusExternal(instanceInfo.pid, false);
if (res.status !== "RUNNING") {
print("start failed - process is gone: " + yaml.safeDump(res));
return 1;
@ -116,67 +161,71 @@ function main (argv) {
}
}
}
var arangoshArgs = {
let arangoshArgs = {
'configuration': fs.join(fs.makeAbsolute(''), 'etc', 'relative', 'arangosh.conf'),
'server.password': "",
'server.endpoint': serverEndpoint,
'javascript.execute': scriptArguments.outputFile
};
res = executeExternalAndWait('bin/arangosh', internal.toArgv(arangoshArgs));
res = executeExternalAndWait(ARANGOSH, internal.toArgv(arangoshArgs));
if (startServer) {
if (typeof(instanceInfo.exitStatus) === 'undefined') {
download(instanceInfo.url+"/_admin/shutdown","");
download(instanceInfo.url + "/_admin/shutdown", "");
print("Waiting for server shut down");
count = 0;
var bar = "[";
let bar = "[";
while (1) {
instanceInfo.exitStatus = statusExternal(instanceInfo.pid, false);
if (instanceInfo.exitStatus.status === "RUNNING") {
count ++;
count++;
if (typeof(options.valgrind) === 'string') {
wait(1);
continue;
}
if (count % 10 ===0) {
if (count % 10 === 0) {
bar = bar + "#";
}
if (count > 600) {
print("forcefully terminating " + yaml.safeDump(instanceInfo.pid) +
" after 600 s grace period; marking crashy.");
" after 600 s grace period; marking crashy.");
serverCrashed = true;
killExternal(instanceInfo.pid);
break;
}
else {
} else {
wait(1);
}
}
else if (instanceInfo.exitStatus.status !== "TERMINATED") {
} else if (instanceInfo.exitStatus.status !== "TERMINATED") {
if (instanceInfo.exitStatus.hasOwnProperty('signal')) {
print("Server shut down with : " +
yaml.safeDump(instanceInfo.exitStatus) +
" marking build as crashy.");
yaml.safeDump(instanceInfo.exitStatus) +
" marking build as crashy.");
serverCrashed = true;
break;
}
if (require("internal").platform.substr(0,3) === 'win') {
if (internal.platform.substr(0, 3) === 'win') {
// Windows: wait for procdump to do its job...
statusExternal(instanceInfo.monitor, true);
}
}
else {
} else {
print("Server shutdown: Success.");
break; // Success.
}
}
if (count > 10) {
print("long Server shutdown: " + bar + ']');
}
}
}
return 0;
}
main(ARGUMENTS);

31
Documentation/Scripts/generateExamples.py → utils/generateExamples.py
View File

@ -1,8 +1,4 @@
################################################################################
### @brief creates examples from documentation files
###
### @file
###
### DISCLAIMER
###
### Copyright by triAGENS GmbH - All rights reserved.
@ -128,14 +124,13 @@ OPTION_OUTPUT_DIR = 2
OPTION_FILTER = 3
OPTION_OUTPUT_FILE = 4
fstate = OPTION_NORMAL
escapeBS = re.compile("\\\\")
doubleBS = "\\\\\\\\"
################################################################################
### @brief generate arangosh example headers with functions etc. needed later
################################################################################
def generateArangoshHeader():
headerF = open("./Documentation/Scripts/exampleHeader.js", "r")
print headerF.read()
@ -144,6 +139,7 @@ def generateArangoshHeader():
################################################################################
### @brief Try to match the start of a command section
################################################################################
regularStartLine = re.compile(r'^(/// )? *@EXAMPLE_ARANGOSH_OUTPUT{([^}]*)}')
runLine = re.compile(r'^(/// )? *@EXAMPLE_ARANGOSH_RUN{([^}]*)}')
@ -160,6 +156,7 @@ def matchStartLine(line, filename):
if name in ArangoshFiles:
print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
sys.exit(1)
# if we match for filters, only output these!
if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
filterTestList.append(name)
@ -184,6 +181,7 @@ def matchStartLine(line, filename):
ArangoshFiles[name] = True
return (name, STATE_ARANGOSH_RUN)
# Not found, remain in STATE_BEGIN
return ("", STATE_BEGIN)
@ -194,9 +192,11 @@ TESTLINES="testlines"
TYPE="type"
LINE_NO="lineNo"
STRING="string"
################################################################################
### @brief loop over the lines of one input file
################################################################################
def analyzeFile(f, filename):
global RunTests, TESTLINES, TYPE, LINE_NO, STRING
strip = None
@ -294,7 +294,6 @@ def generateSetupFunction():
print "(function () {\n%s}());" % ArangoshSetup
print
################################################################################
### @brief generate arangosh example
################################################################################
@ -302,6 +301,7 @@ def generateSetupFunction():
loopDetectRE = re.compile(r'^[ \n]*(while|if|var|throw|for) ')
expectErrorRE = re.compile(r'.*// *xpError\((.*)\).*')
#expectErrorRE = re.compile(r'.*//\s*xpError\(([^)]*)\)/')
def generateArangoshOutput(testName):
value = RunTests[testName]
#print value
@ -370,7 +370,6 @@ def generateArangoshOutput(testName):
}());
'''
################################################################################
### @brief generate arangosh run
################################################################################
@ -429,6 +428,7 @@ def generateArangoshRun(testName):
################################################################################
### @brief generate arangosh run
################################################################################
def generateArangoshShutdown():
print '''
if (allErrors.length > 0) {
@ -437,29 +437,35 @@ if (allErrors.length > 0) {
}
'''
################################################################################
### @brief get file names
################################################################################
def loopDirectories():
global ArangoshSetup, OutputDir, FilterForTestcase
argv = sys.argv
argv.pop(0)
filenames = []
fstate = OPTION_NORMAL
for filename in argv:
if filename == "--arangoshSetup":
fstate = OPTION_ARANGOSH_SETUP
continue
if filename == "--onlyThisOne":
fstate = OPTION_FILTER
continue
if filename == "--outputDir":
fstate = OPTION_OUTPUT_DIR
continue
if filename == "--outputFile":
fstate = OPTION_OUTPUT_FILE
continue
if fstate == OPTION_NORMAL:
if os.path.isdir(filename):
for root, dirs, files in os.walk(filename):
@ -468,11 +474,12 @@ def loopDirectories():
filenames.append(os.path.join(root, file))
else:
filenames.append(filename)
elif fstate == OPTION_FILTER:
fstate = OPTION_NORMAL
if (len(filename) > 0):
FilterForTestcase = re.compile(filename);
elif fstate == OPTION_ARANGOSH_SETUP:
fstate = OPTION_NORMAL
f = open(filename, "r")
@ -486,11 +493,11 @@ def loopDirectories():
elif fstate == OPTION_OUTPUT_DIR:
fstate = OPTION_NORMAL
OutputDir = filename
elif fstate == OPTION_OUTPUT_FILE:
fstate = OPTION_NORMAL
sys.stdout = open(filename, 'w')
print >> sys.stderr, repr(filenames)
for filename in filenames:
if (filename.find("#") < 0):
f = open(filename, "r")
@ -512,10 +519,10 @@ def generateTestCases():
elif RunTests[thisTest][TYPE] == STATE_ARANGOSH_RUN:
generateArangoshRun(thisTest)
################################################################################
### @brief main
################################################################################
loopDirectories()
print >> sys.stderr, "filtering test cases %s" %(filterTestList)