diff --git a/Documentation/Books/AQL/Examples/Join.md b/Documentation/Books/AQL/Examples/Join.md
index 908da8e6c0..20dc8b9c37 100644
--- a/Documentation/Books/AQL/Examples/Join.md
+++ b/Documentation/Books/AQL/Examples/Join.md
@@ -16,54 +16,24 @@ Join tuples
We'll start with a SQL-ish result set and return each tuple (user name, friends userId)
separately. The AQL query to generate such result is:
-```js
-FOR u IN users
- FILTER u.active == true
- LIMIT 0, 4
- FOR f IN relations
- FILTER f.type == "friend" && f.friendOf == u.userId
- RETURN {
- "user" : u.name,
- "friendId" : f.thisUser
- }
-```
-```json
-[
- {
- "user" : "Abigail",
- "friendId" : 108
- },
- {
- "user" : "Abigail",
- "friendId" : 102
- },
- {
- "user" : "Abigail",
- "friendId" : 106
- },
- {
- "user" : "Fred",
- "friendId" : 209
- },
- {
- "user" : "Mary",
- "friendId" : 207
- },
- {
- "user" : "Mary",
- "friendId" : 104
- },
- {
- "user" : "Mariah",
- "friendId" : 203
- },
- {
- "user" : "Mariah",
- "friendId" : 205
- }
-]
-```
+ @startDocuBlockInline joinTuples
+ @EXAMPLE_AQL{joinTuples}
+ @DATASET{joinSampleDataset}
+ FOR u IN users
+ FILTER u.active == true
+ LIMIT 0, 4
+ FOR f IN relations
+ FILTER f.type == @friend && f.friendOf == u.userId
+ RETURN {
+ "user" : u.name,
+ "friendId" : f.thisUser
+ }
+ @BV {
+ friend: "friend"
+ }
+ @END_EXAMPLE_AQL
+ @endDocuBlock joinTuples
We iterate over the collection users. Only the 'active' users will be examined.
For each of these users we will search for up to 4 friends. We locate friends
diff --git a/Documentation/Examples/joinTuples.generated b/Documentation/Examples/joinTuples.generated
new file mode 100644
index 0000000000..10bcc3f955
--- /dev/null
+++ b/Documentation/Examples/joinTuples.generated
@@ -0,0 +1,53 @@
+@Q:
+ FOR u IN users
+ FILTER u.active == true
+ LIMIT 0, 4
+ FOR f IN relations
+ FILTER f.type == @friend && f.friendOf == u.userId
+ RETURN {
+ "user" : u.name,
+ "friendId" : f.thisUser
+ }
+@B
+{
+ "friend": "friend"
+}
+@R
+[
+ {
+ "user": "Abigail",
+ "friendId": 3
+ },
+ {
+ "user": "Abigail",
+ "friendId": 4
+ },
+ {
+ "user": "Abigail",
+ "friendId": 2
+ },
+ {
+ "user": "Mary",
+ "friendId": 1
+ },
+ {
+ "user": "Mary",
+ "friendId": 4
+ },
+ {
+ "user": "Mariah",
+ "friendId": 2
+ },
+ {
+ "user": "Mariah",
+ "friendId": 1
+ },
+ {
+ "user": "Fred",
+ "friendId": 2
+ },
+ {
+ "user": "Fred",
+ "friendId": 5
+ }
+]
\ No newline at end of file
diff --git a/Documentation/Scripts/codeBlockReader.py b/Documentation/Scripts/codeBlockReader.py
index e3fcc75b20..f4d482c2cc 100644
--- a/Documentation/Scripts/codeBlockReader.py
+++ b/Documentation/Scripts/codeBlockReader.py
@@ -60,15 +60,14 @@ def file_content(filepath):
return comments
-def example_content(filepath, fh, tag):
+def example_content(filepath, fh, tag, blockType, placeIntoFilePath):
""" Fetches an example file and inserts it using code
"""
- arangosh = False
- curl = False
first = True
+ aqlResult = False
lastline = None
- long = ""
+ longText = ""
longLines = 0
short = ""
shortLines = 0
@@ -81,26 +80,32 @@ def example_content(filepath, fh, tag):
curlState = CURL_STATE_CMD
+ AQL_STATE_QUERY = 1
+ AQL_STATE_BINDV = 2
+ AQL_STATE_RESULT = 3
+
+ aqlState = AQL_STATE_QUERY
+ blockCount = 0;
+
# read in the context, split into long and short
infile = open(filepath, 'r')
-
for line in infile:
if first:
- arangosh = line.startswith("arangosh>")
- curl = line.startswith("shell> curl")
+ if blockType == "arangosh" and not line.startswith("arangosh>"):
+ raise Exception ("mismatching blocktype - expecting 'arangosh' to start with 'arangosh>' - in %s while inpecting %s - referenced via %s have '%s'" %(filepath, tag, placeIntoFilePath, line))
+ if blockType == "curl" and not line.startswith("shell> curl"):
+ raise Exception("mismatching blocktype - expecting 'curl' to start with 'shell > curl' in %s while inpecting %s - referenced via %s have '%s'" %(filepath, tag, placeIntoFilePath, line))
first = False
- if not curl and not arangosh:
- raise Exception("failed to detect curl or arangosh example in %s while inpecting %s", filepath, tag)
- if arangosh:
+ if blockType == "arangosh":
if line.startswith("arangosh>") or line.startswith("........>"):
if lastline != None:
# short = short + lastline
# shortLines = shortLines + 1
lastline = None
- short = short + line
- shortLines = shortLines + 1
+ short += line
+ shortLines += 1
showdots = True
else:
if showdots:
@@ -116,7 +121,7 @@ def example_content(filepath, fh, tag):
showdots = False
lastline = None
- if curl:
+ if blockType == "curl":
if line.startswith("shell> curl"):
curlState = CURL_STATE_CMD
elif curlState == CURL_STATE_CMD and line.startswith("HTTP/"):
@@ -126,16 +131,42 @@ def example_content(filepath, fh, tag):
if curlState == CURL_STATE_CMD or curlState == CURL_STATE_HEADER:
short = short + line
- shortLines = shortLines + 1
+ shortLines += 1
else:
shortable = True
- long = long + line
- longLines = longLines + 1
+ if blockType == "AQL":
+ if line.startswith("@Q"): # query part
+ blockCount = 0;
+ aqlState = AQL_STATE_QUERY
+ short += "Query:\n\n"
+ longText += "Query:\n\n"
+ continue # skip this line - its only here for this.
+ elif line.startswith("@B"): # bind values part
+ short += "\nBind Values:\n\n"
+ longText += "
\nBind Values:\n\n"
+ blockCount = 0;
+ aqlState = AQL_STATE_BINDV
+ continue # skip this line - its only here for this.
+ elif line.startswith("@R"): # result part
+ shortable = True
+ longText += "\nResults:\n\n"
+ blockCount = 0;
+ aqlState = AQL_STATE_RESULT
+ continue # skip this line - its only here for this.
+
+ if aqlState == AQL_STATE_QUERY or aqlState == AQL_STATE_BINDV:
+ short = short + line
+ shortLines += 1
+
+ blockCount += 1
+
+ longText += line
+ longLines += 1
if lastline != None:
- short = short + lastline
- shortLines = shortLines + 1
+ short += lastline
+ shortLines += 1
infile.close()
@@ -157,24 +188,24 @@ def example_content(filepath, fh, tag):
else:
fh.write("\n" % longTag)
- fh.write("
\n")
-# fh.write("```\n")
- fh.write("%s" % long)
-# fh.write("```\n")
+ if blockType != "AQL":
+ fh.write("\n")
+ fh.write("%s" % longText)
fh.write("\n")
fh.write("\n" % (shortTag, shortToggle))
- fh.write("
\n")
-# fh.write("```\n")
+ if blockType != "AQL":
+ fh.write("\n")
fh.write("%s" % short)
-# fh.write("```\n")
- if arangosh:
+ if blockType == "arangosh":
fh.write("show execution results
\n")
- elif curl:
+ elif blockType == "curl":
fh.write("show response body
\n")
+ elif blockType == "AQL":
+ fh.write("
show query result
\n")
else:
fh.write("
show
\n")
@@ -212,7 +243,16 @@ def fetch_comments(dirpath):
("@endDocuBlock" in _text):
fh.write("%s\n\n" % _text)
elif ("@EXAMPLE_ARANGOSH_OUTPUT" in _text or \
- "@EXAMPLE_ARANGOSH_RUN" in _text):
+ "@EXAMPLE_ARANGOSH_RUN" in _text or \
+ "@EXAMPLE_AQL" in _text):
+ blockType=""
+ if "@EXAMPLE_ARANGOSH_OUTPUT" in _text:
+ blockType = "arangosh"
+ elif "@EXAMPLE_ARANGOSH_RUN" in _text:
+ blockType = "curl"
+ elif "@EXAMPLE_AQL" in _text:
+ blockType = "AQL"
+
shouldIgnoreLine = True
try:
_filename = re.search("{(.*)}", _text).group(1)
@@ -221,14 +261,15 @@ def fetch_comments(dirpath):
raise x
dirpath = os.path.abspath(os.path.join(os.path.dirname( __file__ ), os.pardir, "Examples", _filename + ".generated"))
if os.path.isfile(dirpath):
- example_content(dirpath, fh, _filename)
+ example_content(dirpath, fh, _filename, blockType, filepath)
else:
fullSuccess = False
print "Could not find the generated example for " + _filename + " found in " + filepath
else:
fh.write("%s\n" % _text)
elif ("@END_EXAMPLE_ARANGOSH_OUTPUT" in _text or \
- "@END_EXAMPLE_ARANGOSH_RUN" in _text):
+ "@END_EXAMPLE_ARANGOSH_RUN" in _text or \
+ "@END_EXAMPLE_AQL" in _text):
shouldIgnoreLine = False
else:
fh.write("\n")
diff --git a/Documentation/Scripts/exampleHeader.js b/Documentation/Scripts/exampleHeader.js
index b790b185b9..6762fd618b 100644
--- a/Documentation/Scripts/exampleHeader.js
+++ b/Documentation/Scripts/exampleHeader.js
@@ -41,6 +41,8 @@ var ignoreCollectionAlreadyThere = [];
var rc;
var j;
+var exds = require("@arangodb/examples/examples").Examples;
+
var hljs = require('highlightjs');
var MAP = {
diff --git a/README_maintainers.md b/README_maintainers.md
index e285109d57..0e976fa33d 100644
--- a/README_maintainers.md
+++ b/README_maintainers.md
@@ -700,8 +700,8 @@ Here is how its details work:
- all code in between is executed as javascript in the **arangosh** while talking to a valid **arangod**.
You may inspect the generated js code in `/tmp/arangosh.examples.js`
-OUTPUT and RUN specifics
----------------------------
+OUTPUT, RUN and AQL specifics
+-----------------------------
By default, Examples should be self contained and thus not depend on each other. They should clean up the collections they create.
Building will fail if resources aren't cleaned.
However, if you intend a set of OUTPUT and RUN to demonstrate interactively and share generated *ids*, you have to use an alphabetical
@@ -739,6 +739,13 @@ sortable naming scheme so they're executed in sequence. Using `
_> sys.stderr, "Arangosh: filtering out testcase '%s'" %name
filterTestList.append(name)
return("", STATE_BEGIN);
@@ -169,7 +178,7 @@ def matchStartLine(line, filename):
if m:
strip = m.group(1)
name = m.group(2)
-
+
if name in ArangoshFiles:
print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
sys.exit(1)
@@ -177,14 +186,52 @@ def matchStartLine(line, filename):
# if we match for filters, only output these!
if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
filterTestList.append(name)
+ print >> sys.stderr, "CuRL: filtering out testcase '%s'" %name
return("", STATE_BEGIN);
ArangoshFiles[name] = True
return (name, STATE_ARANGOSH_RUN)
+ m = aqlLine.match(line)
+ if m:
+ strip = m.group(1)
+ name = m.group(2)
+
+ if name in AQLFiles:
+ print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
+ sys.exit(1)
+
+ # if we match for filters, only output these!
+ if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
+ print >> sys.stderr, "AQL: filtering out testcase '%s'" %name
+ filterTestList.append(name)
+ return("", STATE_BEGIN);
+
+ AQLFiles[name] = True
+ return (name, STATE_AQL)
+
# Not found, remain in STATE_BEGIN
return ("", STATE_BEGIN)
+def matchAqlLine(line, filename, oldState):
+ global aqlDataSetLine, aqlBindvaluesLine
+ m = aqlDataSetLine.match(line)
+
+ if m:
+ strip = m.group(1)
+ name = m.group(2)
+ return (name, STATE_AQL_DS)
+
+ m = aqlBindvaluesLine.match(line)
+
+ if m:
+ strip = m.group(1)
+ bindvalueStart = m.group(2)
+ return (bindvalueStart, STATE_AQL_BIND)
+
+ # Not found, remain in STATE_AQL
+ return (line, oldState)
+
endExample = re.compile(r'^(/// )? *@END_EXAMPLE_')
#r5 = re.compile(r'^ +')
@@ -192,6 +239,9 @@ TESTLINES="testlines"
TYPE="type"
LINE_NO="lineNo"
STRING="string"
+AQL="aql"
+AQLDS="aql_dataset"
+AQLBV="aql_bindvalues"
################################################################################
### @brief loop over the lines of one input file
@@ -229,8 +279,24 @@ def analyzeFile(f, filename):
if state == STATE_ARANGOSH_RUN:
RunTests[name][LINE_NO] = lineNo;
RunTests[name][STRING] = "";
+
+ if state == STATE_AQL:
+ RunTests[name][LINE_NO] = lineNo;
+ RunTests[name][AQL] = "";
continue
+ if state == STATE_AQL:
+ (data, aqlState) = matchAqlLine(line, filename, state)
+ if aqlState == STATE_AQL_BIND:
+ RunTests[name][AQLBV] = data
+ state = aqlState
+ continue
+ if aqlState == STATE_AQL_DS:
+ RunTests[name][AQLDS] = data
+ # flip back to aql - query will come.
+ state = STATE_AQL
+ continue
+
# we are within a example
line = line[len(strip):]
showCmd = True
@@ -246,7 +312,8 @@ def analyzeFile(f, filename):
continue
line = line.lstrip('/');
- line = line.lstrip(' ');
+ if state != STATE_AQL:
+ line = line.lstrip(' ');
if state == STATE_ARANGOSH_OUTPUT:
line = line.replace("\\", "\\\\").replace("'", "\\'")
#print line
@@ -287,6 +354,10 @@ def analyzeFile(f, filename):
RunTests[name][TESTLINES].append([line, showCmd, lineNo])
elif state == STATE_ARANGOSH_RUN:
RunTests[name][STRING] += line + "\n"
+ elif state == STATE_AQL:
+ RunTests[name][AQL] += line + "\n"
+ elif state == STATE_AQL_BIND:
+ RunTests[name][AQLBV] += line + "\n"
def generateSetupFunction():
@@ -332,9 +403,9 @@ def generateArangoshOutput(testName):
escapeBS.sub(doubleBS, MapSourceFiles[testName])
)
except Exception as x:
- print x
- print testName
- print value
+ print >> sys.stderr,x
+ print >> sys.stderr,testName
+ print >> sys.stderr,value
raise
for l in value[TESTLINES]:
@@ -429,6 +500,62 @@ def generateArangoshRun(testName):
### @brief generate arangosh run
################################################################################
+def generateAQL(testName):
+ value = RunTests[testName]
+ startLineNo = RunTests[testName][LINE_NO]
+ if not AQLBV in value:
+ value[AQLBV] = "{}"
+
+ print '''
+%s
+/// %s
+(function() {
+ internal.startPrettyPrint(true);
+ internal.stopColorPrint(true);
+ var testName = '%s';
+ var lineCount = 0;
+ var startLineCount = %d;
+ var outputDir = '%s';
+ var sourceFile = '%s';
+ var startTime = time();
+ output = '';
+''' % (
+ ('/'*80),
+ testName,
+ testName,
+ startLineNo,
+ escapeBS.sub(doubleBS, OutputDir),
+ escapeBS.sub(doubleBS, MapSourceFiles[testName])
+ )
+ print " let query = `" + value[AQL] + "`;"
+ print " let bv = " + value[AQLBV] + ";"
+ print " let ds = '" + value[AQLDS] + "';"
+ print '''
+ exds.%s.removeDS();
+ exds.%s.createDS();
+ let result = db._query(query, bv).toArray();
+
+ output += "@Q:\\n"
+ output += highlight("js", query);
+ if (Object.keys(bv).length > 0) {
+ output += "@B\\n"
+ jsonAppender(JSON.stringify(bv, null, 2));
+ }
+ output += "\\n@R\\n";
+ jsonAppender(JSON.stringify(result, null, 2));
+
+ exds.%s.removeDS();
+ fs.write(outputDir + fs.pathSeparator + testName + '.generated', output);
+ print("[" + (time () - startTime) + "s] done with " + testName);
+ checkForOrphanTestCollections('not all collections were cleaned up after ' + sourceFile + ' Line[' + startLineCount + '] [' + testName + ']:');
+}());
+
+''' % (value[AQLDS], value[AQLDS], value[AQLDS])
+
+################################################################################
+### @brief generate arangosh run
+################################################################################
+
def generateArangoshShutdown():
print '''
if (allErrors.length > 0) {
@@ -443,7 +570,6 @@ if (allErrors.length > 0) {
def loopDirectories():
global ArangoshSetup, OutputDir, FilterForTestcase
-
argv = sys.argv
argv.pop(0)
filenames = []
@@ -501,7 +627,6 @@ def loopDirectories():
for filename in filenames:
if (filename.find("#") < 0):
f = open(filename, "r")
-
analyzeFile(f, filename)
f.close()
@@ -518,12 +643,18 @@ def generateTestCases():
generateArangoshOutput(thisTest)
elif RunTests[thisTest][TYPE] == STATE_ARANGOSH_RUN:
generateArangoshRun(thisTest)
+ elif RunTests[thisTest][TYPE] == STATE_AQL:
+ generateAQL(thisTest)
+
################################################################################
### @brief main
################################################################################
loopDirectories()
+if len(RunTests) == 0:
+ print >> sys.stderr, "no testcases generated - bailing out"
+ raise Exception("no Testcases")
print >> sys.stderr, "filtering test %d cases" %(len(filterTestList))
generateArangoshHeader()