1
0
Fork 0

Doc - Interactively generated AQL examples (#4757)

* interactively generated AQL examples
This commit is contained in:
Wilfried Goesgens 2018-03-05 19:17:16 +01:00 committed by sleto-it
parent 76e7461aa9
commit c7a9b41cf5
7 changed files with 369 additions and 88 deletions

View File

@ -16,54 +16,24 @@ Join tuples
We'll start with a SQL-ish result set and return each tuple (user name, friends userId)
separately. The AQL query to generate such result is:
```js
FOR u IN users
FILTER u.active == true
LIMIT 0, 4
FOR f IN relations
FILTER f.type == "friend" && f.friendOf == u.userId
RETURN {
"user" : u.name,
"friendId" : f.thisUser
}
```
```json
[
{
"user" : "Abigail",
"friendId" : 108
},
{
"user" : "Abigail",
"friendId" : 102
},
{
"user" : "Abigail",
"friendId" : 106
},
{
"user" : "Fred",
"friendId" : 209
},
{
"user" : "Mary",
"friendId" : 207
},
{
"user" : "Mary",
"friendId" : 104
},
{
"user" : "Mariah",
"friendId" : 203
},
{
"user" : "Mariah",
"friendId" : 205
}
]
```
@startDocuBlockInline joinTuples
@EXAMPLE_AQL{joinTuples}
@DATASET{joinSampleDataset}
FOR u IN users
FILTER u.active == true
LIMIT 0, 4
FOR f IN relations
FILTER f.type == @friend && f.friendOf == u.userId
RETURN {
"user" : u.name,
"friendId" : f.thisUser
}
@BV {
friend: "friend"
}
@END_EXAMPLE_AQL
@endDocuBlock joinTuples
We iterate over the collection users. Only the 'active' users will be examined.
For each of these users we will search for up to 4 friends. We locate friends

View File

@ -0,0 +1,53 @@
@Q:
FOR u IN users
FILTER u.active == <span class="hljs-literal">true</span>
LIMIT <span class="hljs-number">0</span>, <span class="hljs-number">4</span>
FOR f IN relations
FILTER f.type == @friend &amp;&amp; f.friendOf == u.userId
RETURN {
<span class="hljs-string">"user"</span> : u.name,
<span class="hljs-string">"friendId"</span> : f.thisUser
}
@B
{
<span class="hljs-string">"friend"</span>: <span class="hljs-string">"friend"</span>
}
@R
[
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Abigail"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">3</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Abigail"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">4</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Abigail"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">2</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Mary"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">1</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Mary"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">4</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Mariah"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">2</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Mariah"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">1</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Fred"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">2</span>
},
{
<span class="hljs-string">"user"</span>: <span class="hljs-string">"Fred"</span>,
<span class="hljs-string">"friendId"</span>: <span class="hljs-number">5</span>
}
]

View File

@ -60,15 +60,14 @@ def file_content(filepath):
return comments
def example_content(filepath, fh, tag):
def example_content(filepath, fh, tag, blockType, placeIntoFilePath):
""" Fetches an example file and inserts it using code
"""
arangosh = False
curl = False
first = True
aqlResult = False
lastline = None
long = ""
longText = ""
longLines = 0
short = ""
shortLines = 0
@ -81,26 +80,32 @@ def example_content(filepath, fh, tag):
curlState = CURL_STATE_CMD
AQL_STATE_QUERY = 1
AQL_STATE_BINDV = 2
AQL_STATE_RESULT = 3
aqlState = AQL_STATE_QUERY
blockCount = 0;
# read in the context, split into long and short
infile = open(filepath, 'r')
for line in infile:
if first:
arangosh = line.startswith("arangosh&gt;")
curl = line.startswith("shell> curl")
if blockType == "arangosh" and not line.startswith("arangosh&gt;"):
raise Exception ("mismatching blocktype - expecting 'arangosh' to start with 'arangosh&gt' - in %s while inpecting %s - referenced via %s have '%s'" %(filepath, tag, placeIntoFilePath, line))
if blockType == "curl" and not line.startswith("shell> curl"):
raise Exception("mismatching blocktype - expecting 'curl' to start with 'shell > curl' in %s while inpecting %s - referenced via %s have '%s'" %(filepath, tag, placeIntoFilePath, line))
first = False
if not curl and not arangosh:
raise Exception("failed to detect curl or arangosh example in %s while inpecting %s", filepath, tag)
if arangosh:
if blockType == "arangosh":
if line.startswith("arangosh&gt;") or line.startswith("........&gt;"):
if lastline != None:
# short = short + lastline
# shortLines = shortLines + 1
lastline = None
short = short + line
shortLines = shortLines + 1
short += line
shortLines += 1
showdots = True
else:
if showdots:
@ -116,7 +121,7 @@ def example_content(filepath, fh, tag):
showdots = False
lastline = None
if curl:
if blockType == "curl":
if line.startswith("shell&gt; curl"):
curlState = CURL_STATE_CMD
elif curlState == CURL_STATE_CMD and line.startswith("HTTP/"):
@ -126,16 +131,42 @@ def example_content(filepath, fh, tag):
if curlState == CURL_STATE_CMD or curlState == CURL_STATE_HEADER:
short = short + line
shortLines = shortLines + 1
shortLines += 1
else:
shortable = True
long = long + line
longLines = longLines + 1
if blockType == "AQL":
if line.startswith("@Q"): # query part
blockCount = 0;
aqlState = AQL_STATE_QUERY
short += "<strong>Query:</strong>\n<pre>\n"
longText += "<strong>Query:</strong>\n<pre>\n"
continue # skip this line - its only here for this.
elif line.startswith("@B"): # bind values part
short += "</pre>\n<strong>Bind Values:</strong>\n<pre>\n"
longText += "</pre>\n<strong>Bind Values:</strong>\n<pre>\n"
blockCount = 0;
aqlState = AQL_STATE_BINDV
continue # skip this line - its only here for this.
elif line.startswith("@R"): # result part
shortable = True
longText += "</pre>\n<strong>Results:</strong>\n<pre>\n"
blockCount = 0;
aqlState = AQL_STATE_RESULT
continue # skip this line - its only here for this.
if aqlState == AQL_STATE_QUERY or aqlState == AQL_STATE_BINDV:
short = short + line
shortLines += 1
blockCount += 1
longText += line
longLines += 1
if lastline != None:
short = short + lastline
shortLines = shortLines + 1
short += lastline
shortLines += 1
infile.close()
@ -157,24 +188,24 @@ def example_content(filepath, fh, tag):
else:
fh.write("<div id=\"%s\">\n" % longTag)
fh.write("<pre>\n")
# fh.write("```\n")
fh.write("%s" % long)
# fh.write("```\n")
if blockType != "AQL":
fh.write("<pre>\n")
fh.write("%s" % longText)
fh.write("</pre>\n")
fh.write("</div>\n")
if shortable:
fh.write("<div id=\"%s\" onclick=\"%s\">\n" % (shortTag, shortToggle))
fh.write("<pre>\n")
# fh.write("```\n")
if blockType != "AQL":
fh.write("<pre>\n")
fh.write("%s" % short)
# fh.write("```\n")
if arangosh:
if blockType == "arangosh":
fh.write("</pre><div class=\"example_show_button\">show execution results</div>\n")
elif curl:
elif blockType == "curl":
fh.write("</pre><div class=\"example_show_button\">show response body</div>\n")
elif blockType == "AQL":
fh.write("</pre><div class=\"example_show_button\">show query result</div>\n")
else:
fh.write("</pre><div class=\"example_show_button\">show</div>\n")
@ -212,7 +243,16 @@ def fetch_comments(dirpath):
("@endDocuBlock" in _text):
fh.write("%s\n\n" % _text)
elif ("@EXAMPLE_ARANGOSH_OUTPUT" in _text or \
"@EXAMPLE_ARANGOSH_RUN" in _text):
"@EXAMPLE_ARANGOSH_RUN" in _text or \
"@EXAMPLE_AQL" in _text):
blockType=""
if "@EXAMPLE_ARANGOSH_OUTPUT" in _text:
blockType = "arangosh"
elif "@EXAMPLE_ARANGOSH_RUN" in _text:
blockType = "curl"
elif "@EXAMPLE_AQL" in _text:
blockType = "AQL"
shouldIgnoreLine = True
try:
_filename = re.search("{(.*)}", _text).group(1)
@ -221,14 +261,15 @@ def fetch_comments(dirpath):
raise x
dirpath = os.path.abspath(os.path.join(os.path.dirname( __file__ ), os.pardir, "Examples", _filename + ".generated"))
if os.path.isfile(dirpath):
example_content(dirpath, fh, _filename)
example_content(dirpath, fh, _filename, blockType, filepath)
else:
fullSuccess = False
print "Could not find the generated example for " + _filename + " found in " + filepath
else:
fh.write("%s\n" % _text)
elif ("@END_EXAMPLE_ARANGOSH_OUTPUT" in _text or \
"@END_EXAMPLE_ARANGOSH_RUN" in _text):
"@END_EXAMPLE_ARANGOSH_RUN" in _text or \
"@END_EXAMPLE_AQL" in _text):
shouldIgnoreLine = False
else:
fh.write("\n")

View File

@ -41,6 +41,8 @@ var ignoreCollectionAlreadyThere = [];
var rc;
var j;
var exds = require("@arangodb/examples/examples").Examples;
var hljs = require('highlightjs');
var MAP = {

View File

@ -700,8 +700,8 @@ Here is how its details work:
- all code in between is executed as javascript in the **arangosh** while talking to a valid **arangod**.
You may inspect the generated js code in `/tmp/arangosh.examples.js`
OUTPUT and RUN specifics
---------------------------
OUTPUT, RUN and AQL specifics
-----------------------------
By default, Examples should be self contained and thus not depend on each other. They should clean up the collections they create.
Building will fail if resources aren't cleaned.
However, if you intend a set of OUTPUT and RUN to demonstrate interactively and share generated *ids*, you have to use an alphabetical
@ -739,6 +739,13 @@ sortable naming scheme so they're executed in sequence. Using `<modulename>_<seq
* output the plain text to dump to the user: `logRawResponse(response);`
* dump the reply to the errorlog for testing (will mark run as failed): `logErrorResponse(response);`
- AQL is intended to contain AQL queries that can be pasted into arangosh or the webinterfaces query editor.
Usually this query references an example dataset generator in `js/common/modules/@arangodb/examples/examples.js`
which the users can also invoke to generate the data in their installation.
This sort of example consists of three parts:
- @DATASET{datasetName} - the name of the dataset in the above mentioned `examples.js` to be instanciated before executing this query.
- A following AQL query which may either end at the end of the comment block, or at the optional next section:
- @BV - verbatim object containing the bind parameters to be passed into the query. Will also be put into the generated snippet.
Swagger integration
===================

View File

@ -0,0 +1,77 @@
// //////////////////////////////////////////////////////////////////////////////
// / @brief example-users
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2014 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Wilfried Goesgens
// / @author Copyright 2018, ArangoDB Inc
// //////////////////////////////////////////////////////////////////////////////
let db = require("internal").db;
exports.Examples = {
'joinSampleDataset': {
createDS: function() {
db._create("users");
db._create("relations");
[ [1, "Abigail", true ],
[2, "Fred", true ],
[3, "Mary", true ],
[4, "Mariah", true ],
[5, "John", false]
].forEach(function (v) {
db.users.save( {
_key: v[1],
name: v[1],
active: v[2],
userId: v[0]
});
});
[
[1,2,"friend"],
[1,3,"friend"],
[1,4,"friend"],
[2,5,"friend"],
[2,2,"friend"],
[3,4,"friend"],
[3,1,"friend"],
[4,1,"friend"],
[4,2,"friend"]
].forEach(function (v) {
db.relations.save( {
type: v[2],
friendOf: v[0],
thisUser: v[1]
});
});
},
removeDS: function() {
try {
db._drop("users");
} catch (e) {}
try {
db._drop("relations");
} catch (e) {}
}
}
};

View File

@ -87,6 +87,7 @@ ArangoshRunLineNo = {}
################################################################################
ArangoshFiles = {}
AQLFiles = {}
################################################################################
### @brief map the source files for error messages
@ -113,6 +114,9 @@ FilterForTestcase = None
STATE_BEGIN = 0
STATE_ARANGOSH_OUTPUT = 'HTTP_LOUTPUT'
STATE_ARANGOSH_RUN = 'ARANGOSH_OUTPUT'
STATE_AQL = 'AQL'
STATE_AQL_DS = 'AQL_DS'
STATE_AQL_BIND = 'AQL_BV'
################################################################################
### @brief option states
@ -142,9 +146,13 @@ def generateArangoshHeader():
regularStartLine = re.compile(r'^(/// )? *@EXAMPLE_ARANGOSH_OUTPUT{([^}]*)}')
runLine = re.compile(r'^(/// )? *@EXAMPLE_ARANGOSH_RUN{([^}]*)}')
aqlLine = re.compile(r'^(/// )? *@EXAMPLE_AQL{([^}]*)}')
aqlDataSetLine = re.compile(r'^(/// )? *@DATASET{([^}]*)}')
aqlBindvaluesLine = re.compile(r'^(/// )? *@BV (.*)')
def matchStartLine(line, filename):
global regularStartLine, errorStartLine, runLine, FilterForTestcase, filterTestList
global regularStartLine, errorStartLine, runLine
global aqlLine, FilterForTestcase, filterTestList
errorName = ""
m = regularStartLine.match(line)
@ -159,6 +167,7 @@ def matchStartLine(line, filename):
# if we match for filters, only output these!
if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
print >> sys.stderr, "Arangosh: filtering out testcase '%s'" %name
filterTestList.append(name)
return("", STATE_BEGIN);
@ -169,7 +178,7 @@ def matchStartLine(line, filename):
if m:
strip = m.group(1)
name = m.group(2)
if name in ArangoshFiles:
print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
sys.exit(1)
@ -177,14 +186,52 @@ def matchStartLine(line, filename):
# if we match for filters, only output these!
if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
filterTestList.append(name)
print >> sys.stderr, "CuRL: filtering out testcase '%s'" %name
return("", STATE_BEGIN);
ArangoshFiles[name] = True
return (name, STATE_ARANGOSH_RUN)
m = aqlLine.match(line)
if m:
strip = m.group(1)
name = m.group(2)
if name in AQLFiles:
print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
sys.exit(1)
# if we match for filters, only output these!
if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
print >> sys.stderr, "AQL: filtering out testcase '%s'" %name
filterTestList.append(name)
return("", STATE_BEGIN);
AQLFiles[name] = True
return (name, STATE_AQL)
# Not found, remain in STATE_BEGIN
return ("", STATE_BEGIN)
def matchAqlLine(line, filename, oldState):
global aqlDataSetLine, aqlBindvaluesLine
m = aqlDataSetLine.match(line)
if m:
strip = m.group(1)
name = m.group(2)
return (name, STATE_AQL_DS)
m = aqlBindvaluesLine.match(line)
if m:
strip = m.group(1)
bindvalueStart = m.group(2)
return (bindvalueStart, STATE_AQL_BIND)
# Not found, remain in STATE_AQL
return (line, oldState)
endExample = re.compile(r'^(/// )? *@END_EXAMPLE_')
#r5 = re.compile(r'^ +')
@ -192,6 +239,9 @@ TESTLINES="testlines"
TYPE="type"
LINE_NO="lineNo"
STRING="string"
AQL="aql"
AQLDS="aql_dataset"
AQLBV="aql_bindvalues"
################################################################################
### @brief loop over the lines of one input file
@ -229,8 +279,24 @@ def analyzeFile(f, filename):
if state == STATE_ARANGOSH_RUN:
RunTests[name][LINE_NO] = lineNo;
RunTests[name][STRING] = "";
if state == STATE_AQL:
RunTests[name][LINE_NO] = lineNo;
RunTests[name][AQL] = "";
continue
if state == STATE_AQL:
(data, aqlState) = matchAqlLine(line, filename, state)
if aqlState == STATE_AQL_BIND:
RunTests[name][AQLBV] = data
state = aqlState
continue
if aqlState == STATE_AQL_DS:
RunTests[name][AQLDS] = data
# flip back to aql - query will come.
state = STATE_AQL
continue
# we are within a example
line = line[len(strip):]
showCmd = True
@ -246,7 +312,8 @@ def analyzeFile(f, filename):
continue
line = line.lstrip('/');
line = line.lstrip(' ');
if state != STATE_AQL:
line = line.lstrip(' ');
if state == STATE_ARANGOSH_OUTPUT:
line = line.replace("\\", "\\\\").replace("'", "\\'")
#print line
@ -287,6 +354,10 @@ def analyzeFile(f, filename):
RunTests[name][TESTLINES].append([line, showCmd, lineNo])
elif state == STATE_ARANGOSH_RUN:
RunTests[name][STRING] += line + "\n"
elif state == STATE_AQL:
RunTests[name][AQL] += line + "\n"
elif state == STATE_AQL_BIND:
RunTests[name][AQLBV] += line + "\n"
def generateSetupFunction():
@ -332,9 +403,9 @@ def generateArangoshOutput(testName):
escapeBS.sub(doubleBS, MapSourceFiles[testName])
)
except Exception as x:
print x
print testName
print value
print >> sys.stderr,x
print >> sys.stderr,testName
print >> sys.stderr,value
raise
for l in value[TESTLINES]:
@ -429,6 +500,62 @@ def generateArangoshRun(testName):
### @brief generate arangosh run
################################################################################
def generateAQL(testName):
value = RunTests[testName]
startLineNo = RunTests[testName][LINE_NO]
if not AQLBV in value:
value[AQLBV] = "{}"
print '''
%s
/// %s
(function() {
internal.startPrettyPrint(true);
internal.stopColorPrint(true);
var testName = '%s';
var lineCount = 0;
var startLineCount = %d;
var outputDir = '%s';
var sourceFile = '%s';
var startTime = time();
output = '';
''' % (
('/'*80),
testName,
testName,
startLineNo,
escapeBS.sub(doubleBS, OutputDir),
escapeBS.sub(doubleBS, MapSourceFiles[testName])
)
print " let query = `" + value[AQL] + "`;"
print " let bv = " + value[AQLBV] + ";"
print " let ds = '" + value[AQLDS] + "';"
print '''
exds.%s.removeDS();
exds.%s.createDS();
let result = db._query(query, bv).toArray();
output += "@Q:\\n"
output += highlight("js", query);
if (Object.keys(bv).length > 0) {
output += "@B\\n"
jsonAppender(JSON.stringify(bv, null, 2));
}
output += "\\n@R\\n";
jsonAppender(JSON.stringify(result, null, 2));
exds.%s.removeDS();
fs.write(outputDir + fs.pathSeparator + testName + '.generated', output);
print("[" + (time () - startTime) + "s] done with " + testName);
checkForOrphanTestCollections('not all collections were cleaned up after ' + sourceFile + ' Line[' + startLineCount + '] [' + testName + ']:');
}());
''' % (value[AQLDS], value[AQLDS], value[AQLDS])
################################################################################
### @brief generate arangosh run
################################################################################
def generateArangoshShutdown():
print '''
if (allErrors.length > 0) {
@ -443,7 +570,6 @@ if (allErrors.length > 0) {
def loopDirectories():
global ArangoshSetup, OutputDir, FilterForTestcase
argv = sys.argv
argv.pop(0)
filenames = []
@ -501,7 +627,6 @@ def loopDirectories():
for filename in filenames:
if (filename.find("#") < 0):
f = open(filename, "r")
analyzeFile(f, filename)
f.close()
@ -518,12 +643,18 @@ def generateTestCases():
generateArangoshOutput(thisTest)
elif RunTests[thisTest][TYPE] == STATE_ARANGOSH_RUN:
generateArangoshRun(thisTest)
elif RunTests[thisTest][TYPE] == STATE_AQL:
generateAQL(thisTest)
################################################################################
### @brief main
################################################################################
loopDirectories()
if len(RunTests) == 0:
print >> sys.stderr, "no testcases generated - bailing out"
raise Exception("no Testcases")
print >> sys.stderr, "filtering test %d cases" %(len(filterTestList))
generateArangoshHeader()