Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Documentation/switch example generation to rocksdb (#9478)

This commit is contained in:
Wilfried Goesgens 2019-07-17 19:17:41 +02:00 committed by Simran
parent 40a5c4f8b8
commit 56fff5c489
33 changed files with 299 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Documentation/DocuBlocks/Rest/Administration/get_engine.md
View File

@ -18,14 +18,25 @@ will be *mmfiles* or *rocksdb*
@EXAMPLES
Return the active storage engine
Return the active storage engine with the MMFiles storage engine in use:
@EXAMPLE_ARANGOSH_RUN{RestEngine}
@EXAMPLE_ARANGOSH_RUN{RestEngine_mmfiles}
var response = logCurlRequest('GET', '/_api/engine');
assert(response.code === 200);
logJsonResponse(response);
@END_EXAMPLE_ARANGOSH_RUN
Return the active storage engine with the RocksDB storage engine in use:
@EXAMPLE_ARANGOSH_RUN{RestEngine_rocksdb}
var response = logCurlRequest('GET', '/_api/engine');
assert(response.code === 200);
logJsonResponse(response);
@END_EXAMPLE_ARANGOSH_RUN
@endDocuBlock

29
Documentation/DocuBlocks/Rest/Collections/get_api_collection_figures.md
View File

@ -18,9 +18,9 @@ The name of the collection.
@RESTDESCRIPTION
In addition to the above, the result also contains the number of documents
and additional statistical information about the collection.
**Note** : This will always load the collection into memory.
**Note**: This will always load the collection into memory.
**Note**: collection data that are stored in the write-ahead log only are
**Note**: collection data that is stored in the write-ahead log only is
not reported in the results. When the write-ahead log is collected, documents
might be added to journals and datafiles of the collection, which may modify
the figures of the collection.
@ -126,6 +126,7 @@ engine (in bytes). This figure does not include the document data but only mappi
from document revision ids to storage engine datafile positions.
@RESTSTRUCT{indexes,collection_figures,object,required,collection_figures_indexes}
@RESTSTRUCT{count,collection_figures_indexes,integer,required,int64}
The total number of indexes defined for the collection, including the pre-defined
indexes (e.g. primary index).
@ -153,6 +154,7 @@ head of the collection's cleanup queue. This information can be used for debuggi
compaction and unload issues.
@RESTSTRUCT{compactionStatus,collection_figures,object,optional,compactionStatus_attributes}
@RESTSTRUCT{message,compactionStatus_attributes,string,optional,string}
The action that was performed when the compaction was last run for the collection.
This information can be used for debugging compaction issues.
@ -174,9 +176,9 @@ is returned.
@EXAMPLES
Using an identifier and requesting the figures of the collection:
Using an identifier and requesting the figures of the collection (MMFiles storage engine):
@EXAMPLE_ARANGOSH_RUN{RestCollectionGetCollectionFigures}
@EXAMPLE_ARANGOSH_RUN{RestCollectionGetCollectionFigures_mmfiles}
var cn = "products";
db._drop(cn);
var coll = db._create(cn);
@ -191,5 +193,24 @@ Using an identifier and requesting the figures of the collection:
logJsonResponse(response);
db._drop(cn);
@END_EXAMPLE_ARANGOSH_RUN
Using an identifier and requesting the figures of the collection (RocksDB storage engine):
@EXAMPLE_ARANGOSH_RUN{RestCollectionGetCollectionFigures_rocksdb}
var cn = "products";
db._drop(cn);
var coll = db._create(cn);
coll.save({"test":"hello"});
require("internal").wal.flush(true, true);
var url = "/_api/collection/"+ coll.name() + "/figures";
var response = logCurlRequest('GET', url);
assert(response.code === 200);
logJsonResponse(response);
db._drop(cn);
@END_EXAMPLE_ARANGOSH_RUN
@endDocuBlock

4
Documentation/DocuBlocks/Rest/Collections/put_api_collection_rotate.md
View File

@ -43,7 +43,7 @@ If the *collection-name* is unknown, then a *HTTP 404* is returned.
Rotating the journal:
@EXAMPLE_ARANGOSH_RUN{RestCollectionRotate}
@EXAMPLE_ARANGOSH_RUN{RestCollectionRotate_mmfiles}
var cn = "products";
db._drop(cn);
var coll = db._create(cn);
@ -62,7 +62,7 @@ Rotating the journal:
Rotating if no journal exists:
@EXAMPLE_ARANGOSH_RUN{RestCollectionRotateNoJournal}
@EXAMPLE_ARANGOSH_RUN{RestCollectionRotateNoJournal_mmfiles}
var cn = "products";
db._drop(cn);
var coll = db._create(cn);

4
Documentation/DocuBlocks/Rest/Replication/get_api_replication_dump.md
View File

@ -104,7 +104,7 @@ is returned if an error occurred while assembling the response.
Empty collection:
@EXAMPLE_ARANGOSH_RUN{RestReplicationDumpEmpty}
@EXAMPLE_ARANGOSH_RUN{RestReplicationDumpEmpty_mmfiles}
db._drop("testCollection");
var c = db._create("testCollection");
var url = "/_api/replication/dump?collection=" + c.name();
@ -118,7 +118,7 @@ Empty collection:
Non-empty collection *(One JSON document per line)*:
@EXAMPLE_ARANGOSH_RUN{RestReplicationDump}
@EXAMPLE_ARANGOSH_RUN{RestReplicationDump_mmfiles}
db._drop("testCollection");
var c = db._create("testCollection");
c.save({ "test" : true, "a" : "abc", "_key" : "abcdef" });

4
Documentation/DocuBlocks/Rest/Replication/put_api_replication_inventory.md
View File

@ -96,7 +96,7 @@ is returned if an error occurred while assembling the response.
@EXAMPLES
@EXAMPLE_ARANGOSH_RUN{RestReplicationInventory}
@EXAMPLE_ARANGOSH_RUN{RestReplicationInventory_mmfiles}
var url = "/_api/replication/inventory";
var response = logCurlRequest('GET', url);
@ -107,7 +107,7 @@ is returned if an error occurred while assembling the response.
With some additional indexes:
@EXAMPLE_ARANGOSH_RUN{RestReplicationInventoryIndexes}
@EXAMPLE_ARANGOSH_RUN{RestReplicationInventoryIndexes_mmfiles}
db._drop("IndexedCollection1");
var c1 = db._create("IndexedCollection1");
c1.ensureHashIndex("name");

2
Documentation/DocuBlocks/Rest/wal/get_admin_wal_properties.md
View File

@ -32,7 +32,7 @@ is returned when an invalid HTTP method is used.
@EXAMPLES
@EXAMPLE_ARANGOSH_RUN{RestWalPropertiesGet}
@EXAMPLE_ARANGOSH_RUN{RestWalPropertiesGet_mmfiles}
var url = "/_admin/wal/properties";
var response = logCurlRequest('GET', url);

2
Documentation/DocuBlocks/Rest/wal/get_admin_wal_transactions.md
View File

@ -26,7 +26,7 @@ is returned when an invalid HTTP method is used.
@EXAMPLES
@EXAMPLE_ARANGOSH_RUN{RestWalTransactionsGet}
@EXAMPLE_ARANGOSH_RUN{RestWalTransactionsGet_mmfiles}
var url = "/_admin/wal/transactions";
var response = logCurlRequest('GET', url);

2
Documentation/DocuBlocks/Rest/wal/put_admin_wal_properties.md
View File

@ -33,7 +33,7 @@ is returned when an invalid HTTP method is used.
@EXAMPLES
@EXAMPLE_ARANGOSH_RUN{RestWalPropertiesPut}
@EXAMPLE_ARANGOSH_RUN{RestWalPropertiesPut_mmfiles}
var url = "/_admin/wal/properties";
var body = {
logfileSize: 32 * 1024 * 1024,

2
Documentation/DocuBlocks/collectionFigures.md
View File

@ -83,7 +83,7 @@ used as a lower bound approximation of the disk usage.
@EXAMPLES
@EXAMPLE_ARANGOSH_OUTPUT{collectionFigures}
@EXAMPLE_ARANGOSH_OUTPUT{collectionFigures_mmfiles}
~ require("internal").wal.flush(true, true);
db.demo.figures()
@END_EXAMPLE_ARANGOSH_OUTPUT

3
Documentation/DocuBlocks/ensureUniqueSkiplist.md
View File

@ -36,6 +36,7 @@ db.ids.save({ "myId": 789 });
db.ids.save({ "myId": 123 }); // xpError(ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED)
~db._drop("ids");
@END_EXAMPLE_ARANGOSH_OUTPUT
@EXAMPLE_ARANGOSH_OUTPUT{ensureUniqueSkiplistMultiColmun}
~db._create("ids");
db.ids.ensureIndex({ type: "skiplist", fields: [ "name.first", "name.last" ], unique: true });
@ -46,5 +47,3 @@ db.ids.save({ "name" : { "first" : "hans", "last": "jensen" }});
~ // xpError(ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED)
~db._drop("ids");
@END_EXAMPLE_ARANGOSH_OUTPUT

2
Documentation/DocuBlocks/walPropertiesGet.md
View File

@ -21,7 +21,7 @@ array with the following attributes:
@EXAMPLES
@EXAMPLE_ARANGOSH_OUTPUT{WalPropertiesGet}
@EXAMPLE_ARANGOSH_OUTPUT{WalPropertiesGet_mmfiles}
require("internal").wal.properties();
@END_EXAMPLE_ARANGOSH_OUTPUT

2
Documentation/DocuBlocks/walPropertiesSet.md
View File

@ -22,7 +22,7 @@ will be ignored and the configuration for them will not be modified.
@EXAMPLES
@EXAMPLE_ARANGOSH_OUTPUT{WalPropertiesSet}
@EXAMPLE_ARANGOSH_OUTPUT{WalPropertiesSet_mmfiles}
| require("internal").wal.properties({
| allowOverSizeEntries: true,
logfileSize: 32 * 1024 * 1024 });

0
Documentation/Examples/RestCollectionGetCollectionFigures.generated → Documentation/Examples/RestCollectionGetCollectionFigures_mmfiles.generated
View File

37
Documentation/Examples/RestCollectionGetCollectionFigures_rocksdb.generated Normal file
View File

@ -0,0 +1,37 @@
<span class="hljs-meta">shell&gt;</span><span class="bash"> curl --header <span class="hljs-string">'accept: application/json'</span> --dump - http://localhost:8529/_api/collection/products/figures</span>
HTTP/<span class="hljs-number">1.1</span> OK
content-type: application/json; charset=utf<span class="hljs-number">-8</span>
location: <span class="hljs-regexp">/_api/</span>collection/products/figures
x-content-type-options: nosniff
{
<span class="hljs-string">"error"</span> : <span class="hljs-literal">false</span>,
<span class="hljs-string">"code"</span> : <span class="hljs-number">200</span>,
<span class="hljs-string">"type"</span> : <span class="hljs-number">2</span>,
<span class="hljs-string">"status"</span> : <span class="hljs-number">3</span>,
<span class="hljs-string">"name"</span> : <span class="hljs-string">"products"</span>,
<span class="hljs-string">"waitForSync"</span> : <span class="hljs-literal">false</span>,
<span class="hljs-string">"objectId"</span> : <span class="hljs-string">"101"</span>,
<span class="hljs-string">"cacheEnabled"</span> : <span class="hljs-literal">false</span>,
<span class="hljs-string">"figures"</span> : {
<span class="hljs-string">"indexes"</span> : {
<span class="hljs-string">"count"</span> : <span class="hljs-number">1</span>,
<span class="hljs-string">"size"</span> : <span class="hljs-number">37</span>
},
<span class="hljs-string">"documentsSize"</span> : <span class="hljs-number">846</span>,
<span class="hljs-string">"cacheInUse"</span> : <span class="hljs-literal">false</span>,
<span class="hljs-string">"cacheSize"</span> : <span class="hljs-number">0</span>,
<span class="hljs-string">"cacheUsage"</span> : <span class="hljs-number">0</span>
},
<span class="hljs-string">"keyOptions"</span> : {
<span class="hljs-string">"allowUserKeys"</span> : <span class="hljs-literal">true</span>,
<span class="hljs-string">"type"</span> : <span class="hljs-string">"traditional"</span>,
<span class="hljs-string">"lastValue"</span> : <span class="hljs-number">107</span>
},
<span class="hljs-string">"globallyUniqueId"</span> : <span class="hljs-string">"h5FFC6746B13/102"</span>,
<span class="hljs-string">"statusString"</span> : <span class="hljs-string">"loaded"</span>,
<span class="hljs-string">"id"</span> : <span class="hljs-string">"102"</span>,
<span class="hljs-string">"count"</span> : <span class="hljs-number">1</span>,
<span class="hljs-string">"isSystem"</span> : <span class="hljs-literal">false</span>
}

0
Documentation/Examples/RestCollectionRotateNoJournal.generated → Documentation/Examples/RestCollectionRotateNoJournal_mmfiles.generated
View File

0
Documentation/Examples/RestCollectionRotate.generated → Documentation/Examples/RestCollectionRotate_mmfiles.generated
View File

0
Documentation/Examples/RestEngine.generated → Documentation/Examples/RestEngine_mmfiles.generated
View File

28
Documentation/Examples/RestEngine_rocksdb.generated Normal file
View File

@ -0,0 +1,28 @@
<span class="hljs-meta">shell&gt;</span><span class="bash"> curl --header <span class="hljs-string">'accept: application/json'</span> --dump - http://localhost:8529/_api/engine</span>
HTTP/<span class="hljs-number">1.1</span> OK
content-type: application/json; charset=utf<span class="hljs-number">-8</span>
x-content-type-options: nosniff
{
<span class="hljs-string">"name"</span> : <span class="hljs-string">"rocksdb"</span>,
<span class="hljs-string">"supports"</span> : {
<span class="hljs-string">"dfdb"</span> : <span class="hljs-literal">false</span>,
<span class="hljs-string">"indexes"</span> : [
<span class="hljs-string">"primary"</span>,
<span class="hljs-string">"edge"</span>,
<span class="hljs-string">"hash"</span>,
<span class="hljs-string">"skiplist"</span>,
<span class="hljs-string">"ttl"</span>,
<span class="hljs-string">"persistent"</span>,
<span class="hljs-string">"geo"</span>,
<span class="hljs-string">"fulltext"</span>
],
<span class="hljs-string">"aliases"</span> : {
<span class="hljs-string">"indexes"</span> : {
<span class="hljs-string">"skiplist"</span> : <span class="hljs-string">"persistent"</span>,
<span class="hljs-string">"hash"</span> : <span class="hljs-string">"persistent"</span>
}
}
}
}

0
Documentation/Examples/RestReplicationDumpEmpty.generated → Documentation/Examples/RestReplicationDumpEmpty_mmfiles.generated
View File

0
Documentation/Examples/RestReplicationDump.generated → Documentation/Examples/RestReplicationDump_mmfiles.generated
View File

0
Documentation/Examples/RestReplicationInventoryIndexes.generated → Documentation/Examples/RestReplicationInventoryIndexes_mmfiles.generated
View File

0
Documentation/Examples/RestReplicationInventory.generated → Documentation/Examples/RestReplicationInventory_mmfiles.generated
View File

0
Documentation/Examples/RestWalPropertiesGet.generated → Documentation/Examples/RestWalPropertiesGet_mmfiles.generated
View File

0
Documentation/Examples/RestWalPropertiesPut.generated → Documentation/Examples/RestWalPropertiesPut_mmfiles.generated
View File

0
Documentation/Examples/RestWalTransactionsGet.generated → Documentation/Examples/RestWalTransactionsGet_mmfiles.generated
View File

10
Documentation/Examples/WalPropertiesGet.generated
View File

@ -1,10 +0,0 @@
arangosh&gt; <span class="hljs-built_in">require</span>(<span class="hljs-string">"internal"</span>).wal.properties();
{
<span class="hljs-string">"allowOversizeEntries"</span> : <span class="hljs-literal">true</span>,
<span class="hljs-string">"logfileSize"</span> : <span class="hljs-number">33554432</span>,
<span class="hljs-string">"historicLogfiles"</span> : <span class="hljs-number">10</span>,
<span class="hljs-string">"reserveLogfiles"</span> : <span class="hljs-number">3</span>,
<span class="hljs-string">"syncInterval"</span> : <span class="hljs-number">100</span>,
<span class="hljs-string">"throttleWait"</span> : <span class="hljs-number">15000</span>,
<span class="hljs-string">"throttleWhenPending"</span> : <span class="hljs-number">0</span>
}

3
Documentation/Examples/WalPropertiesGet_mmfiles.generated Normal file
View File

@ -0,0 +1,3 @@
arangosh&gt; <span class="hljs-built_in">require</span>(<span class="hljs-string">"internal"</span>).wal.properties();
{
}

12
Documentation/Examples/WalPropertiesSet.generated
View File

@ -1,12 +0,0 @@
arangosh&gt; <span class="hljs-built_in">require</span>(<span class="hljs-string">"internal"</span>).wal.properties({
........&gt; allowOverSizeEntries: <span class="hljs-literal">true</span>,
........&gt; logfileSize: <span class="hljs-number">32</span> * <span class="hljs-number">1024</span> * <span class="hljs-number">1024</span> });
{
<span class="hljs-string">"allowOversizeEntries"</span> : <span class="hljs-literal">true</span>,
<span class="hljs-string">"logfileSize"</span> : <span class="hljs-number">33554432</span>,
<span class="hljs-string">"historicLogfiles"</span> : <span class="hljs-number">10</span>,
<span class="hljs-string">"reserveLogfiles"</span> : <span class="hljs-number">3</span>,
<span class="hljs-string">"syncInterval"</span> : <span class="hljs-number">100</span>,
<span class="hljs-string">"throttleWait"</span> : <span class="hljs-number">15000</span>,
<span class="hljs-string">"throttleWhenPending"</span> : <span class="hljs-number">0</span>
}

5
Documentation/Examples/WalPropertiesSet_mmfiles.generated Normal file
View File

@ -0,0 +1,5 @@
arangosh&gt; <span class="hljs-built_in">require</span>(<span class="hljs-string">"internal"</span>).wal.properties({
........&gt; allowOverSizeEntries: <span class="hljs-literal">true</span>,
........&gt; logfileSize: <span class="hljs-number">32</span> * <span class="hljs-number">1024</span> * <span class="hljs-number">1024</span> });
{
}

0
Documentation/Examples/collectionFigures.generated → Documentation/Examples/collectionFigures_mmfiles.generated
View File

2
Documentation/README_maintainers.md
View File

@ -237,8 +237,10 @@ Here is how its details work:
- an example start is marked with *@EXAMPLE_ARANGOSH_OUTPUT* or *@EXAMPLE_ARANGOSH_RUN*
- the example is named by the string provided in brackets after the above key
- the output is written to `Documentation/Examples/<name>.generated`
- if your example depends on a storage engine, prepend `<name>` with `_rocksdb` or `_mmfiles` to run it against such a server
- examples end with *@END_EXAMPLE_[OUTPUT|RUN|AQL]*
- all code in between is executed as javascript in the **arangosh** while talking to a valid **arangod**.
- you should strive to group your examples by naming them with a common prefix per topic.
You may inspect the generated js code in `/tmp/arangosh.examples.js`
## OUTPUT, RUN and AQL specifics

237
utils/generateExamples.js
View File

@ -2,6 +2,7 @@
/*global start_pretty_print */
'use strict';
const _ = require("lodash");
const fs = require("fs");
const internal = require("internal");
const executeExternal = internal.executeExternal;
@ -119,134 +120,144 @@ function main(argv) {
args = args.concat(['--arangoshSetup']);
args = args.concat(documentationSourceDirs);
let res = executeExternalAndWait(thePython, args);
let storageEngines = [['mmfiles', false], ['rocksdb', true]];
let res;
if (res.exit !== 0) {
print("parsing the examples failed - aborting!");
print(res);
return -1;
}
storageEngines.forEach(function (engine) {
let pyArgs = _.clone(args);
pyArgs.push('--storageEngine');
pyArgs.push(engine[0]);
pyArgs.push('--storageEngineAgnostic');
pyArgs.push(engine[1]);
print(pyArgs)
res = executeExternalAndWait(thePython, pyArgs);
if (startServer) {
let port = findFreePort();
instanceInfo.port = port;
serverEndpoint = protocol + "://127.0.0.1:" + port;
instanceInfo.url = endpointToURL(serverEndpoint);
fs.makeDirectoryRecursive(fs.join(tmpDataDir, "data"));
let serverArgs = {};
fs.makeDirectoryRecursive(fs.join(tmpDataDir, "apps"));
serverArgs["configuration"] = "none";
serverArgs["database.directory"] = fs.join(tmpDataDir, "data");
serverArgs["javascript.app-path"] = fs.join(tmpDataDir, "apps");
serverArgs["javascript.startup-directory"] = "js";
serverArgs["javascript.module-directory"] = "enterprise/js";
serverArgs["log.file"] = fs.join(tmpDataDir, "log");
serverArgs["server.authentication"] = "false";
serverArgs["server.endpoint"] = serverEndpoint;
serverArgs["server.storage-engine"] = "mmfiles"; // examples depend on it
print("================================================================================");
ARANGOD = locateProgram("arangod", "Cannot find arangod to execute tests against");
print(ARANGOD);
print(toArgv(serverArgs));
instanceInfo.pid = executeExternal(ARANGOD, toArgv(serverArgs)).pid;
// Wait until the server is up:
count = 0;
instanceInfo.endpoint = serverEndpoint;
while (true) {
wait(0.5, false);
let r = download(instanceInfo.url + "/_api/version", "");
if (!r.error && r.code === 200) {
break;
}
count++;
if (count % 60 === 0) {
res = statusExternal(instanceInfo.pid, false);
if (res.status !== "RUNNING") {
print("start failed - process is gone: " + yaml.safeDump(res));
return 1;
}
}
if (res.exit !== 0) {
print("parsing the examples failed - aborting!");
print(res);
return -1;
}
}
let arangoshArgs = {
'configuration': fs.join(fs.makeAbsolute(''), 'etc', 'relative', 'arangosh.conf'),
'server.password': "",
'server.endpoint': serverEndpoint,
'javascript.execute': scriptArguments.outputFile
};
if (startServer) {
let port = findFreePort();
instanceInfo.port = port;
serverEndpoint = protocol + "://127.0.0.1:" + port;
print("--------------------------------------------------------------------------------");
ARANGOSH = locateProgram("arangosh", "Cannot find arangosh to run tests with");
print(ARANGOSH);
print(internal.toArgv(arangoshArgs));
res = executeExternalAndWait(ARANGOSH, internal.toArgv(arangoshArgs));
instanceInfo.url = endpointToURL(serverEndpoint);
if (startServer) {
if (typeof(instanceInfo.exitStatus) === 'undefined') {
download(instanceInfo.url + "/_admin/shutdown", "", {method: "DELETE"});
fs.makeDirectoryRecursive(fs.join(tmpDataDir, engine[0], "data"));
print("Waiting for server shut down");
let serverArgs = {};
fs.makeDirectoryRecursive(fs.join(tmpDataDir, engine[0], "apps"));
serverArgs["configuration"] = "none";
serverArgs["database.directory"] = fs.join(tmpDataDir, engine[0], "data");
serverArgs["javascript.app-path"] = fs.join(tmpDataDir, engine[0], "apps");
serverArgs["javascript.startup-directory"] = "js";
serverArgs["javascript.module-directory"] = "enterprise/js";
serverArgs["log.file"] = fs.join(tmpDataDir, engine[0], "log");
serverArgs["server.authentication"] = "false";
serverArgs["server.endpoint"] = serverEndpoint;
serverArgs["server.storage-engine"] = engine[0]
print("================================================================================");
ARANGOD = locateProgram("arangod", "Cannot find arangod to execute tests against");
print(ARANGOD);
print(toArgv(serverArgs));
instanceInfo.pid = executeExternal(ARANGOD, toArgv(serverArgs)).pid;
// Wait until the server is up:
count = 0;
let bar = "[";
instanceInfo.endpoint = serverEndpoint;
while (1) {
instanceInfo.exitStatus = statusExternal(instanceInfo.pid, false);
while (true) {
wait(0.5, false);
let r = download(instanceInfo.url + "/_api/version", "");
if (instanceInfo.exitStatus.status === "RUNNING") {
count++;
if (typeof(options.valgrind) === 'string') {
wait(1);
continue;
if (!r.error && r.code === 200) {
break;
}
count++;
if (count % 60 === 0) {
res = statusExternal(instanceInfo.pid, false);
if (res.status !== "RUNNING") {
print("start failed - process is gone: " + yaml.safeDump(res));
return 1;
}
if (count % 10 === 0) {
bar = bar + "#";
}
if (count > 600) {
print("forcefully terminating " + yaml.safeDump(instanceInfo.pid) +
" after 600 s grace period; marking crashy.");
serverCrashed = true;
killExternal(instanceInfo.pid);
break;
} else {
wait(1);
}
} else if (instanceInfo.exitStatus.status !== "TERMINATED") {
if (instanceInfo.exitStatus.hasOwnProperty('signal')) {
print("Server shut down with : " +
yaml.safeDump(instanceInfo.exitStatus) +
" marking build as crashy.");
serverCrashed = true;
break;
}
if (internal.platform.substr(0, 3) === 'win') {
// Windows: wait for procdump to do its job...
statusExternal(instanceInfo.monitor, true);
}
} else {
print("Server shutdown: Success.");
break; // Success.
}
}
if (count > 10) {
print("long Server shutdown: " + bar + ']');
}
}
}
let arangoshArgs = {
'configuration': fs.join(fs.makeAbsolute(''), 'etc', 'relative', 'arangosh.conf'),
'server.password': "",
'server.endpoint': serverEndpoint,
'javascript.execute': scriptArguments.outputFile
};
print("--------------------------------------------------------------------------------");
ARANGOSH = locateProgram("arangosh", "Cannot find arangosh to run tests with");
print(ARANGOSH);
print(internal.toArgv(arangoshArgs));
res = executeExternalAndWait(ARANGOSH, internal.toArgv(arangoshArgs));
if (startServer) {
if (typeof(instanceInfo.exitStatus) === 'undefined') {
download(instanceInfo.url + "/_admin/shutdown", "", {method: "DELETE"});
print("Waiting for server shut down");
count = 0;
let bar = "[";
while (1) {
instanceInfo.exitStatus = statusExternal(instanceInfo.pid, false);
if (instanceInfo.exitStatus.status === "RUNNING") {
count++;
if (typeof(options.valgrind) === 'string') {
wait(1);
continue;
}
if (count % 10 === 0) {
bar = bar + "#";
}
if (count > 600) {
print("forcefully terminating " + yaml.safeDump(instanceInfo.pid) +
" after 600 s grace period; marking crashy.");
serverCrashed = true;
killExternal(instanceInfo.pid);
break;
} else {
wait(1);
}
} else if (instanceInfo.exitStatus.status !== "TERMINATED") {
if (instanceInfo.exitStatus.hasOwnProperty('signal')) {
print("Server shut down with : " +
yaml.safeDump(instanceInfo.exitStatus) +
" marking build as crashy.");
serverCrashed = true;
break;
}
if (internal.platform.substr(0, 3) === 'win') {
// Windows: wait for procdump to do its job...
statusExternal(instanceInfo.monitor, true);
}
} else {
print("Server shutdown: Success.");
break; // Success.
}
}
if (count > 10) {
print("long Server shutdown: " + bar + ']');
}
}
}
});
if (res.exit != 0) {
throw("generating examples failed!");
}

64
utils/generateExamples.py
View File

@ -128,7 +128,13 @@ OPTION_ARANGOSH_SETUP = 1
OPTION_OUTPUT_DIR = 2
OPTION_FILTER = 3
OPTION_OUTPUT_FILE = 4
OPTION_OUTPUT_ENGINE = 5
OPTION_OUTPUT_FILTER_NONMATCHING = 6
engines = ["mmfiles", "rocksdb"]
engine = "mmfiles"
otherEngine = "mmfiles"
storageEngineAgnostic = True
escapeBS = re.compile("\\\\")
doubleBS = "\\\\\\\\"
@ -198,17 +204,17 @@ def matchStartLine(line, filename):
if m:
strip = m.group(1)
name = m.group(2)
if name in AQLFiles:
print >> sys.stderr, "%s\nduplicate test name '%s' in file %s!\n%s\n" % ('#' * 80, name, filename, '#' * 80)
sys.exit(1)
# if we match for filters, only output these!
if ((FilterForTestcase != None) and not FilterForTestcase.match(name)):
print >> sys.stderr, "AQL: filtering out testcase '%s'" %name
filterTestList.append(name)
return("", STATE_BEGIN);
AQLFiles[name] = True
return (name, STATE_AQL)
@ -257,10 +263,10 @@ AQLEXPLAIN="aql_explain"
### @brief loop over the lines of one input file
################################################################################
def analyzeFile(f, filename):
def analyzeFile(f, filename):
global RunTests, TESTLINES, TYPE, LINE_NO, STRING
strip = None
name = ""
partialCmd = ""
partialLine = ""
@ -280,7 +286,7 @@ def analyzeFile(f, filename):
if state == STATE_BEGIN:
(name, state) = matchStartLine(line, filename)
if state != STATE_BEGIN:
if state != STATE_BEGIN:
MapSourceFiles[name] = filename
RunTests[name] = {}
RunTests[name][TYPE] = state
@ -315,7 +321,7 @@ def analyzeFile(f, filename):
# we are within a example
line = line[len(strip):]
showCmd = True
# end-example test
m = endExample.match(line)
@ -601,18 +607,18 @@ if (allErrors.length > 0) {
################################################################################
def loopDirectories():
global ArangoshSetup, OutputDir, FilterForTestcase
global ArangoshSetup, OutputDir, FilterForTestcase, storageEngineAgnostic, engine, otherEngine
argv = sys.argv
argv.pop(0)
filenames = []
fstate = OPTION_NORMAL
for filename in argv:
if filename == "--arangoshSetup":
fstate = OPTION_ARANGOSH_SETUP
continue
if filename == "--onlyThisOne":
if filename == "--onlyThisOne":
fstate = OPTION_FILTER
continue
@ -624,6 +630,14 @@ def loopDirectories():
fstate = OPTION_OUTPUT_FILE
continue
if filename == "--storageEngine":
fstate = OPTION_OUTPUT_ENGINE
continue
if filename == "--storageEngineAgnostic":
fstate = OPTION_OUTPUT_FILTER_NONMATCHING
continue
if fstate == OPTION_NORMAL:
if os.path.isdir(filename):
for root, dirs, files in os.walk(filename):
@ -635,17 +649,17 @@ def loopDirectories():
elif fstate == OPTION_FILTER:
fstate = OPTION_NORMAL
if (len(filename) > 0):
if (len(filename) > 0):
FilterForTestcase = re.compile(filename);
elif fstate == OPTION_ARANGOSH_SETUP:
fstate = OPTION_NORMAL
f = open(filename, "r")
for line in f:
line = line.rstrip('\n')
ArangoshSetup += line + "\n"
f.close()
elif fstate == OPTION_OUTPUT_DIR:
@ -656,21 +670,41 @@ def loopDirectories():
fstate = OPTION_NORMAL
sys.stdout = open(filename, 'w')
elif fstate == OPTION_OUTPUT_ENGINE:
fstate = OPTION_NORMAL
engine = filename
if engine == engines[0]:
otherEngine = engines[1]
else:
otherEngine = engines[0]
elif fstate == OPTION_OUTPUT_FILTER_NONMATCHING:
fstate = OPTION_NORMAL
storageEngineAgnostic = filename == "true"
for filename in filenames:
if (filename.find("#") < 0):
f = open(filename, "r")
analyzeFile(f, filename)
f.close()
else:
print >> sys.stderr, "skipping %s\n" % (filename)
def generateTestCases():
global TESTLINES, TYPE, LINE_NO, STRING, RunTests
global TESTLINES, TYPE, LINE_NO, STRING, RunTests, storageEngineAgnostic, engine, otherEngine
testNames = RunTests.keys()
testNames.sort()
for thisTest in testNames:
if thisTest.endswith(otherEngine):
print >> sys.stderr, "skipping " + thisTest
continue
if not storageEngineAgnostic and not thisTest.endswith(engine):
print >> sys.stderr, "skipping " + thisTest
continue
if RunTests[thisTest][TYPE] == STATE_ARANGOSH_OUTPUT:
generateArangoshOutput(thisTest)
elif RunTests[thisTest][TYPE] == STATE_ARANGOSH_RUN: