diff --git a/Documentation/Books/Makefile b/Documentation/Books/Makefile index bfe51aaee3..6823631bf6 100644 --- a/Documentation/Books/Makefile +++ b/Documentation/Books/Makefile @@ -45,6 +45,16 @@ check-docublocks: exit 1; \ fi +check-summary: + @find . -name \*.md |grep -v books/ |sed -e "s;./Users/;;" |grep -v ^SUMMARY.md$ |grep -v ^README.md$ |sort > /tmp/is_md.txt + @cat Users/SUMMARY.md |sed -e "s;.*(;;" -e "s;).*;;" |sort |grep -v '# Summary' > /tmp/is_summary.txt + @if test "`comm -3 /tmp/is_md.txt /tmp/is_summary.txt|wc -l`" -ne 0; then \ + echo "not all files are mapped to the summary!"; \ + echo " files found | files in summary"; \ + comm -3 /tmp/is_md.txt /tmp/is_summary.txt; \ + exit 1; \ + fi + clean: clean-md-files clean-books rm -f allComments.txt diff --git a/Documentation/Books/Users/Installing/README.mdpp b/Documentation/Books/Users/Installing/README.mdpp index 4e2f133179..ed539681e9 100644 --- a/Documentation/Books/Users/Installing/README.mdpp +++ b/Documentation/Books/Users/Installing/README.mdpp @@ -7,4 +7,4 @@ You can find packages for various operation systems at our [install](https://www In this chapter you will also learn how to compile ArangoDB from scratch. -You also get help if you want to update your ArangoDB version to the newest one! +You also get help if you want to update your ArangoDB version to the newest one! diff --git a/Documentation/Books/Users/ModuleFs/README.mdpp b/Documentation/Books/Users/ModuleFs/README.mdpp index dc858b122e..973900b30e 100644 --- a/Documentation/Books/Users/ModuleFs/README.mdpp +++ b/Documentation/Books/Users/ModuleFs/README.mdpp @@ -1,106 +1,80 @@ !CHAPTER Module "fs" -!SUBSECTION File System Module - The implementation tries to follow the CommonJS specification where possible. [Filesystem/A/0](http://wiki.commonjs.org/wiki/Filesystem/A/0). +!SECTION Single File Directory Manipulation - - +!SUBSUBSECTION exists @startDocuBlock JS_Exists - - +!SUBSUBSECTION isFile @startDocuBlock JS_IsFile - - +!SUBSUBSECTION isDirectory @startDocuBlock JS_IsDirectory - - +!SUBSUBSECTION size @startDocuBlock JS_Size - - +!SUBSUBSECTION mtime @startDocuBlock JS_MTime - +!SUBSUBSECTION pathSeparator `fs.pathSeparator` If you want to combine two paths you can use fs.pathSeparator instead of */* or *\\*. +!SUBSUBSECTION join `fs.join(path, filename)` The function returns the combination of the path and filename, e.g. fs.join(Hello/World, foo.bar) would return Hello/World/foo.bar. - - - +!SUBSUBSECTION getTempFile @startDocuBlock JS_GetTempFile - - - +!SUBSUBSECTION getTempPath @startDocuBlock JS_GetTempPath - - +!SUBSUBSECTION makeAbsolute @startDocuBlock JS_MakeAbsolute - - +!SUBSUBSECTION chmod @startDocuBlock JS_Chmod - - - +!SUBSUBSECTION list @startDocuBlock JS_List - - - +!SUBSUBSECTION listTree @startDocuBlock JS_ListTree - - - +!SUBSUBSECTION makeDirectory @startDocuBlock JS_MakeDirectory - - - +!SUBSUBSECTION makeDirectoryRecursive @startDocuBlock JS_MakeDirectoryRecursive - - +!SUBSUBSECTION remove @startDocuBlock JS_Remove - - +!SUBSUBSECTION removeDirectory @startDocuBlock JS_RemoveDirectory - - +!SUBSUBSECTION removeDirectoryRecursive @startDocuBlock JS_RemoveDirectoryRecursive - +!SECTION File IO +!SUBSUBSECTION read @startDocuBlock JS_Read - - - +!SUBSUBSECTION read64 @startDocuBlock JS_Read64 - - - +!SUBSUBSECTION readBuffer @startDocuBlock JS_ReadBuffer - - +!SUBSUBSECTION readFileSync `fs.readFileSync(filename, encoding)` Reads the contents of the file specified in `filename`. If `encoding` is specified, @@ -116,34 +90,29 @@ If no `encoding` is specified, the file contents will be returned in a Buffer object. - - +!SUBSUBSECTION save @startDocuBlock JS_Save - - - +!SUBSUBSECTION writeFileSync `fs.writeFileSync(filename, content)` This is an alias for `fs.write(filename, content)`. +!SECTION Recursive Manipulation - - +!SUBSUBSECTION copyRecursive @startDocuBlock JS_CopyDirectoryRecursive - - +!SUBSUBSECTION CopyFile @startDocuBlock JS_CopyFile - - +!SUBSUBSECTION move @startDocuBlock JS_MoveFile - +!SECTION ZIP +!SUBSUBSECTION unzipFile @startDocuBlock JS_Unzip - - -@startDocuBlock JS_Zip +!SUBSUBSECTION zipFile +startDocuBlock JS_Zip diff --git a/Documentation/Books/Users/NewFeatures/NewFeatures26.mdpp b/Documentation/Books/Users/NewFeatures/NewFeatures26.mdpp index b339d779f6..f298fd07ad 100644 --- a/Documentation/Books/Users/NewFeatures/NewFeatures26.mdpp +++ b/Documentation/Books/Users/NewFeatures/NewFeatures26.mdpp @@ -4,15 +4,166 @@ The following list shows in detail which features have been added or improved in ArangoDB 2.6. ArangoDB 2.6 also contains several bugfixes that are not listed here. For a list of bugfixes, please consult the [CHANGELOG](https://github.com/arangodb/arangodb/blob/devel/CHANGELOG). +!SECTION -!SECTION Admin Frontend Authentication +* added batch document removal and lookup commands: -!SECTION Foxx Configuration + collection.lookupByKeys(keys) +collection.removeByKeys(keys) -!SECTION Foxx Dependencies + These commands can be used to perform multi-document lookup and removal operations efficiently + from the ArangoShell. The argument to these operations is an array of document keys. -!SECTION Foxx API Documentation + Also added HTTP APIs for batch document commands: -!SECTION Foxx Scripts + * PUT /\_api/simple/lookup-by-keys + * PUT /\_api/simple/remove-by-keys -!SECTION Mocha Tests + + * added dedicated collection export HTTP REST API + + ArangoDB now provides a dedicated collection export API, which can take snapshots of entire + collections more efficiently than the general-purpose cursor API. The export API is useful + to transfer the contents of an entire collection to a client application. It provides optional + filtering on specific attributes. + + Tht export API is available at endpoint `POST /_api/export?collection=...`. The API has the + same return value structure as the already established cursor API (`POST /_api/cursor`). + + An introduction to the export API is given in this blog post: + http://jsteemann.github.io/blog/2015/04/04/more-efficient-data-exports/ + +!SECTION AQL improvements + +!SUBSECTION subquery optimizations for AQL queries + +This optimization avoids copying intermediate results into subqueries that are not required +by the subquery. + +A brief description can be found here: +http://jsteemann.github.io/blog/2015/05/04/subquery-optimizations/ + +!SUBSECTION Return value optimization for AQL queries + +This optimization avoids copying the final query result inside the query's main `ReturnNode`. + +A brief description can be found here: +http://jsteemann.github.io/blog/2015/05/04/return-value-optimization-for-aql/ + +!SUBSECTION Speed up AQL queries containing big `IN` lists for index lookups + +`IN` lists used for index lookups had performance issues in previous versions of ArangoDB. +These issues have been addressed in 2.6 so using bigger `IN` lists for filtering is much +faster. + +A brief description can be found here: +http://jsteemann.github.io/blog/2015/05/07/in-list-improvements/ + +!SUBSECTION Added alternative implementation for AQL COLLECT + +The alternative method uses a hash table for grouping and does not require its input elements +to be sorted. It will be taken into account by the optimizer for `COLLECT` statements that do +not use an `INTO` clause. + +In case a `COLLECT` statement can use the hash table variant, the optimizer will create an extra +plan for it at the beginning of the planning phase. In this plan, no extra `SORT` node will be +added in front of the `COLLECT` because the hash table variant of `COLLECT` does not require +sorted input. Instead, a `SORT` node will be added after it to sort its output. This `SORT` node +may be optimized away again in later stages. If the sort order of the result is irrelevant to +the user, adding an extra `SORT null` after a hash `COLLECT` operation will allow the optimizer to +remove the sorts altogether. + +In addition to the hash table variant of `COLLECT`, the optimizer will modify the original plan +to use the regular `COLLECT` implementation. As this implementation requires sorted input, the +optimizer will insert a `SORT` node in front of the `COLLECT`. This `SORT` node may be optimized +away in later stages. + +The created plans will then be shipped through the regular optimization pipeline. In the end, +the optimizer will pick the plan with the lowest estimated total cost as usual. The hash table +variant does not require an up-front sort of the input, and will thus be preferred over the +regular `COLLECT` if the optimizer estimates many input elements for the `COLLECT` node and +cannot use an index to sort them. + +The optimizer can be explicitly told to use the regular *sorted* variant of `COLLECT` by +suffixing a `COLLECT` statement with `OPTIONS { "method" : "sorted" }`. This will override the +optimizer guesswork and only produce the *sorted* variant of `COLLECT`. + +A blog post on the new `COLLECT` implementation can be found here: +http://jsteemann.github.io/blog/2015/04/22/collecting-with-a-hash-table/ + +!SUBSECTION simplified return value syntax for data-modification AQL queries + +ArangoDB 2.4 since version allows to return results from data-modification AQL queries. The +syntax for this was quite limited and verbose: + +``` +FOR i IN 1..10 + INSERT { value: i } IN test + LET inserted = NEW + RETURN inserted +``` +The `LET inserted = NEW RETURN inserted` was required literally to return the inserted +documents. No calculations could be made using the inserted documents. + +This is now more flexible. After a data-modification clause (e.g. `INSERT`, `UPDATE`, `REPLACE`, +`REMOVE`, `UPSERT`) there can follow any number of `LET` calculations. These calculations can +refer to the pseudo-values `OLD` and `NEW` that are created by the data-modification statements. + +This allows returning projections of inserted or updated documents, e.g.: + +``` +FOR i IN 1..10 + INSERT { value: i } IN test + RETURN { _key: NEW._key, value: i } +``` + +Still not every construct is allowed after a data-modification clause. For example, no functions +can be called that may access documents. + +More information can be found here: +http://jsteemann.github.io/blog/2015/03/27/improvements-for-data-modification-queries/ + +!SUBSECTION added AQL `UPSERT` statement + +This adds an `UPSERT` statement to AQL that is a combination of both `INSERT` and `UPDATE` / +`REPLACE`. The `UPSERT` will search for a matching document using a user-provided example. +If no document matches the example, the *insert* part of the `UPSERT` statement will be +executed. If there is a match, the *update* / *replace* part will be carried out: + +``` +UPSERT { page: 'index.html' } /* search example */ +INSERT { page: 'index.html', pageViews: 1 } /* insert part */ +UPDATE { pageViews: OLD.pageViews + 1 } /* update part */ +IN pageViews +``` + +`UPSERT` can be used with an `UPDATE` or `REPLACE` clause. The `UPDATE` clause will perform +a partial update of the found document, whereas the `REPLACE` clause will replace the found +document entirely. The `UPDATE` or `REPLACE` parts can refer to the pseudo-value `OLD`, which +contains all attributes of the found document. + +`UPSERT` statements can optionally return values. In the following query, the return +attribute `found` will return the found document before the `UPDATE` was applied. If no +document was found, `found` will contain a value of `null`. The `updated` result attribute will +contain the inserted / updated document: + +``` +UPSERT { page: 'index.html' } /* search example */ +INSERT { page: 'index.html', pageViews: 1 } /* insert part */ +UPDATE { pageViews: OLD.pageViews + 1 } /* update part */ +IN pageViews +RETURN { found: OLD, updated: NEW } +``` + +A more detailed description of `UPSERT` can be found here: +http://jsteemann.github.io/blog/2015/03/27/preview-of-the-upsert-command/ + +!SECTION Web Interface + +There have been some minor changes in the web interface: + +- The query execution time is displayed in the AQL editor now +- The AQL query button **submit** is now renamed to **execute** +- We've added a query explain feature in the AQL editor +- We've added a demo page. This only is shown if demo data is available otherwise it is hidden +- ArangoDB's built-in web interface now uses sessions. Session information ids are stored in cookies, so clients using the web interface must accept cookies in order to use it diff --git a/Documentation/Books/Users/SUMMARY.md b/Documentation/Books/Users/SUMMARY.md index 19058090a2..59835d5292 100644 --- a/Documentation/Books/Users/SUMMARY.md +++ b/Documentation/Books/Users/SUMMARY.md @@ -14,9 +14,9 @@ * [Incompatible changes in 2.3](Upgrading/UpgradingChanges23.md) * [Upgrading to 2.3](Upgrading/Upgrading23.md) * [Upgrading to 2.2](Upgrading/Upgrading22.md) - * [Upgrading in general](Installing/Upgrading.md) * [Cluster setup](Installing/Cluster.md) -* [Whats New](NewFeatures/NewFeatures25.md) +* [Whats New](NewFeatures/NewFeatures26.md) + * [Whats New in 2.5](NewFeatures/NewFeatures25.md) * [Whats New in 2.4](NewFeatures/NewFeatures24.md) * [Whats New in 2.3](NewFeatures/NewFeatures23.md) * [Whats New in 2.2](NewFeatures/NewFeatures22.md) diff --git a/js/actions/api-graph.js b/js/actions/api-graph.js index b09242706b..eaf7097214 100644 --- a/js/actions/api-graph.js +++ b/js/actions/api-graph.js @@ -1096,7 +1096,7 @@ function process_labels_filter (data, labels, collname) { /// - `batchSize`: the batch size of the returned cursor /// - `limit`: limit the result size /// - `count`: return the total number of results (default "false") -/// - `filter`: a optional filter +/// - `filter`: an optional filter /// /// The attributes of filter /// - `properties`: filter by an array of vertex properties @@ -1161,7 +1161,7 @@ function post_graph_all_vertices (req, res, g) { var query = "FOR v IN @@vertexColl" + data.filter + limit + " RETURN v"; var options = { - count: json.count, + count: json.count || false, batchSize: json.batchSize || 1000 }; @@ -1322,13 +1322,14 @@ function post_graph_vertex_vertices (req, res, g) { } // build aql query - var query = 'FOR n IN NEIGHBORS( @@vertexColl, @@edgeColl, @id, "' + direction + '") ' + - data.filter + limit + " RETURN n.vertex "; + var query = 'FOR n IN NEIGHBORS( @@vertexColl, @@edgeColl, @id, "' + direction + '", null, ' + + '{ includeData: true }) ' + data.filter + limit + " RETURN n.vertex "; var options = { count: json.count, batchSize: json.batchSize || 1000 }; + var cursor = AQL_EXECUTE(query, data.bindVars, options); // error occurred diff --git a/js/client/client.js b/js/client/client.js index 309db04560..94d73400ed 100644 --- a/js/client/client.js +++ b/js/client/client.js @@ -152,6 +152,8 @@ global.tutorial = require("org/arangodb/tutorial"); } if (internal.quiet !== true) { + require("org/arangodb").checkAvailableVersions(); + if (internal.arango && internal.arango.isConnected && internal.arango.isConnected()) { internal.print("Type 'tutorial' for a tutorial or 'help' to see common examples"); } diff --git a/js/client/modules/org/arangodb.js b/js/client/modules/org/arangodb.js index 076db20b67..0c25632a40 100644 --- a/js/client/modules/org/arangodb.js +++ b/js/client/modules/org/arangodb.js @@ -38,6 +38,18 @@ Object.keys(common).forEach(function (key) { // --SECTION-- MODULE EXPORTS // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief isServer +//////////////////////////////////////////////////////////////////////////////// + +exports.isServer = false; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief isClient +//////////////////////////////////////////////////////////////////////////////// + +exports.isClient = true; + //////////////////////////////////////////////////////////////////////////////// /// @brief class "ArangoCollection" //////////////////////////////////////////////////////////////////////////////// diff --git a/js/common/modules/org/arangodb-common.js b/js/common/modules/org/arangodb-common.js index a99ecbd373..61564b6d7e 100644 --- a/js/common/modules/org/arangodb-common.js +++ b/js/common/modules/org/arangodb-common.js @@ -494,6 +494,52 @@ exports.checkParameter = function (usage, descs, vars) { } }; +//////////////////////////////////////////////////////////////////////////////// +/// @brief generate info message for newer version(s) available +//////////////////////////////////////////////////////////////////////////////// + +exports.checkAvailableVersions = function (version) { + var console = require("console"); + var log; + + if (require("org/arangodb").isServer) { + log = console.info; + } + else { + log = internal.print; + } + + if (version === undefined) { + version = internal.version; + } + + if (version.match(/beta|alpha|preview|devel/) !== null) { + log("Your are using an alpha/beta/preview version ('" + version + "') of ArangoDB"); + return; + } + + try { + var u = "https://www.arangodb.com/repositories/versions.php?version="; + var d = internal.download(u + version, "", {timeout: 300}); + var v = JSON.parse(d.body); + + if (v.hasOwnProperty("bugfix")) { + log("Please note that a new bugfix version '" + v.bugfix.version + "' is available"); + } + + if (v.hasOwnProperty("minor")) { + log("Please note that a new minor version '" + v.minor.version + "' is available"); + } + + if (v.hasOwnProperty("major")) { + log("Please note that a new major version '" + v.major.version + "' is available"); + } + } + catch (err) { + console.debug("cannot check for newer version: ", err.stack); + } +}; + // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // ----------------------------------------------------------------------------- diff --git a/js/server/modules/org/arangodb.js b/js/server/modules/org/arangodb.js index 7ef4361860..082af8675f 100644 --- a/js/server/modules/org/arangodb.js +++ b/js/server/modules/org/arangodb.js @@ -43,6 +43,18 @@ var ShapedJson = require("org/arangodb/shaped-json").ShapedJson; // --SECTION-- module "arangodb" // ----------------------------------------------------------------------------- +//////////////////////////////////////////////////////////////////////////////// +/// @brief isServer +//////////////////////////////////////////////////////////////////////////////// + +exports.isServer = true; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief isClient +//////////////////////////////////////////////////////////////////////////////// + +exports.isClient = false; + //////////////////////////////////////////////////////////////////////////////// /// @brief class "ArangoCollection" //////////////////////////////////////////////////////////////////////////////// diff --git a/js/server/modules/org/arangodb/aql.js b/js/server/modules/org/arangodb/aql.js index e47ce740ee..76f3777eb5 100644 --- a/js/server/modules/org/arangodb/aql.js +++ b/js/server/modules/org/arangodb/aql.js @@ -6525,7 +6525,7 @@ function AQL_NEIGHBORS (vertexCollection, options.direction = direction; if (examples === undefined || (Array.isArray(examples) && examples.length <= 1)) { - if (examples.length === 1) { + if (Array.isArray(examples) && examples.length === 1) { options.examples = examples[0]; } if (typeof options.distance === "number" && options.distance > 1) { diff --git a/js/server/modules/org/arangodb/testing.js b/js/server/modules/org/arangodb/testing.js index c9d058ecfc..e51f2ab470 100644 --- a/js/server/modules/org/arangodb/testing.js +++ b/js/server/modules/org/arangodb/testing.js @@ -245,7 +245,6 @@ function makeTestingArgs (appDir) { "database.force-sync-properties": "false", "javascript.app-path": appDir, "javascript.startup-directory": fs.join(topDir, "js"), - "ruby.modules-path": fs.join(topDir,"mr", "common", "modules"), "server.threads": "20", "javascript.v8-contexts": "5", "server.disable-authentication": "true", @@ -826,7 +825,7 @@ function runThere (options, instanceInfo, file) { } var o = makeAuthorisationHeaders(options); o.method = "POST"; - o.timeout = 24 * 3600; + o.timeout = 3600; o.returnBodyOnError = true; r = download(instanceInfo.url + "/_admin/execute?returnAsJSON=true",t,o); if (! r.error && r.code === 200) { diff --git a/js/server/server.js b/js/server/server.js index 4ab2b32d59..786340c67e 100644 --- a/js/server/server.js +++ b/js/server/server.js @@ -69,6 +69,11 @@ require('org/arangodb/foxx/queues/manager').run(); } + // check available versions + if (internal.threadNumber === 0 && internal.quiet !== true) { + require("org/arangodb").checkAvailableVersions(); + } + return true; }());