diff --git a/Documentation/Books/Users/HttpAqlQueryCursor/README.mdpp b/Documentation/Books/Users/HttpAqlQueryCursor/README.mdpp index 9b7e1f9053..98a7bb1f4e 100644 --- a/Documentation/Books/Users/HttpAqlQueryCursor/README.mdpp +++ b/Documentation/Books/Users/HttpAqlQueryCursor/README.mdpp @@ -5,7 +5,7 @@ This is an introduction to ArangoDB's HTTP Interface for Queries. Results of AQL and simple queries are returned as cursors in order to batch the communication between server and client. Each call returns a number of documents in a batch -and an indication, if the current batch has been the final batch. Depending on +and an indication if the current batch has been the final batch. Depending on the query, the total number of documents in the result set might or might not be known in advance. In order to free server resources the client should delete the cursor as soon as it is no longer needed. diff --git a/Documentation/Books/Users/HttpExport/README.mdpp b/Documentation/Books/Users/HttpExport/README.mdpp new file mode 100644 index 0000000000..e5cc87fc1c --- /dev/null +++ b/Documentation/Books/Users/HttpExport/README.mdpp @@ -0,0 +1,3 @@ +!CHAPTER HTTP Interface for Exporting Documents + +@startDocuBlock JSF_post_api_export diff --git a/Documentation/Books/Users/SUMMARY.md b/Documentation/Books/Users/SUMMARY.md index c34e0dddda..afabaa16c7 100644 --- a/Documentation/Books/Users/SUMMARY.md +++ b/Documentation/Books/Users/SUMMARY.md @@ -164,6 +164,7 @@ * [Edges](HttpEdge/README.md) * [Address and ETag](HttpEdge/AddressAndEtag.md) * [Working with Edges](HttpEdge/WorkingWithEdges.md) + * [Exporting data](HttpExport/README.md) * [AQL Query Cursors](HttpAqlQueryCursor/README.md) * [Query Results](HttpAqlQueryCursor/QueryResults.md) * [Accessing Cursors](HttpAqlQueryCursor/AccessingCursors.md) diff --git a/Documentation/Scripts/generateSwaggerApi.py b/Documentation/Scripts/generateSwaggerApi.py index 1fd4ec463c..4c76310713 100755 --- a/Documentation/Scripts/generateSwaggerApi.py +++ b/Documentation/Scripts/generateSwaggerApi.py @@ -44,11 +44,12 @@ import sys, os, json, string files = { "js/actions/api-aqlfunction.js" : "aqlfunction", "arangod/RestHandler/RestBatchHandler.cpp" : "batch", - "js/actions/api-collection.js" : "collection", - "js/actions/api-cursor.js" : "cursor", - "js/actions/api-database.js" : "database", + "js/actions/_api/collection/app.js" : "collection", + "js/actions/_admin/database/app.js" : "database", + "arangod/RestHandler/RestCursorHandler.cpp" : "cursor", "arangod/RestHandler/RestDocumentHandler.cpp" : "document", "arangod/RestHandler/RestEdgeHandler.cpp" : "edge", + "arangod/RestHandler/RestExportHandler.cpp" : "export", "js/actions/api-edges.js" : "edges", "js/actions/api-endpoint.js" : "endpoint", "js/actions/api-explain.js" : "explain", @@ -57,7 +58,7 @@ files = { "js/actions/api-index.js" : "index", "lib/HttpServer/AsyncJobManager.h" : "job", "lib/Admin/RestAdminLogHandler.cpp" : "log", - "js/actions/api-query.js" : "query", + "arangod/RestHandler/RestQueryHandler.cpp" : "query", "arangod/RestHandler/RestReplicationHandler.cpp" : "replication", "js/actions/api-simple.js" : "simple", "js/actions/api-structure.js" : "structure", @@ -65,8 +66,9 @@ files = { "js/actions/api-tasks.js" : "tasks", "js/actions/api-transaction.js" : "transaction", "js/actions/api-traversal.js" : "traversal", - "js/actions/api-user.js" : "user", - "lib/Admin/RestVersionHandler.cpp" : "version" + "js/actions/_api/user/app.js" : "user", + "lib/Admin/RestVersionHandler.cpp" : "version", + "js/actions/_admin/wal/app.js" : "wal" } if len(sys.argv) < 3: diff --git a/arangod/Aql/Query.cpp b/arangod/Aql/Query.cpp index 58a5109d97..4f388abc52 100644 --- a/arangod/Aql/Query.cpp +++ b/arangod/Aql/Query.cpp @@ -480,7 +480,6 @@ void Query::registerErrorCustom (int code, errorMessage.append(": "); errorMessage.append(details); -std::cout << "REGISTER ERROR CUSTOM: " << errorMessage << "\n"; THROW_ARANGO_EXCEPTION_MESSAGE(code, errorMessage); } diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 5206dc9523..20c692ff88 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -105,6 +105,7 @@ add_executable( RestHandler/RestCursorHandler.cpp RestHandler/RestDocumentHandler.cpp RestHandler/RestEdgeHandler.cpp + RestHandler/RestExportHandler.cpp RestHandler/RestImportHandler.cpp RestHandler/RestPleaseUpgradeHandler.cpp RestHandler/RestQueryHandler.cpp @@ -116,6 +117,7 @@ add_executable( RestServer/VocbaseContext.cpp RestServer/arangod.cpp SkipLists/skiplistIndex.cpp + Utils/CollectionExport.cpp Utils/Cursor.cpp Utils/CursorRepository.cpp Utils/DocumentHelper.cpp diff --git a/arangod/Makefile.files b/arangod/Makefile.files index f82b7e3722..fa8ac9c839 100644 --- a/arangod/Makefile.files +++ b/arangod/Makefile.files @@ -78,6 +78,7 @@ arangod_libarangod_a_SOURCES = \ arangod/RestHandler/RestCursorHandler.cpp \ arangod/RestHandler/RestDocumentHandler.cpp \ arangod/RestHandler/RestEdgeHandler.cpp \ + arangod/RestHandler/RestExportHandler.cpp \ arangod/RestHandler/RestImportHandler.cpp \ arangod/RestHandler/RestPleaseUpgradeHandler.cpp \ arangod/RestHandler/RestQueryHandler.cpp \ @@ -89,6 +90,7 @@ arangod_libarangod_a_SOURCES = \ arangod/RestServer/VocbaseContext.cpp \ arangod/RestServer/arangod.cpp \ arangod/SkipLists/skiplistIndex.cpp \ + arangod/Utils/CollectionExport.cpp \ arangod/Utils/Cursor.cpp \ arangod/Utils/CursorRepository.cpp \ arangod/Utils/DocumentHelper.cpp \ diff --git a/arangod/RestHandler/RestCursorHandler.cpp b/arangod/RestHandler/RestCursorHandler.cpp index a522e1ff9c..e0585b5868 100644 --- a/arangod/RestHandler/RestCursorHandler.cpp +++ b/arangod/RestHandler/RestCursorHandler.cpp @@ -236,67 +236,6 @@ triagens::basics::Json RestCursorHandler::buildExtra (triagens::aql::QueryResult return extra; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief append the contents of the cursor into the response body -//////////////////////////////////////////////////////////////////////////////// - -void RestCursorHandler::dumpCursor (Cursor* cursor) { - _response->body().appendText("{\"result\":["); - - size_t const n = cursor->batchSize(); - - for (size_t i = 0; i < n; ++i) { - if (! cursor->hasNext()) { - break; - } - - if (i > 0) { - _response->body().appendChar(','); - } - - auto row = cursor->next(); - if (row == nullptr) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); - } - - int res = TRI_StringifyJson(_response->body().stringBuffer(), row); - - if (res != TRI_ERROR_NO_ERROR) { - THROW_ARANGO_EXCEPTION(res); - } - } - - bool hasCount = cursor->hasCount(); - size_t count = cursor->count(); - bool hasNext = cursor->hasNext(); - TRI_json_t* extra = cursor->extra(); - - _response->body().appendText("],\"hasMore\":"); - _response->body().appendText(hasNext ? "true" : "false"); - - if (hasNext) { - // only return cursor id if there are more documents - _response->body().appendText(",\"id\":\""); - _response->body().appendInteger(cursor->id()); - _response->body().appendText("\""); - } - - if (hasCount) { - _response->body().appendText(",\"count\":"); - _response->body().appendInteger(static_cast(count)); - } - - if (TRI_IsObjectJson(extra)) { - _response->body().appendText(",\"extra\":"); - TRI_StringifyJson(_response->body().stringBuffer(), extra); - } - - _response->body().appendText(",\"error\":false,\"code\":"); - _response->body().appendInteger(static_cast(_response->responseCode())); - - _response->body().appendChar('}'); -} - //////////////////////////////////////////////////////////////////////////////// /// @startDocuBlock JSF_post_api_cursor /// @brief create a cursor and return the first results @@ -735,13 +674,18 @@ void RestCursorHandler::createCursor () { double ttl = triagens::basics::JsonHelper::getNumericValue(options.json(), "ttl", 30); bool count = triagens::basics::JsonHelper::getBooleanValue(options.json(), "count", false); - // steal the query JSON, cursor will take owner the ownership + // steal the query JSON, cursor will take over the ownership auto j = queryResult.json; + triagens::arango::JsonCursor* cursor = cursors->createFromJson(j, batchSize, extra.steal(), ttl, count); queryResult.json = nullptr; - triagens::arango::Cursor* cursor = cursors->createFromJson(j, batchSize, extra.steal(), ttl, count); try { - dumpCursor(cursor); + _response->body().appendChar('{'); + cursor->dump(_response->body()); + _response->body().appendText(",\"error\":false,\"code\":"); + _response->body().appendInteger(static_cast(_response->responseCode())); + _response->body().appendChar('}'); + cursors->release(cursor); } catch (...) { @@ -889,7 +833,12 @@ void RestCursorHandler::modifyCursor () { _response = createResponse(HttpResponse::OK); _response->setContentType("application/json; charset=utf-8"); - dumpCursor(cursor); + _response->body().appendChar('{'); + cursor->dump(_response->body()); + _response->body().appendText(",\"error\":false,\"code\":"); + _response->body().appendInteger(static_cast(_response->responseCode())); + _response->body().appendChar('}'); + cursors->release(cursor); } catch (triagens::basics::Exception const& ex) { diff --git a/arangod/RestHandler/RestCursorHandler.h b/arangod/RestHandler/RestCursorHandler.h index b6b8ab2d5d..cedda7070f 100644 --- a/arangod/RestHandler/RestCursorHandler.h +++ b/arangod/RestHandler/RestCursorHandler.h @@ -50,7 +50,7 @@ namespace triagens { class Cursor; //////////////////////////////////////////////////////////////////////////////// -/// @brief document request handler +/// @brief cursor request handler //////////////////////////////////////////////////////////////////////////////// class RestCursorHandler : public RestVocbaseBaseHandler { diff --git a/arangod/RestHandler/RestExportHandler.cpp b/arangod/RestHandler/RestExportHandler.cpp new file mode 100644 index 0000000000..e150d4e992 --- /dev/null +++ b/arangod/RestHandler/RestExportHandler.cpp @@ -0,0 +1,294 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief export request handler +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2014 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany +/// @author Copyright 2010-2014, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "RestExportHandler.h" +#include "Basics/Exceptions.h" +#include "Basics/json.h" +#include "Basics/MutexLocker.h" +#include "Utils/CollectionExport.h" +#include "Utils/Cursor.h" +#include "Utils/CursorRepository.h" +#include "Wal/LogfileManager.h" + +using namespace triagens::arango; +using namespace triagens::rest; + +// ----------------------------------------------------------------------------- +// --SECTION-- constructors and destructors +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief constructor +//////////////////////////////////////////////////////////////////////////////// + +RestExportHandler::RestExportHandler (HttpRequest* request) + : RestVocbaseBaseHandler(request) { + +} + +// ----------------------------------------------------------------------------- +// --SECTION-- Handler methods +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// {@inheritDoc} +//////////////////////////////////////////////////////////////////////////////// + +HttpHandler::status_t RestExportHandler::execute () { + // extract the sub-request type + HttpRequest::HttpRequestType type = _request->requestType(); + + if (type == HttpRequest::HTTP_REQUEST_POST) { + createCursor(); + return status_t(HANDLER_DONE); + } + + generateError(HttpResponse::METHOD_NOT_ALLOWED, TRI_ERROR_HTTP_METHOD_NOT_ALLOWED); + return status_t(HANDLER_DONE); +} + +// ----------------------------------------------------------------------------- +// --SECTION-- private methods +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief build options for the query as JSON +//////////////////////////////////////////////////////////////////////////////// + +triagens::basics::Json RestExportHandler::buildOptions (TRI_json_t const* json) { + auto getAttribute = [&json] (char const* name) { + return TRI_LookupObjectJson(json, name); + }; + + triagens::basics::Json options(triagens::basics::Json::Object); + + auto attribute = getAttribute("count"); + options.set("count", triagens::basics::Json(TRI_IsBooleanJson(attribute) ? attribute->_value._boolean : false)); + + attribute = getAttribute("batchSize"); + options.set("batchSize", triagens::basics::Json(TRI_IsNumberJson(attribute) ? attribute->_value._number : 1000.0)); + + if (TRI_IsNumberJson(attribute) && static_cast(attribute->_value._number) == 0) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_TYPE_ERROR, "expecting non-zero value for "); + } + + if (! options.has("ttl")) { + attribute = getAttribute("ttl"); + options.set("ttl", triagens::basics::Json(TRI_IsNumberJson(attribute) ? attribute->_value._number : 30.0)); + } + + return options; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @startDocuBlock JSF_post_api_export +/// @brief export all documents from a collection, using a cursor +/// +/// @RESTHEADER{POST /_api/export, Create export cursor} +/// +/// @RESTBODYPARAM{options,json,optional} +/// A JSON object with export options. +/// +/// @RESTQUERYPARAMETERS +/// +/// @RESTQUERYPARAM{collection,string,required} +/// The name of the collection to export. +/// +/// @RESTDESCRIPTION +/// A call to this method creates a cursor containing all documents in the +/// specified collection. In contrast to other data-producing APIs, the internal +/// data structures produced by the export API are more lightweight, so it is +/// the preferred way to retrieve all documents from a collection. +/// +/// The following attributes can be used inside the JSON object: +/// +/// - *count*: boolean flag that indicates whether the number of documents +/// in the result set should be returned in the "count" attribute of the result (optional). +/// Calculating the "count" attribute might in the future have a performance +/// impact so this option is turned off by default, and "count" is only returned +/// when requested. +/// +/// - *batchSize*: maximum number of result documents to be transferred from +/// the server to the client in one roundtrip (optional). If this attribute is +/// not set, a server-controlled default value will be used. +/// +/// - *ttl*: an optional time-to-live for the cursor (in seconds). The cursor will be +/// removed on the server automatically after the specified amount of time. This +/// is useful to ensure garbage collection of cursors that are not fully fetched +/// by clients. If not set, a server-defined value will be used. +/// +/// If the result set can be created by the server, the server will respond with +/// *HTTP 201*. The body of the response will contain a JSON object with the +/// result set. +/// +/// The returned JSON object has the following properties: +/// +/// - *error*: boolean flag to indicate that an error occurred (*false* +/// in this case) +/// +/// - *code*: the HTTP status code +/// +/// - *result*: an array of result documents (might be empty if the collection was empty) +/// +/// - *hasMore*: a boolean indicator whether there are more results +/// available for the cursor on the server +/// +/// - *count*: the total number of result documents available (only +/// available if the query was executed with the *count* attribute set) +/// +/// - *id*: id of temporary cursor created on the server (optional, see above) +/// +/// If the JSON representation is malformed or the query specification is +/// missing from the request, the server will respond with *HTTP 400*. +/// +/// The body of the response will contain a JSON object with additional error +/// details. The object has the following attributes: +/// +/// - *error*: boolean flag to indicate that an error occurred (*true* in this case) +/// +/// - *code*: the HTTP status code +/// +/// - *errorNum*: the server error number +/// +/// - *errorMessage*: a descriptive error message +/// +/// @RESTRETURNCODES +/// +/// @RESTRETURNCODE{201} +/// is returned if the result set can be created by the server. +/// +/// @RESTRETURNCODE{400} +/// is returned if the JSON representation is malformed or the query specification is +/// missing from the request. +/// +/// @RESTRETURNCODE{404} +/// The server will respond with *HTTP 404* in case a non-existing collection is +/// accessed in the query. +/// +/// @RESTRETURNCODE{405} +/// The server will respond with *HTTP 405* if an unsupported HTTP method is used. +/// +/// @endDocuBlock +//////////////////////////////////////////////////////////////////////////////// + +void RestExportHandler::createCursor () { + std::vector const& suffix = _request->suffix(); + + if (suffix.size() != 0) { + generateError(HttpResponse::BAD, + TRI_ERROR_HTTP_BAD_PARAMETER, + "expecting POST /_api/export"); + return; + } + + // extract the cid + bool found; + char const* name = _request->value("collection", found); + + if (! found || *name == '\0') { + generateError(HttpResponse::BAD, + TRI_ERROR_ARANGO_COLLECTION_PARAMETER_MISSING, + "'collection' is missing, expecting " + EXPORT_PATH + "?collection="); + return; + } + + try { + std::unique_ptr json(parseJsonBody()); + + triagens::basics::Json options; + + if (json.get() != nullptr) { + if (! TRI_IsObjectJson(json.get())) { + generateError(HttpResponse::BAD, TRI_ERROR_QUERY_EMPTY); + return; + } + + options = buildOptions(json.get()); + } + else { + // create an empty options object + options = triagens::basics::Json(triagens::basics::Json::Object); + } + + bool flush = triagens::basics::JsonHelper::getBooleanValue(options.json(), "flush", true); + + if (flush) { + // flush the logfiles so the export can fetch all documents + int res = triagens::wal::LogfileManager::instance()->flush(true, true, false); + + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + } + + // this may throw! + std::unique_ptr collectionExport(new CollectionExport(_vocbase, name)); + collectionExport->run(); + + { + _response = createResponse(HttpResponse::CREATED); + _response->setContentType("application/json; charset=utf-8"); + + size_t batchSize = triagens::basics::JsonHelper::getNumericValue(options.json(), "batchSize", 1000); + double ttl = triagens::basics::JsonHelper::getNumericValue(options.json(), "ttl", 30); + bool count = triagens::basics::JsonHelper::getBooleanValue(options.json(), "count", false); + + auto cursors = static_cast(_vocbase->_cursorRepository); + TRI_ASSERT(cursors != nullptr); + + // create a cursor from the result + triagens::arango::Cursor* cursor = cursors->createFromExport(collectionExport.get(), batchSize, ttl, count); + collectionExport.release(); + + try { + cursor->dump(_response->body()); + cursors->release(cursor); + } + catch (...) { + cursors->release(cursor); + throw; + } + } + } + catch (triagens::basics::Exception const& ex) { + generateError(HttpResponse::responseCode(ex.code()), ex.code(), ex.what()); + } + catch (...) { + generateError(HttpResponse::SERVER_ERROR, TRI_ERROR_INTERNAL); + } +} + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + +// Local Variables: +// mode: outline-minor +// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" +// End: diff --git a/arangod/RestHandler/RestExportHandler.h b/arangod/RestHandler/RestExportHandler.h new file mode 100644 index 0000000000..9fb3376006 --- /dev/null +++ b/arangod/RestHandler/RestExportHandler.h @@ -0,0 +1,113 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief export request handler +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2014 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany +/// @author Copyright 2010-2014, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_REST_HANDLER_REST_EXPORT_HANDLER_H +#define ARANGODB_REST_HANDLER_REST_EXPORT_HANDLER_H 1 + +#include "Basics/Common.h" +#include "Basics/Mutex.h" +#include "RestHandler/RestVocbaseBaseHandler.h" + +// ----------------------------------------------------------------------------- +// --SECTION-- class RestExportHandler +// ----------------------------------------------------------------------------- + +namespace triagens { + + namespace arango { + class Cursor; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief document request handler +//////////////////////////////////////////////////////////////////////////////// + + class RestExportHandler : public RestVocbaseBaseHandler { + +// ----------------------------------------------------------------------------- +// --SECTION-- constructors and destructors +// ----------------------------------------------------------------------------- + + public: + +//////////////////////////////////////////////////////////////////////////////// +/// @brief constructor +//////////////////////////////////////////////////////////////////////////////// + + RestExportHandler (rest::HttpRequest*); + +// ----------------------------------------------------------------------------- +// --SECTION-- Handler methods +// ----------------------------------------------------------------------------- + + public: + +//////////////////////////////////////////////////////////////////////////////// +/// {@inheritDoc} +//////////////////////////////////////////////////////////////////////////////// + + status_t execute () override; + +// ----------------------------------------------------------------------------- +// --SECTION-- private methods +// ----------------------------------------------------------------------------- + + private: + +//////////////////////////////////////////////////////////////////////////////// +/// @brief build options for the query as JSON +//////////////////////////////////////////////////////////////////////////////// + + triagens::basics::Json buildOptions (TRI_json_t const*); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief create an export cursor and return the first results +//////////////////////////////////////////////////////////////////////////////// + + void createCursor (); + +// ----------------------------------------------------------------------------- +// --SECTION-- private variables +// ----------------------------------------------------------------------------- + + private: + + }; + } +} + +#endif + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + +// Local Variables: +// mode: outline-minor +// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" +// End: diff --git a/arangod/RestHandler/RestImportHandler.cpp b/arangod/RestHandler/RestImportHandler.cpp index 7bfbd7f2aa..5a36451bb3 100644 --- a/arangod/RestHandler/RestImportHandler.cpp +++ b/arangod/RestHandler/RestImportHandler.cpp @@ -94,7 +94,7 @@ HttpHandler::status_t RestImportHandler::execute () { } default: - generateNotImplemented("ILLEGAL " + DOCUMENT_IMPORT_PATH); + generateNotImplemented("ILLEGAL " + IMPORT_PATH); break; } @@ -593,7 +593,7 @@ bool RestImportHandler::createFromJson (string const& type) { if (suffix.size() != 0) { generateError(HttpResponse::BAD, TRI_ERROR_HTTP_SUPERFLUOUS_SUFFICES, - "superfluous suffix, expecting " + DOCUMENT_IMPORT_PATH + "?collection="); + "superfluous suffix, expecting " + IMPORT_PATH + "?collection="); return false; } @@ -608,7 +608,7 @@ bool RestImportHandler::createFromJson (string const& type) { if (! found || collection.empty()) { generateError(HttpResponse::BAD, TRI_ERROR_ARANGO_COLLECTION_PARAMETER_MISSING, - "'collection' is missing, expecting " + DOCUMENT_IMPORT_PATH + "?collection="); + "'collection' is missing, expecting " + IMPORT_PATH + "?collection="); return false; } @@ -1061,7 +1061,7 @@ bool RestImportHandler::createFromKeyValueList () { if (suffix.size() != 0) { generateError(HttpResponse::BAD, TRI_ERROR_HTTP_SUPERFLUOUS_SUFFICES, - "superfluous suffix, expecting " + DOCUMENT_IMPORT_PATH + "?collection="); + "superfluous suffix, expecting " + IMPORT_PATH + "?collection="); return false; } @@ -1076,7 +1076,7 @@ bool RestImportHandler::createFromKeyValueList () { if (! found || collection.empty()) { generateError(HttpResponse::BAD, TRI_ERROR_ARANGO_COLLECTION_PARAMETER_MISSING, - "'collection' is missing, expecting " + DOCUMENT_IMPORT_PATH + "?collection="); + "'collection' is missing, expecting " + IMPORT_PATH + "?collection="); return false; } diff --git a/arangod/RestHandler/RestVocbaseBaseHandler.cpp b/arangod/RestHandler/RestVocbaseBaseHandler.cpp index e79ab1c61f..adab53a15f 100644 --- a/arangod/RestHandler/RestVocbaseBaseHandler.cpp +++ b/arangod/RestHandler/RestVocbaseBaseHandler.cpp @@ -56,43 +56,49 @@ using namespace triagens::arango; /// @brief batch path //////////////////////////////////////////////////////////////////////////////// -const string RestVocbaseBaseHandler::BATCH_PATH = "/_api/batch"; +const string RestVocbaseBaseHandler::BATCH_PATH = "/_api/batch"; //////////////////////////////////////////////////////////////////////////////// /// @brief cursor path //////////////////////////////////////////////////////////////////////////////// -const string RestVocbaseBaseHandler::CURSOR_PATH = "/_api/cursor"; +const string RestVocbaseBaseHandler::CURSOR_PATH = "/_api/cursor"; //////////////////////////////////////////////////////////////////////////////// /// @brief document path //////////////////////////////////////////////////////////////////////////////// -const string RestVocbaseBaseHandler::DOCUMENT_PATH = "/_api/document"; +const string RestVocbaseBaseHandler::DOCUMENT_PATH = "/_api/document"; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief document path +//////////////////////////////////////////////////////////////////////////////// + +const string RestVocbaseBaseHandler::EDGE_PATH = "/_api/edge"; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief export path +//////////////////////////////////////////////////////////////////////////////// + +const string RestVocbaseBaseHandler::EXPORT_PATH = "/_api/export"; //////////////////////////////////////////////////////////////////////////////// /// @brief documents import path //////////////////////////////////////////////////////////////////////////////// -const string RestVocbaseBaseHandler::DOCUMENT_IMPORT_PATH = "/_api/import"; - -//////////////////////////////////////////////////////////////////////////////// -/// @brief document path -//////////////////////////////////////////////////////////////////////////////// - -const string RestVocbaseBaseHandler::EDGE_PATH = "/_api/edge"; +const string RestVocbaseBaseHandler::IMPORT_PATH = "/_api/import"; //////////////////////////////////////////////////////////////////////////////// /// @brief replication path //////////////////////////////////////////////////////////////////////////////// -const string RestVocbaseBaseHandler::REPLICATION_PATH = "/_api/replication"; +const string RestVocbaseBaseHandler::REPLICATION_PATH = "/_api/replication"; //////////////////////////////////////////////////////////////////////////////// /// @brief upload path //////////////////////////////////////////////////////////////////////////////// -const string RestVocbaseBaseHandler::UPLOAD_PATH = "/_api/upload"; +const string RestVocbaseBaseHandler::UPLOAD_PATH = "/_api/upload"; //////////////////////////////////////////////////////////////////////////////// /// @brief name of the queue diff --git a/arangod/RestHandler/RestVocbaseBaseHandler.h b/arangod/RestHandler/RestVocbaseBaseHandler.h index 3eb7b40755..cdae6b0da7 100644 --- a/arangod/RestHandler/RestVocbaseBaseHandler.h +++ b/arangod/RestHandler/RestVocbaseBaseHandler.h @@ -33,14 +33,12 @@ #include "Basics/Common.h" #include "Admin/RestBaseHandler.h" - #include "Basics/json.h" #include "Basics/logging.h" #include "Basics/json-utilities.h" - #include "Rest/HttpResponse.h" -#include "Utils/transactions.h" #include "RestServer/VocbaseContext.h" +#include "Utils/transactions.h" // ----------------------------------------------------------------------------- // --SECTION-- forward declarations @@ -64,9 +62,10 @@ namespace triagens { //////////////////////////////////////////////////////////////////////////////// class RestVocbaseBaseHandler : public admin::RestBaseHandler { + private: - RestVocbaseBaseHandler (RestVocbaseBaseHandler const&); - RestVocbaseBaseHandler& operator= (RestVocbaseBaseHandler const&); + RestVocbaseBaseHandler (RestVocbaseBaseHandler const&) = delete; + RestVocbaseBaseHandler& operator= (RestVocbaseBaseHandler const&) = delete; // ----------------------------------------------------------------------------- // --SECTION-- public constants @@ -92,18 +91,24 @@ namespace triagens { static const std::string DOCUMENT_PATH; -//////////////////////////////////////////////////////////////////////////////// -/// @brief document import path -//////////////////////////////////////////////////////////////////////////////// - - static const std::string DOCUMENT_IMPORT_PATH; - //////////////////////////////////////////////////////////////////////////////// /// @brief edge path //////////////////////////////////////////////////////////////////////////////// static const std::string EDGE_PATH; +//////////////////////////////////////////////////////////////////////////////// +/// @brief document export path +//////////////////////////////////////////////////////////////////////////////// + + static const std::string EXPORT_PATH; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief document import path +//////////////////////////////////////////////////////////////////////////////// + + static const std::string IMPORT_PATH; + //////////////////////////////////////////////////////////////////////////////// /// @brief replication path //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestServer/ArangoServer.cpp b/arangod/RestServer/ArangoServer.cpp index 787709e8b0..1ced2e4ed3 100644 --- a/arangod/RestServer/ArangoServer.cpp +++ b/arangod/RestServer/ArangoServer.cpp @@ -37,6 +37,7 @@ #include "Admin/RestHandlerCreator.h" #include "Admin/RestShutdownHandler.h" #include "Aql/Query.h" +#include "Aql/RestAqlHandler.h" #include "Basics/FileUtils.h" #include "Basics/Nonce.h" #include "Basics/ProgramOptions.h" @@ -49,7 +50,10 @@ #include "Basics/messages.h" #include "Basics/ThreadPool.h" #include "Basics/tri-strings.h" +#include "Cluster/ApplicationCluster.h" +#include "Cluster/ClusterComm.h" #include "Cluster/HeartbeatThread.h" +#include "Cluster/RestShardHandler.h" #include "Dispatcher/ApplicationDispatcher.h" #include "Dispatcher/Dispatcher.h" #include "HttpServer/ApplicationEndpointServer.h" @@ -61,6 +65,7 @@ #include "RestHandler/RestCursorHandler.h" #include "RestHandler/RestDocumentHandler.h" #include "RestHandler/RestEdgeHandler.h" +#include "RestHandler/RestExportHandler.h" #include "RestHandler/RestImportHandler.h" #include "RestHandler/RestPleaseUpgradeHandler.h" #include "RestHandler/RestQueryHandler.h" @@ -77,10 +82,6 @@ #include "VocBase/auth.h" #include "VocBase/server.h" #include "Wal/LogfileManager.h" -#include "Cluster/ApplicationCluster.h" -#include "Cluster/RestShardHandler.h" -#include "Cluster/ClusterComm.h" -#include "Aql/RestAqlHandler.h" using namespace std; using namespace triagens::basics; @@ -130,9 +131,13 @@ void ArangoServer::defineHandlers (HttpHandlerFactory* factory) { // add "/edge" handler factory->addPrefixHandler(RestVocbaseBaseHandler::EDGE_PATH, RestHandlerCreator::createNoData); + + // add "/export" handler + factory->addPrefixHandler(RestVocbaseBaseHandler::EXPORT_PATH, + RestHandlerCreator::createNoData); // add "/import" handler - factory->addPrefixHandler(RestVocbaseBaseHandler::DOCUMENT_IMPORT_PATH, + factory->addPrefixHandler(RestVocbaseBaseHandler::IMPORT_PATH, RestHandlerCreator::createNoData); // add "/replication" handler diff --git a/arangod/Utils/CollectionExport.cpp b/arangod/Utils/CollectionExport.cpp new file mode 100644 index 0000000000..c046c75078 --- /dev/null +++ b/arangod/Utils/CollectionExport.cpp @@ -0,0 +1,120 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief collection export result container +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2014 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany +/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#include "Utils/CollectionExport.h" +#include "Basics/JsonHelper.h" +#include "Utils/CollectionGuard.h" +#include "Utils/CollectionReadLocker.h" +#include "VocBase/barrier.h" +#include "VocBase/vocbase.h" + +using namespace triagens::arango; + +// ----------------------------------------------------------------------------- +// --SECTION-- class CollectionExport +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// --SECTION-- constructors / destructors +// ----------------------------------------------------------------------------- + +CollectionExport::CollectionExport (TRI_vocbase_t* vocbase, + std::string const& name) + : _guard(nullptr), + _document(nullptr), + _barrier(nullptr), + _resolver(vocbase), + _documents(nullptr) { + + // this may throw + _guard = new triagens::arango::CollectionGuard(vocbase, name.c_str(), false); + + _document = _guard->collection()->_collection; + TRI_ASSERT(_document != nullptr); +} + +CollectionExport::~CollectionExport () { + delete _documents; + + if (_barrier != nullptr) { + TRI_FreeBarrier(_barrier); + } + + delete _guard; +} + +// ----------------------------------------------------------------------------- +// --SECTION-- public functions +// ----------------------------------------------------------------------------- + +void CollectionExport::run () { + // create a fake transaction for iterating over the collection + TransactionBase trx(true); + + _barrier = TRI_CreateBarrierElement(&_document->_barrierList); + + if (_barrier == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + } + + + TRI_ASSERT(_documents == nullptr); + _documents = new std::vector(); + + // RAII read-lock + { + triagens::arango::CollectionReadLocker lock(_document, true); + + size_t const n = _document->_primaryIndex._nrUsed; + + _documents->reserve(n); + + for (size_t i = 0; i < n; ++i) { + auto ptr = _document->_primaryIndex._table[i]; + + if (ptr != nullptr) { + void const* marker = static_cast(ptr)->getDataPtr(); + + // it is only safe to use the markers from the datafiles, not the WAL + if (! TRI_IsWalDataMarkerDatafile(marker)) { + _documents->emplace_back(marker); + } + } + } + } +} + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + +// Local Variables: +// mode: outline-minor +// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" +// End: diff --git a/arangod/Utils/CollectionExport.h b/arangod/Utils/CollectionExport.h new file mode 100644 index 0000000000..35f495f227 --- /dev/null +++ b/arangod/Utils/CollectionExport.h @@ -0,0 +1,97 @@ +//////////////////////////////////////////////////////////////////////////////// +/// @brief collection export result container +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2014 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany +/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGODB_ARANGO_COLLECTION_EXPORT_H +#define ARANGODB_ARANGO_COLLECTION_EXPORT_H 1 + +#include "Basics/Common.h" +#include "Utils/CollectionNameResolver.h" +#include "VocBase/voc-types.h" + +struct TRI_barrier_s; +struct TRI_document_collection_t; +struct TRI_vocbase_s; + +namespace triagens { + namespace arango { + + class CollectionGuard; + +// ----------------------------------------------------------------------------- +// --SECTION-- class CollectionExport +// ----------------------------------------------------------------------------- + + class CollectionExport { + + friend class ExportCursor; + + public: + + CollectionExport (CollectionExport const&) = delete; + CollectionExport& operator= (CollectionExport const&) = delete; + + CollectionExport (TRI_vocbase_s*, + std::string const&); + + ~CollectionExport (); + +// ----------------------------------------------------------------------------- +// --SECTION-- public functions +// ----------------------------------------------------------------------------- + + public: + + void run (); + +// ----------------------------------------------------------------------------- +// --SECTION-- private variables +// ----------------------------------------------------------------------------- + + private: + + triagens::arango::CollectionGuard* _guard; + struct TRI_document_collection_t* _document; + struct TRI_barrier_s* _barrier; + triagens::arango::CollectionNameResolver _resolver; + std::vector* _documents; + }; + + } +} + +#endif + +// ----------------------------------------------------------------------------- +// --SECTION-- END-OF-FILE +// ----------------------------------------------------------------------------- + +// Local Variables: +// mode: outline-minor +// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}" +// End: diff --git a/arangod/Utils/Cursor.cpp b/arangod/Utils/Cursor.cpp index f97c08c3f2..2c6e55f3d2 100644 --- a/arangod/Utils/Cursor.cpp +++ b/arangod/Utils/Cursor.cpp @@ -28,8 +28,12 @@ //////////////////////////////////////////////////////////////////////////////// #include "Utils/Cursor.h" -#include "Basics/json.h" +#include "Basics/JsonHelper.h" +#include "ShapedJson/shaped-json.h" +#include "Utils/CollectionExport.h" +#include "VocBase/document-collection.h" #include "VocBase/vocbase.h" +#include "VocBase/voc-shaper.h" using namespace triagens::arango; @@ -88,9 +92,15 @@ JsonCursor::JsonCursor (TRI_vocbase_t* vocbase, } JsonCursor::~JsonCursor () { + freeJson(); + TRI_ReleaseVocBase(_vocbase); } +// ----------------------------------------------------------------------------- +// --SECTION-- public methods +// ----------------------------------------------------------------------------- + //////////////////////////////////////////////////////////////////////////////// /// @brief check whether the cursor contains more data //////////////////////////////////////////////////////////////////////////////// @@ -123,6 +133,59 @@ size_t JsonCursor::count () const { return _size; } +//////////////////////////////////////////////////////////////////////////////// +/// @brief dump the cursor contents into a string buffer +//////////////////////////////////////////////////////////////////////////////// + +void JsonCursor::dump (triagens::basics::StringBuffer& buffer) { + buffer.appendText("\"result\":["); + + size_t const n = batchSize(); + + for (size_t i = 0; i < n; ++i) { + if (! hasNext()) { + break; + } + + if (i > 0) { + buffer.appendChar(','); + } + + auto row = next(); + if (row == nullptr) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY); + } + + int res = TRI_StringifyJson(buffer.stringBuffer(), row); + + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + } + + buffer.appendText("],\"hasMore\":"); + buffer.appendText(hasNext() ? "true" : "false"); + + if (hasNext()) { + // only return cursor id if there are more documents + buffer.appendText(",\"id\":\""); + buffer.appendInteger(id()); + buffer.appendText("\""); + } + + if (hasCount()) { + buffer.appendText(",\"count\":"); + buffer.appendInteger(static_cast(count())); + } + + TRI_json_t const* extraJson = extra(); + + if (TRI_IsObjectJson(extraJson)) { + buffer.appendText(",\"extra\":"); + TRI_StringifyJson(buffer.stringBuffer(), extraJson); + } +} + // ----------------------------------------------------------------------------- // --SECTION-- private functions // ----------------------------------------------------------------------------- @@ -140,6 +203,133 @@ void JsonCursor::freeJson () { _isDeleted = true; } +// ----------------------------------------------------------------------------- +// --SECTION-- class ExportCursor +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// --SECTION-- constructors / destructors +// ----------------------------------------------------------------------------- + +ExportCursor::ExportCursor (TRI_vocbase_t* vocbase, + CursorId id, + triagens::arango::CollectionExport* ex, + size_t batchSize, + double ttl, + bool hasCount) + : Cursor(id, batchSize, nullptr, ttl, hasCount), + _vocbase(vocbase), + _ex(ex) { + + TRI_UseVocBase(vocbase); +} + +ExportCursor::~ExportCursor () { + TRI_ReleaseVocBase(_vocbase); +} + +// ----------------------------------------------------------------------------- +// --SECTION-- public methods +// ----------------------------------------------------------------------------- + +//////////////////////////////////////////////////////////////////////////////// +/// @brief check whether the cursor contains more data +//////////////////////////////////////////////////////////////////////////////// + +bool ExportCursor::hasNext () { + return (_position < _ex->_documents->size()); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the next element (not implemented) +//////////////////////////////////////////////////////////////////////////////// + +TRI_json_t* ExportCursor::next () { + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief return the cursor size +//////////////////////////////////////////////////////////////////////////////// + +size_t ExportCursor::count () const { + return _ex->_documents->size(); +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief dump the cursor contents into a string buffer +//////////////////////////////////////////////////////////////////////////////// + +void ExportCursor::dump (triagens::basics::StringBuffer& buffer) { + TRI_shaper_t* shaper = _ex->_document->getShaper(); + + buffer.appendText("\"result\":["); + + size_t const n = batchSize(); + + for (size_t i = 0; i < n; ++i) { + if (! hasNext()) { + break; + } + + if (i > 0) { + buffer.appendChar(','); + } + + auto marker = static_cast(_ex->_documents->at(_position++)); + + TRI_shaped_json_t shaped; + TRI_EXTRACT_SHAPED_JSON_MARKER(shaped, marker); + triagens::basics::Json json(shaper->_memoryZone, TRI_JsonShapedJson(shaper, &shaped)); + + // append the internal attributes + + // _id, _key, _rev + char const* key = TRI_EXTRACT_MARKER_KEY(marker); + std::string id(_ex->_resolver.getCollectionName(_ex->_document->_info._cid)); + id.push_back('/'); + id.append(key); + json(TRI_VOC_ATTRIBUTE_ID, triagens::basics::Json(id)); + json(TRI_VOC_ATTRIBUTE_REV, triagens::basics::Json(std::to_string(TRI_EXTRACT_MARKER_RID(marker)))); + json(TRI_VOC_ATTRIBUTE_KEY, triagens::basics::Json(key)); + + if (TRI_IS_EDGE_MARKER(marker)) { + // _from + std::string from(_ex->_resolver.getCollectionNameCluster(TRI_EXTRACT_MARKER_FROM_CID(marker))); + from.push_back('/'); + from.append(TRI_EXTRACT_MARKER_FROM_KEY(marker)); + json(TRI_VOC_ATTRIBUTE_FROM, triagens::basics::Json(from)); + + // _to + std::string to(_ex->_resolver.getCollectionNameCluster(TRI_EXTRACT_MARKER_TO_CID(marker))); + to.push_back('/'); + to.append(TRI_EXTRACT_MARKER_TO_KEY(marker)); + json(TRI_VOC_ATTRIBUTE_TO, triagens::basics::Json(to)); + } + + int res = TRI_StringifyJson(buffer.stringBuffer(), json.json()); + + if (res != TRI_ERROR_NO_ERROR) { + THROW_ARANGO_EXCEPTION(res); + } + } + + buffer.appendText("],\"hasMore\":"); + buffer.appendText(hasNext() ? "true" : "false"); + + if (hasNext()) { + // only return cursor id if there are more documents + buffer.appendText(",\"id\":\""); + buffer.appendInteger(id()); + buffer.appendText("\""); + } + + if (hasCount()) { + buffer.appendText(",\"count\":"); + buffer.appendInteger(static_cast(count())); + } +} + // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // ----------------------------------------------------------------------------- diff --git a/arangod/Utils/Cursor.h b/arangod/Utils/Cursor.h index b074ef1848..c19328dd06 100644 --- a/arangod/Utils/Cursor.h +++ b/arangod/Utils/Cursor.h @@ -31,6 +31,7 @@ #define ARANGODB_ARANGO_CURSOR_H 1 #include "Basics/Common.h" +#include "Basics/StringBuffer.h" #include "VocBase/voc-types.h" struct TRI_json_t; @@ -39,6 +40,8 @@ struct TRI_vocbase_s; namespace triagens { namespace arango { + class CollectionExport; + // ----------------------------------------------------------------------------- // --SECTION-- class Cursor // ----------------------------------------------------------------------------- @@ -120,29 +123,23 @@ namespace triagens { virtual size_t count () const = 0; -// ----------------------------------------------------------------------------- -// --SECTION-- private functions -// ----------------------------------------------------------------------------- - - private: - - void freeJson (); + virtual void dump (triagens::basics::StringBuffer&) = 0; // ----------------------------------------------------------------------------- -// --SECTION-- private variables +// --SECTION-- protected variables // ----------------------------------------------------------------------------- protected: - CursorId const _id; - size_t const _batchSize; - size_t _position; - struct TRI_json_t* _extra; - double _ttl; - double _expires; - bool const _hasCount; - bool _isDeleted; - bool _isUsed; + CursorId const _id; + size_t const _batchSize; + size_t _position; + struct TRI_json_t* _extra; + double _ttl; + double _expires; + bool const _hasCount; + bool _isDeleted; + bool _isUsed; }; // ----------------------------------------------------------------------------- @@ -174,6 +171,8 @@ namespace triagens { size_t count () const override final; + void dump (triagens::basics::StringBuffer&) override final; + // ----------------------------------------------------------------------------- // --SECTION-- private functions // ----------------------------------------------------------------------------- @@ -188,10 +187,51 @@ namespace triagens { private: - struct TRI_vocbase_s* _vocbase; - struct TRI_json_t* _json; - size_t const _size; + struct TRI_vocbase_s* _vocbase; + struct TRI_json_t* _json; + size_t const _size; }; + +// ----------------------------------------------------------------------------- +// --SECTION-- class ExportCursor +// ----------------------------------------------------------------------------- + + class ExportCursor : public Cursor { + public: + + ExportCursor (struct TRI_vocbase_s*, + CursorId, + triagens::arango::CollectionExport*, + size_t, + double, + bool); + + ~ExportCursor (); + +// ----------------------------------------------------------------------------- +// --SECTION-- public functions +// ----------------------------------------------------------------------------- + + public: + + bool hasNext () override final; + + struct TRI_json_t* next () override final; + + size_t count () const override final; + + void dump (triagens::basics::StringBuffer&) override final; + +// ----------------------------------------------------------------------------- +// --SECTION-- private variables +// ----------------------------------------------------------------------------- + + private: + + struct TRI_vocbase_s* _vocbase; + triagens::arango::CollectionExport* _ex; + }; + } } diff --git a/arangod/Utils/CursorRepository.cpp b/arangod/Utils/CursorRepository.cpp index d76e81e96a..7771d4b6c3 100644 --- a/arangod/Utils/CursorRepository.cpp +++ b/arangod/Utils/CursorRepository.cpp @@ -30,6 +30,7 @@ #include "Utils/CursorRepository.h" #include "Basics/json.h" #include "Basics/MutexLocker.h" +#include "Utils/CollectionExport.h" #include "VocBase/server.h" #include "VocBase/vocbase.h" @@ -81,15 +82,15 @@ CursorRepository::~CursorRepository () { /// the cursor will take ownership of both json and extra //////////////////////////////////////////////////////////////////////////////// -Cursor* CursorRepository::createFromJson (TRI_json_t* json, - size_t batchSize, - TRI_json_t* extra, - double ttl, - bool count) { +JsonCursor* CursorRepository::createFromJson (TRI_json_t* json, + size_t batchSize, + TRI_json_t* extra, + double ttl, + bool count) { TRI_ASSERT(json != nullptr); CursorId const id = TRI_NewTickServer(); - triagens::arango::Cursor* cursor = nullptr; + triagens::arango::JsonCursor* cursor = nullptr; try { cursor = new triagens::arango::JsonCursor(_vocbase, id, json, batchSize, extra, ttl, count); @@ -115,6 +116,32 @@ Cursor* CursorRepository::createFromJson (TRI_json_t* json, } } +//////////////////////////////////////////////////////////////////////////////// +/// @brief creates a cursor and stores it in the registry +//////////////////////////////////////////////////////////////////////////////// + +ExportCursor* CursorRepository::createFromExport (triagens::arango::CollectionExport* ex, + size_t batchSize, + double ttl, + bool count) { + TRI_ASSERT(ex != nullptr); + + CursorId const id = TRI_NewTickServer(); + triagens::arango::ExportCursor* cursor = new triagens::arango::ExportCursor(_vocbase, id, ex, batchSize, ttl, count); + + cursor->use(); + + try { + MUTEX_LOCKER(_lock); + _cursors.emplace(std::make_pair(id, cursor)); + return cursor; + } + catch (...) { + delete cursor; + throw; + } +} + //////////////////////////////////////////////////////////////////////////////// /// @brief remove a cursor by id //////////////////////////////////////////////////////////////////////////////// @@ -227,7 +254,6 @@ bool CursorRepository::garbageCollect (bool force) { { MUTEX_LOCKER(_lock); - std::cout << "CLEANUP COUNT: " << _cursors.size() << "\n"; for (auto it = _cursors.begin(); it != _cursors.end(); /* no hoisting */) { auto cursor = (*it).second; diff --git a/arangod/Utils/CursorRepository.h b/arangod/Utils/CursorRepository.h index 3b7e45aeea..073b753927 100644 --- a/arangod/Utils/CursorRepository.h +++ b/arangod/Utils/CursorRepository.h @@ -41,6 +41,8 @@ struct TRI_vocbase_s; namespace triagens { namespace arango { + class CollectionExport; + // ----------------------------------------------------------------------------- // --SECTION-- class CursorRepository // ----------------------------------------------------------------------------- @@ -78,11 +80,20 @@ namespace triagens { /// the cursor will take ownership of both json and extra //////////////////////////////////////////////////////////////////////////////// - Cursor* createFromJson (struct TRI_json_t*, - size_t, - struct TRI_json_t*, - double, - bool); + JsonCursor* createFromJson (struct TRI_json_t*, + size_t, + struct TRI_json_t*, + double, + bool); + +//////////////////////////////////////////////////////////////////////////////// +/// @brief creates a cursor and stores it in the registry +//////////////////////////////////////////////////////////////////////////////// + + ExportCursor* createFromExport (triagens::arango::CollectionExport*, + size_t, + double, + bool); //////////////////////////////////////////////////////////////////////////////// /// @brief remove a cursor by id diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index d05770003a..2708a32bd1 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -1471,6 +1471,10 @@ void TRI_DestroyInitialVocBase (TRI_vocbase_t* vocbase) { TRI_DestroyVectorPointer(&vocbase->_deadCollections); TRI_DestroySpin(&vocbase->_usage._lock); + + if (vocbase->_cursorRepository != nullptr) { + delete static_cast(vocbase->_cursorRepository); + } if (vocbase->_queries != nullptr) { delete static_cast(vocbase->_queries); diff --git a/js/server/tests/aql-modify-noncluster.js b/js/server/tests/aql-modify-noncluster.js index 84d091f409..5ae2cee32c 100644 --- a/js/server/tests/aql-modify-noncluster.js +++ b/js/server/tests/aql-modify-noncluster.js @@ -249,20 +249,20 @@ function ahuacatlModifySuite () { assertQueryError(errors.ERROR_QUERY_PARSE.code, "FOR d IN @@cn REMOVE d IN @@cn OPTIONS 'foo'", { "@cn": cn1 }); }, -//////////////////////////////////////////////////////////////////////////////// -/// @brief test usage of OLD -//////////////////////////////////////////////////////////////////////////////// - - testInvalidUsageOfNew : function () { - assertQueryError(errors.ERROR_QUERY_ACCESS_AFTER_MODIFICATION.code, "REMOVE 'abc' IN @@cn LET removed = NEW RETURN removed", { "@cn": cn1 }); - }, - //////////////////////////////////////////////////////////////////////////////// /// @brief test usage of NEW +//////////////////////////////////////////////////////////////////////////////// + + testInvalidUsageOfNew : function () { + assertQueryError(errors.ERROR_ARANGO_COLLECTION_NOT_FOUND.code, "REMOVE 'abc' IN @@cn LET removed = NEW RETURN removed", { "@cn": cn1 }); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test usage of OLD //////////////////////////////////////////////////////////////////////////////// testInvalidUsageOfOld : function () { - assertQueryError(errors.ERROR_QUERY_ACCESS_AFTER_MODIFICATION.code, "INSERT { } IN @@cn LET inserted = OLD RETURN inserted", { "@cn": cn1 }); + assertQueryError(errors.ERROR_ARANGO_COLLECTION_NOT_FOUND.code, "INSERT { } IN @@cn LET inserted = OLD RETURN inserted", { "@cn": cn1 }); }, //////////////////////////////////////////////////////////////////////////////// @@ -270,7 +270,7 @@ function ahuacatlModifySuite () { //////////////////////////////////////////////////////////////////////////////// testInvalidVariableNames1 : function () { - assertQueryError(errors.ERROR_QUERY_ACCESS_AFTER_MODIFICATION.code, "REMOVE 'abc' IN @@cn LET removed1 = OLD RETURN removed2", { "@cn": cn1 }); + assertQueryError(errors.ERROR_QUERY_ACCESS_AFTER_MODIFICATION.code, "REMOVE 'abc' IN @@cn LET removed1 = OLD RETURN @@cn", { "@cn": cn1 }); }, //////////////////////////////////////////////////////////////////////////////// @@ -278,7 +278,23 @@ function ahuacatlModifySuite () { //////////////////////////////////////////////////////////////////////////////// testInvalidVariableNames2 : function () { - assertQueryError(errors.ERROR_QUERY_ACCESS_AFTER_MODIFICATION.code, "UPDATE 'abc' WITH { } IN @@cn LET updated = NEW RETURN foo", { "@cn": cn1 }); + assertQueryError(errors.ERROR_ARANGO_COLLECTION_NOT_FOUND.code, "REMOVE 'abc' IN @@cn LET removed1 = OLD RETURN removed2", { "@cn": cn1 }); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test variable names +//////////////////////////////////////////////////////////////////////////////// + + testInvalidVariableNames3 : function () { + assertQueryError(errors.ERROR_QUERY_ACCESS_AFTER_MODIFICATION.code, "UPDATE 'abc' WITH { } IN @@cn LET updated = NEW RETURN @@cn", { "@cn": cn1 }); + }, + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test variable names +//////////////////////////////////////////////////////////////////////////////// + + testInvalidVariableNames4 : function () { + assertQueryError(errors.ERROR_ARANGO_COLLECTION_NOT_FOUND.code, "UPDATE 'abc' WITH { } IN @@cn LET updated = NEW RETURN foo", { "@cn": cn1 }); }, ////////////////////////////////////////////////////////////////////////////////