From 6eb44f229ee17dc4564c8edb80b604232e1e7156 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 8 Dec 2015 18:10:13 +0100 Subject: [PATCH 1/4] issue #1598: improved error reporting for dump/restore --- .../RestHandler/RestReplicationHandler.cpp | 52 ++++++++------- arangod/VocBase/replication-dump.cpp | 24 +++++-- arangod/VocBase/replication-dump.h | 1 + arangosh/V8Client/arangodump.cpp | 41 +++++++----- arangosh/V8Client/arangorestore.cpp | 63 +++++++++++-------- 5 files changed, 115 insertions(+), 66 deletions(-) diff --git a/arangod/RestHandler/RestReplicationHandler.cpp b/arangod/RestHandler/RestReplicationHandler.cpp index 3ddd3f598c..d0872c30ea 100644 --- a/arangod/RestHandler/RestReplicationHandler.cpp +++ b/arangod/RestHandler/RestReplicationHandler.cpp @@ -2461,7 +2461,7 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co const TRI_voc_key_t key, const TRI_voc_rid_t rid, TRI_json_t const* json, - string& errorMsg) { + std::string& errorMsg) { if (type == REPLICATION_MARKER_DOCUMENT || type == REPLICATION_MARKER_EDGE) { @@ -2474,8 +2474,6 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co TRI_shaped_json_t* shaped = TRI_ShapedJsonJson(document->getShaper(), json, true); // PROTECTED by trx in trxCollection if (shaped == nullptr) { - errorMsg = TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY); - return TRI_ERROR_OUT_OF_MEMORY; } @@ -2491,6 +2489,7 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co // edge if (document->_info._type != TRI_COL_TYPE_EDGE) { res = TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID; + errorMsg = "expecting edge collection, got document collection"; } else { res = TRI_ERROR_NO_ERROR; @@ -2503,11 +2502,13 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co TRI_document_edge_t edge; if (! DocumentHelper::parseDocumentId(resolver, from.c_str(), edge._fromCid, &edge._fromKey)) { res = TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD; + errorMsg = std::string("handle bad or collection unknown '") + from.c_str() + "'"; } // parse _to if (! DocumentHelper::parseDocumentId(resolver, to.c_str(), edge._toCid, &edge._toKey)) { res = TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD; + errorMsg = std::string("handle bad or collection unknown '") + to.c_str() + "'"; } if (res == TRI_ERROR_NO_ERROR) { @@ -2519,6 +2520,7 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co // document if (document->_info._type != TRI_COL_TYPE_DOCUMENT) { res = TRI_ERROR_ARANGO_COLLECTION_TYPE_INVALID; + errorMsg = std::string(TRI_errno_string(res)) + ": expecting document collection, got edge collection"; } else { res = TRI_InsertShapedJsonDocumentCollection(trxCollection, key, rid, nullptr, &mptr, shaped, nullptr, false, false, true); @@ -2610,13 +2612,9 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons if (pos - ptr > 1) { // found something - TRI_json_t* json = TRI_JsonString(TRI_CORE_MEM_ZONE, ptr); - - if (! JsonHelper::isObject(json)) { - if (json != nullptr) { - TRI_FreeJson(TRI_CORE_MEM_ZONE, json); - } + std::unique_ptr json(TRI_JsonString(TRI_CORE_MEM_ZONE, ptr)); + if (! JsonHelper::isObject(json.get())) { errorMsg = invalidMsg; return TRI_ERROR_HTTP_CORRUPTED_JSON; @@ -2627,20 +2625,19 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons TRI_voc_rid_t rid = 0; TRI_json_t const* doc = nullptr; - size_t const n = TRI_LengthVector(&json->_value._objects); + size_t const n = TRI_LengthVector(&(json.get()->_value._objects)); for (size_t i = 0; i < n; i += 2) { - TRI_json_t const* element = static_cast(TRI_AtVector(&json->_value._objects, i)); + auto const* element = static_cast(TRI_AtVector(&(json.get()->_value._objects), i)); if (! JsonHelper::isString(element)) { - TRI_FreeJson(TRI_CORE_MEM_ZONE, json); errorMsg = invalidMsg; return TRI_ERROR_HTTP_CORRUPTED_JSON; } - const char* attributeName = element->_value._string.data; - TRI_json_t const* value = static_cast(TRI_AtVector(&json->_value._objects, i + 1)); + char const* attributeName = element->_value._string.data; + auto const* value = static_cast(TRI_AtVector(&(json.get()->_value._objects), i + 1)); if (TRI_EqualString(attributeName, "type")) { if (JsonHelper::isNumber(value)) { @@ -2669,7 +2666,6 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons // key must not be 0, but doc can be 0! if (key == nullptr) { - TRI_FreeJson(TRI_CORE_MEM_ZONE, json); errorMsg = invalidMsg; return TRI_ERROR_HTTP_BAD_PARAMETER; @@ -2677,8 +2673,6 @@ int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver cons int res = applyCollectionDumpMarker(resolver, trxCollection, type, (const TRI_voc_key_t) key, rid, doc, errorMsg); - TRI_FreeJson(TRI_CORE_MEM_ZONE, json); - if (res != TRI_ERROR_NO_ERROR && ! force) { return res; } @@ -2698,7 +2692,7 @@ int RestReplicationHandler::processRestoreData (CollectionNameResolver const& re TRI_voc_cid_t cid, bool useRevision, bool force, - string& errorMsg) { + std::string& errorMsg) { SingleCollectionWriteTransaction trx(new StandaloneTransactionContext(), _vocbase, cid); @@ -2767,12 +2761,17 @@ void RestReplicationHandler::handleCommandRestoreData () { force = StringUtils::boolean(value); } - string errorMsg; + std::string errorMsg; int res = processRestoreData(resolver, cid, recycleIds, force, errorMsg); if (res != TRI_ERROR_NO_ERROR) { - generateError(HttpResponse::responseCode(res), res); + if (errorMsg.empty()) { + generateError(HttpResponse::responseCode(res), res); + } + else { + generateError(HttpResponse::responseCode(res), res, std::string(TRI_errno_string(res)) + ": " + errorMsg); + } } else { TRI_json_t result; @@ -3413,6 +3412,9 @@ void RestReplicationHandler::handleCommandRemoveKeys () { /// @RESTQUERYPARAM{includeSystem,boolean,optional} /// Include system collections in the result. The default value is *true*. /// +/// @RESTQUERYPARAM{failOnUnknown,boolean,optional} +/// Produce an error when dumped edges refer to now-unknown collections. +/// /// @RESTQUERYPARAM{ticks,boolean,optional} /// Whether or not to include tick values in the dump. The default value is *true*. /// @@ -3545,6 +3547,7 @@ void RestReplicationHandler::handleCommandDump () { bool flush = true; // flush WAL before dumping? bool withTicks = true; bool translateCollectionIds = true; + bool failOnUnknown = false; uint64_t flushWait = 0; bool found; @@ -3557,6 +3560,13 @@ void RestReplicationHandler::handleCommandDump () { flush = StringUtils::boolean(value); } + // fail on unknown collection names referenced in edges + value = _request->value("failOnUnknown", found); + + if (found) { + failOnUnknown = StringUtils::boolean(value); + } + // determine flush WAL wait time value value = _request->value("flushWait", found); @@ -3639,7 +3649,7 @@ void RestReplicationHandler::handleCommandDump () { // initialize the dump container TRI_replication_dump_t dump(_vocbase, (size_t) determineChunkSize(), includeSystem); - res = TRI_DumpCollectionReplication(&dump, col, tickStart, tickEnd, withTicks, translateCollectionIds); + res = TRI_DumpCollectionReplication(&dump, col, tickStart, tickEnd, withTicks, translateCollectionIds, failOnUnknown); if (res != TRI_ERROR_NO_ERROR) { THROW_ARANGO_EXCEPTION(res); diff --git a/arangod/VocBase/replication-dump.cpp b/arangod/VocBase/replication-dump.cpp index e05dc932c8..d75cb1d304 100644 --- a/arangod/VocBase/replication-dump.cpp +++ b/arangod/VocBase/replication-dump.cpp @@ -153,6 +153,7 @@ static char const* NameFromCid (TRI_replication_dump_t* dump, static int AppendCollection (TRI_replication_dump_t* dump, TRI_voc_cid_t cid, bool translateCollectionIds, + bool failOnUnknown, triagens::arango::CollectionNameResolver* resolver) { if (translateCollectionIds) { if (cid > 0) { @@ -164,10 +165,20 @@ static int AppendCollection (TRI_replication_dump_t* dump, name = resolver->getCollectionName(cid); } APPEND_STRING(dump->_buffer, name.c_str()); + if (failOnUnknown && + name[0] == '_' && + name == "_unknown") { + return TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD; + } + return TRI_ERROR_NO_ERROR; } APPEND_STRING(dump->_buffer, "_unknown"); + + if (failOnUnknown) { + return TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD; + } } else { APPEND_UINT64(dump->_buffer, (uint64_t) cid); @@ -291,6 +302,7 @@ static int StringifyMarkerDump (TRI_replication_dump_t* dump, TRI_df_marker_t const* marker, bool withTicks, bool translateCollectionIds, + bool failOnUnknown, triagens::arango::CollectionNameResolver* resolver) { // This covers two cases: // 1. document is not nullptr and marker points into a data file @@ -427,7 +439,7 @@ static int StringifyMarkerDump (TRI_replication_dump_t* dump, int res; APPEND_STRING(buffer, ",\"" TRI_VOC_ATTRIBUTE_FROM "\":\""); - res = AppendCollection(dump, fromCid, translateCollectionIds, resolver); + res = AppendCollection(dump, fromCid, translateCollectionIds, failOnUnknown, resolver); if (res != TRI_ERROR_NO_ERROR) { return res; @@ -436,7 +448,7 @@ static int StringifyMarkerDump (TRI_replication_dump_t* dump, APPEND_STRING(buffer, "\\/"); APPEND_STRING(buffer, fromKey); APPEND_STRING(buffer, "\",\"" TRI_VOC_ATTRIBUTE_TO "\":\""); - res = AppendCollection(dump, toCid, translateCollectionIds, resolver); + res = AppendCollection(dump, toCid, translateCollectionIds, failOnUnknown, resolver); if (res != TRI_ERROR_NO_ERROR) { return res; @@ -1178,6 +1190,7 @@ static int DumpCollection (TRI_replication_dump_t* dump, TRI_voc_tick_t dataMax, bool withTicks, bool translateCollectionIds, + bool failOnUnknown, triagens::arango::CollectionNameResolver* resolver) { TRI_string_buffer_t* buffer; TRI_voc_tick_t lastFoundTick; @@ -1332,7 +1345,7 @@ static int DumpCollection (TRI_replication_dump_t* dump, } - res = StringifyMarkerDump(dump, document, marker, withTicks, translateCollectionIds, resolver); + res = StringifyMarkerDump(dump, document, marker, withTicks, translateCollectionIds, failOnUnknown, resolver); if (res != TRI_ERROR_NO_ERROR) { break; // will go to NEXT_DF @@ -1395,7 +1408,8 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump, TRI_voc_tick_t dataMin, TRI_voc_tick_t dataMax, bool withTicks, - bool translateCollectionIds) { + bool translateCollectionIds, + bool failOnUnknown) { TRI_ASSERT(col != nullptr); TRI_ASSERT(col->_collection != nullptr); @@ -1416,7 +1430,7 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump, try { res = DumpCollection(dump, document, dataMin, dataMax, withTicks, - translateCollectionIds, &resolver); + translateCollectionIds, failOnUnknown, &resolver); } catch (...) { res = TRI_ERROR_INTERNAL; diff --git a/arangod/VocBase/replication-dump.h b/arangod/VocBase/replication-dump.h index 561cec40e6..6a195bd078 100644 --- a/arangod/VocBase/replication-dump.h +++ b/arangod/VocBase/replication-dump.h @@ -120,6 +120,7 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t*, TRI_voc_tick_t, TRI_voc_tick_t, bool, + bool, bool); //////////////////////////////////////////////////////////////////////////////// diff --git a/arangosh/V8Client/arangodump.cpp b/arangosh/V8Client/arangodump.cpp index b125266713..f892b10dfc 100644 --- a/arangosh/V8Client/arangodump.cpp +++ b/arangosh/V8Client/arangodump.cpp @@ -282,7 +282,7 @@ static void LocalExitFunction (int exitCode, void* data) { static string GetHttpErrorMessage (SimpleHttpResult* result) { StringBuffer const& body = result->getBody(); - string details; + std::string details; std::unique_ptr json(JsonHelper::fromString(body.c_str(), body.length())); @@ -505,6 +505,13 @@ static int DumpCollection (int fd, url += "&to=" + StringUtils::itoa(maxTick); } + if (Force) { + url += "&failOnUnknown=false"; + } + else { + url += "&failOnUnknown=true"; + } + Stats._totalBatches++; std::unique_ptr response(Client->request(HttpRequest::HTTP_REQUEST_GET, @@ -746,9 +753,11 @@ static int RunDump (string& errorMsg) { return TRI_ERROR_INTERNAL; } - const string cid = JsonHelper::getStringValue(parameters, "cid", ""); - const string name = JsonHelper::getStringValue(parameters, "name", ""); - const bool deleted = JsonHelper::getBooleanValue(parameters, "deleted", false); + std::string const cid = JsonHelper::getStringValue(parameters, "cid", ""); + std::string const name = JsonHelper::getStringValue(parameters, "name", ""); + bool const deleted = JsonHelper::getBooleanValue(parameters, "deleted", false); + int type = JsonHelper::getNumericValue(parameters, "type", 2); + std::string const collectionType(type == 2 ? "document" : "edge"); if (cid == "" || name == "") { errorMsg = "got malformed JSON response from server"; @@ -774,7 +783,7 @@ static int RunDump (string& errorMsg) { // found a collection! if (Progress) { - cout << "dumping collection '" << name << "'..." << endl; + cout << "# Dumping " << collectionType << " collection '" << name << "'..." << endl; } // now save the collection meta data and/or the actual data @@ -1054,7 +1063,7 @@ static int RunClusterDump (string& errorMsg) { // found a collection! if (Progress) { - cout << "dumping collection '" << name << "'..." << endl; + cout << "# Dumping collection '" << name << "'..." << endl; } // now save the collection meta data and/or the actual data @@ -1124,7 +1133,7 @@ static int RunClusterDump (string& errorMsg) { string shardName = it->first; string DBserver = it->second; if (Progress) { - cout << "dumping shard '" << shardName << "' from DBserver '" + cout << "# Dumping shard '" << shardName << "' from DBserver '" << DBserver << "' ..." << endl; } res = StartBatch(DBserver, errorMsg); @@ -1380,6 +1389,17 @@ int main (int argc, char* argv[]) { cerr << "Error: caught unknown exception" << endl; res = TRI_ERROR_INTERNAL; } + + if (res != TRI_ERROR_NO_ERROR) { + if (! errorMsg.empty()) { + cerr << "Error: " << errorMsg << endl; + } + else { + cerr << "An error occurred" << endl; + } + ret = EXIT_FAILURE; + } + if (Progress) { if (DumpData) { @@ -1392,13 +1412,6 @@ int main (int argc, char* argv[]) { } } - if (res != TRI_ERROR_NO_ERROR) { - if (! errorMsg.empty()) { - cerr << "Error: " << errorMsg << endl; - } - ret = EXIT_FAILURE; - } - delete Client; TRIAGENS_REST_SHUTDOWN; diff --git a/arangosh/V8Client/arangorestore.cpp b/arangosh/V8Client/arangorestore.cpp index 1aeb511a4a..f3476fa5c4 100644 --- a/arangosh/V8Client/arangorestore.cpp +++ b/arangosh/V8Client/arangorestore.cpp @@ -281,18 +281,16 @@ static void LocalExitFunction (int exitCode, void* data) { /// @brief extract an error message from a response //////////////////////////////////////////////////////////////////////////////// -static string GetHttpErrorMessage (SimpleHttpResult* result) { - const StringBuffer& body = result->getBody(); - string details; +static std::string GetHttpErrorMessage (SimpleHttpResult* result) { + StringBuffer const& body = result->getBody(); + std::string details; LastErrorCode = TRI_ERROR_NO_ERROR; - TRI_json_t* json = JsonHelper::fromString(body.c_str(), body.length()); + std::unique_ptr json(JsonHelper::fromString(body.c_str(), body.length())); if (json != nullptr) { - const string& errorMessage = JsonHelper::getStringValue(json, "errorMessage", ""); - const int errorNum = JsonHelper::getNumericValue(json, "errorNum", 0); - - TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json); + std::string const& errorMessage = JsonHelper::getStringValue(json.get(), "errorMessage", ""); + int const errorNum = JsonHelper::getNumericValue(json.get(), "errorNum", 0); if (errorMessage != "" && errorNum > 0) { details = ": ArangoError " + StringUtils::itoa(errorNum) + ": " + errorMessage; @@ -311,7 +309,6 @@ static string GetHttpErrorMessage (SimpleHttpResult* result) { //////////////////////////////////////////////////////////////////////////////// static int TryCreateDatabase (std::string const& name) { - triagens::basics::Json json(triagens::basics::Json::Object); json("name", triagens::basics::Json(name)); @@ -466,6 +463,9 @@ static int SendRestoreCollection (TRI_json_t const* json, if (response->wasHttpError()) { errorMsg = GetHttpErrorMessage(response.get()); + if (LastErrorCode != TRI_ERROR_NO_ERROR) { + return LastErrorCode; + } return TRI_ERROR_INTERNAL; } @@ -495,6 +495,9 @@ static int SendRestoreIndexes (TRI_json_t const* json, if (response->wasHttpError()) { errorMsg = GetHttpErrorMessage(response.get()); + if (LastErrorCode != TRI_ERROR_NO_ERROR) { + return LastErrorCode; + } return TRI_ERROR_INTERNAL; } @@ -528,6 +531,9 @@ static int SendRestoreData (string const& cname, if (response->wasHttpError()) { errorMsg = GetHttpErrorMessage(response.get()); + if (LastErrorCode != TRI_ERROR_NO_ERROR) { + return LastErrorCode; + } return TRI_ERROR_INTERNAL; } @@ -563,7 +569,7 @@ static int SortCollections (const void* l, /// @brief process all files from the input directory //////////////////////////////////////////////////////////////////////////////// -static int ProcessInputDirectory (string& errorMsg) { +static int ProcessInputDirectory (std::string& errorMsg) { // create a lookup table for collections map restrictList; for (size_t i = 0; i < Collections.size(); ++i) { @@ -674,20 +680,23 @@ static int ProcessInputDirectory (string& errorMsg) { // step2: run the actual import { for (size_t i = 0; i < n; ++i) { - TRI_json_t const* json = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i); + TRI_json_t const* json = static_cast(TRI_AtVector(&collections->_value._objects, i)); TRI_json_t const* parameters = JsonHelper::getObjectElement(json, "parameters"); TRI_json_t const* indexes = JsonHelper::getObjectElement(json, "indexes"); - const string cname = JsonHelper::getStringValue(parameters, "name", ""); - const string cid = JsonHelper::getStringValue(parameters, "cid", ""); + std::string const cname = JsonHelper::getStringValue(parameters, "name", ""); + std::string const cid = JsonHelper::getStringValue(parameters, "cid", ""); + + int type = JsonHelper::getNumericValue(parameters, "type", 2); + std::string const collectionType(type == 2 ? "document" : "edge"); if (ImportStructure) { // re-create collection if (Progress) { if (Overwrite) { - cout << "Re-creating collection '" << cname << "'..." << endl; + cout << "# Re-creating " << collectionType << " collection '" << cname << "'..." << endl; } else { - cout << "Creating collection '" << cname << "'..." << endl; + cout << "# Creating " << collectionType << " collection '" << cname << "'..." << endl; } } @@ -718,7 +727,7 @@ static int ProcessInputDirectory (string& errorMsg) { // found a datafile if (Progress) { - cout << "Loading data into collection '" << cname << "'..." << endl; + cout << "# Loading data into " << collectionType << " collection '" << cname << "'..." << endl; } int fd = TRI_OPEN(datafile.c_str(), O_RDONLY); @@ -829,7 +838,7 @@ static int ProcessInputDirectory (string& errorMsg) { if (TRI_LengthVector(&indexes->_value._objects) > 0) { // we actually have indexes if (Progress) { - cout << "Creating indexes for collection '" << cname << "'..." << endl; + cout << "# Creating indexes for collection '" << cname << "'..." << endl; } int res = SendRestoreIndexes(json, errorMsg); @@ -1005,7 +1014,7 @@ int main (int argc, char* argv[]) { } if (Progress) { - cout << "Connected to ArangoDB '" << BaseClient.endpointServer()->getSpecification() << endl; + cout << "# Connected to ArangoDB '" << BaseClient.endpointServer()->getSpecification() << "'" << endl; } memset(&Stats, 0, sizeof(Stats)); @@ -1025,6 +1034,16 @@ int main (int argc, char* argv[]) { res = TRI_ERROR_INTERNAL; } + if (res != TRI_ERROR_NO_ERROR) { + if (! errorMsg.empty()) { + cerr << "Error: " << errorMsg << endl; + } + else { + cerr << "An error occurred" << endl; + } + ret = EXIT_FAILURE; + } + if (Progress) { if (ImportData) { @@ -1037,14 +1056,6 @@ int main (int argc, char* argv[]) { } } - - if (res != TRI_ERROR_NO_ERROR) { - if (! errorMsg.empty()) { - cerr << "Error: " << errorMsg << endl; - } - ret = EXIT_FAILURE; - } - if (Client != nullptr) { delete Client; } From fcf3c392c976b92e15c109c6d90f6b139638bbe8 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Tue, 8 Dec 2015 18:15:18 +0100 Subject: [PATCH 2/4] Add geo locations to the city navigator. --- .../org/arangodb/graph-examples/example-graph.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/js/common/modules/org/arangodb/graph-examples/example-graph.js b/js/common/modules/org/arangodb/graph-examples/example-graph.js index e4ee20f1fb..f179c905b6 100644 --- a/js/common/modules/org/arangodb/graph-examples/example-graph.js +++ b/js/common/modules/org/arangodb/graph-examples/example-graph.js @@ -81,11 +81,13 @@ var createRoutePlannerGraph = function() { ); var g = Graph._create("routeplanner", edgeDefinition); - var berlin = g.germanCity.save({_key: "Berlin", population : 3000000, isCapital : true}); - var cologne = g.germanCity.save({_key: "Cologne", population : 1000000, isCapital : false}); - var hamburg = g.germanCity.save({_key: "Hamburg", population : 1000000, isCapital : false}); - var lyon = g.frenchCity.save({_key: "Lyon", population : 80000, isCapital : false}); - var paris = g.frenchCity.save({_key: "Paris", population : 4000000, isCapital : true}); + var berlin = g.germanCity.save({_key: "Berlin", population : 3000000, isCapital : true, loc: [52.5167, 13.3833]}); + var cologne = g.germanCity.save({_key: "Cologne", population : 1000000, isCapital : false, loc: [50.9364, 6.9528]}); + var hamburg = g.germanCity.save({_key: "Hamburg", population : 1000000, isCapital : false, loc: [53.5653, 10.0014]}); + var lyon = g.frenchCity.save({_key: "Lyon", population : 80000, isCapital : false, loc: [45.7600, 4.8400]}); + var paris = g.frenchCity.save({_key: "Paris", population : 4000000, isCapital : true, loc: [48.8567, 2.3508]}); + g.germanCity.ensureGeoIndex("loc"); + g.frenchCity.ensureGeoIndex("loc"); g.germanHighway.save(berlin._id, cologne._id, {distance: 850}); g.germanHighway.save(berlin._id, hamburg._id, {distance: 400}); g.germanHighway.save(hamburg._id, cologne._id, {distance: 500}); From fb38aac8763d4825924c95e4a9d0b207acfb75d1 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Tue, 8 Dec 2015 18:20:37 +0100 Subject: [PATCH 3/4] updated CHANGELOG --- CHANGELOG | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 9137123b5e..68232054ed 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,12 @@ v2.8.0 (XXXX-XX-XX) ------------------- +* better error reporting for arangodump and arangorestore + +* arangodump will now fail by default when trying to dump edges that + refer to already dropped collections. This can be circumvented by + specifying the option `--force true` when invoking arangodump + * fixed cluster upgrade procedure From 6b4fbdc5a7c93c9dba98675c89aac0312f89f3fc Mon Sep 17 00:00:00 2001 From: Alan Plum Date: Tue, 8 Dec 2015 18:41:53 +0100 Subject: [PATCH 4/4] Make the Foxx console behave as documented --- .../Books/Users/Foxx/Develop/Console.mdpp | 36 +++++++--- js/common/bootstrap/modules/console.js | 1 + .../modules/org/arangodb/foxx/console.js | 67 +++++++++++++++++-- .../modules/org/arangodb/foxx/service.js | 1 + 4 files changed, 90 insertions(+), 15 deletions(-) diff --git a/Documentation/Books/Users/Foxx/Develop/Console.mdpp b/Documentation/Books/Users/Foxx/Develop/Console.mdpp index c6666cdb1a..cc4b463023 100644 --- a/Documentation/Books/Users/Foxx/Develop/Console.mdpp +++ b/Documentation/Books/Users/Foxx/Develop/Console.mdpp @@ -1,9 +1,13 @@ !CHAPTER Foxx console -Foxx injects a **console** object into each Foxx app that allows writing log entries to the database and querying them from within the app itself. +Foxx injects a **console** object into each Foxx app that allows writing log entries to the database in addition to the ArangoDB log file and querying them from within the app itself. The **console** object supports the CommonJS Console API found in Node.js and modern browsers, while also providing some ArangoDB-specific additions. +ArangoDB also provides [the `console` module](../../ModuleConsole/README.md) which only supports the CommonJS Console API and only writes log entries to the ArangoDB log. + +When working with transactions, keep in mind that the Foxx console will attempt to write to the `_foxxlog` system collection. This behaviour can be disabled using the `setDatabaseLogging` method if you don't want to explicitly allow writing to the log collection during transactions or for performance reasons. + !SECTION Logging !SUBSECTION Logging console messages @@ -19,8 +23,6 @@ If the first argument is not a formatting string or any of the additional argume **Examples** ```js -var console = require("console"); - console.log("%s, %s!", "Hello", "World"); // => "Hello, World!" console.log("%s, World!", "Hello", "extra"); // => "Hello, World! extra" console.log("Hello,", "beautiful", "world!"); // => "Hello, beautiful world!" @@ -40,11 +42,11 @@ By default, `console.log` uses log level **INFO**, making it functionally equiva The built-in log levels are: -* -2: **TRACE** -* -1: **DEBUG** +* -200: **TRACE** +* -100: **DEBUG** * 0: **INFO** -* 1: **WARN** -* 2: **ERROR** +* 100: **WARN** +* 200: **ERROR** !SUBSECTION Logging with timers @@ -160,7 +162,7 @@ This method returns a function that logs messages with the given log level (e.g. **Parameter** * **name**: name of the log level as it appears in the database, usually all-uppercase -* **value** (optional): value of the log level. Default: `999` +* **value** (optional): value of the log level. Default: `50` The **value** is used when determining whether a log entry meets the minimum log level that can be defined in various places. For a list of the built-in log levels and their values see the section on logging with different log levels above. @@ -188,6 +190,24 @@ If **trace** is set to `true`, all log entries will be logged with a parsed stac Because this results in every logging call creating a stack trace (which may have a significant performance impact), this option is disabled by default. +!SUBSECTION Disabling logging to the ArangoDB console + +You can toggle whether logs should be written to the ArangoDB console. + +`console.setNativeLogging(nativeLogging)` + +If **nativeLogging** is set to `false`, log entries will not be logged to the ArangoDB console (which usually writes to the file system). + +!SUBSECTION Disabling logging to the database + +You can toggle whether logs should be written to the database. + +`console.setDatabaseLogging(databaseLogging)` + +If **databaseLogging** is set to `false`, log entries will not be logged to the internal `_foxxlog` collection. + +This is only useful if logging to the ArangoDB console is not also disabled. + !SUBSECTION Enabling assertion errors You can toggle whether console assertions should throw if they fail. diff --git a/js/common/bootstrap/modules/console.js b/js/common/bootstrap/modules/console.js index 41b5251a6a..157c4323d8 100644 --- a/js/common/bootstrap/modules/console.js +++ b/js/common/bootstrap/modules/console.js @@ -337,6 +337,7 @@ exports.infoLines = function () { //////////////////////////////////////////////////////////////////////////////// exports.log = exports.info; +exports._log = log; //////////////////////////////////////////////////////////////////////////////// /// @brief logLines diff --git a/js/server/modules/org/arangodb/foxx/console.js b/js/server/modules/org/arangodb/foxx/console.js index 1e6bff7157..1703ec2a05 100644 --- a/js/server/modules/org/arangodb/foxx/console.js +++ b/js/server/modules/org/arangodb/foxx/console.js @@ -31,11 +31,38 @@ var qb = require('aqb'); var util = require('util'); var extend = require('underscore').extend; +var arangoConsole = require('console'); var ErrorStackParser = require('error-stack-parser'); var AssertionError = require('assert').AssertionError; var exists = require('org/arangodb/is').existy; var db = require('org/arangodb').db; +const NATIVE_LOG_LEVELS = ['debug', 'info', 'warn', 'error']; + +function nativeLogger(level, levelNum, mount) { + let logLevel = String(level).toLowerCase(); + if (logLevel === 'trace' && levelNum === -200) { + logLevel = 'info'; // require('console').trace also uses INFO level + } + if (NATIVE_LOG_LEVELS.indexOf(logLevel) !== -1) { + return function (message) { + arangoConsole._log(logLevel, `${mount} ${message}`); + }; + } + if (levelNum >= 200) { + logLevel = 'error'; + } else if (levelNum >= 100) { + logLevel = 'warn'; + } else if (levelNum <= -100) { + logLevel = 'debug'; + } else { + logLevel = 'info'; + } + return function (message) { + arangoConsole._log(logLevel, `(${level}) ${mount} ${message}`); + }; +} + function ConsoleLogs(console) { this._console = console; this.defaultMaxAge = 2 * 60 * 60 * 1000; @@ -131,8 +158,10 @@ function Console(mount, tracing) { this._mount = mount; this._timers = Object.create(null); this._tracing = Boolean(tracing); + this._nativeLogging = true; + this._databaseLogging = true; this._logLevel = -999; - this._logLevels = {TRACE: -2}; + this._logLevels = {TRACE: -200}; this._assertThrows = false; this.logs = new ConsoleLogs(this); @@ -142,10 +171,10 @@ function Console(mount, tracing) { } }.bind(this)); - this.debug = this.custom('DEBUG', -1); + this.debug = this.custom('DEBUG', -100); this.info = this.custom('INFO', 0); - this.warn = this.custom('WARN', 1); - this.error = this.custom('ERROR', 2); + this.warn = this.custom('WARN', 100); + this.error = this.custom('ERROR', 200); this.assert.level = 'ERROR'; this.dir.level = 'INFO'; @@ -170,14 +199,28 @@ extend(Console.prototype, { level: level, levelNum: this._logLevels[level], time: Date.now(), - message: message + message: String(message) }; + let logLine; + + if (this._nativeLogging) { + logLine = nativeLogger(level, doc.levelNum, doc.mount); + doc.message.split('\n').forEach(logLine); + } + if (this._tracing) { - var e = new Error(); + let e = new Error(); Error.captureStackTrace(e, callee || this._log); e.stack = e.stack.replace(/\n+$/, ''); doc.stack = ErrorStackParser.parse(e).slice(1); + if (this._nativeLogging) { + e.stack.split('\n').slice(2).forEach(logLine); + } + } + + if (!this._databaseLogging) { + return; } if (!db._foxxlog) { @@ -240,7 +283,7 @@ extend(Console.prototype, { custom: function (level, weight) { level = String(level); weight = Number(weight); - weight = weight === weight ? weight : 999; + weight = weight === weight ? weight : 50; this._logLevels[level] = weight; var logWithLevel = function() { this._log(level, util.format.apply(null, arguments), logWithLevel); @@ -264,6 +307,16 @@ extend(Console.prototype, { return this._tracing; }, + setNativeLogging: function (nativeLogging) { + this._nativeLogging = Boolean(nativeLogging); + return this._nativeLogging; + }, + + setDatabaseLogging: function (databaseLogging) { + this._databaseLogging = Boolean(databaseLogging); + return this._databaseLogging; + }, + setAssertThrows: function (assertThrows) { this._assertThrows = Boolean(assertThrows); return this._assertThrows; diff --git a/js/server/modules/org/arangodb/foxx/service.js b/js/server/modules/org/arangodb/foxx/service.js index 28fd17a060..efd4d08c28 100644 --- a/js/server/modules/org/arangodb/foxx/service.js +++ b/js/server/modules/org/arangodb/foxx/service.js @@ -346,6 +346,7 @@ class FoxxService { filename = path.resolve(this.main.context.__dirname, filename); var module = new Module(filename, this.main); + module.context.console = this.main.context.console; module.context.applicationContext = _.extend( new AppContext(this.main.context.applicationContext._service), this.main.context.applicationContext,