From 1726c53805513014d865f35dae6dc7100daaf9f1 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Wed, 15 Jun 2016 18:16:49 +0200 Subject: [PATCH] fixed some import edge cases --- arangosh/Import/ImportFeature.cpp | 15 ++++++++++++--- arangosh/Import/ImportHelper.cpp | 14 +++++++++----- js/common/bootstrap/errors.js | 2 +- lib/Basics/errors.dat | 2 +- lib/Basics/voc-errors.cpp | 2 +- lib/Basics/voc-errors.h | 4 ++-- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/arangosh/Import/ImportFeature.cpp b/arangosh/Import/ImportFeature.cpp index 1d86bbd60e..0fadc1f490 100644 --- a/arangosh/Import/ImportFeature.cpp +++ b/arangosh/Import/ImportFeature.cpp @@ -51,7 +51,7 @@ ImportFeature::ImportFeature(application_features::ApplicationServer* server, _typeImport("json"), _overwrite(false), _quote("\""), - _separator(","), + _separator(""), _progress(true), _onDuplicateAction("error"), _result(result) { @@ -116,7 +116,7 @@ void ImportFeature::collectOptions( options->addOption("--quote", "quote character(s), used for csv", new StringParameter(&_quote)); - options->addOption("--separator", "field separator, used for csv", + options->addOption("--separator", "field separator, used for csv and tsv", new StringParameter(&_separator)); options->addOption("--progress", "show progress", @@ -205,6 +205,8 @@ void ImportFeature::start() { if (_typeImport == "csv") { std::cout << "quote: " << _quote << std::endl; + } + if (_typeImport == "csv" || _typeImport == "tsv") { std::cout << "separator: " << _separator << std::endl; } @@ -234,6 +236,14 @@ void ImportFeature::start() { FATAL_ERROR_EXIT(); } + if (_separator.empty()) { + if (_typeImport == "csv") { + _separator = ","; + } else if (_typeImport == "tsv") { + _separator = "\\t"; + } + } + // separator if (_separator.length() == 1 || _separator == "\\r" || _separator == "\\n" || _separator == "\\t") { ih.setSeparator(_separator); @@ -298,7 +308,6 @@ void ImportFeature::start() { else if (_typeImport == "tsv") { std::cout << "Starting TSV import..." << std::endl; ih.setQuote(""); - ih.setSeparator("\\t"); ok = ih.importDelimited(_collectionName, _filename, arangodb::import::ImportHelper::TSV); } diff --git a/arangosh/Import/ImportHelper.cpp b/arangosh/Import/ImportHelper.cpp index e1c2249015..716e2576a4 100644 --- a/arangosh/Import/ImportHelper.cpp +++ b/arangosh/Import/ImportHelper.cpp @@ -518,11 +518,15 @@ void ImportHelper::addField(char const* field, size_t fieldLength, size_t row, // double value // conversion might fail with out-of-range error try { - double num = StringUtils::doubleDecimal(field, fieldLength); - bool failed = (num != num || num == HUGE_VAL || num == -HUGE_VAL); - if (!failed) { - _lineBuffer.appendDecimal(num); - return; + std::string tmp(field, fieldLength); + size_t pos = 0; + double num = std::stod(tmp, &pos); + if (pos == fieldLength) { + bool failed = (num != num || num == HUGE_VAL || num == -HUGE_VAL); + if (!failed) { + _lineBuffer.appendDecimal(num); + return; + } } // NaN, +inf, -inf // fall-through to appending the number as a string diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index 8e6c063be0..2ea53fad3a 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -97,7 +97,7 @@ "ERROR_ARANGO_USE_SYSTEM_DATABASE" : { "code" : 1230, "message" : "operation only allowed in system database" }, "ERROR_ARANGO_ENDPOINT_NOT_FOUND" : { "code" : 1231, "message" : "endpoint not found" }, "ERROR_ARANGO_INVALID_KEY_GENERATOR" : { "code" : 1232, "message" : "invalid key generator" }, - "ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE" : { "code" : 1233, "message" : "edge attribute missing" }, + "ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE" : { "code" : 1233, "message" : "edge attribute missing or invalid" }, "ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING" : { "code" : 1234, "message" : "index insertion warning - attribute missing in document" }, "ERROR_ARANGO_INDEX_CREATION_FAILED" : { "code" : 1235, "message" : "index creation failed" }, "ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT" : { "code" : 1236, "message" : "write-throttling timeout" }, diff --git a/lib/Basics/errors.dat b/lib/Basics/errors.dat index 435d29c40b..ae21639e1e 100755 --- a/lib/Basics/errors.dat +++ b/lib/Basics/errors.dat @@ -117,7 +117,7 @@ ERROR_ARANGO_DATABASE_NAME_INVALID,1229,"database name invalid","Will be raised ERROR_ARANGO_USE_SYSTEM_DATABASE,1230,"operation only allowed in system database","Will be raised when an operation is requested in a database other than the system database." ERROR_ARANGO_ENDPOINT_NOT_FOUND,1231,"endpoint not found","Will be raised when there is an attempt to delete a non-existing endpoint." ERROR_ARANGO_INVALID_KEY_GENERATOR,1232,"invalid key generator","Will be raised when an invalid key generator description is used." -ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE,1233,"edge attribute missing","will be raised when the _from or _to values of an edge are undefined or contain an invalid value." +ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE,1233,"edge attribute missing or invalid","will be raised when the _from or _to values of an edge are undefined or contain an invalid value." ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING,1234,"index insertion warning - attribute missing in document","Will be raised when an attempt to insert a document into an index is caused by in the document not having one or more attributes which the index is built on." ERROR_ARANGO_INDEX_CREATION_FAILED,1235,"index creation failed","Will be raised when an attempt to create an index has failed." ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT,1236,"write-throttling timeout","Will be raised when the server is write-throttled and a write operation has waited too long for the server to process queued operations." diff --git a/lib/Basics/voc-errors.cpp b/lib/Basics/voc-errors.cpp index 7cb2159098..bec9b05c2b 100644 --- a/lib/Basics/voc-errors.cpp +++ b/lib/Basics/voc-errors.cpp @@ -93,7 +93,7 @@ void TRI_InitializeErrorMessages () { REG_ERROR(ERROR_ARANGO_USE_SYSTEM_DATABASE, "operation only allowed in system database"); REG_ERROR(ERROR_ARANGO_ENDPOINT_NOT_FOUND, "endpoint not found"); REG_ERROR(ERROR_ARANGO_INVALID_KEY_GENERATOR, "invalid key generator"); - REG_ERROR(ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE, "edge attribute missing"); + REG_ERROR(ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE, "edge attribute missing or invalid"); REG_ERROR(ERROR_ARANGO_INDEX_DOCUMENT_ATTRIBUTE_MISSING, "index insertion warning - attribute missing in document"); REG_ERROR(ERROR_ARANGO_INDEX_CREATION_FAILED, "index creation failed"); REG_ERROR(ERROR_ARANGO_WRITE_THROTTLE_TIMEOUT, "write-throttling timeout"); diff --git a/lib/Basics/voc-errors.h b/lib/Basics/voc-errors.h index c8d8777590..facbedc27b 100644 --- a/lib/Basics/voc-errors.h +++ b/lib/Basics/voc-errors.h @@ -203,7 +203,7 @@ /// Will be raised when there is an attempt to delete a non-existing endpoint. /// - 1232: @LIT{invalid key generator} /// Will be raised when an invalid key generator description is used. -/// - 1233: @LIT{edge attribute missing} +/// - 1233: @LIT{edge attribute missing or invalid} /// will be raised when the _from or _to values of an edge are undefined or /// contain an invalid value. /// - 1234: @LIT{index insertion warning - attribute missing in document} @@ -1534,7 +1534,7 @@ void TRI_InitializeErrorMessages (); //////////////////////////////////////////////////////////////////////////////// /// @brief 1233: ERROR_ARANGO_INVALID_EDGE_ATTRIBUTE /// -/// edge attribute missing +/// edge attribute missing or invalid /// /// will be raised when the _from or _to values of an edge are undefined or /// contain an invalid value.