diff --git a/arangod/RestHandler/RestImportHandler.cpp b/arangod/RestHandler/RestImportHandler.cpp index 9cbe50620d..0127283809 100644 --- a/arangod/RestHandler/RestImportHandler.cpp +++ b/arangod/RestHandler/RestImportHandler.cpp @@ -1073,27 +1073,35 @@ std::shared_ptr RestImportHandler::createVelocyPackObject( } TRI_ASSERT(keys.isArray()); - VPackValueLength const n = keys.length(); - VPackValueLength const m = values.length(); - - if (n != m) { + + VPackArrayIterator itKeys(keys); + VPackArrayIterator itValues(values); + + if (itKeys.size() != itValues.size()) { errorMsg = positionize(lineNumber) + "wrong number of JSON values (got " + - std::to_string(m) + ", expected " + std::to_string(n) + ")"; + std::to_string(itKeys.size()) + ", expected " + std::to_string(itValues.size()) + ")"; THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, errorMsg); } auto result = std::make_shared(); result->openObject(); - for (size_t i = 0; i < n; ++i) { - VPackSlice const key = keys.at(i); - VPackSlice const value = values.at(i); + while (itKeys.valid()) { + TRI_ASSERT(itValues.valid()); + + VPackSlice const key = itKeys.value(); + VPackSlice const value = itValues.value(); if (key.isString() && !value.isNone() && !value.isNull()) { - std::string tmp = key.copyString(); - result->add(tmp, value); + VPackValueLength l; + char const* p = key.getString(l); + result->add(p, l, value); } + + itKeys.next(); + itValues.next(); } + result->close(); return result; diff --git a/arangosh/Import/ImportHelper.cpp b/arangosh/Import/ImportHelper.cpp index d751a7e8b7..631feac23f 100644 --- a/arangosh/Import/ImportHelper.cpp +++ b/arangosh/Import/ImportHelper.cpp @@ -502,12 +502,7 @@ void ImportHelper::addField(char const* field, size_t fieldLength, size_t row, return; } - if (!_convert) { - _lineBuffer.appendText(field, fieldLength); - return; - } - - if (*field == '\0') { + if (*field == '\0' || fieldLength == 0) { // do nothing _lineBuffer.appendText(TRI_CHAR_LENGTH_PAIR("null")); return; @@ -523,50 +518,60 @@ void ImportHelper::addField(char const* field, size_t fieldLength, size_t row, return; } - if (IsInteger(field, fieldLength)) { - // integer value - // conversion might fail with out-of-range error - try { - if (fieldLength > 8) { - // long integer numbers might be problematic. check if we get out of - // range - (void) std::stoll(std::string( - field, - fieldLength)); // this will fail if the number cannot be converted + if (_convert) { + if (IsInteger(field, fieldLength)) { + // integer value + // conversion might fail with out-of-range error + try { + if (fieldLength > 8) { + // long integer numbers might be problematic. check if we get out of + // range + (void) std::stoll(std::string( + field, + fieldLength)); // this will fail if the number cannot be converted + } + + int64_t num = StringUtils::int64(field, fieldLength); + _lineBuffer.appendInteger(num); + } catch (...) { + // conversion failed + _lineBuffer.appendJsonEncoded(field, fieldLength); + } + } else if (IsDecimal(field, fieldLength)) { + // double value + // conversion might fail with out-of-range error + try { + std::string tmp(field, fieldLength); + size_t pos = 0; + double num = std::stod(tmp, &pos); + if (pos == fieldLength) { + bool failed = (num != num || num == HUGE_VAL || num == -HUGE_VAL); + if (!failed) { + _lineBuffer.appendDecimal(num); + return; + } + } + // NaN, +inf, -inf + // fall-through to appending the number as a string + } catch (...) { + // conversion failed + // fall-through to appending the number as a string } - int64_t num = StringUtils::int64(field, fieldLength); - _lineBuffer.appendInteger(num); - } catch (...) { - // conversion failed + _lineBuffer.appendChar('"'); + _lineBuffer.appendText(field, fieldLength); + _lineBuffer.appendChar('"'); + } else { _lineBuffer.appendJsonEncoded(field, fieldLength); } - } else if (IsDecimal(field, fieldLength)) { - // double value - // conversion might fail with out-of-range error - try { - std::string tmp(field, fieldLength); - size_t pos = 0; - double num = std::stod(tmp, &pos); - if (pos == fieldLength) { - bool failed = (num != num || num == HUGE_VAL || num == -HUGE_VAL); - if (!failed) { - _lineBuffer.appendDecimal(num); - return; - } - } - // NaN, +inf, -inf - // fall-through to appending the number as a string - } catch (...) { - // conversion failed - // fall-through to appending the number as a string - } - - _lineBuffer.appendChar('"'); - _lineBuffer.appendText(field, fieldLength); - _lineBuffer.appendChar('"'); } else { - _lineBuffer.appendJsonEncoded(field, fieldLength); + if (IsInteger(field, fieldLength) || IsDecimal(field, fieldLength)) { + // numeric value. don't convert + _lineBuffer.appendText(field, fieldLength); + } else { + // non-numeric value + _lineBuffer.appendJsonEncoded(field, fieldLength); + } } } diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index ad14978c90..dad3328a05 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -1163,6 +1163,10 @@ function runArangoImp (options, instanceInfo, what) { if (what.separator !== undefined) { args['separator'] = what.separator; } + + if (what.convert !== undefined) { + args['convert'] = what.convert ? 'true' : 'false'; + } return executeAndWait(ARANGOIMP_BIN, toArgv(args), options); } @@ -3227,6 +3231,15 @@ const impTodos = [{ create: 'true', separator: ';', backslash: true +}, { + id: 'csvnoconvert', + data: makePathUnix('js/common/test-data/import/import-noconvert.csv'), + coll: 'UnitTestsImportCsvNoConvert', + type: 'csv', + create: 'true', + separator: ',', + convert: true, + backslash: true }, { id: 'csvnoeol', data: makePathUnix('js/common/test-data/import/import-noeol.csv'), diff --git a/js/common/test-data/import/import-noconvert.csv b/js/common/test-data/import/import-noconvert.csv new file mode 100644 index 0000000000..8258348f88 --- /dev/null +++ b/js/common/test-data/import/import-noconvert.csv @@ -0,0 +1,17 @@ +"value1","value2" +1,null +2,false +3,true +4,1 +5,2 +6,3 +7,a +8,b +9, a +10,-1 +11,-.5 +12,3.566 +13,0 +14, +15, c +16, 1 diff --git a/js/server/tests/import/import-setup.js b/js/server/tests/import/import-setup.js index 037ced3569..306bb66142 100644 --- a/js/server/tests/import/import-setup.js +++ b/js/server/tests/import/import-setup.js @@ -42,6 +42,7 @@ db._drop("UnitTestsImportCsv3"); db._drop("UnitTestsImportCsv4"); db._drop("UnitTestsImportCsv5"); + db._drop("UnitTestsImportCsvNoConvert"); db._drop("UnitTestsImportCsvNoEol"); db._drop("UnitTestsImportTsv1"); db._drop("UnitTestsImportTsv2"); diff --git a/js/server/tests/import/import-teardown.js b/js/server/tests/import/import-teardown.js index 936737d637..ee727b6416 100644 --- a/js/server/tests/import/import-teardown.js +++ b/js/server/tests/import/import-teardown.js @@ -42,6 +42,7 @@ db._drop("UnitTestsImportCsv3"); db._drop("UnitTestsImportCsv4"); db._drop("UnitTestsImportCsv5"); + db._drop("UnitTestsImportCsvNoConvert"); db._drop("UnitTestsImportCsvNoEol"); db._drop("UnitTestsImportTsv1"); db._drop("UnitTestsImportTsv2"); diff --git a/js/server/tests/import/import.js b/js/server/tests/import/import.js index c704b1993e..214b29bc67 100644 --- a/js/server/tests/import/import.js +++ b/js/server/tests/import/import.js @@ -285,6 +285,34 @@ function importTestSuite () { assertEqual(expected, actual); }, +//////////////////////////////////////////////////////////////////////////////// +/// @brief test csv import without converting +//////////////////////////////////////////////////////////////////////////////// + + testCsvImportNoConvert : function () { + var expected = [ + { value1: 1 }, + { value1: 2, value2: false }, + { value1: 3, value2: true }, + { value1: 4, value2: 1 }, + { value1: 5, value2: 2 }, + { value1: 6, value2: 3 }, + { value1: 7, value2: "a" }, + { value1: 8, value2: "b" }, + { value1: 9, value2: " a" }, + { value1: 10, value2: -1 }, + { value1: 11, value2: -0.5 }, + { value1: 12, value2: 3.566 }, + { value1: 13, value2: 0 }, + { value1: 14 }, + { value1: 15, value2: " c" }, + { value1: 16, value2: " 1" } + ]; + + var actual = getQueryResults("FOR i IN UnitTestsImportCsvNoConvert SORT i.value1 RETURN i"); + assertEqual(expected, actual); + }, + //////////////////////////////////////////////////////////////////////////////// /// @brief test csv import without trailing eol ////////////////////////////////////////////////////////////////////////////////