fixed --convert false in arangoimp

arangod/RestHandler/RestImportHandler.cpp

@@ -1073,27 +1073,35 @@ std::shared_ptr<VPackBuilder> RestImportHandler::createVelocyPackObject(
   }
 
   TRI_ASSERT(keys.isArray());
-  VPackValueLength const n = keys.length();
-  VPackValueLength const m = values.length();
-
-  if (n != m) {
+  
+  VPackArrayIterator itKeys(keys);
+  VPackArrayIterator itValues(values);
+ 
+  if (itKeys.size() != itValues.size()) { 
     errorMsg = positionize(lineNumber) + "wrong number of JSON values (got " +
-               std::to_string(m) + ", expected " + std::to_string(n) + ")";
+               std::to_string(itKeys.size()) + ", expected " + std::to_string(itValues.size()) + ")";
     THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, errorMsg);
   }
 
   auto result = std::make_shared<VPackBuilder>();
   result->openObject();
 
-  for (size_t i = 0; i < n; ++i) {
-    VPackSlice const key = keys.at(i);
-    VPackSlice const value = values.at(i);
+  while (itKeys.valid()) {
+    TRI_ASSERT(itValues.valid());
+    
+    VPackSlice const key = itKeys.value();
+    VPackSlice const value = itValues.value();
 
     if (key.isString() && !value.isNone() && !value.isNull()) {
-      std::string tmp = key.copyString();
-      result->add(tmp, value);
+      VPackValueLength l;
+      char const* p = key.getString(l);
+      result->add(p, l, value);
     }
+
+    itKeys.next();
+    itValues.next();
   }
+
   result->close();
 
   return result;

arangosh/Import/ImportHelper.cpp

@@ -502,12 +502,7 @@ void ImportHelper::addField(char const* field, size_t fieldLength, size_t row,
     return;
   }
 
-  if (!_convert) {
-    _lineBuffer.appendText(field, fieldLength);
-    return;
-  }
-
-  if (*field == '\0') {
+  if (*field == '\0' || fieldLength == 0) {
     // do nothing
     _lineBuffer.appendText(TRI_CHAR_LENGTH_PAIR("null"));
     return;
@@ -523,50 +518,60 @@ void ImportHelper::addField(char const* field, size_t fieldLength, size_t row,
     return;
   }
 
-  if (IsInteger(field, fieldLength)) {
-    // integer value
-    // conversion might fail with out-of-range error
-    try {
-      if (fieldLength > 8) {
-        // long integer numbers might be problematic. check if we get out of
-        // range
-        (void) std::stoll(std::string(
-            field,
-            fieldLength));  // this will fail if the number cannot be converted
+  if (_convert) {
+    if (IsInteger(field, fieldLength)) {
+      // integer value
+      // conversion might fail with out-of-range error
+      try {
+        if (fieldLength > 8) {
+          // long integer numbers might be problematic. check if we get out of
+          // range
+          (void) std::stoll(std::string(
+              field,
+              fieldLength));  // this will fail if the number cannot be converted
+        }
+
+        int64_t num = StringUtils::int64(field, fieldLength);
+        _lineBuffer.appendInteger(num);
+      } catch (...) {
+        // conversion failed
+        _lineBuffer.appendJsonEncoded(field, fieldLength);
+      }
+    } else if (IsDecimal(field, fieldLength)) {
+      // double value
+      // conversion might fail with out-of-range error
+      try {
+        std::string tmp(field, fieldLength);
+        size_t pos = 0;
+        double num = std::stod(tmp, &pos);
+        if (pos == fieldLength) {
+          bool failed = (num != num || num == HUGE_VAL || num == -HUGE_VAL);
+          if (!failed) {
+            _lineBuffer.appendDecimal(num);
+            return;
+          }
+        }
+        // NaN, +inf, -inf
+        // fall-through to appending the number as a string
+      } catch (...) {
+        // conversion failed
+        // fall-through to appending the number as a string
       }
 
-      int64_t num = StringUtils::int64(field, fieldLength);
-      _lineBuffer.appendInteger(num);
-    } catch (...) {
-      // conversion failed
+      _lineBuffer.appendChar('"');
+      _lineBuffer.appendText(field, fieldLength);
+      _lineBuffer.appendChar('"');
+    } else {
       _lineBuffer.appendJsonEncoded(field, fieldLength);
     }
-  } else if (IsDecimal(field, fieldLength)) {
-    // double value
-    // conversion might fail with out-of-range error
-    try {
-      std::string tmp(field, fieldLength);
-      size_t pos = 0;
-      double num = std::stod(tmp, &pos);
-      if (pos == fieldLength) {
-        bool failed = (num != num || num == HUGE_VAL || num == -HUGE_VAL);
-        if (!failed) {
-          _lineBuffer.appendDecimal(num);
-          return;
-        }
-      }
-      // NaN, +inf, -inf
-      // fall-through to appending the number as a string
-    } catch (...) {
-      // conversion failed
-      // fall-through to appending the number as a string
-    }
-
-    _lineBuffer.appendChar('"');
-    _lineBuffer.appendText(field, fieldLength);
-    _lineBuffer.appendChar('"');
   } else {
-    _lineBuffer.appendJsonEncoded(field, fieldLength);
+    if (IsInteger(field, fieldLength) || IsDecimal(field, fieldLength)) {
+      // numeric value. don't convert
+      _lineBuffer.appendText(field, fieldLength);
+    } else {
+      // non-numeric value
+      _lineBuffer.appendJsonEncoded(field, fieldLength);
+    }
   }
 }
 

js/client/modules/@arangodb/testing.js

@@ -1163,6 +1163,10 @@ function runArangoImp (options, instanceInfo, what) {
   if (what.separator !== undefined) {
     args['separator'] = what.separator;
   }
+  
+  if (what.convert !== undefined) {
+    args['convert'] = what.convert ? 'true' : 'false';
+  }
 
   return executeAndWait(ARANGOIMP_BIN, toArgv(args), options);
 }
@@ -3227,6 +3231,15 @@ const impTodos = [{
   create: 'true',
   separator: ';',
   backslash: true
+}, {
+  id: 'csvnoconvert',
+  data: makePathUnix('js/common/test-data/import/import-noconvert.csv'),
+  coll: 'UnitTestsImportCsvNoConvert',
+  type: 'csv',
+  create: 'true',
+  separator: ',',
+  convert: true,
+  backslash: true
 }, {
   id: 'csvnoeol',
   data: makePathUnix('js/common/test-data/import/import-noeol.csv'),

js/common/test-data/import/import-noconvert.csv

@@ -0,0 +1,17 @@
+"value1","value2"
+1,null
+2,false
+3,true
+4,1
+5,2
+6,3
+7,a
+8,b
+9, a
+10,-1
+11,-.5
+12,3.566
+13,0
+14,
+15,       c
+16,       1

js/server/tests/import/import-setup.js

@@ -42,6 +42,7 @@
   db._drop("UnitTestsImportCsv3");
   db._drop("UnitTestsImportCsv4");
   db._drop("UnitTestsImportCsv5");
+  db._drop("UnitTestsImportCsvNoConvert");
   db._drop("UnitTestsImportCsvNoEol");
   db._drop("UnitTestsImportTsv1");
   db._drop("UnitTestsImportTsv2");

js/server/tests/import/import-teardown.js

@@ -42,6 +42,7 @@
   db._drop("UnitTestsImportCsv3");
   db._drop("UnitTestsImportCsv4");
   db._drop("UnitTestsImportCsv5");
+  db._drop("UnitTestsImportCsvNoConvert");
   db._drop("UnitTestsImportCsvNoEol");
   db._drop("UnitTestsImportTsv1");
   db._drop("UnitTestsImportTsv2");

js/server/tests/import/import.js

@@ -285,6 +285,34 @@ function importTestSuite () {
       assertEqual(expected, actual);
     },
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief test csv import without converting
+////////////////////////////////////////////////////////////////////////////////
+    
+    testCsvImportNoConvert : function () {
+      var expected = [ 
+        { value1: 1 },
+        { value1: 2, value2: false },
+        { value1: 3, value2: true },
+        { value1: 4, value2: 1 },
+        { value1: 5, value2: 2 },
+        { value1: 6, value2: 3 },
+        { value1: 7, value2: "a" },
+        { value1: 8, value2: "b" },
+        { value1: 9, value2: " a" },
+        { value1: 10, value2: -1 },
+        { value1: 11, value2: -0.5 },
+        { value1: 12, value2: 3.566 },
+        { value1: 13, value2: 0 },
+        { value1: 14 },
+        { value1: 15, value2: "       c" },
+        { value1: 16, value2: "       1" }
+      ];
+
+      var actual = getQueryResults("FOR i IN UnitTestsImportCsvNoConvert SORT i.value1 RETURN i");
+      assertEqual(expected, actual);
+    },
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief test csv import without trailing eol
 ////////////////////////////////////////////////////////////////////////////////