Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

added input type jsonl

This commit is contained in:
jsteemann 2017-03-06 17:39:44 +01:00
parent d55e60cb2e
commit aab138a4e6
4 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
arangosh/Import/ImportFeature.cpp
View File

@ -115,7 +115,7 @@ void ImportFeature::collectOptions(
new DiscreteValuesParameter<StringParameter>(&_createCollectionType, new DiscreteValuesParameter<StringParameter>(&_createCollectionType,
types)); types));
std::unordered_set<std::string> imports = {"csv", "tsv", "json", "auto"}; std::unordered_set<std::string> imports = {"csv", "tsv", "json", "jsonl", "auto"};
options->addOption( options->addOption(
"--type", "type of import file", "--type", "type of import file",
@ -230,12 +230,8 @@ void ImportFeature::start() {
} }
std::string extension = match[1].str(); std::string extension = match[1].str();
if (extension == "json") { if (extension == "json" || extension == "jsonl" || extension == "csv" || extension == "tsv") {
_typeImport = "json"; _typeImport = extension;
} else if (extension == "csv") {
_typeImport = "csv";
} else if (extension == "tsv") {
_typeImport = "tsv";
} else { } else {
LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "Unsupported file extension '" << extension << "'"; LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "Unsupported file extension '" << extension << "'";
FATAL_ERROR_EXIT(); FATAL_ERROR_EXIT();
@ -386,9 +382,9 @@ void ImportFeature::start() {
arangodb::import::ImportHelper::TSV); arangodb::import::ImportHelper::TSV);
} }
else if (_typeImport == "json") { else if (_typeImport == "json" || _typeImport == "jsonl") {
std::cout << "Starting JSON import..." << std::endl; std::cout << "Starting JSON import..." << std::endl;
ok = ih.importJson(_collectionName, _filename); ok = ih.importJson(_collectionName, _filename, (_typeImport == "jsonl"));
} }
else { else {

11
arangosh/Import/ImportHelper.cpp
View File

@ -280,7 +280,8 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
} }
bool ImportHelper::importJson(std::string const& collectionName, bool ImportHelper::importJson(std::string const& collectionName,
std::string const& fileName) { std::string const& fileName,
bool assumeLinewise) {
_collectionName = collectionName; _collectionName = collectionName;
_firstLine = ""; _firstLine = "";
_outputBuffer.clear(); _outputBuffer.clear();
@ -309,6 +310,11 @@ bool ImportHelper::importJson(std::string const& collectionName,
bool isObject = false; bool isObject = false;
bool checkedFront = false; bool checkedFront = false;
if (assumeLinewise) {
checkedFront = true;
isObject = false;
}
// progress display control variables // progress display control variables
int64_t totalRead = 0; int64_t totalRead = 0;
double nextProgress = ProgressStep; double nextProgress = ProgressStep;
@ -345,8 +351,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
if (!checkedFront) { if (!checkedFront) {
// detect the import file format (single lines with individual JSON // detect the import file format (single lines with individual JSON
// objects // objects or a JSON array with all documents)
// or a JSON array with all documents)
char const* p = _outputBuffer.begin(); char const* p = _outputBuffer.begin();
char const* e = _outputBuffer.end(); char const* e = _outputBuffer.end();

3
arangosh/Import/ImportHelper.h
View File

@ -79,7 +79,8 @@ class ImportHelper {
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
bool importJson(std::string const& collectionName, bool importJson(std::string const& collectionName,
std::string const& fileName); std::string const& fileName,
bool assumeLinewise);
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
/// @brief sets the action to carry out on duplicate _key /// @brief sets the action to carry out on duplicate _key

2
arangosh/Shell/V8ClientConnection.cpp
View File

@ -1024,7 +1024,7 @@ static void ClientConnection_importJson(
std::string fileName = TRI_ObjectToString(isolate, args[0]); std::string fileName = TRI_ObjectToString(isolate, args[0]);
std::string collectionName = TRI_ObjectToString(isolate, args[1]); std::string collectionName = TRI_ObjectToString(isolate, args[1]);
if (ih.importJson(collectionName, fileName)) { if (ih.importJson(collectionName, fileName, false)) {
v8::Handle<v8::Object> result = v8::Object::New(isolate); v8::Handle<v8::Object> result = v8::Object::New(isolate);
result->Set(TRI_V8_ASCII_STRING("lines"), result->Set(TRI_V8_ASCII_STRING("lines"),