1
0
Fork 0

Renamed arangoimp to arangoimport (with alias for compatibility.) (#4040)

This commit is contained in:
Dan Larkin 2017-12-14 15:31:21 -05:00 committed by Jan
parent db9b7189f6
commit 9cd8c47eb7
29 changed files with 212 additions and 200 deletions

View File

@ -99,7 +99,7 @@ set(ARANGO_BENCH_FRIENDLY_STRING "arangobench - stress test program")
set(ARANGO_DUMP_FRIENDLY_STRING "arangodump - export") set(ARANGO_DUMP_FRIENDLY_STRING "arangodump - export")
set(ARANGO_RESTORE_FRIENDLY_STRING "arangrestore - importer") set(ARANGO_RESTORE_FRIENDLY_STRING "arangrestore - importer")
set(ARANGO_EXPORT_FRIENDLY_STRING "arangoexport - datae xporter") set(ARANGO_EXPORT_FRIENDLY_STRING "arangoexport - datae xporter")
set(ARANGO_IMP_FRIENDLY_STRING "arangoimp - TSV/CSV/JSON importer") set(ARANGO_IMPORT_FRIENDLY_STRING "arangoimport - TSV/CSV/JSON importer")
set(ARANGOSH_FRIENDLY_STRING "arangosh - commandline client") set(ARANGOSH_FRIENDLY_STRING "arangosh - commandline client")
set(ARANGO_VPACK_FRIENDLY_STRING "arangovpack - vpack printer") set(ARANGO_VPACK_FRIENDLY_STRING "arangovpack - vpack printer")
@ -112,7 +112,7 @@ set(BIN_ARANGOBENCH arangobench)
set(BIN_ARANGOD arangod) set(BIN_ARANGOD arangod)
set(BIN_ARANGODUMP arangodump) set(BIN_ARANGODUMP arangodump)
set(BIN_ARANGOEXPORT arangoexport) set(BIN_ARANGOEXPORT arangoexport)
set(BIN_ARANGOIMP arangoimp) set(BIN_ARANGOIMPORT arangoimport)
set(BIN_ARANGORESTORE arangorestore) set(BIN_ARANGORESTORE arangorestore)
set(BIN_ARANGOSH arangosh) set(BIN_ARANGOSH arangosh)
set(BIN_ARANGOVPACK arangovpack) set(BIN_ARANGOVPACK arangovpack)
@ -1009,7 +1009,7 @@ add_dependencies(arangobench zlibstatic)
add_dependencies(arangod zlibstatic) add_dependencies(arangod zlibstatic)
add_dependencies(arangodump zlibstatic) add_dependencies(arangodump zlibstatic)
add_dependencies(arangoexport zlibstatic) add_dependencies(arangoexport zlibstatic)
add_dependencies(arangoimp zlibstatic) add_dependencies(arangoimport zlibstatic)
add_dependencies(arangorestore zlibstatic) add_dependencies(arangorestore zlibstatic)
add_dependencies(arangosh zlibstatic) add_dependencies(arangosh zlibstatic)
@ -1019,7 +1019,7 @@ if (NOT USE_PRECOMPILED_V8)
add_dependencies(arangod v8_build) add_dependencies(arangod v8_build)
add_dependencies(arangodump v8_build) add_dependencies(arangodump v8_build)
add_dependencies(arangoexport v8_build) add_dependencies(arangoexport v8_build)
add_dependencies(arangoimp v8_build) add_dependencies(arangoimport v8_build)
add_dependencies(arangorestore v8_build) add_dependencies(arangorestore v8_build)
add_dependencies(arangosh v8_build) add_dependencies(arangosh v8_build)
if (USE_CATCH_TESTS) if (USE_CATCH_TESTS)

View File

@ -9,10 +9,10 @@ I want to import data from a file into ArangoDB.
Solution Solution
-------- --------
ArangoDB comes with a command-line tool utility named `arangoimp`. This utility can be ArangoDB comes with a command-line tool utility named `arangoimport`. This utility can be
used for importing JSON-encoded, CSV, and tab-separated files into ArangoDB. used for importing JSON-encoded, CSV, and tab-separated files into ArangoDB.
`arangoimp` needs to be invoked from the command-line once for each import file. `arangoimport` needs to be invoked from the command-line once for each import file.
The target collection can already exist or can be created by the import run. The target collection can already exist or can be created by the import run.
### Importing JSON-encoded data ### Importing JSON-encoded data
@ -59,7 +59,7 @@ Here's an example for the same data in **array format**:
#### Importing JSON data in line-by-line format #### Importing JSON data in line-by-line format
An example data file in **line-by-line format** can be downloaded An example data file in **line-by-line format** can be downloaded
[here](http://jsteemann.github.io/downloads/code/git-commits-single-line.json). The example [here](http://jsteemann.github.io/downloads/code/git-commits-single-line.json). The example
file contains all the commits to the ArangoDB repository as shown by `git log --reverse`. file contains all the commits to the ArangoDB repository as shown by `git log --reverse`.
@ -70,10 +70,10 @@ The following commands will import the data from the file into a collection name
wget http://jsteemann.github.io/downloads/code/git-commits-single-line.json wget http://jsteemann.github.io/downloads/code/git-commits-single-line.json
# actually import data # actually import data
arangoimp --file git-commits-single-line.json --collection commits --create-collection true arangoimport --file git-commits-single-line.json --collection commits --create-collection true
``` ```
Note that no file type has been specified when `arangoimp` was invoked. This is because `json` Note that no file type has been specified when `arangoimport` was invoked. This is because `json`
is its default input format. is its default input format.
The other parameters used have the following meanings: The other parameters used have the following meanings:
@ -82,7 +82,7 @@ The other parameters used have the following meanings:
- `collection`: name of the target collection - `collection`: name of the target collection
- `create-collection`: whether or not the collection should be created if it does not exist - `create-collection`: whether or not the collection should be created if it does not exist
The result of the import printed by `arangoimp` should be: The result of the import printed by `arangoimport` should be:
``` ```
created: 20039 created: 20039
@ -103,13 +103,13 @@ The command for importing JSON data in **array format** is similar to what we've
wget http://jsteemann.github.io/downloads/code/git-commits-array.json wget http://jsteemann.github.io/downloads/code/git-commits-array.json
# actually import data # actually import data
arangoimp --file git-commits-array.json --collection commits --create-collection true arangoimport --file git-commits-array.json --collection commits --create-collection true
``` ```
Though the import command is the same (except the filename), there is a notable difference between the Though the import command is the same (except the filename), there is a notable difference between the
two JSON formats: for the **array format**, `arangoimp` will read and parse the JSON in its entirety two JSON formats: for the **array format**, `arangoimport` will read and parse the JSON in its entirety
before it sends any data to the ArangoDB server. That means the whole input file must fit into before it sends any data to the ArangoDB server. That means the whole input file must fit into
`arangoimp`'s buffer. By default, `arangoimp` will allocate a 16 MiB internal buffer, and input files bigger `arangoimport`'s buffer. By default, `arangoimport` will allocate a 16 MiB internal buffer, and input files bigger
than that will be rejected with the following message: than that will be rejected with the following message:
``` ```
@ -132,7 +132,7 @@ The `--type` parameter for the import command must now be set to `csv`:
wget http://jsteemann.github.io/downloads/code/git-commits.csv wget http://jsteemann.github.io/downloads/code/git-commits.csv
# actually import data # actually import data
arangoimp --file git-commits.csv --type csv --collection commits --create-collection true arangoimport --file git-commits.csv --type csv --collection commits --create-collection true
``` ```
For the CSV import, the first line in the input file has a special meaning: every value listed in the For the CSV import, the first line in the input file has a special meaning: every value listed in the
@ -141,9 +141,9 @@ lines should also have the same number of "columns".
"columns" inside the CSV input file can be left empty though. If a "column" is left empty in a line, "columns" inside the CSV input file can be left empty though. If a "column" is left empty in a line,
then this value will be omitted for the import so the respective attribute will not be set in the imported then this value will be omitted for the import so the respective attribute will not be set in the imported
document. Note that values from the input file that are enclosed in double quotes will always be imported as document. Note that values from the input file that are enclosed in double quotes will always be imported as
strings. To import numeric values, boolean values or the `null` value, don't enclose these values in quotes in strings. To import numeric values, boolean values or the `null` value, don't enclose these values in quotes in
the input file. Note that leading zeros in numeric values will be removed. Importing numbers with leading the input file. Note that leading zeros in numeric values will be removed. Importing numbers with leading
zeros will only work when putting the numbers into strings. zeros will only work when putting the numbers into strings.
Here is an example CSV file: Here is an example CSV file:
@ -155,11 +155,11 @@ Here is an example CSV file:
... ...
``` ```
`arangoimp` supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might also occur inside values `arangoimport` supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might also occur inside values
that are enclosed with the quote character. that are enclosed with the quote character.
The default separator for CSV files is the comma. It can be changed using the `--separator` parameter The default separator for CSV files is the comma. It can be changed using the `--separator` parameter
when invoking `arangoimp`. The quote character defaults to the double quote (**"**). To use a literal double when invoking `arangoimport`. The quote character defaults to the double quote (**"**). To use a literal double
quote inside a "column" in the import data, use two double quotes. To change the quote character, use the quote inside a "column" in the import data, use two double quotes. To change the quote character, use the
`--quote` parameter. To use a backslash for escaping quote characters, please set the option `--backslash-escape` `--quote` parameter. To use a backslash for escaping quote characters, please set the option `--backslash-escape`
to `true`. to `true`.
@ -167,7 +167,7 @@ to `true`.
### Changing the database and server endpoint ### Changing the database and server endpoint
By default, `arangoimp` will connect to the default database on `127.0.0.1:8529` with a user named By default, `arangoimport` will connect to the default database on `127.0.0.1:8529` with a user named
`root`. To change this, use the following parameters: `root`. To change this, use the following parameters:
- `server.database`: name of the database to use when importing (default: `_system`) - `server.database`: name of the database to use when importing (default: `_system`)
@ -176,8 +176,8 @@ By default, `arangoimp` will connect to the default database on `127.0.0.1:8529`
### Using authentication ### Using authentication
`arangoimp` will by default send an username `root` and an empty password to the ArangoDB `arangoimport` will by default send an username `root` and an empty password to the ArangoDB
server. This is ArangoDB's default configuration, and it should be changed. To make `arangoimp` server. This is ArangoDB's default configuration, and it should be changed. To make `arangoimport`
use a different username or password, the following command-line arguments can be used: use a different username or password, the following command-line arguments can be used:
- `server.username`: username, used if authentication is enabled on server - `server.username`: username, used if authentication is enabled on server
@ -185,16 +185,16 @@ use a different username or password, the following command-line arguments can b
The password argument can also be omitted in order to avoid having it saved in the shell's The password argument can also be omitted in order to avoid having it saved in the shell's
command-line history. When specifying a username but omitting the password parameter, command-line history. When specifying a username but omitting the password parameter,
`arangoimp` will prompt for a password. `arangoimport` will prompt for a password.
### Additional parameters ### Additional parameters
By default, `arangoimp` will import data into the specified collection but will not touch By default, `arangoimport` will import data into the specified collection but will not touch
existing data. Often it is convenient to first remove all data from a collection and then run existing data. Often it is convenient to first remove all data from a collection and then run
the import. `arangoimp` supports this with the optional `--overwrite` flag. When setting it to the import. `arangoimport` supports this with the optional `--overwrite` flag. When setting it to
`true`, all documents in the collection will be removed prior to the import. `true`, all documents in the collection will be removed prior to the import.
**Author:** [Jan Steemann](https://github.com/jsteemann) **Author:** [Jan Steemann](https://github.com/jsteemann)
**Tags**: #arangoimp #import **Tags**: #arangoimport #import

View File

@ -17,8 +17,8 @@ First of all we need some data in an ArangoDB collection. For this example we wi
wget https://jsteemann.github.io/downloads/code/users-100000.json.tar.gz wget https://jsteemann.github.io/downloads/code/users-100000.json.tar.gz
# uncompress it # uncompress it
tar xvfz users-100000.json.tar.gz tar xvfz users-100000.json.tar.gz
# import into ArangoDB # import into ArangoDB
arangoimp --file users-100000.json --collection users --create-collection true arangoimport --file users-100000.json --collection users --create-collection true
``` ```
### Setting up ArangoDB-PHP ### Setting up ArangoDB-PHP
@ -54,7 +54,7 @@ $connectionOptions = array(
ConnectionOptions::OPTION_AUTH_PASSWD => '', ConnectionOptions::OPTION_AUTH_PASSWD => '',
// timeout in seconds // timeout in seconds
ConnectionOptions::OPTION_TIMEOUT => 30, ConnectionOptions::OPTION_TIMEOUT => 30,
// database name // database name
ConnectionOptions::OPTION_DATABASE => '_system' ConnectionOptions::OPTION_DATABASE => '_system'
); );
@ -262,7 +262,7 @@ function export($collection, Connection $connection) {
$output .= '"' . implode('","', $values) . '"' . PHP_EOL; $output .= '"' . implode('","', $values) . '"' . PHP_EOL;
} }
// print out the data directly // print out the data directly
print $output; print $output;
} }
} }

View File

@ -1,11 +1,11 @@
Arangoimp Arangoimport
========= ============
This manual describes the ArangoDB importer _arangoimp_, which can be used for This manual describes the ArangoDB importer _arangoimport_, which can be used for
bulk imports. bulk imports.
The most convenient method to import a lot of data into ArangoDB is to use the The most convenient method to import a lot of data into ArangoDB is to use the
*arangoimp* command-line tool. It allows you to import data records from a file *arangoimport* command-line tool. It allows you to import data records from a file
into an existing database collection. into an existing database collection.
It is possible to import [document keys](../Appendix/Glossary.md#document-key) with the documents using the *_key* It is possible to import [document keys](../Appendix/Glossary.md#document-key) with the documents using the *_key*
@ -13,7 +13,7 @@ attribute. When importing into an [edge collection](../Appendix/Glossary.md#edge
imported documents have the *_from* and *_to* attributes, and that they contain imported documents have the *_from* and *_to* attributes, and that they contain
valid references. valid references.
Let's assume for the following examples you want to import user data into an Let's assume for the following examples you want to import user data into an
existing collection named "users" on the server. existing collection named "users" on the server.
@ -34,25 +34,25 @@ example user records to import:
To import these records, all you need to do is to put them into a file (with one To import these records, all you need to do is to put them into a file (with one
line for each record to import) and run the following command: line for each record to import) and run the following command:
> arangoimp --file "data.json" --type jsonl --collection "users" > arangoimport --file "data.json" --type jsonl --collection "users"
This will transfer the data to the server, import the records, and print a This will transfer the data to the server, import the records, and print a
status summary. To show the intermediate progress during the import process, the status summary. To show the intermediate progress during the import process, the
option *--progress* can be added. This option will show the percentage of the option *--progress* can be added. This option will show the percentage of the
input file that has been sent to the server. This will only be useful for big input file that has been sent to the server. This will only be useful for big
import files. import files.
> arangoimp --file "data.json" --type json --collection users --progress true > arangoimport --file "data.json" --type json --collection users --progress true
It is also possible to use the output of another command as an input for arangoimp. It is also possible to use the output of another command as an input for arangoimport.
For example, the following shell command can be used to pipe data from the `cat` For example, the following shell command can be used to pipe data from the `cat`
process to arangoimp: process to arangoimport:
> cat data.json | arangoimp --file - --type json --collection users > cat data.json | arangoimport --file - --type json --collection users
Note that you have to use `--file -` if you want to use another command as input Note that you have to use `--file -` if you want to use another command as input
for arangoimp. No progress can be reported for such imports as the size of the input for arangoimport. No progress can be reported for such imports as the size of the input
will be unknown to arangoimp. will be unknown to arangoimport.
By default, the endpoint *tcp://127.0.0.1:8529* will be used. If you want to By default, the endpoint *tcp://127.0.0.1:8529* will be used. If you want to
specify a different endpoint, you can use the *--server.endpoint* option. You specify a different endpoint, you can use the *--server.endpoint* option. You
@ -60,43 +60,43 @@ probably want to specify a database user and password as well. You can do so by
using the options *--server.username* and *--server.password*. If you do not using the options *--server.username* and *--server.password*. If you do not
specify a password, you will be prompted for one. specify a password, you will be prompted for one.
> arangoimp --server.endpoint tcp://127.0.0.1:8529 --server.username root --file "data.json" --type json --collection "users" > arangoimport --server.endpoint tcp://127.0.0.1:8529 --server.username root --file "data.json" --type json --collection "users"
Note that the collection (*users* in this case) must already exist or the import Note that the collection (*users* in this case) must already exist or the import
will fail. If you want to create a new collection with the import data, you need will fail. If you want to create a new collection with the import data, you need
to specify the *--create-collection* option. Note that by default it will create to specify the *--create-collection* option. Note that by default it will create
a document collection and no edge collection. a document collection and no edge collection.
> arangoimp --file "data.json" --type json --collection "users" --create-collection true > arangoimport --file "data.json" --type json --collection "users" --create-collection true
To create an edge collection instead, use the *--create-collection-type* option To create an edge collection instead, use the *--create-collection-type* option
and set it to *edge*: and set it to *edge*:
> arangoimp --file "data.json" --collection "myedges" --create-collection true --create-collection-type edge > arangoimport --file "data.json" --collection "myedges" --create-collection true --create-collection-type edge
When importing data into an existing collection it is often convenient to first When importing data into an existing collection it is often convenient to first
remove all data from the collection and then start the import. This can be achieved remove all data from the collection and then start the import. This can be achieved
by passing the *--overwrite* parameter to _arangoimp_. If it is set to *true*, by passing the *--overwrite* parameter to _arangoimport_. If it is set to *true*,
any existing data in the collection will be removed prior to the import. Note any existing data in the collection will be removed prior to the import. Note
that any existing index definitions for the collection will be preserved even if that any existing index definitions for the collection will be preserved even if
*--overwrite* is set to true. *--overwrite* is set to true.
> arangoimp --file "data.json" --type json --collection "users" --overwrite true > arangoimport --file "data.json" --type json --collection "users" --overwrite true
As the import file already contains the data in JSON format, attribute names and As the import file already contains the data in JSON format, attribute names and
data types are fully preserved. As can be seen in the example data, there is no data types are fully preserved. As can be seen in the example data, there is no
need for all data records to have the same attribute names or types. Records can need for all data records to have the same attribute names or types. Records can
be inhomogeneous. be inhomogeneous.
Please note that by default, _arangoimp_ will import data into the specified Please note that by default, _arangoimport_ will import data into the specified
collection in the default database (*_system*). To specify a different database, collection in the default database (*_system*). To specify a different database,
use the *--server.database* option when invoking _arangoimp_. use the *--server.database* option when invoking _arangoimport_.
The tool also supports parallel imports, with multiple threads. Using multiple The tool also supports parallel imports, with multiple threads. Using multiple
threads may provide a speedup, especially when using the RocksDB storage engine. threads may provide a speedup, especially when using the RocksDB storage engine.
To specify the number of parallel threads use the `--threads` option: To specify the number of parallel threads use the `--threads` option:
> arangoimp --threads 4 --file "data.json" --type json --collection "users" > arangoimport --threads 4 --file "data.json" --type json --collection "users"
Note that using multiple threads may lead to a non-sequential import of the input Note that using multiple threads may lead to a non-sequential import of the input
data. Data that appears later in the input file may be imported earlier than data data. Data that appears later in the input file may be imported earlier than data
@ -106,7 +106,7 @@ this case, the number of threads should be set to 1.
### JSON input file formats ### JSON input file formats
**Note**: *arangoimp* supports two formats when importing JSON data from **Note**: *arangoimport* supports two formats when importing JSON data from
a file. The first format that we also used above is commonly known as [jsonl](http://jsonlines.org)). a file. The first format that we also used above is commonly known as [jsonl](http://jsonlines.org)).
However, in contrast to the JSONL specification it requires the input file to contain However, in contrast to the JSONL specification it requires the input file to contain
one complete JSON document in each line, e.g. one complete JSON document in each line, e.g.
@ -120,7 +120,7 @@ one complete JSON document in each line, e.g.
So one could argue that this is only a subset of JSONL. So one could argue that this is only a subset of JSONL.
The above format can be imported sequentially by _arangoimp_. It will read data The above format can be imported sequentially by _arangoimport_. It will read data
from the input file in chunks and send it in batches to the server. Each batch from the input file in chunks and send it in batches to the server. Each batch
will be about as big as specified in the command-line parameter *--batch-size*. will be about as big as specified in the command-line parameter *--batch-size*.
@ -135,17 +135,17 @@ An alternative is to put one big JSON document into the input file like this:
] ]
``` ```
This format allows line breaks within the input file as required. The downside This format allows line breaks within the input file as required. The downside
is that the whole input file will need to be read by _arangoimp_ before it can is that the whole input file will need to be read by _arangoimport_ before it can
send the first batch. This might be a problem if the input file is big. By send the first batch. This might be a problem if the input file is big. By
default, _arangoimp_ will allow importing such files up to a size of about 16 MB. default, _arangoimport_ will allow importing such files up to a size of about 16 MB.
If you want to allow your _arangoimp_ instance to use more memory, you may want If you want to allow your _arangoimport_ instance to use more memory, you may want
to increase the maximum file size by specifying the command-line option to increase the maximum file size by specifying the command-line option
*--batch-size*. For example, to set the batch size to 32 MB, use the following *--batch-size*. For example, to set the batch size to 32 MB, use the following
command: command:
> arangoimp --file "data.json" --type json --collection "users" --batch-size 33554432 > arangoimport --file "data.json" --type json --collection "users" --batch-size 33554432
Please also note that you may need to increase the value of *--batch-size* if Please also note that you may need to increase the value of *--batch-size* if
a single document inside the input file is bigger than the value of *--batch-size*. a single document inside the input file is bigger than the value of *--batch-size*.
@ -153,7 +153,7 @@ a single document inside the input file is bigger than the value of *--batch-siz
### Importing CSV Data ### Importing CSV Data
_arangoimp_ also offers the possibility to import data from CSV files. This _arangoimport_ also offers the possibility to import data from CSV files. This
comes handy when the data at hand is in CSV format already and you don't want to comes handy when the data at hand is in CSV format already and you don't want to
spend time converting them to JSON for the import. spend time converting them to JSON for the import.
@ -183,16 +183,16 @@ Wayne,Brewer,,false,
The command line to execute the import is: The command line to execute the import is:
> arangoimp --file "data.csv" --type csv --collection "users" > arangoimport --file "data.csv" --type csv --collection "users"
The above data will be imported into 5 documents which will look as follows: The above data will be imported into 5 documents which will look as follows:
```js ```js
{ "first" : "John", "last" : "Connor", "active" : true, "age" : 25 } { "first" : "John", "last" : "Connor", "active" : true, "age" : 25 }
{ "first" : "Jim", "last" : "O'Brady", "age" : 19 } { "first" : "Jim", "last" : "O'Brady", "age" : 19 }
{ "first" : "Lisa", "last" : "Jones", "dob" : "1981-04-09" } { "first" : "Lisa", "last" : "Jones", "dob" : "1981-04-09" }
{ "first" : "Hans", "last" : "dos Santos", "age" : 123 } { "first" : "Hans", "last" : "dos Santos", "age" : 123 }
{ "first" : "Wayne", "last" : "Brewer", "active" : false } { "first" : "Wayne", "last" : "Brewer", "active" : false }
``` ```
As can be seen, values left completely empty in the input file will be treated As can be seen, values left completely empty in the input file will be treated
@ -210,10 +210,10 @@ escaped with another quote character (or with a backslash if the *--backslash-es
option is used). option is used).
Note that the quote and separator characters can be adjusted via the Note that the quote and separator characters can be adjusted via the
*--quote* and *--separator* arguments when invoking _arangoimp_. The quote *--quote* and *--separator* arguments when invoking _arangoimport_. The quote
character defaults to the double quote (*"*). To use a literal quote in a character defaults to the double quote (*"*). To use a literal quote in a
string, you can use two quote characters. string, you can use two quote characters.
To use backslash for escaping quote characters, please set the option To use backslash for escaping quote characters, please set the option
*--backslash-escape* to *true*. *--backslash-escape* to *true*.
The importer supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might The importer supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might
@ -230,8 +230,8 @@ multine password!"
"Bartholomew ""Bart"" Simpson","Milhouse" "Bartholomew ""Bart"" Simpson","Milhouse"
``` ```
Extra whitespace at the end of each line will be ignored. Whitespace at the Extra whitespace at the end of each line will be ignored. Whitespace at the
start of lines or between field values will not be ignored, so please make sure start of lines or between field values will not be ignored, so please make sure
that there is no extra whitespace in front of values or between them. that there is no extra whitespace in front of values or between them.
@ -246,11 +246,11 @@ As with CSV, the first line in the TSV file must contain the attribute names,
and all lines must have an identical number of values. and all lines must have an identical number of values.
If a different separator character or string should be used, it can be specified If a different separator character or string should be used, it can be specified
with the *--separator* argument. with the *--separator* argument.
An example command line to execute the TSV import is: An example command line to execute the TSV import is:
> arangoimp --file "data.tsv" --type tsv --collection "users" > arangoimport --file "data.tsv" --type tsv --collection "users"
### Attribute Name Translation ### Attribute Name Translation
@ -261,41 +261,41 @@ that should be used in ArangoDB.
A common use case is to rename an "id" column from the input file into "_key" as A common use case is to rename an "id" column from the input file into "_key" as
it is expected by ArangoDB. To do this, specify the following translation when it is expected by ArangoDB. To do this, specify the following translation when
invoking arangoimp: invoking arangoimport:
> arangoimp --file "data.csv" --type csv --translate "id=_key" > arangoimport --file "data.csv" --type csv --translate "id=_key"
Other common cases are to rename columns in the input file to *_from* and *_to*: Other common cases are to rename columns in the input file to *_from* and *_to*:
> arangoimp --file "data.csv" --type csv --translate "from=_from" --translate "to=_to" > arangoimport --file "data.csv" --type csv --translate "from=_from" --translate "to=_to"
The *translate* option can be specified multiple types. The source attribute name The *translate* option can be specified multiple types. The source attribute name
and the target attribute must be separated with a *=*. and the target attribute must be separated with a *=*.
### Ignoring Attributes ### Ignoring Attributes
For the CSV and TSV input formats, certain attribute names can be ignored on imports. For the CSV and TSV input formats, certain attribute names can be ignored on imports.
In an ArangoDB cluster there are cases where this can come in handy, In an ArangoDB cluster there are cases where this can come in handy,
when your documents already contain a `_key` attribute when your documents already contain a `_key` attribute
and your collection has a sharding attribute other than `_key`: In the cluster this and your collection has a sharding attribute other than `_key`: In the cluster this
configuration is not supported, because ArangoDB needs to guarantee the uniqueness of the `_key` configuration is not supported, because ArangoDB needs to guarantee the uniqueness of the `_key`
attribute in *all* shards of the collection. attribute in *all* shards of the collection.
> arangoimp --file "data.csv" --type csv --remove-attribute "_key" > arangoimport --file "data.csv" --type csv --remove-attribute "_key"
The same thing would apply if your data contains an *_id* attribute: The same thing would apply if your data contains an *_id* attribute:
> arangoimp --file "data.csv" --type csv --remove-attribute "_id" > arangoimport --file "data.csv" --type csv --remove-attribute "_id"
### Importing into an Edge Collection ### Importing into an Edge Collection
arangoimp can also be used to import data into an existing edge collection. arangoimport can also be used to import data into an existing edge collection.
The import data must, for each edge to import, contain at least the *_from* and The import data must, for each edge to import, contain at least the *_from* and
*_to* attributes. These indicate which other two documents the edge should connect. *_to* attributes. These indicate which other two documents the edge should connect.
It is necessary that these attributes are set for all records, and point to It is necessary that these attributes are set for all records, and point to
valid document ids in existing collections. valid document ids in existing collections.
*Examples* *Examples*
@ -304,20 +304,20 @@ valid document ids in existing collections.
{ "_from" : "users/1234", "_to" : "users/4321", "desc" : "1234 is connected to 4321" } { "_from" : "users/1234", "_to" : "users/4321", "desc" : "1234 is connected to 4321" }
``` ```
**Note**: The edge collection must already exist when the import is started. Using **Note**: The edge collection must already exist when the import is started. Using
the *--create-collection* flag will not work because arangoimp will always try to the *--create-collection* flag will not work because arangoimport will always try to
create a regular document collection if the target collection does not exist. create a regular document collection if the target collection does not exist.
### Updating existing documents ### Updating existing documents
By default, arangoimp will try to insert all documents from the import file into the By default, arangoimport will try to insert all documents from the import file into the
specified collection. In case the import file contains documents that are already present specified collection. In case the import file contains documents that are already present
in the target collection (matching is done via the *_key* attributes), then a default in the target collection (matching is done via the *_key* attributes), then a default
arangoimp run will not import these documents and complain about unique key constraint arangoimport run will not import these documents and complain about unique key constraint
violations. violations.
However, arangoimp can be used to update or replace existing documents in case they However, arangoimport can be used to update or replace existing documents in case they
already exist in the target collection. It provides the command-line option *--on-duplicate* already exist in the target collection. It provides the command-line option *--on-duplicate*
to control the behavior in case a document is already present in the database. to control the behavior in case a document is already present in the database.
@ -329,47 +329,47 @@ the database will not be modified.
Other possible values for *--on-duplicate* are: Other possible values for *--on-duplicate* are:
- *update*: each document present in the import file that is also present in the target - *update*: each document present in the import file that is also present in the target
collection already will be updated by arangoimp. *update* will perform a partial update collection already will be updated by arangoimport. *update* will perform a partial update
of the existing document, modifying only the attributes that are present in the import of the existing document, modifying only the attributes that are present in the import
file and leaving all other attributes untouched. file and leaving all other attributes untouched.
The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be
updated or replaced in existing documents. updated or replaced in existing documents.
- *replace*: each document present in the import file that is also present in the target - *replace*: each document present in the import file that is also present in the target
collection already will be replace by arangoimp. *replace* will replace the existing collection already will be replace by arangoimport. *replace* will replace the existing
document entirely, resulting in a document with only the attributes specified in the import document entirely, resulting in a document with only the attributes specified in the import
file. file.
The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be
updated or replaced in existing documents. updated or replaced in existing documents.
- *ignore*: each document present in the import file that is also present in the target - *ignore*: each document present in the import file that is also present in the target
collection already will be ignored and not modified in the target collection. collection already will be ignored and not modified in the target collection.
When *--on-duplicate* is set to either *update* or *replace*, arangoimp will return the When *--on-duplicate* is set to either *update* or *replace*, arangoimport will return the
number of documents updated/replaced in the *updated* return value. When set to another number of documents updated/replaced in the *updated* return value. When set to another
value, the value of *updated* will always be zero. When *--on-duplicate* is set to *ignore*, value, the value of *updated* will always be zero. When *--on-duplicate* is set to *ignore*,
arangoimp will return the number of ignored documents in the *ignored* return value. arangoimport will return the number of ignored documents in the *ignored* return value.
When set to another value, *ignored* will always be zero. When set to another value, *ignored* will always be zero.
It is possible to perform a combination of inserts and updates/replaces with a single It is possible to perform a combination of inserts and updates/replaces with a single
arangoimp run. When *--on-duplicate* is set to *update* or *replace*, all documents present arangoimport run. When *--on-duplicate* is set to *update* or *replace*, all documents present
in the import file will be inserted into the target collection provided they are valid in the import file will be inserted into the target collection provided they are valid
and do not already exist with the specified *_key*. Documents that are already present and do not already exist with the specified *_key*. Documents that are already present
in the target collection (identified by *_key* attribute) will instead be updated/replaced. in the target collection (identified by *_key* attribute) will instead be updated/replaced.
### Arangoimp result output ### Arangoimport result output
An _arangoimp_ import run will print out the final results on the command line. An _arangoimport_ import run will print out the final results on the command line.
It will show the It will show the
* number of documents created (*created*) * number of documents created (*created*)
* number of documents updated/replaced (*updated/replaced*, only non-zero if * number of documents updated/replaced (*updated/replaced*, only non-zero if
*--on-duplicate* was set to *update* or *replace*, see below) *--on-duplicate* was set to *update* or *replace*, see below)
* number of warnings or errors that occurred on the server side (*warnings/errors*) * number of warnings or errors that occurred on the server side (*warnings/errors*)
* number of ignored documents (only non-zero if *--on-duplicate* was set to *ignore*). * number of ignored documents (only non-zero if *--on-duplicate* was set to *ignore*).
*Example* *Example*
@ -383,18 +383,18 @@ ignored: 0
For CSV and TSV imports, the total number of input file lines read will also be printed For CSV and TSV imports, the total number of input file lines read will also be printed
(*lines read*). (*lines read*).
_arangoimp_ will also print out details about warnings and errors that happened on the _arangoimport_ will also print out details about warnings and errors that happened on the
server-side (if any). server-side (if any).
### Attribute Naming and Special Attributes ### Attribute Naming and Special Attributes
Attributes whose names start with an underscore are treated in a special way by Attributes whose names start with an underscore are treated in a special way by
ArangoDB: ArangoDB:
- the optional *_key* attribute contains the document's key. If specified, the value - the optional *_key* attribute contains the document's key. If specified, the value
must be formally valid (e.g. must be a string and conform to the naming conventions). must be formally valid (e.g. must be a string and conform to the naming conventions).
Additionally, the key value must be unique within the Additionally, the key value must be unique within the
collection the import is run for. collection the import is run for.
- *_from*: when importing into an edge collection, this attribute contains the id - *_from*: when importing into an edge collection, this attribute contains the id
of one of the documents connected by the edge. The value of *_from* must be a of one of the documents connected by the edge. The value of *_from* must be a
@ -403,7 +403,7 @@ ArangoDB:
of the other document connected by the edge. The value of *_to* must be a of the other document connected by the edge. The value of *_to* must be a
syntactically valid document id and the referred collection must exist. syntactically valid document id and the referred collection must exist.
- *_rev*: this attribute contains the revision number of a document. However, the - *_rev*: this attribute contains the revision number of a document. However, the
revision numbers are managed by ArangoDB and cannot be specified on import. Thus revision numbers are managed by ArangoDB and cannot be specified on import. Thus
any value in this attribute is ignored on import. any value in this attribute is ignored on import.
If you import values into *_key*, you should make sure they are valid and unique. If you import values into *_key*, you should make sure they are valid and unique.
@ -419,7 +419,7 @@ inside *_from* and/or *_to*.
*Example* *Example*
> arangoimp --from-collection-prefix users --to-collection-prefix products ... > arangoimport --from-collection-prefix users --to-collection-prefix products ...
Importing the following document will then create an edge between *users/1234* and Importing the following document will then create an edge between *users/1234* and
*products/4321*: *products/4321*:

View File

@ -79,7 +79,7 @@ database directory [to the new default paths](#custom-install-paths). Upgrading
will keep your old data, password and choice of storage engine as it is. will keep your old data, password and choice of storage engine as it is.
Switching to the RocksDB storage engine requires a Switching to the RocksDB storage engine requires a
[export](../../Administration/Arangoexport.md) and [export](../../Administration/Arangoexport.md) and
[reimport](../../Administration/Arangoimp.md) of your data. [reimport](../../Administration/Arangoimport.md) of your data.
Starting Starting
-------- --------
@ -101,7 +101,7 @@ business. Have fun!` at the end of its output.
We now wish to check that the installation is working correctly and to do this We now wish to check that the installation is working correctly and to do this
we will be using the administration web interface. Execute *arangod.exe* if you we will be using the administration web interface. Execute *arangod.exe* if you
have not already done so, then open up your web browser and point it to the have not already done so, then open up your web browser and point it to the
page: page:
http://127.0.0.1:8529/ http://127.0.0.1:8529/

View File

@ -358,6 +358,7 @@ if we use the `NEW` pseudo-variable:
"age": 40, "age": 40,
"name": "Katie Foster" "name": "Katie Foster"
} }
]
``` ```
If we used `REPLACE` instead, the name attribute would be gone. With `UPDATE`, If we used `REPLACE` instead, the name attribute would be gone. With `UPDATE`,
@ -581,22 +582,22 @@ The ArangoDB package comes with the following programs:
- `arangod`: The [ArangoDB database daemon](../Administration/Configuration/GeneralArangod.md). - `arangod`: The [ArangoDB database daemon](../Administration/Configuration/GeneralArangod.md).
This server program is intended to run as a daemon process and to serve the This server program is intended to run as a daemon process and to serve the
various clients connection to the server via TCP / HTTP. various clients connection to the server via TCP / HTTP.
- `arangosh`: The [ArangoDB shell](../Administration/Arangosh/README.md). - `arangosh`: The [ArangoDB shell](../Administration/Arangosh/README.md).
A client that implements a read-eval-print loop (REPL) and provides functions A client that implements a read-eval-print loop (REPL) and provides functions
to access and administrate the ArangoDB server. to access and administrate the ArangoDB server.
- `arangoimp`: A [bulk importer](../Administration/Arangoimp.md) for the - `arangoimport`: A [bulk importer](../Administration/Arangoimport.md) for the
ArangoDB server. It supports JSON and CSV. ArangoDB server. It supports JSON and CSV.
- `arangodump`: A tool to [create backups](../Administration/Arangodump.md) - `arangodump`: A tool to [create backups](../Administration/Arangodump.md)
of an ArangoDB database in JSON format. of an ArangoDB database in JSON format.
- `arangorestore`: A tool to [load data of a backup](../Administration/Arangorestore.md) - `arangorestore`: A tool to [load data of a backup](../Administration/Arangorestore.md)
back into an ArangoDB database. back into an ArangoDB database.
- `arango-dfdb`: A [datafile debugger](../Troubleshooting/DatafileDebugger.md) for - `arango-dfdb`: A [datafile debugger](../Troubleshooting/DatafileDebugger.md) for
ArangoDB. It is primarily intended to be used during development of ArangoDB. ArangoDB. It is primarily intended to be used during development of ArangoDB.
- `arangobench`: A [benchmark and test tool](../Troubleshooting/Arangobench.md). - `arangobench`: A [benchmark and test tool](../Troubleshooting/Arangobench.md).
It can be used for performance and server function testing. It can be used for performance and server function testing.

View File

@ -149,7 +149,7 @@
* [Configuration](Administration/Arangosh/Configuration.md) * [Configuration](Administration/Arangosh/Configuration.md)
# relocate file? # relocate file?
* [Details](GettingStarted/Arangosh.md) * [Details](GettingStarted/Arangosh.md)
* [Arangoimp](Administration/Arangoimp.md) * [Arangoimport](Administration/Arangoimport.md)
* [Arangodump](Administration/Arangodump.md) * [Arangodump](Administration/Arangodump.md)
* [Arangorestore](Administration/Arangorestore.md) * [Arangorestore](Administration/Arangorestore.md)
* [Arangoexport](Administration/Arangoexport.md) * [Arangoexport](Administration/Arangoexport.md)

View File

@ -15,7 +15,7 @@ if (USE_MAINTAINER_MODE AND NOT(MSVC))
set(MAN_NAMES set(MAN_NAMES
man1/arangobench.1 man1/arangobench.1
man1/arangodump.1 man1/arangodump.1
man1/arangoimp.1 man1/arangoimport.1
man1/arangorestore.1 man1/arangorestore.1
man1/arangoexport.1 man1/arangoexport.1
man1/arangosh.1 man1/arangosh.1
@ -61,7 +61,7 @@ if (USE_MAINTAINER_MODE AND NOT(MSVC))
add_custom_target(clean_man_autogenerated add_custom_target(clean_man_autogenerated
COMMAND rm -f ${CMAKE_SOURCE_DIR}/README COMMAND rm -f ${CMAKE_SOURCE_DIR}/README
COMMAND rm -f ${MAN_FILES}) COMMAND rm -f ${MAN_FILES})
list(APPEND CLEAN_AUTOGENERATED_FILES clean_man_autogenerated) list(APPEND CLEAN_AUTOGENERATED_FILES clean_man_autogenerated)
set(CLEAN_AUTOGENERATED_FILES ${CLEAN_AUTOGENERATED_FILES} PARENT_SCOPE) set(CLEAN_AUTOGENERATED_FILES ${CLEAN_AUTOGENERATED_FILES} PARENT_SCOPE)

View File

@ -2,7 +2,7 @@
.SH NAME .SH NAME
arangoexport - a tool to export collections of an ArangoDB database arangoexport - a tool to export collections of an ArangoDB database
.SH SYNOPSIS .SH SYNOPSIS
arangoexport [options] arangoexport [options]
.SH DESCRIPTION .SH DESCRIPTION
The arangoexport binary can be used to export collections of an ArangoDB The arangoexport binary can be used to export collections of an ArangoDB
database to json and jsonl. It can also export a graph or collections database to json and jsonl. It can also export a graph or collections
@ -12,7 +12,7 @@ arangoexport will work on the specified database only. If no database name
is specified, arangoexport will work on the default database ("_system"). is specified, arangoexport will work on the default database ("_system").
The exported jsonl files can be re-imported in an ArangoDB database The exported jsonl files can be re-imported in an ArangoDB database
using the arangoimp tool. using the arangoimport tool.
.SH OPTIONS .SH OPTIONS
The arangoexport binary has many options that can be used to control its The arangoexport binary has many options that can be used to control its
behavior. For a complete list of options, please refer to the behavior. For a complete list of options, please refer to the
@ -21,4 +21,3 @@ arangoexport --help.
.SH AUTHOR .SH AUTHOR
Copyright ArangoDB GmbH, Cologne, Germany Copyright ArangoDB GmbH, Cologne, Germany

View File

@ -1,19 +1,18 @@
.TH arangoimp 1 "3.3.devel" "ArangoDB" "ArangoDB" .TH arangoimport 1 "3.3.devel" "ArangoDB" "ArangoDB"
.SH NAME .SH NAME
arangoimp - a bulk importer for the ArangoDB database arangoimport - a bulk importer for the ArangoDB database
.SH SYNOPSIS .SH SYNOPSIS
arangoimp [options] arangoimport [options]
.SH DESCRIPTION .SH DESCRIPTION
The arangoimp binary can be used to bulk import data from a file into the The arangoimport binary can be used to bulk import data from a file into the
ArangoDB database. Input data be present in the input file in either CSV ArangoDB database. Input data be present in the input file in either CSV
format with column headlines, or in JSON format. If JSON format is used, format with column headlines, or in JSON format. If JSON format is used,
each line in the input file must contain exactly one JSON document with each line in the input file must contain exactly one JSON document with
the attribute name/value pairs to import. the attribute name/value pairs to import.
.SH OPTIONS .SH OPTIONS
The arangoimp binary has many options that can be used to control its The arangoimport binary has many options that can be used to control its
behavior. For a complete list of options, please refer to the behavior. For a complete list of options, please refer to the
ArangoDB online manual, available at https://www.arangodb.com/ ArangoDB online manual, available at https://www.arangodb.com/
.SH AUTHOR .SH AUTHOR
Copyright ArangoDB GmbH, Cologne, Germany Copyright ArangoDB GmbH, Cologne, Germany

View File

@ -1,7 +1,7 @@
NAME NAME
<COMMAND> - a tool to export collections of an ArangoDB database <COMMAND> - a tool to export collections of an ArangoDB database
SYNOPSIS SYNOPSIS
<COMMAND> [options] <COMMAND> [options]
DESCRIPTION DESCRIPTION
The <COMMAND> binary can be used to export collections of an ArangoDB The <COMMAND> binary can be used to export collections of an ArangoDB
database to json and jsonl. It can also export a graph or collections database to json and jsonl. It can also export a graph or collections
@ -11,7 +11,7 @@ DESCRIPTION
is specified, <COMMAND> will work on the default database ("_system"). is specified, <COMMAND> will work on the default database ("_system").
The exported jsonl files can be re-imported in an ArangoDB database The exported jsonl files can be re-imported in an ArangoDB database
using the arangoimp tool. using the arangoimport tool.
OPTIONS OPTIONS
The <COMMAND> binary has many options that can be used to control its The <COMMAND> binary has many options that can be used to control its
behavior. For a complete list of options, please refer to the behavior. For a complete list of options, please refer to the

View File

@ -1,12 +1,12 @@
NAME NAME
<COMMAND> - a bulk importer for the ArangoDB database <COMMAND> - a bulk importer for the ArangoDB database
SYNOPSIS SYNOPSIS
<COMMAND> [options] <COMMAND> [options]
DESCRIPTION DESCRIPTION
The <COMMAND> binary can be used to bulk import data from a file into the The <COMMAND> binary can be used to bulk import data from a file into the
ArangoDB database. Input data be present in the input file in either CSV ArangoDB database. Input data be present in the input file in either CSV
format with column headlines, or in JSON format. If JSON format is used, format with column headlines, or in JSON format. If JSON format is used,
each line in the input file must contain exactly one JSON document with each line in the input file must contain exactly one JSON document with
the attribute name/value pairs to import. the attribute name/value pairs to import.
OPTIONS OPTIONS
The <COMMAND> binary has many options that can be used to control its The <COMMAND> binary has many options that can be used to control its

View File

@ -60,8 +60,8 @@ apps:
plugs: # enable interfaces plugs: # enable interfaces
- network - network
arangoimp: arangoimport:
command: arangoimp \ command: arangoimport \
--server.endpoint=tcp://127.0.0.1:@SNAP_PORT@ \ --server.endpoint=tcp://127.0.0.1:@SNAP_PORT@ \
--temp.path=${SNAP_DATA} --temp.path=${SNAP_DATA}
plugs: # enable interfaces plugs: # enable interfaces

View File

@ -97,7 +97,7 @@ It is written in C/C++.
%description client %description client
The ArangoDB shell as stand-alone program. It also contains the utility programs: arangobench (benchmark), arangorestore & arangodump (backup), arangoimp (import). The ArangoDB shell as stand-alone program. It also contains the utility programs: arangobench (benchmark), arangorestore & arangodump (backup), arangoimport (import).
%description debuginfo %description debuginfo
Debug symbols for the arangodb database Debug symbols for the arangodb database
@ -170,13 +170,13 @@ mkdir -p %{buildroot}%{_piddir}
#%doc LICENSES-OTHER-COMPONENTS.md #%doc LICENSES-OTHER-COMPONENTS.md
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangosh.conf %config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangosh.conf
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangoimp.conf %config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangoimport.conf
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangodump.conf %config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangodump.conf
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangorestore.conf %config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangorestore.conf
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangobench.conf %config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangobench.conf
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/foxx-manager.conf %config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/foxx-manager.conf
%{_bindir}/arangosh %{_bindir}/arangosh
%{_bindir}/arangoimp %{_bindir}/arangoimport
%{_bindir}/arangodump %{_bindir}/arangodump
%{_bindir}/arangorestore %{_bindir}/arangorestore
%{_bindir}/arangobench %{_bindir}/arangobench
@ -185,7 +185,7 @@ mkdir -p %{buildroot}%{_piddir}
%{_mandir}/*/arangorestore* %{_mandir}/*/arangorestore*
%{_mandir}/*/arangobench* %{_mandir}/*/arangobench*
%{_mandir}/*/arangodump* %{_mandir}/*/arangodump*
%{_mandir}/*/arangoimp* %{_mandir}/*/arangoimport*
%{_datadir}/arangodb3/js/common %{_datadir}/arangodb3/js/common
%{_datadir}/arangodb3/js/client %{_datadir}/arangodb3/js/client
%{_datadir}/arangodb3/js/node %{_datadir}/arangodb3/js/node

View File

@ -3,7 +3,7 @@
Pregel Subsystem Pregel Subsystem
======== ========
The pregel subsystem implements a variety of different grapg algorithms, The pregel subsystem implements a variety of different grapg algorithms,
this readme is more intended for internal use. this readme is more intended for internal use.
#### Protocol #### Protocol
@ -12,9 +12,9 @@ Message format between DBServers:
{sender:"someid", {sender:"someid",
executionNumber:1337, executionNumber:1337,
globalSuperstep:123, globalSuperstep:123,
messages: [<vertexID1>, <slice1>, vertexID2, <slice2>] messages: [<vertexID1>, <slice1>, vertexID2, <slice2>]
} }
Any type of slice is supported Any type of slice is supported
@ -23,7 +23,7 @@ Any type of slice is supported
### Useful Commands ### Useful Commands
Import graph e.g. https://github.com/arangodb/example-datasets/tree/master/Graphs/1000 Import graph e.g. https://github.com/arangodb/example-datasets/tree/master/Graphs/1000
First rename the columns '_key', '_from', '_to' arangoimp will keep those. First rename the columns '_key', '_from', '_to' arangoimport will keep those.
In arangosh: In arangosh:
@ -31,19 +31,19 @@ In arangosh:
db._createEdgeCollection('alt_edges'); db._createEdgeCollection('alt_edges');
db._createEdgeCollection('edges', {numberOfShards: 2, shardKeys:["_vertex"], distributeShardsLike:'vertices'}); db._createEdgeCollection('edges', {numberOfShards: 2, shardKeys:["_vertex"], distributeShardsLike:'vertices'});
arangoimp --file generated_vertices.csv --type csv --collection vertices --overwrite true --server.endpoint http+tcp://127.0.0.1:8530 arangoimport --file generated_vertices.csv --type csv --collection vertices --overwrite true --server.endpoint http+tcp://127.0.0.1:8530
Or: Or:
for(var i=0; i < 5000; i++) db.vertices.save({_key:i+""}); for(var i=0; i < 5000; i++) db.vertices.save({_key:i+""});
arangoimp --file generated_edges.csv --type csv --collection alt_edges --overwrite true --from-collection-prefix "vertices" --to-collection-prefix "vertices" --convert false --server.endpoint http+tcp://127.0.0.1:8530 arangoimport --file generated_edges.csv --type csv --collection alt_edges --overwrite true --from-collection-prefix "vertices" --to-collection-prefix "vertices" --convert false --server.endpoint http+tcp://127.0.0.1:8530
AQL script to copy edge collection into one with '_vertex': AQL script to copy edge collection into one with '_vertex':
FOR doc IN alt_edges FOR doc IN alt_edges
INSERT {_vertex:SUBSTRING(doc._from,FIND_FIRST(doc._from,"/")+1), INSERT {_vertex:SUBSTRING(doc._from,FIND_FIRST(doc._from,"/")+1),
_from:doc._from, _from:doc._from,
_to:doc._to} IN edges _to:doc._to} IN edges
LET values = ( LET values = (
@ -63,5 +63,5 @@ Make CSV file with arango compatible edges
cat edges.csv | awk -F" " '{print "profiles/" $1 "\tprofiles/" $2 "\t" $1}' >> arango-edges.csv cat edges.csv | awk -F" " '{print "profiles/" $1 "\tprofiles/" $2 "\t" $1}' >> arango-edges.csv
arangoimp --file vertices.csv --type csv --collection twitter_v --overwrite true --convert false --server.endpoint http+tcp://127.0.0.1:8530 -c none arangoimport --file vertices.csv --type csv --collection twitter_v --overwrite true --convert false --server.endpoint http+tcp://127.0.0.1:8530 -c none
arangoimp --file arango-edges.csv --type csv --collection twitter_e --overwrite true --convert false --separator "\t" --server.endpoint http+tcp://127.0.0.1:8530 -c none arangoimport --file arango-edges.csv --type csv --collection twitter_e --overwrite true --convert false --separator "\t" --server.endpoint http+tcp://127.0.0.1:8530 -c none

View File

@ -155,13 +155,13 @@ if (USE_JEMALLOC)
endif () endif ()
################################################################################ ################################################################################
## arangoimp ## arangoimport
################################################################################ ################################################################################
if (MSVC AND NOT(SKIP_PACKAGING)) if (MSVC AND NOT(SKIP_PACKAGING))
generate_product_version(ProductVersionFiles_arangoimp generate_product_version(ProductVersionFiles_arangoimport
NAME arangoimp NAME arangoimport
FILE_DESCRIPTION ${ARANGO_IMP_FRIENDLY_STRING} FILE_DESCRIPTION ${ARANGO_IMPORT_FRIENDLY_STRING}
ICON ${ARANGO_ICON} ICON ${ARANGO_ICON}
VERSION_MAJOR ${CPACK_PACKAGE_VERSION_MAJOR} VERSION_MAJOR ${CPACK_PACKAGE_VERSION_MAJOR}
VERSION_MINOR ${CPACK_PACKAGE_VERSION_MINOR} VERSION_MINOR ${CPACK_PACKAGE_VERSION_MINOR}
@ -170,19 +170,19 @@ if (MSVC AND NOT(SKIP_PACKAGING))
) )
endif () endif ()
add_executable(${BIN_ARANGOIMP} add_executable(${BIN_ARANGOIMPORT}
${ProductVersionFiles_arangoimp} ${ProductVersionFiles_arangoimport}
${PROJECT_SOURCE_DIR}/lib/Basics/WorkMonitorDummy.cpp ${PROJECT_SOURCE_DIR}/lib/Basics/WorkMonitorDummy.cpp
Import/ImportFeature.cpp Import/ImportFeature.cpp
Import/ImportHelper.cpp Import/ImportHelper.cpp
Import/SenderThread.cpp Import/SenderThread.cpp
Import/arangoimp.cpp Import/arangoimport.cpp
Shell/ClientFeature.cpp Shell/ClientFeature.cpp
Shell/ConsoleFeature.cpp Shell/ConsoleFeature.cpp
V8Client/ArangoClientHelper.cpp V8Client/ArangoClientHelper.cpp
) )
target_link_libraries(${BIN_ARANGOIMP} target_link_libraries(${BIN_ARANGOIMPORT}
${LIB_ARANGO} ${LIB_ARANGO}
${MSVC_LIBS} ${MSVC_LIBS}
${SYSTEM_LIBRARIES} ${SYSTEM_LIBRARIES}
@ -191,21 +191,26 @@ target_link_libraries(${BIN_ARANGOIMP}
) )
install( install(
TARGETS ${BIN_ARANGOIMP} TARGETS ${BIN_ARANGOIMPORT}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
install_config(arangoimp) install_config(arangoimport)
if (NOT USE_PRECOMPILED_V8) if (NOT USE_PRECOMPILED_V8)
add_dependencies(arangoimp zlibstatic v8_build) # v8_build includes ICU build add_dependencies(arangoimport zlibstatic v8_build) # v8_build includes ICU
# build
else () else ()
add_dependencies(arangoimp zlibstatic) # v8_build includes ICU build add_dependencies(arangoimport zlibstatic) # v8_build includes ICU build
endif () endif ()
if (USE_JEMALLOC) if (USE_JEMALLOC)
add_dependencies(arangoimp jemalloc) add_dependencies(arangoimport jemalloc)
endif () endif ()
install_command_alias(${BIN_ARANGOIMPORT}
${CMAKE_INSTALL_BINDIR}
arangoimp
)
################################################################################ ################################################################################
## arangorestore ## arangorestore
################################################################################ ################################################################################

View File

@ -47,21 +47,21 @@ int main(int argc, char* argv[]) {
context.installHup(); context.installHup();
std::shared_ptr<options::ProgramOptions> options(new options::ProgramOptions( std::shared_ptr<options::ProgramOptions> options(new options::ProgramOptions(
argv[0], "Usage: arangoimp [<options>]", "For more information use:", BIN_DIRECTORY)); argv[0], "Usage: arangoimport [<options>]", "For more information use:", BIN_DIRECTORY));
ApplicationServer server(options, BIN_DIRECTORY); ApplicationServer server(options, BIN_DIRECTORY);
int ret; int ret;
server.addFeature(new ClientFeature(&server)); server.addFeature(new ClientFeature(&server));
server.addFeature(new ConfigFeature(&server, "arangoimp")); server.addFeature(new ConfigFeature(&server, "arangoimport"));
server.addFeature(new ImportFeature(&server, &ret)); server.addFeature(new ImportFeature(&server, &ret));
server.addFeature(new LoggerFeature(&server, false)); server.addFeature(new LoggerFeature(&server, false));
server.addFeature(new RandomFeature(&server)); server.addFeature(new RandomFeature(&server));
server.addFeature(new ShellColorsFeature(&server)); server.addFeature(new ShellColorsFeature(&server));
server.addFeature(new ShutdownFeature(&server, {"Import"})); server.addFeature(new ShutdownFeature(&server, {"Import"}));
server.addFeature(new SslFeature(&server)); server.addFeature(new SslFeature(&server));
server.addFeature(new TempFeature(&server, "arangoimp")); server.addFeature(new TempFeature(&server, "arangoimport"));
server.addFeature(new VersionFeature(&server)); server.addFeature(new VersionFeature(&server));
try { try {
@ -72,11 +72,11 @@ int main(int argc, char* argv[]) {
} }
} catch (std::exception const& ex) { } catch (std::exception const& ex) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "arangoimp terminated because of an unhandled exception: " << ex.what(); << "arangoimport terminated because of an unhandled exception: " << ex.what();
ret = EXIT_FAILURE; ret = EXIT_FAILURE;
} catch (...) { } catch (...) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "arangoimp terminated because of an unhandled exception of unknown type"; << "arangoimport terminated because of an unhandled exception of unknown type";
ret = EXIT_FAILURE; ret = EXIT_FAILURE;
} }

View File

@ -18,7 +18,7 @@ install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}" "${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}" "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}"
"${BIN_ARANGOIMP}") "${BIN_ARANGOIMPORT}")
install_debinfo( install_debinfo(
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip" "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"

View File

@ -6,7 +6,7 @@ set(STRIP_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/cstrip")
add_custom_target(strip_install_client ALL) add_custom_target(strip_install_client ALL)
strip_install_bin_and_config(arangobench ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client) strip_install_bin_and_config(arangobench ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
strip_install_bin_and_config(arangodump ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client) strip_install_bin_and_config(arangodump ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
strip_install_bin_and_config(arangoimp ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client) strip_install_bin_and_config(arangoimport ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
strip_install_bin_and_config(arangorestore ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client) strip_install_bin_and_config(arangorestore ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
strip_install_bin_and_config(arangoexport ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client) strip_install_bin_and_config(arangoexport ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
strip_install_bin_and_config(arangosh ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client) strip_install_bin_and_config(arangosh ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
@ -16,4 +16,3 @@ install_command_alias(${BIN_ARANGOSH}
${CMAKE_INSTALL_BINDIR} ${CMAKE_INSTALL_BINDIR}
foxx-manager) foxx-manager)
install_config(foxx-manager) install_config(foxx-manager)

View File

@ -69,7 +69,7 @@ let BIN_DIR;
let ARANGOBENCH_BIN; let ARANGOBENCH_BIN;
let ARANGODUMP_BIN; let ARANGODUMP_BIN;
let ARANGOD_BIN; let ARANGOD_BIN;
let ARANGOIMP_BIN; let ARANGOIMPORT_BIN;
let ARANGORESTORE_BIN; let ARANGORESTORE_BIN;
let ARANGOEXPORT_BIN; let ARANGOEXPORT_BIN;
let ARANGOSH_BIN; let ARANGOSH_BIN;
@ -130,7 +130,7 @@ function setupBinaries (builddir, buildType, configDir) {
ARANGOBENCH_BIN = fs.join(BIN_DIR, 'arangobench' + executableExt); ARANGOBENCH_BIN = fs.join(BIN_DIR, 'arangobench' + executableExt);
ARANGODUMP_BIN = fs.join(BIN_DIR, 'arangodump' + executableExt); ARANGODUMP_BIN = fs.join(BIN_DIR, 'arangodump' + executableExt);
ARANGOD_BIN = fs.join(BIN_DIR, 'arangod' + executableExt); ARANGOD_BIN = fs.join(BIN_DIR, 'arangod' + executableExt);
ARANGOIMP_BIN = fs.join(BIN_DIR, 'arangoimp' + executableExt); ARANGOIMPORT_BIN = fs.join(BIN_DIR, 'arangoimport' + executableExt);
ARANGORESTORE_BIN = fs.join(BIN_DIR, 'arangorestore' + executableExt); ARANGORESTORE_BIN = fs.join(BIN_DIR, 'arangorestore' + executableExt);
ARANGOEXPORT_BIN = fs.join(BIN_DIR, 'arangoexport' + executableExt); ARANGOEXPORT_BIN = fs.join(BIN_DIR, 'arangoexport' + executableExt);
ARANGOSH_BIN = fs.join(BIN_DIR, 'arangosh' + executableExt); ARANGOSH_BIN = fs.join(BIN_DIR, 'arangosh' + executableExt);
@ -152,7 +152,7 @@ function setupBinaries (builddir, buildType, configDir) {
ARANGOBENCH_BIN, ARANGOBENCH_BIN,
ARANGODUMP_BIN, ARANGODUMP_BIN,
ARANGOD_BIN, ARANGOD_BIN,
ARANGOIMP_BIN, ARANGOIMPORT_BIN,
ARANGORESTORE_BIN, ARANGORESTORE_BIN,
ARANGOEXPORT_BIN, ARANGOEXPORT_BIN,
ARANGOSH_BIN]; ARANGOSH_BIN];
@ -529,10 +529,10 @@ function runArangoshCmd (options, instanceInfo, addArgs, cmds) {
} }
// ////////////////////////////////////////////////////////////////////////////// // //////////////////////////////////////////////////////////////////////////////
// / @brief runs arangoimp // / @brief runs arangoimport
// ////////////////////////////////////////////////////////////////////////////// // //////////////////////////////////////////////////////////////////////////////
function runArangoImp (options, instanceInfo, what) { function runArangoImport (options, instanceInfo, what) {
let args = { let args = {
'server.username': options.username, 'server.username': options.username,
'server.password': options.password, 'server.password': options.password,
@ -566,7 +566,7 @@ function runArangoImp (options, instanceInfo, what) {
args['remove-attribute'] = what.removeAttribute; args['remove-attribute'] = what.removeAttribute;
} }
return executeAndWait(ARANGOIMP_BIN, toArgv(args), options, 'arangoimp', instanceInfo.rootDir); return executeAndWait(ARANGOIMPORT_BIN, toArgv(args), options, 'arangoimport', instanceInfo.rootDir);
} }
// ////////////////////////////////////////////////////////////////////////////// // //////////////////////////////////////////////////////////////////////////////
@ -585,7 +585,7 @@ function runArangoDumpRestore (options, instanceInfo, which, database, rootDir,
let exe; let exe;
rootDir = rootDir || instanceInfo.rootDir; rootDir = rootDir || instanceInfo.rootDir;
if (which === 'dump') { if (which === 'dump') {
args['output-directory'] = fs.join(rootDir, dumpDir); args['output-directory'] = fs.join(rootDir, dumpDir);
exe = ARANGODUMP_BIN; exe = ARANGODUMP_BIN;
@ -594,9 +594,9 @@ function runArangoDumpRestore (options, instanceInfo, which, database, rootDir,
args['input-directory'] = fs.join(rootDir, dumpDir); args['input-directory'] = fs.join(rootDir, dumpDir);
exe = ARANGORESTORE_BIN; exe = ARANGORESTORE_BIN;
} }
if (options.encrypted) { if (options.encrypted) {
args['encryption.keyfile'] = fs.join(rootDir, 'secret-key'); args['encryption.keyfile'] = fs.join(rootDir, 'secret-key');
} }
if (options.extremeVerbosity === true) { if (options.extremeVerbosity === true) {
@ -970,7 +970,7 @@ function startInstanceCluster (instanceInfo, protocol, options,
} }
} else if (options.singleresilient) { } else if (options.singleresilient) {
// for now start just two (TODO config parameter) // for now start just two (TODO config parameter)
for (i = 0; i < 2; i++) { for (i = 0; i < 2; i++) {
let port = findFreePort(options.minPort, options.maxPort, usedPorts); let port = findFreePort(options.minPort, options.maxPort, usedPorts);
usedPorts.push(port); usedPorts.push(port);
let endpoint = protocol + '://127.0.0.1:' + port; let endpoint = protocol + '://127.0.0.1:' + port;
@ -1047,7 +1047,7 @@ function startInstanceCluster (instanceInfo, protocol, options,
const reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', makeAuthorizationHeaders(authOpts)); const reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', makeAuthorizationHeaders(authOpts));
if (!reply.error && reply.code === 200) { if (!reply.error && reply.code === 200) {
let res = JSON.parse(reply.body); let res = JSON.parse(reply.body);
internal.print("Response ====> " + reply.body); internal.print("Response ====> " + reply.body);
let leader = res.endpoints[0].endpoint; let leader = res.endpoints[0].endpoint;
instanceInfo.arangods.forEach(d => { instanceInfo.arangods.forEach(d => {
if (d.endpoint === leader) { if (d.endpoint === leader) {
@ -1056,7 +1056,7 @@ function startInstanceCluster (instanceInfo, protocol, options,
} }
}); });
} }
} }
arango.reconnect(instanceInfo.endpoint, '_system', 'root', ''); arango.reconnect(instanceInfo.endpoint, '_system', 'root', '');
return true; return true;
@ -1327,7 +1327,7 @@ exports.executeAndWait = executeAndWait;
exports.run = { exports.run = {
arangoshCmd: runArangoshCmd, arangoshCmd: runArangoshCmd,
arangoImp: runArangoImp, arangoImport: runArangoImport,
arangoDumpRestore: runArangoDumpRestore, arangoDumpRestore: runArangoDumpRestore,
arangoBenchmark: runArangoBenchmark arangoBenchmark: runArangoBenchmark
}; };

View File

@ -81,7 +81,7 @@ function config (options) {
'arangod', 'arangod',
'arangobench', 'arangobench',
'arangodump', 'arangodump',
'arangoimp', 'arangoimport',
'arangorestore', 'arangorestore',
'arangoexport', 'arangoexport',
'arangosh', 'arangosh',

View File

@ -214,7 +214,7 @@ function importing (options) {
for (let i = 0; i < impTodos.length; i++) { for (let i = 0; i < impTodos.length; i++) {
const impTodo = impTodos[i]; const impTodo = impTodos[i];
result[impTodo.id] = pu.run.arangoImp(options, instanceInfo, impTodo); result[impTodo.id] = pu.run.arangoImport(options, instanceInfo, impTodo);
result[impTodo.id].failed = 0; result[impTodo.id].failed = 0;
if (result[impTodo.id].status !== true && !options.force) { if (result[impTodo.id].status !== true && !options.force) {

View File

@ -143,6 +143,7 @@ void ConfigFeature::loadConfigFile(std::shared_ptr<ProgramOptions> options,
auto context = ArangoGlobalContext::CONTEXT; auto context = ArangoGlobalContext::CONTEXT;
std::string basename = progname; std::string basename = progname;
bool checkArangoImp = (progname == "arangoimport");
if (!StringUtils::isSuffix(basename, ".conf")) { if (!StringUtils::isSuffix(basename, ".conf")) {
basename += ".conf"; basename += ".conf";
@ -177,6 +178,14 @@ void ConfigFeature::loadConfigFile(std::shared_ptr<ProgramOptions> options,
LOG_TOPIC(DEBUG, Logger::CONFIG) << "found config file '" << name << "'"; LOG_TOPIC(DEBUG, Logger::CONFIG) << "found config file '" << name << "'";
filename = name; filename = name;
break; break;
} else if (checkArangoImp) {
name = FileUtils::buildFilename(location, "arangoimp");
LOG_TOPIC(TRACE, Logger::CONFIG) << "checking config file'" << name << "'";
if (FileUtils::exists(name)) {
LOG_TOPIC(DEBUG, Logger::CONFIG) << "found config file '" << name << "'";
filename = name;
break;
}
} }
} }

View File

@ -35,7 +35,7 @@ EOF
echo "" > $out echo "" > $out
progs="arangobench arangosh arangoimp arangodump arangorestore arangod" progs="arangobench arangosh arangoimport arangodump arangorestore arangod"
for progname in $progs for progname in $progs
do do
@ -43,14 +43,14 @@ for progname in $progs
# check if the executable exists # check if the executable exists
if [[ -f "build/bin/$progname" ]]; then if [[ -f "build/bin/$progname" ]]; then
executable="build/bin/$progname" executable="build/bin/$progname"
# setup the help command string # setup the help command string
command="--help-all" command="--help-all"
# set up the list of completions for the executable # set up the list of completions for the executable
completions="`\"$executable\" $command | grep -o \"^\\ \\+--[a-z-]\\+\\(\\.[a-z0-9-]\\+\\)\\?\" | xargs`" completions="`\"$executable\" $command | grep -o \"^\\ \\+--[a-z-]\\+\\(\\.[a-z0-9-]\\+\\)\\?\" | xargs`"
sed -e "s/PROGNAME/$progname/g" -e "s/PROGOPTS/$completions/g" /tmp/completions-template >> $out sed -e "s/PROGNAME/$progname/g" -e "s/PROGOPTS/$completions/g" /tmp/completions-template >> $out
fi fi
done done

View File

@ -8,7 +8,7 @@ out="$1"
echo "" > $out echo "" > $out
progs="arangobench arangosh arangoimp arangodump arangorestore arangod" progs="arangobench arangosh arangoimport arangodump arangorestore arangod"
for progname in $progs for progname in $progs
do do
@ -16,9 +16,9 @@ for progname in $progs
# check if the executable exists # check if the executable exists
if [[ -f "build/bin/$progname" ]]; then if [[ -f "build/bin/$progname" ]]; then
executable="build/bin/$progname" executable="build/bin/$progname"
# setup the help command string # setup the help command string
command="--help-all" command="--help-all"
# set up the list of completions for the executable # set up the list of completions for the executable
echo "# completions for $progname" >> "$out" echo "# completions for $progname" >> "$out"