mirror of https://gitee.com/bigwinds/arangodb
Renamed arangoimp to arangoimport (with alias for compatibility.) (#4040)
This commit is contained in:
parent
db9b7189f6
commit
9cd8c47eb7
|
@ -99,7 +99,7 @@ set(ARANGO_BENCH_FRIENDLY_STRING "arangobench - stress test program")
|
||||||
set(ARANGO_DUMP_FRIENDLY_STRING "arangodump - export")
|
set(ARANGO_DUMP_FRIENDLY_STRING "arangodump - export")
|
||||||
set(ARANGO_RESTORE_FRIENDLY_STRING "arangrestore - importer")
|
set(ARANGO_RESTORE_FRIENDLY_STRING "arangrestore - importer")
|
||||||
set(ARANGO_EXPORT_FRIENDLY_STRING "arangoexport - datae xporter")
|
set(ARANGO_EXPORT_FRIENDLY_STRING "arangoexport - datae xporter")
|
||||||
set(ARANGO_IMP_FRIENDLY_STRING "arangoimp - TSV/CSV/JSON importer")
|
set(ARANGO_IMPORT_FRIENDLY_STRING "arangoimport - TSV/CSV/JSON importer")
|
||||||
set(ARANGOSH_FRIENDLY_STRING "arangosh - commandline client")
|
set(ARANGOSH_FRIENDLY_STRING "arangosh - commandline client")
|
||||||
set(ARANGO_VPACK_FRIENDLY_STRING "arangovpack - vpack printer")
|
set(ARANGO_VPACK_FRIENDLY_STRING "arangovpack - vpack printer")
|
||||||
|
|
||||||
|
@ -112,7 +112,7 @@ set(BIN_ARANGOBENCH arangobench)
|
||||||
set(BIN_ARANGOD arangod)
|
set(BIN_ARANGOD arangod)
|
||||||
set(BIN_ARANGODUMP arangodump)
|
set(BIN_ARANGODUMP arangodump)
|
||||||
set(BIN_ARANGOEXPORT arangoexport)
|
set(BIN_ARANGOEXPORT arangoexport)
|
||||||
set(BIN_ARANGOIMP arangoimp)
|
set(BIN_ARANGOIMPORT arangoimport)
|
||||||
set(BIN_ARANGORESTORE arangorestore)
|
set(BIN_ARANGORESTORE arangorestore)
|
||||||
set(BIN_ARANGOSH arangosh)
|
set(BIN_ARANGOSH arangosh)
|
||||||
set(BIN_ARANGOVPACK arangovpack)
|
set(BIN_ARANGOVPACK arangovpack)
|
||||||
|
@ -1009,7 +1009,7 @@ add_dependencies(arangobench zlibstatic)
|
||||||
add_dependencies(arangod zlibstatic)
|
add_dependencies(arangod zlibstatic)
|
||||||
add_dependencies(arangodump zlibstatic)
|
add_dependencies(arangodump zlibstatic)
|
||||||
add_dependencies(arangoexport zlibstatic)
|
add_dependencies(arangoexport zlibstatic)
|
||||||
add_dependencies(arangoimp zlibstatic)
|
add_dependencies(arangoimport zlibstatic)
|
||||||
add_dependencies(arangorestore zlibstatic)
|
add_dependencies(arangorestore zlibstatic)
|
||||||
add_dependencies(arangosh zlibstatic)
|
add_dependencies(arangosh zlibstatic)
|
||||||
|
|
||||||
|
@ -1019,7 +1019,7 @@ if (NOT USE_PRECOMPILED_V8)
|
||||||
add_dependencies(arangod v8_build)
|
add_dependencies(arangod v8_build)
|
||||||
add_dependencies(arangodump v8_build)
|
add_dependencies(arangodump v8_build)
|
||||||
add_dependencies(arangoexport v8_build)
|
add_dependencies(arangoexport v8_build)
|
||||||
add_dependencies(arangoimp v8_build)
|
add_dependencies(arangoimport v8_build)
|
||||||
add_dependencies(arangorestore v8_build)
|
add_dependencies(arangorestore v8_build)
|
||||||
add_dependencies(arangosh v8_build)
|
add_dependencies(arangosh v8_build)
|
||||||
if (USE_CATCH_TESTS)
|
if (USE_CATCH_TESTS)
|
||||||
|
|
|
@ -9,10 +9,10 @@ I want to import data from a file into ArangoDB.
|
||||||
Solution
|
Solution
|
||||||
--------
|
--------
|
||||||
|
|
||||||
ArangoDB comes with a command-line tool utility named `arangoimp`. This utility can be
|
ArangoDB comes with a command-line tool utility named `arangoimport`. This utility can be
|
||||||
used for importing JSON-encoded, CSV, and tab-separated files into ArangoDB.
|
used for importing JSON-encoded, CSV, and tab-separated files into ArangoDB.
|
||||||
|
|
||||||
`arangoimp` needs to be invoked from the command-line once for each import file.
|
`arangoimport` needs to be invoked from the command-line once for each import file.
|
||||||
The target collection can already exist or can be created by the import run.
|
The target collection can already exist or can be created by the import run.
|
||||||
|
|
||||||
### Importing JSON-encoded data
|
### Importing JSON-encoded data
|
||||||
|
@ -59,7 +59,7 @@ Here's an example for the same data in **array format**:
|
||||||
|
|
||||||
#### Importing JSON data in line-by-line format
|
#### Importing JSON data in line-by-line format
|
||||||
|
|
||||||
An example data file in **line-by-line format** can be downloaded
|
An example data file in **line-by-line format** can be downloaded
|
||||||
[here](http://jsteemann.github.io/downloads/code/git-commits-single-line.json). The example
|
[here](http://jsteemann.github.io/downloads/code/git-commits-single-line.json). The example
|
||||||
file contains all the commits to the ArangoDB repository as shown by `git log --reverse`.
|
file contains all the commits to the ArangoDB repository as shown by `git log --reverse`.
|
||||||
|
|
||||||
|
@ -70,10 +70,10 @@ The following commands will import the data from the file into a collection name
|
||||||
wget http://jsteemann.github.io/downloads/code/git-commits-single-line.json
|
wget http://jsteemann.github.io/downloads/code/git-commits-single-line.json
|
||||||
|
|
||||||
# actually import data
|
# actually import data
|
||||||
arangoimp --file git-commits-single-line.json --collection commits --create-collection true
|
arangoimport --file git-commits-single-line.json --collection commits --create-collection true
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that no file type has been specified when `arangoimp` was invoked. This is because `json`
|
Note that no file type has been specified when `arangoimport` was invoked. This is because `json`
|
||||||
is its default input format.
|
is its default input format.
|
||||||
|
|
||||||
The other parameters used have the following meanings:
|
The other parameters used have the following meanings:
|
||||||
|
@ -82,7 +82,7 @@ The other parameters used have the following meanings:
|
||||||
- `collection`: name of the target collection
|
- `collection`: name of the target collection
|
||||||
- `create-collection`: whether or not the collection should be created if it does not exist
|
- `create-collection`: whether or not the collection should be created if it does not exist
|
||||||
|
|
||||||
The result of the import printed by `arangoimp` should be:
|
The result of the import printed by `arangoimport` should be:
|
||||||
|
|
||||||
```
|
```
|
||||||
created: 20039
|
created: 20039
|
||||||
|
@ -103,13 +103,13 @@ The command for importing JSON data in **array format** is similar to what we've
|
||||||
wget http://jsteemann.github.io/downloads/code/git-commits-array.json
|
wget http://jsteemann.github.io/downloads/code/git-commits-array.json
|
||||||
|
|
||||||
# actually import data
|
# actually import data
|
||||||
arangoimp --file git-commits-array.json --collection commits --create-collection true
|
arangoimport --file git-commits-array.json --collection commits --create-collection true
|
||||||
```
|
```
|
||||||
|
|
||||||
Though the import command is the same (except the filename), there is a notable difference between the
|
Though the import command is the same (except the filename), there is a notable difference between the
|
||||||
two JSON formats: for the **array format**, `arangoimp` will read and parse the JSON in its entirety
|
two JSON formats: for the **array format**, `arangoimport` will read and parse the JSON in its entirety
|
||||||
before it sends any data to the ArangoDB server. That means the whole input file must fit into
|
before it sends any data to the ArangoDB server. That means the whole input file must fit into
|
||||||
`arangoimp`'s buffer. By default, `arangoimp` will allocate a 16 MiB internal buffer, and input files bigger
|
`arangoimport`'s buffer. By default, `arangoimport` will allocate a 16 MiB internal buffer, and input files bigger
|
||||||
than that will be rejected with the following message:
|
than that will be rejected with the following message:
|
||||||
|
|
||||||
```
|
```
|
||||||
|
@ -132,7 +132,7 @@ The `--type` parameter for the import command must now be set to `csv`:
|
||||||
wget http://jsteemann.github.io/downloads/code/git-commits.csv
|
wget http://jsteemann.github.io/downloads/code/git-commits.csv
|
||||||
|
|
||||||
# actually import data
|
# actually import data
|
||||||
arangoimp --file git-commits.csv --type csv --collection commits --create-collection true
|
arangoimport --file git-commits.csv --type csv --collection commits --create-collection true
|
||||||
```
|
```
|
||||||
|
|
||||||
For the CSV import, the first line in the input file has a special meaning: every value listed in the
|
For the CSV import, the first line in the input file has a special meaning: every value listed in the
|
||||||
|
@ -141,9 +141,9 @@ lines should also have the same number of "columns".
|
||||||
|
|
||||||
"columns" inside the CSV input file can be left empty though. If a "column" is left empty in a line,
|
"columns" inside the CSV input file can be left empty though. If a "column" is left empty in a line,
|
||||||
then this value will be omitted for the import so the respective attribute will not be set in the imported
|
then this value will be omitted for the import so the respective attribute will not be set in the imported
|
||||||
document. Note that values from the input file that are enclosed in double quotes will always be imported as
|
document. Note that values from the input file that are enclosed in double quotes will always be imported as
|
||||||
strings. To import numeric values, boolean values or the `null` value, don't enclose these values in quotes in
|
strings. To import numeric values, boolean values or the `null` value, don't enclose these values in quotes in
|
||||||
the input file. Note that leading zeros in numeric values will be removed. Importing numbers with leading
|
the input file. Note that leading zeros in numeric values will be removed. Importing numbers with leading
|
||||||
zeros will only work when putting the numbers into strings.
|
zeros will only work when putting the numbers into strings.
|
||||||
|
|
||||||
Here is an example CSV file:
|
Here is an example CSV file:
|
||||||
|
@ -155,11 +155,11 @@ Here is an example CSV file:
|
||||||
...
|
...
|
||||||
```
|
```
|
||||||
|
|
||||||
`arangoimp` supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might also occur inside values
|
`arangoimport` supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might also occur inside values
|
||||||
that are enclosed with the quote character.
|
that are enclosed with the quote character.
|
||||||
|
|
||||||
The default separator for CSV files is the comma. It can be changed using the `--separator` parameter
|
The default separator for CSV files is the comma. It can be changed using the `--separator` parameter
|
||||||
when invoking `arangoimp`. The quote character defaults to the double quote (**"**). To use a literal double
|
when invoking `arangoimport`. The quote character defaults to the double quote (**"**). To use a literal double
|
||||||
quote inside a "column" in the import data, use two double quotes. To change the quote character, use the
|
quote inside a "column" in the import data, use two double quotes. To change the quote character, use the
|
||||||
`--quote` parameter. To use a backslash for escaping quote characters, please set the option `--backslash-escape`
|
`--quote` parameter. To use a backslash for escaping quote characters, please set the option `--backslash-escape`
|
||||||
to `true`.
|
to `true`.
|
||||||
|
@ -167,7 +167,7 @@ to `true`.
|
||||||
|
|
||||||
### Changing the database and server endpoint
|
### Changing the database and server endpoint
|
||||||
|
|
||||||
By default, `arangoimp` will connect to the default database on `127.0.0.1:8529` with a user named
|
By default, `arangoimport` will connect to the default database on `127.0.0.1:8529` with a user named
|
||||||
`root`. To change this, use the following parameters:
|
`root`. To change this, use the following parameters:
|
||||||
|
|
||||||
- `server.database`: name of the database to use when importing (default: `_system`)
|
- `server.database`: name of the database to use when importing (default: `_system`)
|
||||||
|
@ -176,8 +176,8 @@ By default, `arangoimp` will connect to the default database on `127.0.0.1:8529`
|
||||||
|
|
||||||
### Using authentication
|
### Using authentication
|
||||||
|
|
||||||
`arangoimp` will by default send an username `root` and an empty password to the ArangoDB
|
`arangoimport` will by default send an username `root` and an empty password to the ArangoDB
|
||||||
server. This is ArangoDB's default configuration, and it should be changed. To make `arangoimp`
|
server. This is ArangoDB's default configuration, and it should be changed. To make `arangoimport`
|
||||||
use a different username or password, the following command-line arguments can be used:
|
use a different username or password, the following command-line arguments can be used:
|
||||||
|
|
||||||
- `server.username`: username, used if authentication is enabled on server
|
- `server.username`: username, used if authentication is enabled on server
|
||||||
|
@ -185,16 +185,16 @@ use a different username or password, the following command-line arguments can b
|
||||||
|
|
||||||
The password argument can also be omitted in order to avoid having it saved in the shell's
|
The password argument can also be omitted in order to avoid having it saved in the shell's
|
||||||
command-line history. When specifying a username but omitting the password parameter,
|
command-line history. When specifying a username but omitting the password parameter,
|
||||||
`arangoimp` will prompt for a password.
|
`arangoimport` will prompt for a password.
|
||||||
|
|
||||||
|
|
||||||
### Additional parameters
|
### Additional parameters
|
||||||
|
|
||||||
By default, `arangoimp` will import data into the specified collection but will not touch
|
By default, `arangoimport` will import data into the specified collection but will not touch
|
||||||
existing data. Often it is convenient to first remove all data from a collection and then run
|
existing data. Often it is convenient to first remove all data from a collection and then run
|
||||||
the import. `arangoimp` supports this with the optional `--overwrite` flag. When setting it to
|
the import. `arangoimport` supports this with the optional `--overwrite` flag. When setting it to
|
||||||
`true`, all documents in the collection will be removed prior to the import.
|
`true`, all documents in the collection will be removed prior to the import.
|
||||||
|
|
||||||
**Author:** [Jan Steemann](https://github.com/jsteemann)
|
**Author:** [Jan Steemann](https://github.com/jsteemann)
|
||||||
|
|
||||||
**Tags**: #arangoimp #import
|
**Tags**: #arangoimport #import
|
||||||
|
|
|
@ -17,8 +17,8 @@ First of all we need some data in an ArangoDB collection. For this example we wi
|
||||||
wget https://jsteemann.github.io/downloads/code/users-100000.json.tar.gz
|
wget https://jsteemann.github.io/downloads/code/users-100000.json.tar.gz
|
||||||
# uncompress it
|
# uncompress it
|
||||||
tar xvfz users-100000.json.tar.gz
|
tar xvfz users-100000.json.tar.gz
|
||||||
# import into ArangoDB
|
# import into ArangoDB
|
||||||
arangoimp --file users-100000.json --collection users --create-collection true
|
arangoimport --file users-100000.json --collection users --create-collection true
|
||||||
```
|
```
|
||||||
|
|
||||||
### Setting up ArangoDB-PHP
|
### Setting up ArangoDB-PHP
|
||||||
|
@ -54,7 +54,7 @@ $connectionOptions = array(
|
||||||
ConnectionOptions::OPTION_AUTH_PASSWD => '',
|
ConnectionOptions::OPTION_AUTH_PASSWD => '',
|
||||||
// timeout in seconds
|
// timeout in seconds
|
||||||
ConnectionOptions::OPTION_TIMEOUT => 30,
|
ConnectionOptions::OPTION_TIMEOUT => 30,
|
||||||
// database name
|
// database name
|
||||||
ConnectionOptions::OPTION_DATABASE => '_system'
|
ConnectionOptions::OPTION_DATABASE => '_system'
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -262,7 +262,7 @@ function export($collection, Connection $connection) {
|
||||||
$output .= '"' . implode('","', $values) . '"' . PHP_EOL;
|
$output .= '"' . implode('","', $values) . '"' . PHP_EOL;
|
||||||
}
|
}
|
||||||
|
|
||||||
// print out the data directly
|
// print out the data directly
|
||||||
print $output;
|
print $output;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
Arangoimp
|
Arangoimport
|
||||||
=========
|
============
|
||||||
|
|
||||||
This manual describes the ArangoDB importer _arangoimp_, which can be used for
|
This manual describes the ArangoDB importer _arangoimport_, which can be used for
|
||||||
bulk imports.
|
bulk imports.
|
||||||
|
|
||||||
The most convenient method to import a lot of data into ArangoDB is to use the
|
The most convenient method to import a lot of data into ArangoDB is to use the
|
||||||
*arangoimp* command-line tool. It allows you to import data records from a file
|
*arangoimport* command-line tool. It allows you to import data records from a file
|
||||||
into an existing database collection.
|
into an existing database collection.
|
||||||
|
|
||||||
It is possible to import [document keys](../Appendix/Glossary.md#document-key) with the documents using the *_key*
|
It is possible to import [document keys](../Appendix/Glossary.md#document-key) with the documents using the *_key*
|
||||||
|
@ -13,7 +13,7 @@ attribute. When importing into an [edge collection](../Appendix/Glossary.md#edge
|
||||||
imported documents have the *_from* and *_to* attributes, and that they contain
|
imported documents have the *_from* and *_to* attributes, and that they contain
|
||||||
valid references.
|
valid references.
|
||||||
|
|
||||||
Let's assume for the following examples you want to import user data into an
|
Let's assume for the following examples you want to import user data into an
|
||||||
existing collection named "users" on the server.
|
existing collection named "users" on the server.
|
||||||
|
|
||||||
|
|
||||||
|
@ -34,25 +34,25 @@ example user records to import:
|
||||||
To import these records, all you need to do is to put them into a file (with one
|
To import these records, all you need to do is to put them into a file (with one
|
||||||
line for each record to import) and run the following command:
|
line for each record to import) and run the following command:
|
||||||
|
|
||||||
> arangoimp --file "data.json" --type jsonl --collection "users"
|
> arangoimport --file "data.json" --type jsonl --collection "users"
|
||||||
|
|
||||||
This will transfer the data to the server, import the records, and print a
|
This will transfer the data to the server, import the records, and print a
|
||||||
status summary. To show the intermediate progress during the import process, the
|
status summary. To show the intermediate progress during the import process, the
|
||||||
option *--progress* can be added. This option will show the percentage of the
|
option *--progress* can be added. This option will show the percentage of the
|
||||||
input file that has been sent to the server. This will only be useful for big
|
input file that has been sent to the server. This will only be useful for big
|
||||||
import files.
|
import files.
|
||||||
|
|
||||||
> arangoimp --file "data.json" --type json --collection users --progress true
|
> arangoimport --file "data.json" --type json --collection users --progress true
|
||||||
|
|
||||||
It is also possible to use the output of another command as an input for arangoimp.
|
It is also possible to use the output of another command as an input for arangoimport.
|
||||||
For example, the following shell command can be used to pipe data from the `cat`
|
For example, the following shell command can be used to pipe data from the `cat`
|
||||||
process to arangoimp:
|
process to arangoimport:
|
||||||
|
|
||||||
> cat data.json | arangoimp --file - --type json --collection users
|
> cat data.json | arangoimport --file - --type json --collection users
|
||||||
|
|
||||||
Note that you have to use `--file -` if you want to use another command as input
|
Note that you have to use `--file -` if you want to use another command as input
|
||||||
for arangoimp. No progress can be reported for such imports as the size of the input
|
for arangoimport. No progress can be reported for such imports as the size of the input
|
||||||
will be unknown to arangoimp.
|
will be unknown to arangoimport.
|
||||||
|
|
||||||
By default, the endpoint *tcp://127.0.0.1:8529* will be used. If you want to
|
By default, the endpoint *tcp://127.0.0.1:8529* will be used. If you want to
|
||||||
specify a different endpoint, you can use the *--server.endpoint* option. You
|
specify a different endpoint, you can use the *--server.endpoint* option. You
|
||||||
|
@ -60,43 +60,43 @@ probably want to specify a database user and password as well. You can do so by
|
||||||
using the options *--server.username* and *--server.password*. If you do not
|
using the options *--server.username* and *--server.password*. If you do not
|
||||||
specify a password, you will be prompted for one.
|
specify a password, you will be prompted for one.
|
||||||
|
|
||||||
> arangoimp --server.endpoint tcp://127.0.0.1:8529 --server.username root --file "data.json" --type json --collection "users"
|
> arangoimport --server.endpoint tcp://127.0.0.1:8529 --server.username root --file "data.json" --type json --collection "users"
|
||||||
|
|
||||||
Note that the collection (*users* in this case) must already exist or the import
|
Note that the collection (*users* in this case) must already exist or the import
|
||||||
will fail. If you want to create a new collection with the import data, you need
|
will fail. If you want to create a new collection with the import data, you need
|
||||||
to specify the *--create-collection* option. Note that by default it will create
|
to specify the *--create-collection* option. Note that by default it will create
|
||||||
a document collection and no edge collection.
|
a document collection and no edge collection.
|
||||||
|
|
||||||
> arangoimp --file "data.json" --type json --collection "users" --create-collection true
|
> arangoimport --file "data.json" --type json --collection "users" --create-collection true
|
||||||
|
|
||||||
To create an edge collection instead, use the *--create-collection-type* option
|
To create an edge collection instead, use the *--create-collection-type* option
|
||||||
and set it to *edge*:
|
and set it to *edge*:
|
||||||
|
|
||||||
> arangoimp --file "data.json" --collection "myedges" --create-collection true --create-collection-type edge
|
> arangoimport --file "data.json" --collection "myedges" --create-collection true --create-collection-type edge
|
||||||
|
|
||||||
When importing data into an existing collection it is often convenient to first
|
When importing data into an existing collection it is often convenient to first
|
||||||
remove all data from the collection and then start the import. This can be achieved
|
remove all data from the collection and then start the import. This can be achieved
|
||||||
by passing the *--overwrite* parameter to _arangoimp_. If it is set to *true*,
|
by passing the *--overwrite* parameter to _arangoimport_. If it is set to *true*,
|
||||||
any existing data in the collection will be removed prior to the import. Note
|
any existing data in the collection will be removed prior to the import. Note
|
||||||
that any existing index definitions for the collection will be preserved even if
|
that any existing index definitions for the collection will be preserved even if
|
||||||
*--overwrite* is set to true.
|
*--overwrite* is set to true.
|
||||||
|
|
||||||
> arangoimp --file "data.json" --type json --collection "users" --overwrite true
|
> arangoimport --file "data.json" --type json --collection "users" --overwrite true
|
||||||
|
|
||||||
As the import file already contains the data in JSON format, attribute names and
|
As the import file already contains the data in JSON format, attribute names and
|
||||||
data types are fully preserved. As can be seen in the example data, there is no
|
data types are fully preserved. As can be seen in the example data, there is no
|
||||||
need for all data records to have the same attribute names or types. Records can
|
need for all data records to have the same attribute names or types. Records can
|
||||||
be inhomogeneous.
|
be inhomogeneous.
|
||||||
|
|
||||||
Please note that by default, _arangoimp_ will import data into the specified
|
Please note that by default, _arangoimport_ will import data into the specified
|
||||||
collection in the default database (*_system*). To specify a different database,
|
collection in the default database (*_system*). To specify a different database,
|
||||||
use the *--server.database* option when invoking _arangoimp_.
|
use the *--server.database* option when invoking _arangoimport_.
|
||||||
|
|
||||||
The tool also supports parallel imports, with multiple threads. Using multiple
|
The tool also supports parallel imports, with multiple threads. Using multiple
|
||||||
threads may provide a speedup, especially when using the RocksDB storage engine.
|
threads may provide a speedup, especially when using the RocksDB storage engine.
|
||||||
To specify the number of parallel threads use the `--threads` option:
|
To specify the number of parallel threads use the `--threads` option:
|
||||||
|
|
||||||
> arangoimp --threads 4 --file "data.json" --type json --collection "users"
|
> arangoimport --threads 4 --file "data.json" --type json --collection "users"
|
||||||
|
|
||||||
Note that using multiple threads may lead to a non-sequential import of the input
|
Note that using multiple threads may lead to a non-sequential import of the input
|
||||||
data. Data that appears later in the input file may be imported earlier than data
|
data. Data that appears later in the input file may be imported earlier than data
|
||||||
|
@ -106,7 +106,7 @@ this case, the number of threads should be set to 1.
|
||||||
|
|
||||||
### JSON input file formats
|
### JSON input file formats
|
||||||
|
|
||||||
**Note**: *arangoimp* supports two formats when importing JSON data from
|
**Note**: *arangoimport* supports two formats when importing JSON data from
|
||||||
a file. The first format that we also used above is commonly known as [jsonl](http://jsonlines.org)).
|
a file. The first format that we also used above is commonly known as [jsonl](http://jsonlines.org)).
|
||||||
However, in contrast to the JSONL specification it requires the input file to contain
|
However, in contrast to the JSONL specification it requires the input file to contain
|
||||||
one complete JSON document in each line, e.g.
|
one complete JSON document in each line, e.g.
|
||||||
|
@ -120,7 +120,7 @@ one complete JSON document in each line, e.g.
|
||||||
|
|
||||||
So one could argue that this is only a subset of JSONL.
|
So one could argue that this is only a subset of JSONL.
|
||||||
|
|
||||||
The above format can be imported sequentially by _arangoimp_. It will read data
|
The above format can be imported sequentially by _arangoimport_. It will read data
|
||||||
from the input file in chunks and send it in batches to the server. Each batch
|
from the input file in chunks and send it in batches to the server. Each batch
|
||||||
will be about as big as specified in the command-line parameter *--batch-size*.
|
will be about as big as specified in the command-line parameter *--batch-size*.
|
||||||
|
|
||||||
|
@ -135,17 +135,17 @@ An alternative is to put one big JSON document into the input file like this:
|
||||||
]
|
]
|
||||||
```
|
```
|
||||||
|
|
||||||
This format allows line breaks within the input file as required. The downside
|
This format allows line breaks within the input file as required. The downside
|
||||||
is that the whole input file will need to be read by _arangoimp_ before it can
|
is that the whole input file will need to be read by _arangoimport_ before it can
|
||||||
send the first batch. This might be a problem if the input file is big. By
|
send the first batch. This might be a problem if the input file is big. By
|
||||||
default, _arangoimp_ will allow importing such files up to a size of about 16 MB.
|
default, _arangoimport_ will allow importing such files up to a size of about 16 MB.
|
||||||
|
|
||||||
If you want to allow your _arangoimp_ instance to use more memory, you may want
|
If you want to allow your _arangoimport_ instance to use more memory, you may want
|
||||||
to increase the maximum file size by specifying the command-line option
|
to increase the maximum file size by specifying the command-line option
|
||||||
*--batch-size*. For example, to set the batch size to 32 MB, use the following
|
*--batch-size*. For example, to set the batch size to 32 MB, use the following
|
||||||
command:
|
command:
|
||||||
|
|
||||||
> arangoimp --file "data.json" --type json --collection "users" --batch-size 33554432
|
> arangoimport --file "data.json" --type json --collection "users" --batch-size 33554432
|
||||||
|
|
||||||
Please also note that you may need to increase the value of *--batch-size* if
|
Please also note that you may need to increase the value of *--batch-size* if
|
||||||
a single document inside the input file is bigger than the value of *--batch-size*.
|
a single document inside the input file is bigger than the value of *--batch-size*.
|
||||||
|
@ -153,7 +153,7 @@ a single document inside the input file is bigger than the value of *--batch-siz
|
||||||
|
|
||||||
### Importing CSV Data
|
### Importing CSV Data
|
||||||
|
|
||||||
_arangoimp_ also offers the possibility to import data from CSV files. This
|
_arangoimport_ also offers the possibility to import data from CSV files. This
|
||||||
comes handy when the data at hand is in CSV format already and you don't want to
|
comes handy when the data at hand is in CSV format already and you don't want to
|
||||||
spend time converting them to JSON for the import.
|
spend time converting them to JSON for the import.
|
||||||
|
|
||||||
|
@ -183,16 +183,16 @@ Wayne,Brewer,,false,
|
||||||
|
|
||||||
The command line to execute the import is:
|
The command line to execute the import is:
|
||||||
|
|
||||||
> arangoimp --file "data.csv" --type csv --collection "users"
|
> arangoimport --file "data.csv" --type csv --collection "users"
|
||||||
|
|
||||||
The above data will be imported into 5 documents which will look as follows:
|
The above data will be imported into 5 documents which will look as follows:
|
||||||
|
|
||||||
```js
|
```js
|
||||||
{ "first" : "John", "last" : "Connor", "active" : true, "age" : 25 }
|
{ "first" : "John", "last" : "Connor", "active" : true, "age" : 25 }
|
||||||
{ "first" : "Jim", "last" : "O'Brady", "age" : 19 }
|
{ "first" : "Jim", "last" : "O'Brady", "age" : 19 }
|
||||||
{ "first" : "Lisa", "last" : "Jones", "dob" : "1981-04-09" }
|
{ "first" : "Lisa", "last" : "Jones", "dob" : "1981-04-09" }
|
||||||
{ "first" : "Hans", "last" : "dos Santos", "age" : 123 }
|
{ "first" : "Hans", "last" : "dos Santos", "age" : 123 }
|
||||||
{ "first" : "Wayne", "last" : "Brewer", "active" : false }
|
{ "first" : "Wayne", "last" : "Brewer", "active" : false }
|
||||||
```
|
```
|
||||||
|
|
||||||
As can be seen, values left completely empty in the input file will be treated
|
As can be seen, values left completely empty in the input file will be treated
|
||||||
|
@ -210,10 +210,10 @@ escaped with another quote character (or with a backslash if the *--backslash-es
|
||||||
option is used).
|
option is used).
|
||||||
|
|
||||||
Note that the quote and separator characters can be adjusted via the
|
Note that the quote and separator characters can be adjusted via the
|
||||||
*--quote* and *--separator* arguments when invoking _arangoimp_. The quote
|
*--quote* and *--separator* arguments when invoking _arangoimport_. The quote
|
||||||
character defaults to the double quote (*"*). To use a literal quote in a
|
character defaults to the double quote (*"*). To use a literal quote in a
|
||||||
string, you can use two quote characters.
|
string, you can use two quote characters.
|
||||||
To use backslash for escaping quote characters, please set the option
|
To use backslash for escaping quote characters, please set the option
|
||||||
*--backslash-escape* to *true*.
|
*--backslash-escape* to *true*.
|
||||||
|
|
||||||
The importer supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might
|
The importer supports Windows (CRLF) and Unix (LF) line breaks. Line breaks might
|
||||||
|
@ -230,8 +230,8 @@ multine password!"
|
||||||
"Bartholomew ""Bart"" Simpson","Milhouse"
|
"Bartholomew ""Bart"" Simpson","Milhouse"
|
||||||
```
|
```
|
||||||
|
|
||||||
Extra whitespace at the end of each line will be ignored. Whitespace at the
|
Extra whitespace at the end of each line will be ignored. Whitespace at the
|
||||||
start of lines or between field values will not be ignored, so please make sure
|
start of lines or between field values will not be ignored, so please make sure
|
||||||
that there is no extra whitespace in front of values or between them.
|
that there is no extra whitespace in front of values or between them.
|
||||||
|
|
||||||
|
|
||||||
|
@ -246,11 +246,11 @@ As with CSV, the first line in the TSV file must contain the attribute names,
|
||||||
and all lines must have an identical number of values.
|
and all lines must have an identical number of values.
|
||||||
|
|
||||||
If a different separator character or string should be used, it can be specified
|
If a different separator character or string should be used, it can be specified
|
||||||
with the *--separator* argument.
|
with the *--separator* argument.
|
||||||
|
|
||||||
An example command line to execute the TSV import is:
|
An example command line to execute the TSV import is:
|
||||||
|
|
||||||
> arangoimp --file "data.tsv" --type tsv --collection "users"
|
> arangoimport --file "data.tsv" --type tsv --collection "users"
|
||||||
|
|
||||||
|
|
||||||
### Attribute Name Translation
|
### Attribute Name Translation
|
||||||
|
@ -261,41 +261,41 @@ that should be used in ArangoDB.
|
||||||
|
|
||||||
A common use case is to rename an "id" column from the input file into "_key" as
|
A common use case is to rename an "id" column from the input file into "_key" as
|
||||||
it is expected by ArangoDB. To do this, specify the following translation when
|
it is expected by ArangoDB. To do this, specify the following translation when
|
||||||
invoking arangoimp:
|
invoking arangoimport:
|
||||||
|
|
||||||
> arangoimp --file "data.csv" --type csv --translate "id=_key"
|
> arangoimport --file "data.csv" --type csv --translate "id=_key"
|
||||||
|
|
||||||
Other common cases are to rename columns in the input file to *_from* and *_to*:
|
Other common cases are to rename columns in the input file to *_from* and *_to*:
|
||||||
|
|
||||||
> arangoimp --file "data.csv" --type csv --translate "from=_from" --translate "to=_to"
|
> arangoimport --file "data.csv" --type csv --translate "from=_from" --translate "to=_to"
|
||||||
|
|
||||||
The *translate* option can be specified multiple types. The source attribute name
|
The *translate* option can be specified multiple types. The source attribute name
|
||||||
and the target attribute must be separated with a *=*.
|
and the target attribute must be separated with a *=*.
|
||||||
|
|
||||||
|
|
||||||
### Ignoring Attributes
|
### Ignoring Attributes
|
||||||
|
|
||||||
|
|
||||||
For the CSV and TSV input formats, certain attribute names can be ignored on imports.
|
For the CSV and TSV input formats, certain attribute names can be ignored on imports.
|
||||||
In an ArangoDB cluster there are cases where this can come in handy,
|
In an ArangoDB cluster there are cases where this can come in handy,
|
||||||
when your documents already contain a `_key` attribute
|
when your documents already contain a `_key` attribute
|
||||||
and your collection has a sharding attribute other than `_key`: In the cluster this
|
and your collection has a sharding attribute other than `_key`: In the cluster this
|
||||||
configuration is not supported, because ArangoDB needs to guarantee the uniqueness of the `_key`
|
configuration is not supported, because ArangoDB needs to guarantee the uniqueness of the `_key`
|
||||||
attribute in *all* shards of the collection.
|
attribute in *all* shards of the collection.
|
||||||
|
|
||||||
> arangoimp --file "data.csv" --type csv --remove-attribute "_key"
|
> arangoimport --file "data.csv" --type csv --remove-attribute "_key"
|
||||||
|
|
||||||
The same thing would apply if your data contains an *_id* attribute:
|
The same thing would apply if your data contains an *_id* attribute:
|
||||||
|
|
||||||
> arangoimp --file "data.csv" --type csv --remove-attribute "_id"
|
> arangoimport --file "data.csv" --type csv --remove-attribute "_id"
|
||||||
|
|
||||||
|
|
||||||
### Importing into an Edge Collection
|
### Importing into an Edge Collection
|
||||||
|
|
||||||
arangoimp can also be used to import data into an existing edge collection.
|
arangoimport can also be used to import data into an existing edge collection.
|
||||||
The import data must, for each edge to import, contain at least the *_from* and
|
The import data must, for each edge to import, contain at least the *_from* and
|
||||||
*_to* attributes. These indicate which other two documents the edge should connect.
|
*_to* attributes. These indicate which other two documents the edge should connect.
|
||||||
It is necessary that these attributes are set for all records, and point to
|
It is necessary that these attributes are set for all records, and point to
|
||||||
valid document ids in existing collections.
|
valid document ids in existing collections.
|
||||||
|
|
||||||
*Examples*
|
*Examples*
|
||||||
|
@ -304,20 +304,20 @@ valid document ids in existing collections.
|
||||||
{ "_from" : "users/1234", "_to" : "users/4321", "desc" : "1234 is connected to 4321" }
|
{ "_from" : "users/1234", "_to" : "users/4321", "desc" : "1234 is connected to 4321" }
|
||||||
```
|
```
|
||||||
|
|
||||||
**Note**: The edge collection must already exist when the import is started. Using
|
**Note**: The edge collection must already exist when the import is started. Using
|
||||||
the *--create-collection* flag will not work because arangoimp will always try to
|
the *--create-collection* flag will not work because arangoimport will always try to
|
||||||
create a regular document collection if the target collection does not exist.
|
create a regular document collection if the target collection does not exist.
|
||||||
|
|
||||||
|
|
||||||
### Updating existing documents
|
### Updating existing documents
|
||||||
|
|
||||||
By default, arangoimp will try to insert all documents from the import file into the
|
By default, arangoimport will try to insert all documents from the import file into the
|
||||||
specified collection. In case the import file contains documents that are already present
|
specified collection. In case the import file contains documents that are already present
|
||||||
in the target collection (matching is done via the *_key* attributes), then a default
|
in the target collection (matching is done via the *_key* attributes), then a default
|
||||||
arangoimp run will not import these documents and complain about unique key constraint
|
arangoimport run will not import these documents and complain about unique key constraint
|
||||||
violations.
|
violations.
|
||||||
|
|
||||||
However, arangoimp can be used to update or replace existing documents in case they
|
However, arangoimport can be used to update or replace existing documents in case they
|
||||||
already exist in the target collection. It provides the command-line option *--on-duplicate*
|
already exist in the target collection. It provides the command-line option *--on-duplicate*
|
||||||
to control the behavior in case a document is already present in the database.
|
to control the behavior in case a document is already present in the database.
|
||||||
|
|
||||||
|
@ -329,47 +329,47 @@ the database will not be modified.
|
||||||
Other possible values for *--on-duplicate* are:
|
Other possible values for *--on-duplicate* are:
|
||||||
|
|
||||||
- *update*: each document present in the import file that is also present in the target
|
- *update*: each document present in the import file that is also present in the target
|
||||||
collection already will be updated by arangoimp. *update* will perform a partial update
|
collection already will be updated by arangoimport. *update* will perform a partial update
|
||||||
of the existing document, modifying only the attributes that are present in the import
|
of the existing document, modifying only the attributes that are present in the import
|
||||||
file and leaving all other attributes untouched.
|
file and leaving all other attributes untouched.
|
||||||
|
|
||||||
The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be
|
The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be
|
||||||
updated or replaced in existing documents.
|
updated or replaced in existing documents.
|
||||||
|
|
||||||
- *replace*: each document present in the import file that is also present in the target
|
- *replace*: each document present in the import file that is also present in the target
|
||||||
collection already will be replace by arangoimp. *replace* will replace the existing
|
collection already will be replace by arangoimport. *replace* will replace the existing
|
||||||
document entirely, resulting in a document with only the attributes specified in the import
|
document entirely, resulting in a document with only the attributes specified in the import
|
||||||
file.
|
file.
|
||||||
|
|
||||||
The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be
|
The values of system attributes *_id*, *_key*, *_rev*, *_from* and *_to* cannot be
|
||||||
updated or replaced in existing documents.
|
updated or replaced in existing documents.
|
||||||
|
|
||||||
- *ignore*: each document present in the import file that is also present in the target
|
- *ignore*: each document present in the import file that is also present in the target
|
||||||
collection already will be ignored and not modified in the target collection.
|
collection already will be ignored and not modified in the target collection.
|
||||||
|
|
||||||
When *--on-duplicate* is set to either *update* or *replace*, arangoimp will return the
|
When *--on-duplicate* is set to either *update* or *replace*, arangoimport will return the
|
||||||
number of documents updated/replaced in the *updated* return value. When set to another
|
number of documents updated/replaced in the *updated* return value. When set to another
|
||||||
value, the value of *updated* will always be zero. When *--on-duplicate* is set to *ignore*,
|
value, the value of *updated* will always be zero. When *--on-duplicate* is set to *ignore*,
|
||||||
arangoimp will return the number of ignored documents in the *ignored* return value.
|
arangoimport will return the number of ignored documents in the *ignored* return value.
|
||||||
When set to another value, *ignored* will always be zero.
|
When set to another value, *ignored* will always be zero.
|
||||||
|
|
||||||
It is possible to perform a combination of inserts and updates/replaces with a single
|
It is possible to perform a combination of inserts and updates/replaces with a single
|
||||||
arangoimp run. When *--on-duplicate* is set to *update* or *replace*, all documents present
|
arangoimport run. When *--on-duplicate* is set to *update* or *replace*, all documents present
|
||||||
in the import file will be inserted into the target collection provided they are valid
|
in the import file will be inserted into the target collection provided they are valid
|
||||||
and do not already exist with the specified *_key*. Documents that are already present
|
and do not already exist with the specified *_key*. Documents that are already present
|
||||||
in the target collection (identified by *_key* attribute) will instead be updated/replaced.
|
in the target collection (identified by *_key* attribute) will instead be updated/replaced.
|
||||||
|
|
||||||
|
|
||||||
### Arangoimp result output
|
### Arangoimport result output
|
||||||
|
|
||||||
An _arangoimp_ import run will print out the final results on the command line.
|
An _arangoimport_ import run will print out the final results on the command line.
|
||||||
It will show the
|
It will show the
|
||||||
|
|
||||||
* number of documents created (*created*)
|
* number of documents created (*created*)
|
||||||
* number of documents updated/replaced (*updated/replaced*, only non-zero if
|
* number of documents updated/replaced (*updated/replaced*, only non-zero if
|
||||||
*--on-duplicate* was set to *update* or *replace*, see below)
|
*--on-duplicate* was set to *update* or *replace*, see below)
|
||||||
* number of warnings or errors that occurred on the server side (*warnings/errors*)
|
* number of warnings or errors that occurred on the server side (*warnings/errors*)
|
||||||
* number of ignored documents (only non-zero if *--on-duplicate* was set to *ignore*).
|
* number of ignored documents (only non-zero if *--on-duplicate* was set to *ignore*).
|
||||||
|
|
||||||
*Example*
|
*Example*
|
||||||
|
|
||||||
|
@ -383,18 +383,18 @@ ignored: 0
|
||||||
For CSV and TSV imports, the total number of input file lines read will also be printed
|
For CSV and TSV imports, the total number of input file lines read will also be printed
|
||||||
(*lines read*).
|
(*lines read*).
|
||||||
|
|
||||||
_arangoimp_ will also print out details about warnings and errors that happened on the
|
_arangoimport_ will also print out details about warnings and errors that happened on the
|
||||||
server-side (if any).
|
server-side (if any).
|
||||||
|
|
||||||
|
|
||||||
### Attribute Naming and Special Attributes
|
### Attribute Naming and Special Attributes
|
||||||
|
|
||||||
Attributes whose names start with an underscore are treated in a special way by
|
Attributes whose names start with an underscore are treated in a special way by
|
||||||
ArangoDB:
|
ArangoDB:
|
||||||
|
|
||||||
- the optional *_key* attribute contains the document's key. If specified, the value
|
- the optional *_key* attribute contains the document's key. If specified, the value
|
||||||
must be formally valid (e.g. must be a string and conform to the naming conventions).
|
must be formally valid (e.g. must be a string and conform to the naming conventions).
|
||||||
Additionally, the key value must be unique within the
|
Additionally, the key value must be unique within the
|
||||||
collection the import is run for.
|
collection the import is run for.
|
||||||
- *_from*: when importing into an edge collection, this attribute contains the id
|
- *_from*: when importing into an edge collection, this attribute contains the id
|
||||||
of one of the documents connected by the edge. The value of *_from* must be a
|
of one of the documents connected by the edge. The value of *_from* must be a
|
||||||
|
@ -403,7 +403,7 @@ ArangoDB:
|
||||||
of the other document connected by the edge. The value of *_to* must be a
|
of the other document connected by the edge. The value of *_to* must be a
|
||||||
syntactically valid document id and the referred collection must exist.
|
syntactically valid document id and the referred collection must exist.
|
||||||
- *_rev*: this attribute contains the revision number of a document. However, the
|
- *_rev*: this attribute contains the revision number of a document. However, the
|
||||||
revision numbers are managed by ArangoDB and cannot be specified on import. Thus
|
revision numbers are managed by ArangoDB and cannot be specified on import. Thus
|
||||||
any value in this attribute is ignored on import.
|
any value in this attribute is ignored on import.
|
||||||
|
|
||||||
If you import values into *_key*, you should make sure they are valid and unique.
|
If you import values into *_key*, you should make sure they are valid and unique.
|
||||||
|
@ -419,7 +419,7 @@ inside *_from* and/or *_to*.
|
||||||
|
|
||||||
*Example*
|
*Example*
|
||||||
|
|
||||||
> arangoimp --from-collection-prefix users --to-collection-prefix products ...
|
> arangoimport --from-collection-prefix users --to-collection-prefix products ...
|
||||||
|
|
||||||
Importing the following document will then create an edge between *users/1234* and
|
Importing the following document will then create an edge between *users/1234* and
|
||||||
*products/4321*:
|
*products/4321*:
|
|
@ -79,7 +79,7 @@ database directory [to the new default paths](#custom-install-paths). Upgrading
|
||||||
will keep your old data, password and choice of storage engine as it is.
|
will keep your old data, password and choice of storage engine as it is.
|
||||||
Switching to the RocksDB storage engine requires a
|
Switching to the RocksDB storage engine requires a
|
||||||
[export](../../Administration/Arangoexport.md) and
|
[export](../../Administration/Arangoexport.md) and
|
||||||
[reimport](../../Administration/Arangoimp.md) of your data.
|
[reimport](../../Administration/Arangoimport.md) of your data.
|
||||||
|
|
||||||
Starting
|
Starting
|
||||||
--------
|
--------
|
||||||
|
@ -101,7 +101,7 @@ business. Have fun!` at the end of its output.
|
||||||
We now wish to check that the installation is working correctly and to do this
|
We now wish to check that the installation is working correctly and to do this
|
||||||
we will be using the administration web interface. Execute *arangod.exe* if you
|
we will be using the administration web interface. Execute *arangod.exe* if you
|
||||||
have not already done so, then open up your web browser and point it to the
|
have not already done so, then open up your web browser and point it to the
|
||||||
page:
|
page:
|
||||||
|
|
||||||
http://127.0.0.1:8529/
|
http://127.0.0.1:8529/
|
||||||
|
|
||||||
|
|
|
@ -358,6 +358,7 @@ if we use the `NEW` pseudo-variable:
|
||||||
"age": 40,
|
"age": 40,
|
||||||
"name": "Katie Foster"
|
"name": "Katie Foster"
|
||||||
}
|
}
|
||||||
|
]
|
||||||
```
|
```
|
||||||
|
|
||||||
If we used `REPLACE` instead, the name attribute would be gone. With `UPDATE`,
|
If we used `REPLACE` instead, the name attribute would be gone. With `UPDATE`,
|
||||||
|
@ -581,22 +582,22 @@ The ArangoDB package comes with the following programs:
|
||||||
- `arangod`: The [ArangoDB database daemon](../Administration/Configuration/GeneralArangod.md).
|
- `arangod`: The [ArangoDB database daemon](../Administration/Configuration/GeneralArangod.md).
|
||||||
This server program is intended to run as a daemon process and to serve the
|
This server program is intended to run as a daemon process and to serve the
|
||||||
various clients connection to the server via TCP / HTTP.
|
various clients connection to the server via TCP / HTTP.
|
||||||
|
|
||||||
- `arangosh`: The [ArangoDB shell](../Administration/Arangosh/README.md).
|
- `arangosh`: The [ArangoDB shell](../Administration/Arangosh/README.md).
|
||||||
A client that implements a read-eval-print loop (REPL) and provides functions
|
A client that implements a read-eval-print loop (REPL) and provides functions
|
||||||
to access and administrate the ArangoDB server.
|
to access and administrate the ArangoDB server.
|
||||||
|
|
||||||
- `arangoimp`: A [bulk importer](../Administration/Arangoimp.md) for the
|
- `arangoimport`: A [bulk importer](../Administration/Arangoimport.md) for the
|
||||||
ArangoDB server. It supports JSON and CSV.
|
ArangoDB server. It supports JSON and CSV.
|
||||||
|
|
||||||
- `arangodump`: A tool to [create backups](../Administration/Arangodump.md)
|
- `arangodump`: A tool to [create backups](../Administration/Arangodump.md)
|
||||||
of an ArangoDB database in JSON format.
|
of an ArangoDB database in JSON format.
|
||||||
|
|
||||||
- `arangorestore`: A tool to [load data of a backup](../Administration/Arangorestore.md)
|
- `arangorestore`: A tool to [load data of a backup](../Administration/Arangorestore.md)
|
||||||
back into an ArangoDB database.
|
back into an ArangoDB database.
|
||||||
|
|
||||||
- `arango-dfdb`: A [datafile debugger](../Troubleshooting/DatafileDebugger.md) for
|
- `arango-dfdb`: A [datafile debugger](../Troubleshooting/DatafileDebugger.md) for
|
||||||
ArangoDB. It is primarily intended to be used during development of ArangoDB.
|
ArangoDB. It is primarily intended to be used during development of ArangoDB.
|
||||||
|
|
||||||
- `arangobench`: A [benchmark and test tool](../Troubleshooting/Arangobench.md).
|
- `arangobench`: A [benchmark and test tool](../Troubleshooting/Arangobench.md).
|
||||||
It can be used for performance and server function testing.
|
It can be used for performance and server function testing.
|
||||||
|
|
|
@ -149,7 +149,7 @@
|
||||||
* [Configuration](Administration/Arangosh/Configuration.md)
|
* [Configuration](Administration/Arangosh/Configuration.md)
|
||||||
# relocate file?
|
# relocate file?
|
||||||
* [Details](GettingStarted/Arangosh.md)
|
* [Details](GettingStarted/Arangosh.md)
|
||||||
* [Arangoimp](Administration/Arangoimp.md)
|
* [Arangoimport](Administration/Arangoimport.md)
|
||||||
* [Arangodump](Administration/Arangodump.md)
|
* [Arangodump](Administration/Arangodump.md)
|
||||||
* [Arangorestore](Administration/Arangorestore.md)
|
* [Arangorestore](Administration/Arangorestore.md)
|
||||||
* [Arangoexport](Administration/Arangoexport.md)
|
* [Arangoexport](Administration/Arangoexport.md)
|
||||||
|
|
|
@ -15,7 +15,7 @@ if (USE_MAINTAINER_MODE AND NOT(MSVC))
|
||||||
set(MAN_NAMES
|
set(MAN_NAMES
|
||||||
man1/arangobench.1
|
man1/arangobench.1
|
||||||
man1/arangodump.1
|
man1/arangodump.1
|
||||||
man1/arangoimp.1
|
man1/arangoimport.1
|
||||||
man1/arangorestore.1
|
man1/arangorestore.1
|
||||||
man1/arangoexport.1
|
man1/arangoexport.1
|
||||||
man1/arangosh.1
|
man1/arangosh.1
|
||||||
|
@ -61,7 +61,7 @@ if (USE_MAINTAINER_MODE AND NOT(MSVC))
|
||||||
add_custom_target(clean_man_autogenerated
|
add_custom_target(clean_man_autogenerated
|
||||||
COMMAND rm -f ${CMAKE_SOURCE_DIR}/README
|
COMMAND rm -f ${CMAKE_SOURCE_DIR}/README
|
||||||
COMMAND rm -f ${MAN_FILES})
|
COMMAND rm -f ${MAN_FILES})
|
||||||
|
|
||||||
list(APPEND CLEAN_AUTOGENERATED_FILES clean_man_autogenerated)
|
list(APPEND CLEAN_AUTOGENERATED_FILES clean_man_autogenerated)
|
||||||
set(CLEAN_AUTOGENERATED_FILES ${CLEAN_AUTOGENERATED_FILES} PARENT_SCOPE)
|
set(CLEAN_AUTOGENERATED_FILES ${CLEAN_AUTOGENERATED_FILES} PARENT_SCOPE)
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
.SH NAME
|
.SH NAME
|
||||||
arangoexport - a tool to export collections of an ArangoDB database
|
arangoexport - a tool to export collections of an ArangoDB database
|
||||||
.SH SYNOPSIS
|
.SH SYNOPSIS
|
||||||
arangoexport [options]
|
arangoexport [options]
|
||||||
.SH DESCRIPTION
|
.SH DESCRIPTION
|
||||||
The arangoexport binary can be used to export collections of an ArangoDB
|
The arangoexport binary can be used to export collections of an ArangoDB
|
||||||
database to json and jsonl. It can also export a graph or collections
|
database to json and jsonl. It can also export a graph or collections
|
||||||
|
@ -12,7 +12,7 @@ arangoexport will work on the specified database only. If no database name
|
||||||
is specified, arangoexport will work on the default database ("_system").
|
is specified, arangoexport will work on the default database ("_system").
|
||||||
|
|
||||||
The exported jsonl files can be re-imported in an ArangoDB database
|
The exported jsonl files can be re-imported in an ArangoDB database
|
||||||
using the arangoimp tool.
|
using the arangoimport tool.
|
||||||
.SH OPTIONS
|
.SH OPTIONS
|
||||||
The arangoexport binary has many options that can be used to control its
|
The arangoexport binary has many options that can be used to control its
|
||||||
behavior. For a complete list of options, please refer to the
|
behavior. For a complete list of options, please refer to the
|
||||||
|
@ -21,4 +21,3 @@ arangoexport --help.
|
||||||
|
|
||||||
.SH AUTHOR
|
.SH AUTHOR
|
||||||
Copyright ArangoDB GmbH, Cologne, Germany
|
Copyright ArangoDB GmbH, Cologne, Germany
|
||||||
|
|
||||||
|
|
|
@ -1,19 +1,18 @@
|
||||||
.TH arangoimp 1 "3.3.devel" "ArangoDB" "ArangoDB"
|
.TH arangoimport 1 "3.3.devel" "ArangoDB" "ArangoDB"
|
||||||
.SH NAME
|
.SH NAME
|
||||||
arangoimp - a bulk importer for the ArangoDB database
|
arangoimport - a bulk importer for the ArangoDB database
|
||||||
.SH SYNOPSIS
|
.SH SYNOPSIS
|
||||||
arangoimp [options]
|
arangoimport [options]
|
||||||
.SH DESCRIPTION
|
.SH DESCRIPTION
|
||||||
The arangoimp binary can be used to bulk import data from a file into the
|
The arangoimport binary can be used to bulk import data from a file into the
|
||||||
ArangoDB database. Input data be present in the input file in either CSV
|
ArangoDB database. Input data be present in the input file in either CSV
|
||||||
format with column headlines, or in JSON format. If JSON format is used,
|
format with column headlines, or in JSON format. If JSON format is used,
|
||||||
each line in the input file must contain exactly one JSON document with
|
each line in the input file must contain exactly one JSON document with
|
||||||
the attribute name/value pairs to import.
|
the attribute name/value pairs to import.
|
||||||
.SH OPTIONS
|
.SH OPTIONS
|
||||||
The arangoimp binary has many options that can be used to control its
|
The arangoimport binary has many options that can be used to control its
|
||||||
behavior. For a complete list of options, please refer to the
|
behavior. For a complete list of options, please refer to the
|
||||||
ArangoDB online manual, available at https://www.arangodb.com/
|
ArangoDB online manual, available at https://www.arangodb.com/
|
||||||
|
|
||||||
.SH AUTHOR
|
.SH AUTHOR
|
||||||
Copyright ArangoDB GmbH, Cologne, Germany
|
Copyright ArangoDB GmbH, Cologne, Germany
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
NAME
|
NAME
|
||||||
<COMMAND> - a tool to export collections of an ArangoDB database
|
<COMMAND> - a tool to export collections of an ArangoDB database
|
||||||
SYNOPSIS
|
SYNOPSIS
|
||||||
<COMMAND> [options]
|
<COMMAND> [options]
|
||||||
DESCRIPTION
|
DESCRIPTION
|
||||||
The <COMMAND> binary can be used to export collections of an ArangoDB
|
The <COMMAND> binary can be used to export collections of an ArangoDB
|
||||||
database to json and jsonl. It can also export a graph or collections
|
database to json and jsonl. It can also export a graph or collections
|
||||||
|
@ -11,7 +11,7 @@ DESCRIPTION
|
||||||
is specified, <COMMAND> will work on the default database ("_system").
|
is specified, <COMMAND> will work on the default database ("_system").
|
||||||
|
|
||||||
The exported jsonl files can be re-imported in an ArangoDB database
|
The exported jsonl files can be re-imported in an ArangoDB database
|
||||||
using the arangoimp tool.
|
using the arangoimport tool.
|
||||||
OPTIONS
|
OPTIONS
|
||||||
The <COMMAND> binary has many options that can be used to control its
|
The <COMMAND> binary has many options that can be used to control its
|
||||||
behavior. For a complete list of options, please refer to the
|
behavior. For a complete list of options, please refer to the
|
||||||
|
|
|
@ -1,12 +1,12 @@
|
||||||
NAME
|
NAME
|
||||||
<COMMAND> - a bulk importer for the ArangoDB database
|
<COMMAND> - a bulk importer for the ArangoDB database
|
||||||
SYNOPSIS
|
SYNOPSIS
|
||||||
<COMMAND> [options]
|
<COMMAND> [options]
|
||||||
DESCRIPTION
|
DESCRIPTION
|
||||||
The <COMMAND> binary can be used to bulk import data from a file into the
|
The <COMMAND> binary can be used to bulk import data from a file into the
|
||||||
ArangoDB database. Input data be present in the input file in either CSV
|
ArangoDB database. Input data be present in the input file in either CSV
|
||||||
format with column headlines, or in JSON format. If JSON format is used,
|
format with column headlines, or in JSON format. If JSON format is used,
|
||||||
each line in the input file must contain exactly one JSON document with
|
each line in the input file must contain exactly one JSON document with
|
||||||
the attribute name/value pairs to import.
|
the attribute name/value pairs to import.
|
||||||
OPTIONS
|
OPTIONS
|
||||||
The <COMMAND> binary has many options that can be used to control its
|
The <COMMAND> binary has many options that can be used to control its
|
|
@ -60,8 +60,8 @@ apps:
|
||||||
plugs: # enable interfaces
|
plugs: # enable interfaces
|
||||||
- network
|
- network
|
||||||
|
|
||||||
arangoimp:
|
arangoimport:
|
||||||
command: arangoimp \
|
command: arangoimport \
|
||||||
--server.endpoint=tcp://127.0.0.1:@SNAP_PORT@ \
|
--server.endpoint=tcp://127.0.0.1:@SNAP_PORT@ \
|
||||||
--temp.path=${SNAP_DATA}
|
--temp.path=${SNAP_DATA}
|
||||||
plugs: # enable interfaces
|
plugs: # enable interfaces
|
||||||
|
|
|
@ -97,7 +97,7 @@ It is written in C/C++.
|
||||||
|
|
||||||
|
|
||||||
%description client
|
%description client
|
||||||
The ArangoDB shell as stand-alone program. It also contains the utility programs: arangobench (benchmark), arangorestore & arangodump (backup), arangoimp (import).
|
The ArangoDB shell as stand-alone program. It also contains the utility programs: arangobench (benchmark), arangorestore & arangodump (backup), arangoimport (import).
|
||||||
|
|
||||||
%description debuginfo
|
%description debuginfo
|
||||||
Debug symbols for the arangodb database
|
Debug symbols for the arangodb database
|
||||||
|
@ -170,13 +170,13 @@ mkdir -p %{buildroot}%{_piddir}
|
||||||
#%doc LICENSES-OTHER-COMPONENTS.md
|
#%doc LICENSES-OTHER-COMPONENTS.md
|
||||||
|
|
||||||
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangosh.conf
|
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangosh.conf
|
||||||
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangoimp.conf
|
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangoimport.conf
|
||||||
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangodump.conf
|
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangodump.conf
|
||||||
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangorestore.conf
|
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangorestore.conf
|
||||||
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangobench.conf
|
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/arangobench.conf
|
||||||
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/foxx-manager.conf
|
%config(noreplace) %attr(-,arangodb,arangodb) %{_cfgdir}/foxx-manager.conf
|
||||||
%{_bindir}/arangosh
|
%{_bindir}/arangosh
|
||||||
%{_bindir}/arangoimp
|
%{_bindir}/arangoimport
|
||||||
%{_bindir}/arangodump
|
%{_bindir}/arangodump
|
||||||
%{_bindir}/arangorestore
|
%{_bindir}/arangorestore
|
||||||
%{_bindir}/arangobench
|
%{_bindir}/arangobench
|
||||||
|
@ -185,7 +185,7 @@ mkdir -p %{buildroot}%{_piddir}
|
||||||
%{_mandir}/*/arangorestore*
|
%{_mandir}/*/arangorestore*
|
||||||
%{_mandir}/*/arangobench*
|
%{_mandir}/*/arangobench*
|
||||||
%{_mandir}/*/arangodump*
|
%{_mandir}/*/arangodump*
|
||||||
%{_mandir}/*/arangoimp*
|
%{_mandir}/*/arangoimport*
|
||||||
%{_datadir}/arangodb3/js/common
|
%{_datadir}/arangodb3/js/common
|
||||||
%{_datadir}/arangodb3/js/client
|
%{_datadir}/arangodb3/js/client
|
||||||
%{_datadir}/arangodb3/js/node
|
%{_datadir}/arangodb3/js/node
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
Pregel Subsystem
|
Pregel Subsystem
|
||||||
========
|
========
|
||||||
|
|
||||||
The pregel subsystem implements a variety of different grapg algorithms,
|
The pregel subsystem implements a variety of different grapg algorithms,
|
||||||
this readme is more intended for internal use.
|
this readme is more intended for internal use.
|
||||||
|
|
||||||
#### Protocol
|
#### Protocol
|
||||||
|
@ -12,9 +12,9 @@ Message format between DBServers:
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
{sender:"someid",
|
{sender:"someid",
|
||||||
executionNumber:1337,
|
executionNumber:1337,
|
||||||
globalSuperstep:123,
|
globalSuperstep:123,
|
||||||
messages: [<vertexID1>, <slice1>, vertexID2, <slice2>]
|
messages: [<vertexID1>, <slice1>, vertexID2, <slice2>]
|
||||||
}
|
}
|
||||||
Any type of slice is supported
|
Any type of slice is supported
|
||||||
|
@ -23,7 +23,7 @@ Any type of slice is supported
|
||||||
### Useful Commands
|
### Useful Commands
|
||||||
|
|
||||||
Import graph e.g. https://github.com/arangodb/example-datasets/tree/master/Graphs/1000
|
Import graph e.g. https://github.com/arangodb/example-datasets/tree/master/Graphs/1000
|
||||||
First rename the columns '_key', '_from', '_to' arangoimp will keep those.
|
First rename the columns '_key', '_from', '_to' arangoimport will keep those.
|
||||||
|
|
||||||
In arangosh:
|
In arangosh:
|
||||||
|
|
||||||
|
@ -31,19 +31,19 @@ In arangosh:
|
||||||
db._createEdgeCollection('alt_edges');
|
db._createEdgeCollection('alt_edges');
|
||||||
db._createEdgeCollection('edges', {numberOfShards: 2, shardKeys:["_vertex"], distributeShardsLike:'vertices'});
|
db._createEdgeCollection('edges', {numberOfShards: 2, shardKeys:["_vertex"], distributeShardsLike:'vertices'});
|
||||||
|
|
||||||
arangoimp --file generated_vertices.csv --type csv --collection vertices --overwrite true --server.endpoint http+tcp://127.0.0.1:8530
|
arangoimport --file generated_vertices.csv --type csv --collection vertices --overwrite true --server.endpoint http+tcp://127.0.0.1:8530
|
||||||
|
|
||||||
Or:
|
Or:
|
||||||
for(var i=0; i < 5000; i++) db.vertices.save({_key:i+""});
|
for(var i=0; i < 5000; i++) db.vertices.save({_key:i+""});
|
||||||
|
|
||||||
arangoimp --file generated_edges.csv --type csv --collection alt_edges --overwrite true --from-collection-prefix "vertices" --to-collection-prefix "vertices" --convert false --server.endpoint http+tcp://127.0.0.1:8530
|
arangoimport --file generated_edges.csv --type csv --collection alt_edges --overwrite true --from-collection-prefix "vertices" --to-collection-prefix "vertices" --convert false --server.endpoint http+tcp://127.0.0.1:8530
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
AQL script to copy edge collection into one with '_vertex':
|
AQL script to copy edge collection into one with '_vertex':
|
||||||
|
|
||||||
FOR doc IN alt_edges
|
FOR doc IN alt_edges
|
||||||
INSERT {_vertex:SUBSTRING(doc._from,FIND_FIRST(doc._from,"/")+1),
|
INSERT {_vertex:SUBSTRING(doc._from,FIND_FIRST(doc._from,"/")+1),
|
||||||
_from:doc._from,
|
_from:doc._from,
|
||||||
_to:doc._to} IN edges
|
_to:doc._to} IN edges
|
||||||
LET values = (
|
LET values = (
|
||||||
|
@ -63,5 +63,5 @@ Make CSV file with arango compatible edges
|
||||||
cat edges.csv | awk -F" " '{print "profiles/" $1 "\tprofiles/" $2 "\t" $1}' >> arango-edges.csv
|
cat edges.csv | awk -F" " '{print "profiles/" $1 "\tprofiles/" $2 "\t" $1}' >> arango-edges.csv
|
||||||
|
|
||||||
|
|
||||||
arangoimp --file vertices.csv --type csv --collection twitter_v --overwrite true --convert false --server.endpoint http+tcp://127.0.0.1:8530 -c none
|
arangoimport --file vertices.csv --type csv --collection twitter_v --overwrite true --convert false --server.endpoint http+tcp://127.0.0.1:8530 -c none
|
||||||
arangoimp --file arango-edges.csv --type csv --collection twitter_e --overwrite true --convert false --separator "\t" --server.endpoint http+tcp://127.0.0.1:8530 -c none
|
arangoimport --file arango-edges.csv --type csv --collection twitter_e --overwrite true --convert false --separator "\t" --server.endpoint http+tcp://127.0.0.1:8530 -c none
|
||||||
|
|
|
@ -155,13 +155,13 @@ if (USE_JEMALLOC)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
## arangoimp
|
## arangoimport
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
if (MSVC AND NOT(SKIP_PACKAGING))
|
if (MSVC AND NOT(SKIP_PACKAGING))
|
||||||
generate_product_version(ProductVersionFiles_arangoimp
|
generate_product_version(ProductVersionFiles_arangoimport
|
||||||
NAME arangoimp
|
NAME arangoimport
|
||||||
FILE_DESCRIPTION ${ARANGO_IMP_FRIENDLY_STRING}
|
FILE_DESCRIPTION ${ARANGO_IMPORT_FRIENDLY_STRING}
|
||||||
ICON ${ARANGO_ICON}
|
ICON ${ARANGO_ICON}
|
||||||
VERSION_MAJOR ${CPACK_PACKAGE_VERSION_MAJOR}
|
VERSION_MAJOR ${CPACK_PACKAGE_VERSION_MAJOR}
|
||||||
VERSION_MINOR ${CPACK_PACKAGE_VERSION_MINOR}
|
VERSION_MINOR ${CPACK_PACKAGE_VERSION_MINOR}
|
||||||
|
@ -170,19 +170,19 @@ if (MSVC AND NOT(SKIP_PACKAGING))
|
||||||
)
|
)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
add_executable(${BIN_ARANGOIMP}
|
add_executable(${BIN_ARANGOIMPORT}
|
||||||
${ProductVersionFiles_arangoimp}
|
${ProductVersionFiles_arangoimport}
|
||||||
${PROJECT_SOURCE_DIR}/lib/Basics/WorkMonitorDummy.cpp
|
${PROJECT_SOURCE_DIR}/lib/Basics/WorkMonitorDummy.cpp
|
||||||
Import/ImportFeature.cpp
|
Import/ImportFeature.cpp
|
||||||
Import/ImportHelper.cpp
|
Import/ImportHelper.cpp
|
||||||
Import/SenderThread.cpp
|
Import/SenderThread.cpp
|
||||||
Import/arangoimp.cpp
|
Import/arangoimport.cpp
|
||||||
Shell/ClientFeature.cpp
|
Shell/ClientFeature.cpp
|
||||||
Shell/ConsoleFeature.cpp
|
Shell/ConsoleFeature.cpp
|
||||||
V8Client/ArangoClientHelper.cpp
|
V8Client/ArangoClientHelper.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
target_link_libraries(${BIN_ARANGOIMP}
|
target_link_libraries(${BIN_ARANGOIMPORT}
|
||||||
${LIB_ARANGO}
|
${LIB_ARANGO}
|
||||||
${MSVC_LIBS}
|
${MSVC_LIBS}
|
||||||
${SYSTEM_LIBRARIES}
|
${SYSTEM_LIBRARIES}
|
||||||
|
@ -191,21 +191,26 @@ target_link_libraries(${BIN_ARANGOIMP}
|
||||||
)
|
)
|
||||||
|
|
||||||
install(
|
install(
|
||||||
TARGETS ${BIN_ARANGOIMP}
|
TARGETS ${BIN_ARANGOIMPORT}
|
||||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||||
|
|
||||||
install_config(arangoimp)
|
install_config(arangoimport)
|
||||||
|
|
||||||
if (NOT USE_PRECOMPILED_V8)
|
if (NOT USE_PRECOMPILED_V8)
|
||||||
add_dependencies(arangoimp zlibstatic v8_build) # v8_build includes ICU build
|
add_dependencies(arangoimport zlibstatic v8_build) # v8_build includes ICU
|
||||||
|
# build
|
||||||
else ()
|
else ()
|
||||||
add_dependencies(arangoimp zlibstatic) # v8_build includes ICU build
|
add_dependencies(arangoimport zlibstatic) # v8_build includes ICU build
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
if (USE_JEMALLOC)
|
if (USE_JEMALLOC)
|
||||||
add_dependencies(arangoimp jemalloc)
|
add_dependencies(arangoimport jemalloc)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
|
install_command_alias(${BIN_ARANGOIMPORT}
|
||||||
|
${CMAKE_INSTALL_BINDIR}
|
||||||
|
arangoimp
|
||||||
|
)
|
||||||
################################################################################
|
################################################################################
|
||||||
## arangorestore
|
## arangorestore
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
|
@ -47,21 +47,21 @@ int main(int argc, char* argv[]) {
|
||||||
context.installHup();
|
context.installHup();
|
||||||
|
|
||||||
std::shared_ptr<options::ProgramOptions> options(new options::ProgramOptions(
|
std::shared_ptr<options::ProgramOptions> options(new options::ProgramOptions(
|
||||||
argv[0], "Usage: arangoimp [<options>]", "For more information use:", BIN_DIRECTORY));
|
argv[0], "Usage: arangoimport [<options>]", "For more information use:", BIN_DIRECTORY));
|
||||||
|
|
||||||
ApplicationServer server(options, BIN_DIRECTORY);
|
ApplicationServer server(options, BIN_DIRECTORY);
|
||||||
|
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
server.addFeature(new ClientFeature(&server));
|
server.addFeature(new ClientFeature(&server));
|
||||||
server.addFeature(new ConfigFeature(&server, "arangoimp"));
|
server.addFeature(new ConfigFeature(&server, "arangoimport"));
|
||||||
server.addFeature(new ImportFeature(&server, &ret));
|
server.addFeature(new ImportFeature(&server, &ret));
|
||||||
server.addFeature(new LoggerFeature(&server, false));
|
server.addFeature(new LoggerFeature(&server, false));
|
||||||
server.addFeature(new RandomFeature(&server));
|
server.addFeature(new RandomFeature(&server));
|
||||||
server.addFeature(new ShellColorsFeature(&server));
|
server.addFeature(new ShellColorsFeature(&server));
|
||||||
server.addFeature(new ShutdownFeature(&server, {"Import"}));
|
server.addFeature(new ShutdownFeature(&server, {"Import"}));
|
||||||
server.addFeature(new SslFeature(&server));
|
server.addFeature(new SslFeature(&server));
|
||||||
server.addFeature(new TempFeature(&server, "arangoimp"));
|
server.addFeature(new TempFeature(&server, "arangoimport"));
|
||||||
server.addFeature(new VersionFeature(&server));
|
server.addFeature(new VersionFeature(&server));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -72,11 +72,11 @@ int main(int argc, char* argv[]) {
|
||||||
}
|
}
|
||||||
} catch (std::exception const& ex) {
|
} catch (std::exception const& ex) {
|
||||||
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
|
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
|
||||||
<< "arangoimp terminated because of an unhandled exception: " << ex.what();
|
<< "arangoimport terminated because of an unhandled exception: " << ex.what();
|
||||||
ret = EXIT_FAILURE;
|
ret = EXIT_FAILURE;
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
|
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
|
||||||
<< "arangoimp terminated because of an unhandled exception of unknown type";
|
<< "arangoimport terminated because of an unhandled exception of unknown type";
|
||||||
ret = EXIT_FAILURE;
|
ret = EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,7 @@ install_debinfo(
|
||||||
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
|
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
|
||||||
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
|
"${CMAKE_PROJECT_NAME}/${CMAKE_INSTALL_BINDIR}"
|
||||||
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}"
|
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}"
|
||||||
"${BIN_ARANGOIMP}")
|
"${BIN_ARANGOIMPORT}")
|
||||||
|
|
||||||
install_debinfo(
|
install_debinfo(
|
||||||
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
|
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/strip"
|
||||||
|
|
|
@ -6,7 +6,7 @@ set(STRIP_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/cstrip")
|
||||||
add_custom_target(strip_install_client ALL)
|
add_custom_target(strip_install_client ALL)
|
||||||
strip_install_bin_and_config(arangobench ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
strip_install_bin_and_config(arangobench ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
||||||
strip_install_bin_and_config(arangodump ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
strip_install_bin_and_config(arangodump ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
||||||
strip_install_bin_and_config(arangoimp ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
strip_install_bin_and_config(arangoimport ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
||||||
strip_install_bin_and_config(arangorestore ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
strip_install_bin_and_config(arangorestore ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
||||||
strip_install_bin_and_config(arangoexport ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
strip_install_bin_and_config(arangoexport ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
||||||
strip_install_bin_and_config(arangosh ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
strip_install_bin_and_config(arangosh ${STRIP_DIR} ${CMAKE_INSTALL_BINDIR} strip_install_client)
|
||||||
|
@ -16,4 +16,3 @@ install_command_alias(${BIN_ARANGOSH}
|
||||||
${CMAKE_INSTALL_BINDIR}
|
${CMAKE_INSTALL_BINDIR}
|
||||||
foxx-manager)
|
foxx-manager)
|
||||||
install_config(foxx-manager)
|
install_config(foxx-manager)
|
||||||
|
|
||||||
|
|
|
@ -69,7 +69,7 @@ let BIN_DIR;
|
||||||
let ARANGOBENCH_BIN;
|
let ARANGOBENCH_BIN;
|
||||||
let ARANGODUMP_BIN;
|
let ARANGODUMP_BIN;
|
||||||
let ARANGOD_BIN;
|
let ARANGOD_BIN;
|
||||||
let ARANGOIMP_BIN;
|
let ARANGOIMPORT_BIN;
|
||||||
let ARANGORESTORE_BIN;
|
let ARANGORESTORE_BIN;
|
||||||
let ARANGOEXPORT_BIN;
|
let ARANGOEXPORT_BIN;
|
||||||
let ARANGOSH_BIN;
|
let ARANGOSH_BIN;
|
||||||
|
@ -130,7 +130,7 @@ function setupBinaries (builddir, buildType, configDir) {
|
||||||
ARANGOBENCH_BIN = fs.join(BIN_DIR, 'arangobench' + executableExt);
|
ARANGOBENCH_BIN = fs.join(BIN_DIR, 'arangobench' + executableExt);
|
||||||
ARANGODUMP_BIN = fs.join(BIN_DIR, 'arangodump' + executableExt);
|
ARANGODUMP_BIN = fs.join(BIN_DIR, 'arangodump' + executableExt);
|
||||||
ARANGOD_BIN = fs.join(BIN_DIR, 'arangod' + executableExt);
|
ARANGOD_BIN = fs.join(BIN_DIR, 'arangod' + executableExt);
|
||||||
ARANGOIMP_BIN = fs.join(BIN_DIR, 'arangoimp' + executableExt);
|
ARANGOIMPORT_BIN = fs.join(BIN_DIR, 'arangoimport' + executableExt);
|
||||||
ARANGORESTORE_BIN = fs.join(BIN_DIR, 'arangorestore' + executableExt);
|
ARANGORESTORE_BIN = fs.join(BIN_DIR, 'arangorestore' + executableExt);
|
||||||
ARANGOEXPORT_BIN = fs.join(BIN_DIR, 'arangoexport' + executableExt);
|
ARANGOEXPORT_BIN = fs.join(BIN_DIR, 'arangoexport' + executableExt);
|
||||||
ARANGOSH_BIN = fs.join(BIN_DIR, 'arangosh' + executableExt);
|
ARANGOSH_BIN = fs.join(BIN_DIR, 'arangosh' + executableExt);
|
||||||
|
@ -152,7 +152,7 @@ function setupBinaries (builddir, buildType, configDir) {
|
||||||
ARANGOBENCH_BIN,
|
ARANGOBENCH_BIN,
|
||||||
ARANGODUMP_BIN,
|
ARANGODUMP_BIN,
|
||||||
ARANGOD_BIN,
|
ARANGOD_BIN,
|
||||||
ARANGOIMP_BIN,
|
ARANGOIMPORT_BIN,
|
||||||
ARANGORESTORE_BIN,
|
ARANGORESTORE_BIN,
|
||||||
ARANGOEXPORT_BIN,
|
ARANGOEXPORT_BIN,
|
||||||
ARANGOSH_BIN];
|
ARANGOSH_BIN];
|
||||||
|
@ -529,10 +529,10 @@ function runArangoshCmd (options, instanceInfo, addArgs, cmds) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// //////////////////////////////////////////////////////////////////////////////
|
// //////////////////////////////////////////////////////////////////////////////
|
||||||
// / @brief runs arangoimp
|
// / @brief runs arangoimport
|
||||||
// //////////////////////////////////////////////////////////////////////////////
|
// //////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
function runArangoImp (options, instanceInfo, what) {
|
function runArangoImport (options, instanceInfo, what) {
|
||||||
let args = {
|
let args = {
|
||||||
'server.username': options.username,
|
'server.username': options.username,
|
||||||
'server.password': options.password,
|
'server.password': options.password,
|
||||||
|
@ -566,7 +566,7 @@ function runArangoImp (options, instanceInfo, what) {
|
||||||
args['remove-attribute'] = what.removeAttribute;
|
args['remove-attribute'] = what.removeAttribute;
|
||||||
}
|
}
|
||||||
|
|
||||||
return executeAndWait(ARANGOIMP_BIN, toArgv(args), options, 'arangoimp', instanceInfo.rootDir);
|
return executeAndWait(ARANGOIMPORT_BIN, toArgv(args), options, 'arangoimport', instanceInfo.rootDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
// //////////////////////////////////////////////////////////////////////////////
|
// //////////////////////////////////////////////////////////////////////////////
|
||||||
|
@ -585,7 +585,7 @@ function runArangoDumpRestore (options, instanceInfo, which, database, rootDir,
|
||||||
|
|
||||||
let exe;
|
let exe;
|
||||||
rootDir = rootDir || instanceInfo.rootDir;
|
rootDir = rootDir || instanceInfo.rootDir;
|
||||||
|
|
||||||
if (which === 'dump') {
|
if (which === 'dump') {
|
||||||
args['output-directory'] = fs.join(rootDir, dumpDir);
|
args['output-directory'] = fs.join(rootDir, dumpDir);
|
||||||
exe = ARANGODUMP_BIN;
|
exe = ARANGODUMP_BIN;
|
||||||
|
@ -594,9 +594,9 @@ function runArangoDumpRestore (options, instanceInfo, which, database, rootDir,
|
||||||
args['input-directory'] = fs.join(rootDir, dumpDir);
|
args['input-directory'] = fs.join(rootDir, dumpDir);
|
||||||
exe = ARANGORESTORE_BIN;
|
exe = ARANGORESTORE_BIN;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options.encrypted) {
|
if (options.encrypted) {
|
||||||
args['encryption.keyfile'] = fs.join(rootDir, 'secret-key');
|
args['encryption.keyfile'] = fs.join(rootDir, 'secret-key');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options.extremeVerbosity === true) {
|
if (options.extremeVerbosity === true) {
|
||||||
|
@ -970,7 +970,7 @@ function startInstanceCluster (instanceInfo, protocol, options,
|
||||||
}
|
}
|
||||||
} else if (options.singleresilient) {
|
} else if (options.singleresilient) {
|
||||||
// for now start just two (TODO config parameter)
|
// for now start just two (TODO config parameter)
|
||||||
for (i = 0; i < 2; i++) {
|
for (i = 0; i < 2; i++) {
|
||||||
let port = findFreePort(options.minPort, options.maxPort, usedPorts);
|
let port = findFreePort(options.minPort, options.maxPort, usedPorts);
|
||||||
usedPorts.push(port);
|
usedPorts.push(port);
|
||||||
let endpoint = protocol + '://127.0.0.1:' + port;
|
let endpoint = protocol + '://127.0.0.1:' + port;
|
||||||
|
@ -1047,7 +1047,7 @@ function startInstanceCluster (instanceInfo, protocol, options,
|
||||||
const reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', makeAuthorizationHeaders(authOpts));
|
const reply = download(instanceInfo.url + '/_api/cluster/endpoints', '', makeAuthorizationHeaders(authOpts));
|
||||||
if (!reply.error && reply.code === 200) {
|
if (!reply.error && reply.code === 200) {
|
||||||
let res = JSON.parse(reply.body);
|
let res = JSON.parse(reply.body);
|
||||||
internal.print("Response ====> " + reply.body);
|
internal.print("Response ====> " + reply.body);
|
||||||
let leader = res.endpoints[0].endpoint;
|
let leader = res.endpoints[0].endpoint;
|
||||||
instanceInfo.arangods.forEach(d => {
|
instanceInfo.arangods.forEach(d => {
|
||||||
if (d.endpoint === leader) {
|
if (d.endpoint === leader) {
|
||||||
|
@ -1056,7 +1056,7 @@ function startInstanceCluster (instanceInfo, protocol, options,
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
arango.reconnect(instanceInfo.endpoint, '_system', 'root', '');
|
arango.reconnect(instanceInfo.endpoint, '_system', 'root', '');
|
||||||
return true;
|
return true;
|
||||||
|
@ -1327,7 +1327,7 @@ exports.executeAndWait = executeAndWait;
|
||||||
|
|
||||||
exports.run = {
|
exports.run = {
|
||||||
arangoshCmd: runArangoshCmd,
|
arangoshCmd: runArangoshCmd,
|
||||||
arangoImp: runArangoImp,
|
arangoImport: runArangoImport,
|
||||||
arangoDumpRestore: runArangoDumpRestore,
|
arangoDumpRestore: runArangoDumpRestore,
|
||||||
arangoBenchmark: runArangoBenchmark
|
arangoBenchmark: runArangoBenchmark
|
||||||
};
|
};
|
||||||
|
|
|
@ -81,7 +81,7 @@ function config (options) {
|
||||||
'arangod',
|
'arangod',
|
||||||
'arangobench',
|
'arangobench',
|
||||||
'arangodump',
|
'arangodump',
|
||||||
'arangoimp',
|
'arangoimport',
|
||||||
'arangorestore',
|
'arangorestore',
|
||||||
'arangoexport',
|
'arangoexport',
|
||||||
'arangosh',
|
'arangosh',
|
||||||
|
|
|
@ -214,7 +214,7 @@ function importing (options) {
|
||||||
for (let i = 0; i < impTodos.length; i++) {
|
for (let i = 0; i < impTodos.length; i++) {
|
||||||
const impTodo = impTodos[i];
|
const impTodo = impTodos[i];
|
||||||
|
|
||||||
result[impTodo.id] = pu.run.arangoImp(options, instanceInfo, impTodo);
|
result[impTodo.id] = pu.run.arangoImport(options, instanceInfo, impTodo);
|
||||||
result[impTodo.id].failed = 0;
|
result[impTodo.id].failed = 0;
|
||||||
|
|
||||||
if (result[impTodo.id].status !== true && !options.force) {
|
if (result[impTodo.id].status !== true && !options.force) {
|
||||||
|
|
|
@ -143,6 +143,7 @@ void ConfigFeature::loadConfigFile(std::shared_ptr<ProgramOptions> options,
|
||||||
|
|
||||||
auto context = ArangoGlobalContext::CONTEXT;
|
auto context = ArangoGlobalContext::CONTEXT;
|
||||||
std::string basename = progname;
|
std::string basename = progname;
|
||||||
|
bool checkArangoImp = (progname == "arangoimport");
|
||||||
|
|
||||||
if (!StringUtils::isSuffix(basename, ".conf")) {
|
if (!StringUtils::isSuffix(basename, ".conf")) {
|
||||||
basename += ".conf";
|
basename += ".conf";
|
||||||
|
@ -177,6 +178,14 @@ void ConfigFeature::loadConfigFile(std::shared_ptr<ProgramOptions> options,
|
||||||
LOG_TOPIC(DEBUG, Logger::CONFIG) << "found config file '" << name << "'";
|
LOG_TOPIC(DEBUG, Logger::CONFIG) << "found config file '" << name << "'";
|
||||||
filename = name;
|
filename = name;
|
||||||
break;
|
break;
|
||||||
|
} else if (checkArangoImp) {
|
||||||
|
name = FileUtils::buildFilename(location, "arangoimp");
|
||||||
|
LOG_TOPIC(TRACE, Logger::CONFIG) << "checking config file'" << name << "'";
|
||||||
|
if (FileUtils::exists(name)) {
|
||||||
|
LOG_TOPIC(DEBUG, Logger::CONFIG) << "found config file '" << name << "'";
|
||||||
|
filename = name;
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -35,7 +35,7 @@ EOF
|
||||||
|
|
||||||
echo "" > $out
|
echo "" > $out
|
||||||
|
|
||||||
progs="arangobench arangosh arangoimp arangodump arangorestore arangod"
|
progs="arangobench arangosh arangoimport arangodump arangorestore arangod"
|
||||||
|
|
||||||
for progname in $progs
|
for progname in $progs
|
||||||
do
|
do
|
||||||
|
@ -43,14 +43,14 @@ for progname in $progs
|
||||||
# check if the executable exists
|
# check if the executable exists
|
||||||
if [[ -f "build/bin/$progname" ]]; then
|
if [[ -f "build/bin/$progname" ]]; then
|
||||||
executable="build/bin/$progname"
|
executable="build/bin/$progname"
|
||||||
|
|
||||||
# setup the help command string
|
# setup the help command string
|
||||||
command="--help-all"
|
command="--help-all"
|
||||||
|
|
||||||
# set up the list of completions for the executable
|
# set up the list of completions for the executable
|
||||||
completions="`\"$executable\" $command | grep -o \"^\\ \\+--[a-z-]\\+\\(\\.[a-z0-9-]\\+\\)\\?\" | xargs`"
|
completions="`\"$executable\" $command | grep -o \"^\\ \\+--[a-z-]\\+\\(\\.[a-z0-9-]\\+\\)\\?\" | xargs`"
|
||||||
|
|
||||||
sed -e "s/PROGNAME/$progname/g" -e "s/PROGOPTS/$completions/g" /tmp/completions-template >> $out
|
sed -e "s/PROGNAME/$progname/g" -e "s/PROGOPTS/$completions/g" /tmp/completions-template >> $out
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,7 @@ out="$1"
|
||||||
|
|
||||||
echo "" > $out
|
echo "" > $out
|
||||||
|
|
||||||
progs="arangobench arangosh arangoimp arangodump arangorestore arangod"
|
progs="arangobench arangosh arangoimport arangodump arangorestore arangod"
|
||||||
|
|
||||||
for progname in $progs
|
for progname in $progs
|
||||||
do
|
do
|
||||||
|
@ -16,9 +16,9 @@ for progname in $progs
|
||||||
# check if the executable exists
|
# check if the executable exists
|
||||||
if [[ -f "build/bin/$progname" ]]; then
|
if [[ -f "build/bin/$progname" ]]; then
|
||||||
executable="build/bin/$progname"
|
executable="build/bin/$progname"
|
||||||
|
|
||||||
# setup the help command string
|
# setup the help command string
|
||||||
command="--help-all"
|
command="--help-all"
|
||||||
|
|
||||||
# set up the list of completions for the executable
|
# set up the list of completions for the executable
|
||||||
echo "# completions for $progname" >> "$out"
|
echo "# completions for $progname" >> "$out"
|
||||||
|
|
Loading…
Reference in New Issue