mirror of https://gitee.com/bigwinds/arangodb
Feature/maskings (#8006)
This commit is contained in:
parent
5502ed2a3a
commit
9927b3a281
|
@ -3,84 +3,111 @@ Arangodump Data Maskings
|
|||
|
||||
*--maskings path-of-config*
|
||||
|
||||
It is possible to mask certain fields during dump. A JSON config file is
|
||||
It is possible to mask certain fields for a dump. A JSON configuration file is
|
||||
used to define which fields should be masked and how.
|
||||
|
||||
The general structure of the config file is
|
||||
The general structure of the config file looks like this:
|
||||
|
||||
{
|
||||
"collection-name": {
|
||||
"type": MASKING_TYPE
|
||||
"maskings" : [
|
||||
MASKING1,
|
||||
MASKING2,
|
||||
...
|
||||
]
|
||||
},
|
||||
```json
|
||||
{
|
||||
"collection-name": {
|
||||
"type": MASKING_TYPE
|
||||
"maskings" : [
|
||||
MASKING1,
|
||||
MASKING2,
|
||||
...
|
||||
}
|
||||
]
|
||||
},
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Using `"*"` as collection name defines a default behavior for collections not
|
||||
listed explicitly.
|
||||
|
||||
Masking Types
|
||||
-------------
|
||||
|
||||
This is a string describing how to mask this collection. Possible values are
|
||||
`type` is a string describing how to mask the given collection.
|
||||
Possible values are:
|
||||
|
||||
- "exclude": the collection is ignored completely and not even the structure data
|
||||
- `"exclude"`: the collection is ignored completely and not even the structure data
|
||||
is dumped.
|
||||
|
||||
- "structure": only the collection structure is dumped, but no data at all
|
||||
- `"structure"`: only the collection structure is dumped, but no data at all
|
||||
|
||||
- "masked": the collection structure and all data is dumped. However, the data
|
||||
is subject to maskings defined in the attribute maskings.
|
||||
- `"masked"`: the collection structure and all data is dumped. However, the data
|
||||
is subject to obfuscation defined in the attribute `maskings`.
|
||||
|
||||
- "full": the collection structure and all data is dumped. No masking at all
|
||||
is done for this collection.
|
||||
- `"full"`: the collection structure and all data is dumped. No masking is
|
||||
applied to this collection at all.
|
||||
|
||||
For example:
|
||||
**Example**
|
||||
|
||||
{
|
||||
"private": {
|
||||
"type": "exclude"
|
||||
```json
|
||||
{
|
||||
"private": {
|
||||
"type": "exclude"
|
||||
},
|
||||
|
||||
"log": {
|
||||
"type": "structure"
|
||||
},
|
||||
|
||||
"person": {
|
||||
"type": "masked",
|
||||
"maskings": [
|
||||
{
|
||||
"path": "name",
|
||||
"type": "xifyFront",
|
||||
"unmaskedLength": 2
|
||||
},
|
||||
|
||||
"log": {
|
||||
"type": "structure"
|
||||
},
|
||||
|
||||
"person": {
|
||||
"type": "masked",
|
||||
"maskings": [
|
||||
{
|
||||
"path": "name",
|
||||
"type": "xify_front",
|
||||
"unmaskedLength": 2
|
||||
},
|
||||
{
|
||||
"path": ".security_id",
|
||||
"type": "xify_front",
|
||||
"unmaskedLength": 2
|
||||
}
|
||||
]
|
||||
{
|
||||
"path": ".security_id",
|
||||
"type": "xifyFront",
|
||||
"unmaskedLength": 2
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In the example the collection "private" is completely ignored. Only the
|
||||
structure of the collection "log" is dumped, but not the data itself.
|
||||
The collection "person" is dumped completely but masking the "name" field if
|
||||
it occurs on the top-level. It masks the field "security_id" anywhere in the
|
||||
document. See below for a complete description of the parameters of
|
||||
"xify_front".
|
||||
In the example the collection _private_ is completely ignored. Only the
|
||||
structure of the collection _log_ is dumped, but not the data itself.
|
||||
The collection _person_ is dumped completely but with the _name_ field masked
|
||||
if it occurs on the top-level. It also masks fields with the name "security_id"
|
||||
anywhere in the document. See below for a complete description of the parameters
|
||||
of [type "xifyFront"](#xify-front).
|
||||
|
||||
### Masking vs. dump-data option
|
||||
|
||||
*arangodump* also supports a very coarse masking with the option
|
||||
`--dump-data false`. This basically removes all data from the dump.
|
||||
|
||||
You can either use `--masking` or `--dump-data false`, but not both.
|
||||
|
||||
### Masking vs. include-collection option
|
||||
|
||||
*arangodump* also supports a very coarse masking with the option
|
||||
`--include-collection`. This will restrict the collections that are
|
||||
dumped to the ones explicitly listed.
|
||||
|
||||
It is possible to combine `--masking` and `--include-collection`.
|
||||
This will take the intersection of exportable collections.
|
||||
|
||||
Path
|
||||
----
|
||||
|
||||
If the path starts with a `.` then it is considered to be a wildcard match.
|
||||
For example, `.name` will match the attribute name `name` everywhere in the
|
||||
document. `name` will only match at top level. `person.name` will match
|
||||
the attribute `name` in the top-level object `person`.
|
||||
If the path starts with a `.` then it is considered to match any path
|
||||
ending in `name`. For example, `.name` will match the attribute name
|
||||
`name` all leaf attributes in the document. Leaf attributes are
|
||||
attributes whose value is `null` or of data type `string`, `number`,
|
||||
`bool` and `array` (see below). `name` will only match leaf attributes
|
||||
at top level. `person.name` will match the attribute `name` of a leaf
|
||||
in the top-level object `person`.
|
||||
|
||||
If you have a attribute name that contains a dot, you need to quote the
|
||||
name with either a tick or a backtick. For example
|
||||
If you have an attribute name that contains a dot, you need to quote the
|
||||
name with either a tick or a backtick. For example:
|
||||
|
||||
"path": "´name.with.dots´"
|
||||
|
||||
|
@ -88,24 +115,233 @@ or
|
|||
|
||||
"path": "`name.with.dots`"
|
||||
|
||||
xify_front
|
||||
----------
|
||||
If the attribute value is an array the masking is applied to all the
|
||||
array elements individually.
|
||||
|
||||
This masking replaces characters with `x` and ` `. Alphanumeric characters,
|
||||
`_` and `-` are replaced by `x`, everything else is replaced by ` `.
|
||||
**Example**
|
||||
|
||||
The following configuration will replace the value of the "name"
|
||||
attribute with an "XXXX"-masked string:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "xifyFront",
|
||||
"path": ".name",
|
||||
"unmaskedLength": 2
|
||||
}
|
||||
```
|
||||
|
||||
The document:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "top-level-name",
|
||||
"age": 42,
|
||||
"nicknames" : [ { "name": "hugo" }, "egon" ],
|
||||
"other": {
|
||||
"name": [ "emil", { "secret": "superman" } ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
… will be changed as follows:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "xxxxxxxxxxxxme",
|
||||
"age": 42,
|
||||
"nicknames" : [ { "name": "xxgo" }, "egon" ],
|
||||
"other": {
|
||||
"name": [ "xxil", { "secret": "superman" } ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The values `"egon"` and `"superman"` are not replaced, because they
|
||||
are not contained in an attribute value of which the attribute name is
|
||||
`name`.
|
||||
|
||||
### Nested objects and arrays
|
||||
|
||||
If you specify a path and the attribute value is an array then the
|
||||
masking decision is applied to each element of the array as if this
|
||||
was the value of the attribute.
|
||||
|
||||
If the attribute value is an object, then the attribute is not masked.
|
||||
Instead the nested object is checked further for leaf attributes.
|
||||
|
||||
**Example**
|
||||
|
||||
Masking `email` will convert:
|
||||
|
||||
|
||||
```json
|
||||
{
|
||||
"email" : "email address"
|
||||
}
|
||||
```
|
||||
|
||||
… into:
|
||||
|
||||
```json
|
||||
{
|
||||
"email" : "xxil xxxxxxss"
|
||||
}
|
||||
```
|
||||
|
||||
because `email` is a leaf attribute. The document:
|
||||
|
||||
```json
|
||||
{
|
||||
"email" : [
|
||||
"address one",
|
||||
"address two"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
… will be converted into:
|
||||
|
||||
```json
|
||||
{
|
||||
"email" : [
|
||||
"xxxxxss xne",
|
||||
"xxxxxss xwo"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
… because the array is "unfolded". The document:
|
||||
|
||||
```json
|
||||
{
|
||||
"email" : {
|
||||
"address" : "email address"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
… will not be changed because `email` is not a leaf attribute.
|
||||
|
||||
|
||||
Masking Functions
|
||||
-----------------
|
||||
|
||||
{% hint 'info' %}
|
||||
The following masking functions are only available in the
|
||||
[**Enterprise Edition**](https://www.arangodb.com/why-arangodb/arangodb-enterprise/)
|
||||
{% endhint %}
|
||||
|
||||
- xify front
|
||||
- zip
|
||||
- datetime
|
||||
- integral number
|
||||
- decimal number
|
||||
- credit card number
|
||||
- phone number
|
||||
- email address
|
||||
|
||||
The function:
|
||||
|
||||
- random string
|
||||
|
||||
… is available on Community Edition and in the Enterprise Edition.
|
||||
|
||||
|
||||
### Random string
|
||||
|
||||
```json
|
||||
{
|
||||
"path": ".name",
|
||||
"type": "randomString"
|
||||
}
|
||||
```
|
||||
|
||||
This masking type will replace all values of attributes with key
|
||||
`name` with an anonymized string. It is not guaranteed that the string
|
||||
will be of the same length.
|
||||
|
||||
A hash of the original string is computed. If the original string is
|
||||
shorter then the hash will be used. This will result in a longer
|
||||
replacement string. If the string is longer than the hash then
|
||||
characters will be repeated as many times as needed to reach the full
|
||||
original string length.
|
||||
|
||||
**Example**
|
||||
|
||||
Masking name as above, the document:
|
||||
|
||||
```json
|
||||
{
|
||||
"_key" : "38937",
|
||||
"_id" : "examplecollection/38937",
|
||||
"_rev" : "_YFaGG1u--_",
|
||||
"name" : [
|
||||
"My Name",
|
||||
{
|
||||
"other" : "Hallo Name"
|
||||
},
|
||||
[
|
||||
"Name One",
|
||||
"Name Two"
|
||||
],
|
||||
true,
|
||||
false,
|
||||
null,
|
||||
1.0,
|
||||
1234,
|
||||
"This is a very long name"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
… will be converted into
|
||||
|
||||
```json
|
||||
{
|
||||
"_key": "38937",
|
||||
"_id": "examplecollection/38937",
|
||||
"_rev": "_YFaGG1u--_",
|
||||
"name": [
|
||||
"+y5OQiYmp/o=",
|
||||
{
|
||||
"path": ".name",
|
||||
"unmaskedLength": 2
|
||||
}
|
||||
"other": "Hallo Name"
|
||||
},
|
||||
[
|
||||
"ihCTrlsKKdk=",
|
||||
"yo/55hfla0U="
|
||||
],
|
||||
true,
|
||||
false,
|
||||
null,
|
||||
1.0,
|
||||
1234,
|
||||
"hwjAfNe5BGw=hwjAfNe5BGw="
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
This will mask all alphanumeric characters of a word except the last 2.
|
||||
Words of length 1 and 2 are unmasked. If the attribute value is not a
|
||||
string the result will be `xxxx`.
|
||||
### Xify front
|
||||
|
||||
This masking type replaces the front characters with `x` and
|
||||
blanks. Alphanumeric characters, `_` and `-` are replaced by `x`,
|
||||
everything else is replaced by a blank.
|
||||
|
||||
```json
|
||||
{
|
||||
"path": ".name",
|
||||
"type": "xifyFront",
|
||||
"unmaskedLength": 2
|
||||
}
|
||||
```
|
||||
|
||||
This will mask all alphanumeric characters of a word except the last
|
||||
two characters. Words of length 1 and 2 are unmasked. If the
|
||||
attribute value is not a string the result will be `xxxx`.
|
||||
|
||||
"This is a test!Do you agree?"
|
||||
|
||||
will become
|
||||
… will become
|
||||
|
||||
"xxis is a xxst Do xou xxxee "
|
||||
|
||||
|
@ -113,30 +349,193 @@ There is a catch. If you have an index on the attribute the masking
|
|||
might distort the index efficiency or even cause errors in case of a
|
||||
unique index.
|
||||
|
||||
{
|
||||
"path": ".name",
|
||||
"unmaskedLength": 2,
|
||||
"hash": true
|
||||
}
|
||||
```json
|
||||
{
|
||||
"type": "xifyFront",
|
||||
"path": ".name",
|
||||
"unmaskedLength": 2,
|
||||
"hash": true
|
||||
}
|
||||
```
|
||||
|
||||
This will add a hash at the end of the string.
|
||||
|
||||
"This is a test!Do you agree?"
|
||||
|
||||
will become
|
||||
… will become
|
||||
|
||||
"xxis is a xxst Do xou xxxee NAATm8c9hVQ="
|
||||
|
||||
Note that the hash is based on a random secrect that is different for
|
||||
each run. This avoids dictionary attacks.
|
||||
each run. This avoids dictionary attacks which can be used to guess
|
||||
values based pre-computations on dictionaries.
|
||||
|
||||
If you need reproducable results, i.e. hash that do not change between
|
||||
different runs of *arangodump*, you need to specify a seed, which must
|
||||
not be `0`.
|
||||
If you need reproducible results, i.e. hashes that do not change between
|
||||
different runs of *arangodump*, you need to specify a secret as seed,
|
||||
a number which must not be `0`.
|
||||
|
||||
{
|
||||
"path": ".name",
|
||||
"unmaskedLength": 2,
|
||||
"hash": true,
|
||||
"seed": 246781478647
|
||||
}
|
||||
```json
|
||||
{
|
||||
"type": "xifyFront",
|
||||
"path": ".name",
|
||||
"unmaskedLength": 2,
|
||||
"hash": true,
|
||||
"seed": 246781478647
|
||||
}
|
||||
```
|
||||
|
||||
### Zip
|
||||
|
||||
This masking type replaces a zip code with a random one. If the
|
||||
attribute value is not a string then the default value of `"12345"` is
|
||||
used as no zip is known. You can change the default value, see below.
|
||||
|
||||
```json
|
||||
{
|
||||
"path": ".code",
|
||||
"type": "zip",
|
||||
}
|
||||
```
|
||||
|
||||
This will replace a real zip code with a random one. It uses the following
|
||||
rule: If a character of the original zip code is a digit it will be replaced
|
||||
by a random digit. If a character of the original zip code is a letter it
|
||||
will be replaced by a random letter keeping the case.
|
||||
|
||||
```json
|
||||
{
|
||||
"path": ".code",
|
||||
"type": "zip",
|
||||
"default": "abcdef"
|
||||
}
|
||||
```
|
||||
|
||||
**Example**
|
||||
|
||||
If the original zip code is:
|
||||
|
||||
50674
|
||||
|
||||
… it will be replaced by e.g.:
|
||||
|
||||
98146
|
||||
|
||||
If the original zip code is:
|
||||
|
||||
SA34-EA
|
||||
|
||||
… it will be replaced by e.g.:
|
||||
|
||||
OW91-JI
|
||||
|
||||
Note that this will generate random zip code. Therefore there is a
|
||||
chance generate the same zip code value multiple times, which can
|
||||
cause unique constraint violations if a unique index is or will be
|
||||
used on the zip code attribute.
|
||||
|
||||
### Datetime
|
||||
|
||||
This masking type replaces the value of the attribute with a random
|
||||
date.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "datetime",
|
||||
"begin" : "2019-01-01",
|
||||
"end": "2019-12-31",
|
||||
"format": "%yyyy-%mm-%dd",
|
||||
}
|
||||
```
|
||||
|
||||
`begin` and `end` are in ISO8601 format.
|
||||
|
||||
The format is described in
|
||||
[DATE_FORMAT](../../../AQL/Functions/Date.html#dateformat).
|
||||
|
||||
### Integral number
|
||||
|
||||
This masking type replaces the value of the attribute with a random
|
||||
integral number. It will replace the value even if it is a string,
|
||||
boolean, or false.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "integer",
|
||||
"lower" : -100,
|
||||
"upper": 100
|
||||
}
|
||||
```
|
||||
|
||||
### Decimal number
|
||||
|
||||
This masking type replaces the value of the attribute with a random
|
||||
decimal. It will replace the value even if it is a string, boolean,
|
||||
or false.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "float",
|
||||
"lower" : -0.3,
|
||||
"upper": 0.3
|
||||
}
|
||||
```
|
||||
|
||||
By default, the decimal has a scale of 2. I.e. it has at most 2
|
||||
decimal digits. The definition:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "float",
|
||||
"lower" : -0.3,
|
||||
"upper": 0.3,
|
||||
"scale": 3
|
||||
}
|
||||
```
|
||||
|
||||
… will generate numbers with at most 3 decimal digits.
|
||||
|
||||
### Credit card number
|
||||
|
||||
This masking type replaces the value of the attribute with a random
|
||||
credit card number.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "creditCard",
|
||||
}
|
||||
```
|
||||
|
||||
See [Luhn](https://en.wikipedia.org/wiki/Luhn_algorithm) for details.
|
||||
|
||||
### Phone number
|
||||
|
||||
This masking type replaces a phone number with a random one. If the
|
||||
attribute value is not a string it is replaced by the string
|
||||
`"+1234567890"`.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "phone",
|
||||
"default": "+4912345123456789"
|
||||
}
|
||||
```
|
||||
|
||||
This will replace an existing phone number with a random one. It uses
|
||||
the following rule: If a character of the original number is a digit
|
||||
it will be replaced by a random digit. If it is a letter it is replaced
|
||||
by a letter. All other characters are unchanged.
|
||||
|
||||
```json
|
||||
{ "type": "zip",
|
||||
"default": "+4912345123456789"
|
||||
}
|
||||
```
|
||||
|
||||
If the attribute value is not a string use the value of default
|
||||
`"+4912345123456789"`.
|
||||
|
||||
### Email address
|
||||
|
||||
This masking type takes an email address, computes a hash value and
|
||||
split it into three equal parts `AAAA`, `BBBB`, and `CCCC`. The
|
||||
resulting email address is `AAAA.BBBB@CCCC.invalid`.
|
||||
|
|
|
@ -29,6 +29,15 @@
|
|||
* License Name: Boost Software License 1.0
|
||||
* License Id: BSL-1.0
|
||||
|
||||
### CreditCardGenerator 2016
|
||||
|
||||
* Name: CreditCardGenerator
|
||||
* Version: 1.8.1
|
||||
* Project Home: https://github.com/stormdark/CreditCardGenerator
|
||||
* License: https://raw.githubusercontent.com/stormdark/CreditCardGenerator/master/LICENSE
|
||||
* License Name: MIT License
|
||||
* License Id: MIT
|
||||
|
||||
### Curl 7.50.3
|
||||
|
||||
* Name: Curl
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
#include "Dump/DumpFeature.h"
|
||||
#include "Logger/Logger.h"
|
||||
#include "Logger/LoggerFeature.h"
|
||||
#include "Maskings/AttributeMasking.h"
|
||||
#include "ProgramOptions/ProgramOptions.h"
|
||||
#include "Random/RandomFeature.h"
|
||||
#include "Shell/ClientFeature.h"
|
||||
|
@ -42,6 +43,7 @@
|
|||
|
||||
#ifdef USE_ENTERPRISE
|
||||
#include "Enterprise/Encryption/EncryptionFeature.h"
|
||||
#include "Enterprise/Maskings/AttributeMaskingEE.h"
|
||||
#endif
|
||||
|
||||
using namespace arangodb;
|
||||
|
@ -53,6 +55,12 @@ int main(int argc, char* argv[]) {
|
|||
ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
|
||||
context.installHup();
|
||||
|
||||
maskings::InstallMaskings();
|
||||
|
||||
#ifdef USE_ENTERPRISE
|
||||
maskings::InstallMaskingsEE();
|
||||
#endif
|
||||
|
||||
std::shared_ptr<options::ProgramOptions> options(
|
||||
new options::ProgramOptions(argv[0], "Usage: arangodump [<options>]",
|
||||
"For more information use:", BIN_DIRECTORY));
|
||||
|
|
|
@ -107,6 +107,12 @@ class ConfigBuilder {
|
|||
this.config['create-database'] = 'false';
|
||||
}
|
||||
}
|
||||
setMaskings(dir) {
|
||||
if (this.type !== 'dump') {
|
||||
throw '"maskings" is not supported for binary: ' + this.type;
|
||||
}
|
||||
this.config['maskings'] = fs.join(TOP_DIR, "tests/js/common/test-data/maskings", dir);
|
||||
}
|
||||
activateEncryption() { this.config['encription.keyfile'] = fs.join(this.rootDir, 'secret-key'); }
|
||||
setRootDir(dir) { this.rootDir = dir; }
|
||||
restrictToCollection(collection) {
|
||||
|
|
|
@ -2,34 +2,36 @@
|
|||
/* global print */
|
||||
'use strict';
|
||||
|
||||
// //////////////////////////////////////////////////////////////////////////////
|
||||
// / DISCLAIMER
|
||||
// /
|
||||
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
|
||||
// / Copyright 2014 triagens GmbH, Cologne, Germany
|
||||
// /
|
||||
// / Licensed under the Apache License, Version 2.0 (the "License")
|
||||
// / you may not use this file except in compliance with the License.
|
||||
// / You may obtain a copy of the License at
|
||||
// /
|
||||
// / http://www.apache.org/licenses/LICENSE-2.0
|
||||
// /
|
||||
// / Unless required by applicable law or agreed to in writing, software
|
||||
// / distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// / See the License for the specific language governing permissions and
|
||||
// / limitations under the License.
|
||||
// /
|
||||
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
|
||||
// /
|
||||
// / @author Max Neunhoeffer
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
// DISCLAIMER
|
||||
//
|
||||
// Copyright 2016-2019 ArangoDB GmbH, Cologne, Germany
|
||||
// Copyright 2014 triagens GmbH, Cologne, Germany
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License")
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Copyright holder is ArangoDB GmbH, Cologne, Germany
|
||||
//
|
||||
// @author Max Neunhoeffer
|
||||
// //////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
const functionsDocumentation = {
|
||||
'dump': 'dump tests',
|
||||
'dump_authentication': 'dump tests with authentication',
|
||||
'dump_encrypted': 'encrypted dump tests',
|
||||
'dump_authentication': 'dump tests with authentication'
|
||||
'dump_maskings': 'masked dump tests'
|
||||
};
|
||||
|
||||
const optionsDocumentation = [
|
||||
' - `skipEncrypted` : if set to true the encryption tests are skipped'
|
||||
];
|
||||
|
@ -48,16 +50,18 @@ const RESET = require('internal').COLORS.COLOR_RESET;
|
|||
|
||||
const testPaths = {
|
||||
'dump': [tu.pathForTesting('server/dump')],
|
||||
'dump_authentication': [tu.pathForTesting('server/dump')],
|
||||
'dump_encrypted': [tu.pathForTesting('server/dump')],
|
||||
'dump_authentication': [tu.pathForTesting('server/dump')]
|
||||
'dump_maskings': [tu.pathForTesting('server/dump')]
|
||||
};
|
||||
|
||||
class DumpRestoreHelper {
|
||||
constructor(instanceInfo, options, clientAuth, dumpOptions, which, afterServerStart) {
|
||||
constructor(instanceInfo, options, clientAuth, dumpOptions, restoreOptions, which, afterServerStart) {
|
||||
this.instanceInfo = instanceInfo;
|
||||
this.options = options;
|
||||
this.clientAuth = clientAuth;
|
||||
this.dumpOptions = dumpOptions;
|
||||
this.restoreOptions = restoreOptions;
|
||||
this.which = which;
|
||||
this.fn = afterServerStart(instanceInfo);
|
||||
this.results = {failed: 1};
|
||||
|
@ -66,11 +70,15 @@ class DumpRestoreHelper {
|
|||
this.dumpConfig.setOutputDirectory('dump');
|
||||
this.dumpConfig.setIncludeSystem(true);
|
||||
|
||||
this.restoreConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo);
|
||||
if (dumpOptions.hasOwnProperty("maskings")) {
|
||||
this.dumpConfig.setMaskings(dumpOptions.maskings);
|
||||
}
|
||||
|
||||
this.restoreConfig = pu.createBaseConfig('restore', this.restoreOptions, this.instanceInfo);
|
||||
this.restoreConfig.setInputDirectory('dump', true);
|
||||
this.restoreConfig.setIncludeSystem(true);
|
||||
|
||||
this.restoreOldConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo);
|
||||
this.restoreOldConfig = pu.createBaseConfig('restore', this.restoreOptions, this.instanceInfo);
|
||||
this.restoreOldConfig.setInputDirectory('dump', true);
|
||||
this.restoreOldConfig.setIncludeSystem(true);
|
||||
this.restoreOldConfig.setDatabase('_system');
|
||||
|
@ -81,8 +89,8 @@ class DumpRestoreHelper {
|
|||
this.restoreOldConfig.activateEncryption();
|
||||
}
|
||||
|
||||
this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.dumpOptions, this.instanceInfo.rootDir);
|
||||
this.arangorestoreOld = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreOldConfig, this.dumpOptions, this.instanceInfo.rootDir);
|
||||
this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.restoreOptions, this.instanceInfo.rootDir);
|
||||
this.arangorestoreOld = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreOldConfig, this.restoreOptions, this.instanceInfo.rootDir);
|
||||
this.arangodump = pu.run.arangoDumpRestoreWithConfig.bind(this, this.dumpConfig, this.dumpOptions, this.instanceInfo.rootDir);
|
||||
}
|
||||
|
||||
|
@ -225,10 +233,7 @@ function getClusterStrings(options)
|
|||
}
|
||||
}
|
||||
|
||||
// //////////////////////////////////////////////////////////////////////////////
|
||||
// / @brief TEST: dump
|
||||
// //////////////////////////////////////////////////////////////////////////////
|
||||
function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which, tstFiles, afterServerStart) {
|
||||
function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restoreOptions, which, tstFiles, afterServerStart) {
|
||||
print(CYAN + which + ' tests...' + RESET);
|
||||
|
||||
let instanceInfo = pu.startInstance('tcp', options, serverAuthInfo, which);
|
||||
|
@ -243,7 +248,7 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which,
|
|||
};
|
||||
return rc;
|
||||
}
|
||||
const helper = new DumpRestoreHelper(instanceInfo, options, clientAuth, dumpOptions, which, afterServerStart);
|
||||
const helper = new DumpRestoreHelper(instanceInfo, options, clientAuth, dumpOptions, restoreOptions, which, afterServerStart);
|
||||
|
||||
const setupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpSetup));
|
||||
const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain));
|
||||
|
@ -267,21 +272,21 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which,
|
|||
}
|
||||
}
|
||||
|
||||
const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest));
|
||||
if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') ||
|
||||
!helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') ||
|
||||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') ||
|
||||
!helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxAppsBundle') ||
|
||||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxBundleApps') ||
|
||||
!helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxBundleApps')) {
|
||||
return helper.extractResults();
|
||||
if (tstFiles.hasOwnProperty("foxxTest")) {
|
||||
const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest));
|
||||
if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') ||
|
||||
!helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') ||
|
||||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') ||
|
||||
!helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxAppsBundle') ||
|
||||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxBundleApps') ||
|
||||
!helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxBundleApps')) {
|
||||
return helper.extractResults();
|
||||
}
|
||||
}
|
||||
|
||||
return helper.extractResults();
|
||||
}
|
||||
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
function dump (options) {
|
||||
let c = getClusterStrings(options);
|
||||
let tstFiles = {
|
||||
|
@ -292,7 +297,7 @@ function dump (options) {
|
|||
foxxTest: 'check-foxx.js'
|
||||
};
|
||||
|
||||
return dump_backend(options, {}, {}, options, 'dump', tstFiles, function(){});
|
||||
return dump_backend(options, {}, {}, options, options, 'dump', tstFiles, function(){});
|
||||
}
|
||||
|
||||
function dumpAuthentication (options) {
|
||||
|
@ -332,7 +337,7 @@ function dumpAuthentication (options) {
|
|||
foxxTest: 'check-foxx.js'
|
||||
};
|
||||
|
||||
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, 'dump_authentication', tstFiles, function(){});
|
||||
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, dumpAuthOpts, 'dump_authentication', tstFiles, function(){});
|
||||
}
|
||||
|
||||
function dumpEncrypted (options) {
|
||||
|
@ -373,20 +378,57 @@ function dumpEncrypted (options) {
|
|||
foxxTest: 'check-foxx.js'
|
||||
};
|
||||
|
||||
return dump_backend(options, {}, {}, dumpOptions, 'dump_encrypted', tstFiles, afterServerStart);
|
||||
return dump_backend(options, {}, {}, dumpOptions, dumpOptions, 'dump_encrypted', tstFiles, afterServerStart);
|
||||
}
|
||||
|
||||
function dumpMaskings (options) {
|
||||
// test is only meaningful in the enterprise version
|
||||
let skip = true;
|
||||
if (global.ARANGODB_CLIENT_VERSION) {
|
||||
let version = global.ARANGODB_CLIENT_VERSION(true);
|
||||
if (version.hasOwnProperty('enterprise-version')) {
|
||||
skip = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (skip) {
|
||||
print('skipping dump_maskings test');
|
||||
return {
|
||||
dump_maskings: {
|
||||
status: true,
|
||||
skipped: true
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let tstFiles = {
|
||||
dumpSetup: 'dump-maskings-setup.js',
|
||||
dumpAgain: 'dump-maskings.js',
|
||||
dumpTearDown: 'dump-teardown.js'
|
||||
};
|
||||
|
||||
let dumpMaskingsOpts = {
|
||||
maskings: 'maskings1.json'
|
||||
};
|
||||
|
||||
_.defaults(dumpMaskingsOpts, options);
|
||||
|
||||
return dump_backend(options, {}, {}, dumpMaskingsOpts, options, 'dump_maskings', tstFiles, function(){});
|
||||
}
|
||||
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {
|
||||
Object.assign(allTestPaths, testPaths);
|
||||
testFns['dump'] = dump;
|
||||
defaultFns.push('dump');
|
||||
|
||||
testFns['dump_authentication'] = dumpAuthentication;
|
||||
defaultFns.push('dump_authentication');
|
||||
|
||||
testFns['dump_encrypted'] = dumpEncrypted;
|
||||
defaultFns.push('dump_encrypted');
|
||||
|
||||
testFns['dump_authentication'] = dumpAuthentication;
|
||||
defaultFns.push('dump_authentication');
|
||||
testFns['dump_maskings'] = dumpMaskings;
|
||||
defaultFns.push('dump_maskings');
|
||||
|
||||
for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; }
|
||||
for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); }
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// DISCLAIMER
|
||||
///
|
||||
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
|
||||
/// Copyright 2014-2019 ArangoDB GmbH, Cologne, Germany
|
||||
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
@ -25,9 +25,10 @@
|
|||
#ifndef ARANGODB_BASICS_UTF8HELPER_H
|
||||
#define ARANGODB_BASICS_UTF8HELPER_H 1
|
||||
|
||||
#include <velocypack/StringRef.h>
|
||||
#include "Basics/Common.h"
|
||||
|
||||
#include <velocypack/StringRef.h>
|
||||
|
||||
#include <unicode/coll.h>
|
||||
#include <unicode/regex.h>
|
||||
#include <unicode/ustring.h>
|
||||
|
@ -40,10 +41,6 @@ class Utf8Helper {
|
|||
Utf8Helper& operator=(Utf8Helper const&) = delete;
|
||||
|
||||
public:
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
/// @brief a default helper
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static Utf8Helper DefaultUtf8Helper;
|
||||
|
||||
public:
|
||||
|
@ -153,6 +150,26 @@ class Utf8Helper {
|
|||
char const* replacement, size_t replacementLength,
|
||||
bool partial, bool& error);
|
||||
|
||||
// append an UTF8 to a string. This will append 1 to 4 bytes.
|
||||
static void appendUtf8Character(std::string& result, uint32_t ch) {
|
||||
if (ch <= 0x7f) {
|
||||
result.push_back((uint8_t)ch);
|
||||
} else {
|
||||
if (ch <= 0x7ff) {
|
||||
result.push_back((uint8_t)((ch >> 6) | 0xc0));
|
||||
} else {
|
||||
if (ch <= 0xffff) {
|
||||
result.push_back((uint8_t)((ch >> 12) | 0xe0));
|
||||
} else {
|
||||
result.push_back((uint8_t)((ch >> 18) | 0xf0));
|
||||
result.push_back((uint8_t)(((ch >> 12) & 0x3f) | 0x80));
|
||||
}
|
||||
result.push_back((uint8_t)(((ch >> 6) & 0x3f) | 0x80));
|
||||
}
|
||||
result.push_back((uint8_t)((ch & 0x3f) | 0x80));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
Collator* _coll;
|
||||
};
|
||||
|
|
|
@ -23,6 +23,8 @@
|
|||
#ifndef ARANGODB_BASICS_DATETIME_H
|
||||
#define ARANGODB_BASICS_DATETIME_H 1
|
||||
|
||||
#include "Basics/Common.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <regex>
|
||||
|
||||
|
|
|
@ -237,7 +237,7 @@ add_library(${LIB_ARANGO} STATIC
|
|||
Maskings/Collection.cpp
|
||||
Maskings/Maskings.cpp
|
||||
Maskings/Path.cpp
|
||||
Maskings/XifyFront.cpp
|
||||
Maskings/RandomStringMask.cpp
|
||||
ProgramOptions/Option.cpp
|
||||
ProgramOptions/ProgramOptions.cpp
|
||||
ProgramOptions/Section.cpp
|
||||
|
|
|
@ -24,11 +24,17 @@
|
|||
|
||||
#include "Basics/StringUtils.h"
|
||||
#include "Logger/Logger.h"
|
||||
#include "Maskings/XifyFront.h"
|
||||
#include "Maskings/RandomStringMask.h"
|
||||
|
||||
using namespace arangodb;
|
||||
using namespace arangodb::maskings;
|
||||
|
||||
void arangodb::maskings::InstallMaskings() {
|
||||
AttributeMasking::installMasking("randomString", RandomStringMask::create);
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, ParseResult<AttributeMasking> (*)(Path, Maskings*, VPackSlice const&)> AttributeMasking::_maskings;
|
||||
|
||||
ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
|
||||
VPackSlice const& def) {
|
||||
if (!def.isObject()) {
|
||||
|
@ -39,9 +45,6 @@ ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
|
|||
|
||||
std::string path = "";
|
||||
std::string type = "";
|
||||
uint64_t length = 2;
|
||||
uint64_t seed = 0;
|
||||
bool hash = false;
|
||||
|
||||
for (auto const& entry : VPackObjectIterator(def, false)) {
|
||||
std::string key = entry.key.copyString();
|
||||
|
@ -60,27 +63,6 @@ ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
|
|||
}
|
||||
|
||||
path = entry.value.copyString();
|
||||
} else if (key == "unmaskedLength") {
|
||||
if (!entry.value.isInteger()) {
|
||||
return ParseResult<AttributeMasking>(ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
|
||||
"length must be an integer");
|
||||
}
|
||||
|
||||
length = entry.value.getInt();
|
||||
} else if (key == "hash") {
|
||||
if (!entry.value.isBool()) {
|
||||
return ParseResult<AttributeMasking>(ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
|
||||
"hash must be an integer");
|
||||
}
|
||||
|
||||
hash = entry.value.getBool();
|
||||
} else if (key == "seed") {
|
||||
if (!entry.value.isInteger()) {
|
||||
return ParseResult<AttributeMasking>(ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
|
||||
"seed must be an integer");
|
||||
}
|
||||
|
||||
seed = entry.value.getInt();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -96,20 +78,15 @@ ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
|
|||
(ParseResult<AttributeMasking>::StatusCode)(int)ap.status, ap.message);
|
||||
}
|
||||
|
||||
if (type == "xify_front") {
|
||||
if (length < 1) {
|
||||
return ParseResult<AttributeMasking>(
|
||||
ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
|
||||
"expecting length to be at least for xify_front");
|
||||
}
|
||||
auto const& it = _maskings.find(type);
|
||||
|
||||
if (it == _maskings.end()) {
|
||||
return ParseResult<AttributeMasking>(
|
||||
AttributeMasking(ap.result, new XifyFront(maskings, length, hash, seed)));
|
||||
ParseResult<AttributeMasking>::UNKNOWN_TYPE,
|
||||
"unknown attribute masking type '" + type + "'");
|
||||
}
|
||||
|
||||
return ParseResult<AttributeMasking>(
|
||||
ParseResult<AttributeMasking>::UNKNOWN_TYPE,
|
||||
"expecting unknown attribute masking type '" + type + "'");
|
||||
return it->second(ap.result, maskings, def);
|
||||
}
|
||||
|
||||
bool AttributeMasking::match(std::vector<std::string> const& path) const {
|
||||
|
|
|
@ -37,9 +37,14 @@
|
|||
|
||||
namespace arangodb {
|
||||
namespace maskings {
|
||||
void InstallMaskings();
|
||||
|
||||
class AttributeMasking {
|
||||
public:
|
||||
static ParseResult<AttributeMasking> parse(Maskings*, VPackSlice const&);
|
||||
static void installMasking(std::string const& name, ParseResult<AttributeMasking> (* func)(Path, Maskings*, VPackSlice const&)) {
|
||||
_maskings[name] = func;
|
||||
}
|
||||
|
||||
public:
|
||||
AttributeMasking() = default;
|
||||
|
@ -52,6 +57,9 @@ class AttributeMasking {
|
|||
|
||||
MaskingFunction* func() const { return _func.get(); }
|
||||
|
||||
private:
|
||||
static std::unordered_map<std::string, ParseResult<AttributeMasking> (*)(Path, Maskings*, VPackSlice const&)> _maskings;
|
||||
|
||||
private:
|
||||
Path _path;
|
||||
std::shared_ptr<MaskingFunction> _func;
|
||||
|
|
|
@ -25,6 +25,8 @@
|
|||
|
||||
#include "Basics/Common.h"
|
||||
|
||||
#include "Basics/Utf8Helper.h"
|
||||
|
||||
#include <velocypack/Builder.h>
|
||||
#include <velocypack/Iterator.h>
|
||||
#include <velocypack/Parser.h>
|
||||
|
@ -37,9 +39,8 @@ class Maskings;
|
|||
|
||||
class MaskingFunction {
|
||||
public:
|
||||
static bool isNameChar(char c) {
|
||||
return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
|
||||
('0' <= c && c <= '9') || c == '_' || c == '-';
|
||||
static bool isNameChar(UChar32 ch) {
|
||||
return u_isalpha(ch) || u_isdigit(ch) || ch == U'_' || ch == U'-';
|
||||
}
|
||||
|
||||
public:
|
||||
|
|
|
@ -85,11 +85,21 @@ ParseResult<Maskings> Maskings::parse(VPackSlice const& def) {
|
|||
|
||||
for (auto const& entry : VPackObjectIterator(def, false)) {
|
||||
std::string key = entry.key.copyString();
|
||||
LOG_TOPIC(TRACE, Logger::CONFIG) << "masking collection '" << key << "'";
|
||||
|
||||
if (_collections.find(key) != _collections.end()) {
|
||||
return ParseResult<Maskings>(ParseResult<Maskings>::DUPLICATE_COLLECTION,
|
||||
"duplicate collection entry '" + key + "'");
|
||||
if (key == "*") {
|
||||
LOG_TOPIC(TRACE, Logger::CONFIG) << "default masking";
|
||||
|
||||
if (_hasDefaultCollection) {
|
||||
return ParseResult<Maskings>(ParseResult<Maskings>::DUPLICATE_COLLECTION,
|
||||
"duplicate default entry");
|
||||
}
|
||||
} else {
|
||||
LOG_TOPIC(TRACE, Logger::CONFIG) << "masking collection '" << key << "'";
|
||||
|
||||
if (_collections.find(key) != _collections.end()) {
|
||||
return ParseResult<Maskings>(ParseResult<Maskings>::DUPLICATE_COLLECTION,
|
||||
"duplicate collection entry '" + key + "'");
|
||||
}
|
||||
}
|
||||
|
||||
ParseResult<Collection> c = Collection::parse(this, entry.value);
|
||||
|
@ -99,20 +109,30 @@ ParseResult<Maskings> Maskings::parse(VPackSlice const& def) {
|
|||
c.message);
|
||||
}
|
||||
|
||||
_collections[key] = c.result;
|
||||
if (key == "*") {
|
||||
_hasDefaultCollection = true;
|
||||
_defaultCollection = c.result;
|
||||
} else {
|
||||
_collections[key] = c.result;
|
||||
}
|
||||
}
|
||||
|
||||
return ParseResult<Maskings>(ParseResult<Maskings>::VALID);
|
||||
}
|
||||
|
||||
bool Maskings::shouldDumpStructure(std::string const& name) {
|
||||
CollectionSelection select = CollectionSelection::EXCLUDE;
|
||||
auto const itr = _collections.find(name);
|
||||
|
||||
if (itr == _collections.end()) {
|
||||
return false;
|
||||
if (_hasDefaultCollection) {
|
||||
select = _defaultCollection.selection();
|
||||
}
|
||||
} else {
|
||||
select = itr->second.selection();
|
||||
}
|
||||
|
||||
switch (itr->second.selection()) {
|
||||
switch (select) {
|
||||
case CollectionSelection::FULL:
|
||||
return true;
|
||||
case CollectionSelection::MASKED:
|
||||
|
@ -129,13 +149,18 @@ bool Maskings::shouldDumpStructure(std::string const& name) {
|
|||
}
|
||||
|
||||
bool Maskings::shouldDumpData(std::string const& name) {
|
||||
CollectionSelection select = CollectionSelection::EXCLUDE;
|
||||
auto const itr = _collections.find(name);
|
||||
|
||||
if (itr == _collections.end()) {
|
||||
return false;
|
||||
if (_hasDefaultCollection) {
|
||||
select = _defaultCollection.selection();
|
||||
}
|
||||
} else {
|
||||
select = itr->second.selection();
|
||||
}
|
||||
|
||||
switch (itr->second.selection()) {
|
||||
switch (select) {
|
||||
case CollectionSelection::FULL:
|
||||
return true;
|
||||
case CollectionSelection::MASKED:
|
||||
|
@ -289,14 +314,21 @@ void Maskings::mask(std::string const& name, basics::StringBuffer const& data,
|
|||
basics::StringBuffer& result) {
|
||||
result.clear();
|
||||
|
||||
Collection* collection;
|
||||
auto const itr = _collections.find(name);
|
||||
|
||||
if (itr == _collections.end()) {
|
||||
result.copy(data);
|
||||
return;
|
||||
if (_hasDefaultCollection) {
|
||||
collection = &_defaultCollection;
|
||||
} else {
|
||||
result.copy(data);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
collection = &(itr->second);
|
||||
}
|
||||
|
||||
if (itr->second.selection() == CollectionSelection::FULL) {
|
||||
if (collection->selection() == CollectionSelection::FULL) {
|
||||
result.copy(data);
|
||||
return;
|
||||
}
|
||||
|
@ -314,7 +346,7 @@ void Maskings::mask(std::string const& name, basics::StringBuffer const& data,
|
|||
|
||||
std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(q, p - q);
|
||||
|
||||
addMasked(itr->second, result, builder->slice());
|
||||
addMasked(*collection, result, builder->slice());
|
||||
|
||||
while (p < e && (*p == '\n' || *p == '\r')) {
|
||||
++p;
|
||||
|
|
|
@ -80,6 +80,8 @@ class Maskings {
|
|||
|
||||
private:
|
||||
std::map<std::string, Collection> _collections;
|
||||
bool _hasDefaultCollection = false;
|
||||
Collection _defaultCollection;
|
||||
uint64_t _randomSeed = 0;
|
||||
};
|
||||
|
||||
|
|
|
@ -23,6 +23,7 @@
|
|||
#include "Collection.h"
|
||||
|
||||
#include "Basics/StringUtils.h"
|
||||
#include "Basics/Utf8Helper.h"
|
||||
#include "Logger/Logger.h"
|
||||
|
||||
using namespace arangodb;
|
||||
|
@ -40,71 +41,66 @@ ParseResult<Path> Path::parse(std::string const& def) {
|
|||
wildcard = true;
|
||||
}
|
||||
|
||||
char const* p = def.c_str();
|
||||
char const* e = p + def.size();
|
||||
uint8_t const* p = reinterpret_cast<uint8_t const*>(def.c_str());
|
||||
int32_t off = 0;
|
||||
int32_t len = def.size();
|
||||
UChar32 ch;
|
||||
|
||||
if (wildcard) {
|
||||
++p;
|
||||
U8_NEXT(p, off, len, ch);
|
||||
}
|
||||
|
||||
std::vector<std::string> components;
|
||||
std::string buffer;
|
||||
|
||||
while (p < e) {
|
||||
if (*p == '.') {
|
||||
while (off < len) {
|
||||
U8_NEXT(p, off, len, ch);
|
||||
|
||||
if (ch < 0) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def + "' contains illegal UTF-8");
|
||||
} else if (ch == 46) {
|
||||
if (buffer.size() == 0) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def +
|
||||
"' contains an empty component");
|
||||
}
|
||||
|
||||
++p;
|
||||
components.push_back(buffer);
|
||||
buffer.clear();
|
||||
} else if (*p == 96) { // backtick `
|
||||
++p;
|
||||
} else if (ch == 96 || ch == 180) { // windows does not like U'`' and U'´'
|
||||
UChar32 quote = ch;
|
||||
U8_NEXT(p, off, len, ch);
|
||||
|
||||
while (p < e && *p != 96) {
|
||||
buffer.push_back(*p++);
|
||||
if (ch < 0) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def + "' contains illegal UTF-8");
|
||||
}
|
||||
|
||||
if (p == e) {
|
||||
while (off < len && ch != quote) {
|
||||
basics::Utf8Helper::appendUtf8Character(buffer, ch);
|
||||
U8_NEXT(p, off, len, ch);
|
||||
|
||||
if (ch < 0) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def + "' contains illegal UTF-8");
|
||||
}
|
||||
}
|
||||
|
||||
if (ch != quote) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def +
|
||||
"' contains an unbalanced quote");
|
||||
}
|
||||
|
||||
++p;
|
||||
} else if (p[0] == -62 && p[1] == -76) { // there is also a 0 at *e, so p[1] is ok
|
||||
p += 2;
|
||||
U8_NEXT(p, off, len, ch);
|
||||
|
||||
while (p < e - 1 && (p[0] != -62 || p[1] != -76)) {
|
||||
buffer.push_back(*p++);
|
||||
}
|
||||
|
||||
if (p == e) {
|
||||
if (ch < 0) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def +
|
||||
"' contains an unbalanced quote");
|
||||
"path '" + def + "' contains illegal UTF-8");
|
||||
}
|
||||
|
||||
p += 2;
|
||||
} else if (p[0] == -76 && p[1] == -62) { // there is also a 0 at *e, so p[1] is ok
|
||||
p += 2;
|
||||
|
||||
while (p < e - 1 && (p[0] != -76 || p[1] != -62)) {
|
||||
buffer.push_back(*p++);
|
||||
}
|
||||
|
||||
if (p == e) {
|
||||
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
|
||||
"path '" + def +
|
||||
"' contains an unbalanced quote");
|
||||
}
|
||||
|
||||
p += 2;
|
||||
} else {
|
||||
buffer.push_back(*p++);
|
||||
basics::Utf8Helper::appendUtf8Character(buffer, ch);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,73 @@
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// DISCLAIMER
|
||||
///
|
||||
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
|
||||
///
|
||||
/// @author Frank Celler
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "RandomStringMask.h"
|
||||
|
||||
#include "Basics/StringUtils.h"
|
||||
#include "Basics/fasthash.h"
|
||||
#include "Maskings/Maskings.h"
|
||||
|
||||
static std::string const xxxx("xxxx");
|
||||
|
||||
using namespace arangodb;
|
||||
using namespace arangodb::maskings;
|
||||
|
||||
ParseResult<AttributeMasking> RandomStringMask::create(Path path, Maskings* maskings,
|
||||
VPackSlice const&) {
|
||||
return ParseResult<AttributeMasking>(AttributeMasking(path, new RandomStringMask(maskings)));
|
||||
}
|
||||
|
||||
VPackValue RandomStringMask::mask(bool value) const {
|
||||
return VPackValue(value);
|
||||
}
|
||||
|
||||
VPackValue RandomStringMask::mask(std::string const& data, std::string& buffer) const {
|
||||
uint64_t len = data.size();
|
||||
uint64_t hash;
|
||||
|
||||
hash = fasthash64(data.c_str(), data.size(), _maskings->randomSeed());
|
||||
|
||||
std::string hash64 = basics::StringUtils::encodeBase64(
|
||||
std::string((char const*)&hash, sizeof(decltype(hash))));
|
||||
|
||||
buffer.clear();
|
||||
buffer.reserve(len);
|
||||
buffer.append(hash64);
|
||||
|
||||
if (buffer.size() < len) {
|
||||
while (buffer.size() < len) {
|
||||
buffer.append(hash64);
|
||||
}
|
||||
|
||||
buffer.resize(len);
|
||||
}
|
||||
|
||||
return VPackValue(buffer);
|
||||
}
|
||||
|
||||
VPackValue RandomStringMask::mask(int64_t value) const {
|
||||
return VPackValue(value);
|
||||
}
|
||||
|
||||
VPackValue RandomStringMask::mask(double value) const {
|
||||
return VPackValue(value);
|
||||
}
|
|
@ -20,30 +20,27 @@
|
|||
/// @author Frank Celler
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef ARANGODB_MASKINGS_ATTRIBUTE_XIFY_FRONT_H
|
||||
#define ARANGODB_MASKINGS_ATTRIBUTE_XIFY_FRONT_H 1
|
||||
#ifndef ARANGODB_MASKINGS_ATTRIBUTE_RANDOM_STRING_MASK_H
|
||||
#define ARANGODB_MASKINGS_ATTRIBUTE_RANDOM_STRING_MASK_H 1
|
||||
|
||||
#include "Maskings/AttributeMasking.h"
|
||||
#include "Maskings/MaskingFunction.h"
|
||||
#include "Maskings/ParseResult.h"
|
||||
|
||||
namespace arangodb {
|
||||
namespace maskings {
|
||||
class XifyFront : public MaskingFunction {
|
||||
class RandomStringMask : public MaskingFunction {
|
||||
public:
|
||||
XifyFront(Maskings* maskings, int64_t length, bool hash, uint64_t seed)
|
||||
: MaskingFunction(maskings),
|
||||
_length((uint64_t)length),
|
||||
_randomSeed(seed),
|
||||
_hash(hash) {}
|
||||
static ParseResult<AttributeMasking> create(Path, Maskings*, VPackSlice const& def);
|
||||
|
||||
public:
|
||||
VPackValue mask(bool) const override;
|
||||
VPackValue mask(std::string const&, std::string& buffer) const override;
|
||||
VPackValue mask(std::string const& data, std::string& buffer) const override;
|
||||
VPackValue mask(int64_t) const override;
|
||||
VPackValue mask(double) const override;
|
||||
|
||||
private:
|
||||
uint64_t _length;
|
||||
uint64_t _randomSeed;
|
||||
bool _hash;
|
||||
explicit RandomStringMask(Maskings* maskings) : MaskingFunction(maskings) {}
|
||||
};
|
||||
} // namespace maskings
|
||||
} // namespace arangodb
|
|
@ -1,92 +0,0 @@
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
/// DISCLAIMER
|
||||
///
|
||||
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
|
||||
///
|
||||
/// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
/// you may not use this file except in compliance with the License.
|
||||
/// You may obtain a copy of the License at
|
||||
///
|
||||
/// http://www.apache.org/licenses/LICENSE-2.0
|
||||
///
|
||||
/// Unless required by applicable law or agreed to in writing, software
|
||||
/// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
/// See the License for the specific language governing permissions and
|
||||
/// limitations under the License.
|
||||
///
|
||||
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
|
||||
///
|
||||
/// @author Frank Celler
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "XifyFront.h"
|
||||
|
||||
#include "Basics/StringUtils.h"
|
||||
#include "Basics/fasthash.h"
|
||||
#include "Maskings/Maskings.h"
|
||||
|
||||
static std::string const xxxx("xxxx");
|
||||
|
||||
using namespace arangodb;
|
||||
using namespace arangodb::maskings;
|
||||
|
||||
VPackValue XifyFront::mask(bool) const { return VPackValue(xxxx); }
|
||||
|
||||
VPackValue XifyFront::mask(std::string const& data, std::string& buffer) const {
|
||||
char const* p = data.c_str();
|
||||
char const* q = p;
|
||||
char const* e = p + data.size();
|
||||
|
||||
buffer.clear();
|
||||
buffer.reserve(data.size());
|
||||
|
||||
while (p < e) {
|
||||
while (p < e && isNameChar(*p)) {
|
||||
++p;
|
||||
}
|
||||
|
||||
if (p != q) {
|
||||
char const* w = p - _length;
|
||||
|
||||
while (q < w) {
|
||||
buffer.push_back('x');
|
||||
++q;
|
||||
}
|
||||
|
||||
while (q < p) {
|
||||
buffer.push_back(*q);
|
||||
++q;
|
||||
}
|
||||
}
|
||||
|
||||
while (p < e && !isNameChar(*p)) {
|
||||
buffer.push_back(' ');
|
||||
++p;
|
||||
}
|
||||
|
||||
q = p;
|
||||
}
|
||||
|
||||
if (_hash) {
|
||||
uint64_t hash;
|
||||
|
||||
if (_randomSeed == 0) {
|
||||
hash = fasthash64(data.c_str(), data.size(), _maskings->randomSeed());
|
||||
} else {
|
||||
hash = fasthash64(data.c_str(), data.size(), _randomSeed);
|
||||
}
|
||||
|
||||
std::string hash64 =
|
||||
basics::StringUtils::encodeBase64(std::string((char const*)&hash, 8));
|
||||
|
||||
buffer.push_back(' ');
|
||||
buffer.append(hash64);
|
||||
}
|
||||
|
||||
return VPackValue(buffer);
|
||||
}
|
||||
|
||||
VPackValue XifyFront::mask(int64_t) const { return VPackValue(xxxx); }
|
||||
|
||||
VPackValue XifyFront::mask(double) const { return VPackValue(xxxx); }
|
|
@ -38,9 +38,15 @@ UniformCharacter::UniformCharacter(std::string const& characters)
|
|||
UniformCharacter::UniformCharacter(size_t length, std::string const& characters)
|
||||
: _length(length), _characters(characters) {}
|
||||
|
||||
std::string UniformCharacter::random() { return random(_length); }
|
||||
char UniformCharacter::randomChar() const {
|
||||
size_t r = RandomGenerator::interval((uint32_t)(_characters.size() - 1));
|
||||
|
||||
std::string UniformCharacter::random(size_t length) {
|
||||
return _characters[r];
|
||||
}
|
||||
|
||||
std::string UniformCharacter::random() const { return random(_length); }
|
||||
|
||||
std::string UniformCharacter::random(size_t length) const {
|
||||
std::string buffer;
|
||||
buffer.reserve(length);
|
||||
|
||||
|
|
|
@ -38,11 +38,12 @@ class UniformCharacter {
|
|||
UniformCharacter(size_t length, std::string const& characters);
|
||||
|
||||
public:
|
||||
std::string random();
|
||||
std::string random(size_t length);
|
||||
std::string random() const;
|
||||
std::string random(size_t length) const;
|
||||
char randomChar() const;
|
||||
|
||||
private:
|
||||
size_t _length;
|
||||
size_t const _length;
|
||||
std::string const _characters;
|
||||
};
|
||||
} // namespace arangodb
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
{ "maskings1": {
|
||||
"type": "masked",
|
||||
"maskings": [
|
||||
{
|
||||
"path": "´name´",
|
||||
"type": "xifyFront",
|
||||
"unmaskedLength": 1
|
||||
},
|
||||
{
|
||||
"path": ".`name`",
|
||||
"type": "xifyFront",
|
||||
"unmaskedLength": 2
|
||||
},
|
||||
{
|
||||
"path": "email",
|
||||
"type": "xifyFront",
|
||||
"unmaskedLength": 3
|
||||
}
|
||||
]
|
||||
},
|
||||
"maskings2": {
|
||||
"type": "masked",
|
||||
"maskings": [
|
||||
{
|
||||
"path": "random",
|
||||
"type": "randomString"
|
||||
},
|
||||
{
|
||||
"path": "zip",
|
||||
"type": "zip"
|
||||
},
|
||||
{
|
||||
"path": "date",
|
||||
"type": "date",
|
||||
"begin": "1900-01-01",
|
||||
"end": "2017-12-31",
|
||||
"format": "%yyyy %mm %dd"
|
||||
},
|
||||
{
|
||||
"path": "integer",
|
||||
"type": "integer",
|
||||
"lower": -10,
|
||||
"upper": 10
|
||||
},
|
||||
{
|
||||
"path": "decimal",
|
||||
"type": "decimal",
|
||||
"lower": -10,
|
||||
"upper": 10,
|
||||
"scale": 2
|
||||
},
|
||||
{
|
||||
"path": "ccard",
|
||||
"type": "creditCard"
|
||||
},
|
||||
{
|
||||
"path": "phone",
|
||||
"type": "phone"
|
||||
},
|
||||
{
|
||||
"path": "email",
|
||||
"type": "email"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
|
@ -0,0 +1,88 @@
|
|||
/*jshint globalstrict:false, strict:false, maxlen:4000, unused:false */
|
||||
/*global arango */
|
||||
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
// @brief tests for dump/reload
|
||||
//
|
||||
// @file
|
||||
//
|
||||
// DISCLAIMER
|
||||
//
|
||||
// Copyright 2019 ArangoDB GmbH, Cologne, Germany
|
||||
// Copyright 2010-2012 triagens GmbH, Cologne, Germany
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Copyright holder is ArangoDB GmbH, Cologne, Germany
|
||||
//
|
||||
// @author Frank Celler
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
(function () {
|
||||
'use strict';
|
||||
var db = require("@arangodb").db;
|
||||
var i, c;
|
||||
|
||||
try {
|
||||
db._dropDatabase("UnitTestsDumpSrc");
|
||||
} catch (err1) {
|
||||
}
|
||||
|
||||
db._createDatabase("UnitTestsDumpSrc");
|
||||
db._useDatabase("UnitTestsDumpSrc");
|
||||
|
||||
db._create("maskings1");
|
||||
|
||||
db.maskings1.save({
|
||||
_key: "1",
|
||||
|
||||
name: "Hallo World! This is a t0st a top-level",
|
||||
|
||||
blub: {
|
||||
name: "Hallo World! This is a t0st in a sub-object",
|
||||
},
|
||||
|
||||
email: [
|
||||
"testing arrays",
|
||||
"this is another one",
|
||||
{ something: "something else" },
|
||||
{ email: "within a subject" },
|
||||
{ name: [ "emails within a subject", "as list" ] }
|
||||
],
|
||||
|
||||
sub: {
|
||||
name: "this is a name leaf attribute",
|
||||
email: [ "in this case as list", "with more than one entry" ]
|
||||
}
|
||||
});
|
||||
|
||||
db._create("maskings2");
|
||||
|
||||
db.maskings2.save({
|
||||
_key: "2",
|
||||
|
||||
random: "a",
|
||||
zip: "12345",
|
||||
date: "2018-01-01",
|
||||
integer: 100,
|
||||
decimal: 100.12,
|
||||
ccard: "1234 1234 1234 1234",
|
||||
phone: "abcd 1234",
|
||||
email: "me@you.here"
|
||||
});
|
||||
})();
|
||||
|
||||
return {
|
||||
status: true
|
||||
};
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*jshint globalstrict:false, strict:false, maxlen:4000 */
|
||||
/*global assertEqual, assertTrue, assertFalse, assertNotNull */
|
||||
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
// @brief tests for dump/reload
|
||||
//
|
||||
// @file
|
||||
//
|
||||
// DISCLAIMER
|
||||
//
|
||||
// Copyright 2019 ArangoDB GmbH, Cologne, Germany
|
||||
// Copyright 2010-2012 triagens GmbH, Cologne, Germany
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Copyright holder is ArangoDB GmbH, Cologne, Germany
|
||||
//
|
||||
// @author Frank Celler
|
||||
// /////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
var internal = require("internal");
|
||||
var jsunity = require("jsunity");
|
||||
let users = require("@arangodb/users");
|
||||
|
||||
function dumpMaskingSuite () {
|
||||
'use strict';
|
||||
var db = internal.db;
|
||||
|
||||
return {
|
||||
setUp : function () {
|
||||
},
|
||||
|
||||
tearDown : function () {
|
||||
},
|
||||
|
||||
testGeneral : function () {
|
||||
var c = db._collection("maskings1");
|
||||
var d = c.document("1");
|
||||
|
||||
assertNotNull(d, "document '1' was restored");
|
||||
assertEqual(d.name, "xxxxo xxxxd xxxs xs a xxxt a xxxxxxxxl");
|
||||
assertEqual(d.blub.name, "xxxlo xxxld xxis is a xxst in a xxxxxxxxct");
|
||||
assertEqual(d.email.length, 5);
|
||||
assertEqual(d.email[0], "xxxxing xxxays");
|
||||
assertEqual(d.email[1], "xhis is xxxxher one");
|
||||
assertEqual(d.email[2].something, "something else");
|
||||
assertEqual(d.email[3].email, "within a subject");
|
||||
assertEqual(d.email[4].name.length, 2);
|
||||
assertEqual(d.email[4].name[0], "xxxxls xxxxin a xxxxxct");
|
||||
assertEqual(d.email[4].name[1], "as xxst");
|
||||
assertEqual(d.sub.name, "xxis is a xxme xxaf xxxxxxxte");
|
||||
assertEqual(d.sub.email.length, 2);
|
||||
assertEqual(d.sub.email[0], "in this case as list");
|
||||
assertEqual(d.sub.email[1], "with more than one entry");
|
||||
},
|
||||
|
||||
testRandomString : function () {
|
||||
var c = db._collection("maskings2");
|
||||
var d = c.document("2");
|
||||
|
||||
assertFalse(d.random === "a");
|
||||
assertFalse(d.zip === "12345");
|
||||
assertFalse(d.date === "2018-01-01");
|
||||
assertFalse(d.integer === 100);
|
||||
assertFalse(d.ccard === "1234 1234 1234 1234");
|
||||
assertFalse(d.phone === "abcd 1234");
|
||||
assertFalse(d.emil === "me@you.here");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
jsunity.run(dumpMaskingSuite);
|
||||
|
||||
return jsunity.done();
|
Loading…
Reference in New Issue