Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/maskings (#8006)

This commit is contained in:
Frank Celler 2019-01-22 22:23:25 +01:00 committed by GitHub
parent 5502ed2a3a
commit 9927b3a281
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1058 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

563
Documentation/Books/Manual/Programs/Arangodump/Maskings.md
View File

@ -3,84 +3,111 @@ Arangodump Data Maskings
*--maskings path-of-config* *--maskings path-of-config*
It is possible to mask certain fields during dump. A JSON config file is It is possible to mask certain fields for a dump. A JSON configuration file is
used to define which fields should be masked and how. used to define which fields should be masked and how.
The general structure of the config file is The general structure of the config file looks like this:
{ ```json
"collection-name": { {
"type": MASKING_TYPE "collection-name": {
"maskings" : [ "type": MASKING_TYPE
MASKING1, "maskings" : [
MASKING2, MASKING1,
... MASKING2,
]
},
... ...
} ]
},
...
}
```
Using `"*"` as collection name defines a default behavior for collections not
listed explicitly.
Masking Types Masking Types
------------- -------------
This is a string describing how to mask this collection. Possible values are `type` is a string describing how to mask the given collection.
Possible values are:
- "exclude": the collection is ignored completely and not even the structure data - `"exclude"`: the collection is ignored completely and not even the structure data
is dumped. is dumped.
- "structure": only the collection structure is dumped, but no data at all - `"structure"`: only the collection structure is dumped, but no data at all
- "masked": the collection structure and all data is dumped. However, the data - `"masked"`: the collection structure and all data is dumped. However, the data
is subject to maskings defined in the attribute maskings. is subject to obfuscation defined in the attribute `maskings`.
- "full": the collection structure and all data is dumped. No masking at all - `"full"`: the collection structure and all data is dumped. No masking is
is done for this collection. applied to this collection at all.
For example: **Example**
{ ```json
"private": { {
"type": "exclude" "private": {
"type": "exclude"
},
"log": {
"type": "structure"
},
"person": {
"type": "masked",
"maskings": [
{
"path": "name",
"type": "xifyFront",
"unmaskedLength": 2
}, },
{
"log": { "path": ".security_id",
"type": "structure" "type": "xifyFront",
}, "unmaskedLength": 2
"person": {
"type": "masked",
"maskings": [
{
"path": "name",
"type": "xify_front",
"unmaskedLength": 2
},
{
"path": ".security_id",
"type": "xify_front",
"unmaskedLength": 2
}
]
} }
} ]
}
}
```
In the example the collection "private" is completely ignored. Only the In the example the collection _private_ is completely ignored. Only the
structure of the collection "log" is dumped, but not the data itself. structure of the collection _log_ is dumped, but not the data itself.
The collection "person" is dumped completely but masking the "name" field if The collection _person_ is dumped completely but with the _name_ field masked
it occurs on the top-level. It masks the field "security_id" anywhere in the if it occurs on the top-level. It also masks fields with the name "security_id"
document. See below for a complete description of the parameters of anywhere in the document. See below for a complete description of the parameters
"xify_front". of [type "xifyFront"](#xify-front).
### Masking vs. dump-data option
*arangodump* also supports a very coarse masking with the option
`--dump-data false`. This basically removes all data from the dump.
You can either use `--masking` or `--dump-data false`, but not both.
### Masking vs. include-collection option
*arangodump* also supports a very coarse masking with the option
`--include-collection`. This will restrict the collections that are
dumped to the ones explicitly listed.
It is possible to combine `--masking` and `--include-collection`.
This will take the intersection of exportable collections.
Path Path
---- ----
If the path starts with a `.` then it is considered to be a wildcard match. If the path starts with a `.` then it is considered to match any path
For example, `.name` will match the attribute name `name` everywhere in the ending in `name`. For example, `.name` will match the attribute name
document. `name` will only match at top level. `person.name` will match `name` all leaf attributes in the document. Leaf attributes are
the attribute `name` in the top-level object `person`. attributes whose value is `null` or of data type `string`, `number`,
`bool` and `array` (see below). `name` will only match leaf attributes
at top level. `person.name` will match the attribute `name` of a leaf
in the top-level object `person`.
If you have a attribute name that contains a dot, you need to quote the If you have an attribute name that contains a dot, you need to quote the
name with either a tick or a backtick. For example name with either a tick or a backtick. For example:
"path": "´name.with.dots´" "path": "´name.with.dots´"
@ -88,24 +115,233 @@ or
"path": "`name.with.dots`" "path": "`name.with.dots`"
xify_front If the attribute value is an array the masking is applied to all the
---------- array elements individually.
This masking replaces characters with `x` and ` `. Alphanumeric characters, **Example**
`_` and `-` are replaced by `x`, everything else is replaced by ` `.
The following configuration will replace the value of the "name"
attribute with an "XXXX"-masked string:
```json
{
"type": "xifyFront",
"path": ".name",
"unmaskedLength": 2
}
```
The document:
```json
{
"name": "top-level-name",
"age": 42,
"nicknames" : [ { "name": "hugo" }, "egon" ],
"other": {
"name": [ "emil", { "secret": "superman" } ]
}
}
```
… will be changed as follows:
```json
{
"name": "xxxxxxxxxxxxme",
"age": 42,
"nicknames" : [ { "name": "xxgo" }, "egon" ],
"other": {
"name": [ "xxil", { "secret": "superman" } ]
}
}
```
The values `"egon"` and `"superman"` are not replaced, because they
are not contained in an attribute value of which the attribute name is
`name`.
### Nested objects and arrays
If you specify a path and the attribute value is an array then the
masking decision is applied to each element of the array as if this
was the value of the attribute.
If the attribute value is an object, then the attribute is not masked.
Instead the nested object is checked further for leaf attributes.
**Example**
Masking `email` will convert:
```json
{
"email" : "email address"
}
```
… into:
```json
{
"email" : "xxil xxxxxxss"
}
```
because `email` is a leaf attribute. The document:
```json
{
"email" : [
"address one",
"address two"
]
}
```
… will be converted into:
```json
{
"email" : [
"xxxxxss xne",
"xxxxxss xwo"
]
}
```
… because the array is "unfolded". The document:
```json
{
"email" : {
"address" : "email address"
}
}
```
… will not be changed because `email` is not a leaf attribute.
Masking Functions
-----------------
{% hint 'info' %}
The following masking functions are only available in the
[**Enterprise Edition**](https://www.arangodb.com/why-arangodb/arangodb-enterprise/)
{% endhint %}
- xify front
- zip
- datetime
- integral number
- decimal number
- credit card number
- phone number
- email address
The function:
- random string
… is available on Community Edition and in the Enterprise Edition.
### Random string
```json
{
"path": ".name",
"type": "randomString"
}
```
This masking type will replace all values of attributes with key
`name` with an anonymized string. It is not guaranteed that the string
will be of the same length.
A hash of the original string is computed. If the original string is
shorter then the hash will be used. This will result in a longer
replacement string. If the string is longer than the hash then
characters will be repeated as many times as needed to reach the full
original string length.
**Example**
Masking name as above, the document:
```json
{
"_key" : "38937",
"_id" : "examplecollection/38937",
"_rev" : "_YFaGG1u--_",
"name" : [
"My Name",
{
"other" : "Hallo Name"
},
[
"Name One",
"Name Two"
],
true,
false,
null,
1.0,
1234,
"This is a very long name"
]
}
```
… will be converted into
```json
{
"_key": "38937",
"_id": "examplecollection/38937",
"_rev": "_YFaGG1u--_",
"name": [
"+y5OQiYmp/o=",
{ {
"path": ".name", "other": "Hallo Name"
"unmaskedLength": 2 },
} [
"ihCTrlsKKdk=",
"yo/55hfla0U="
],
true,
false,
null,
1.0,
1234,
"hwjAfNe5BGw=hwjAfNe5BGw="
]
}
```
This will mask all alphanumeric characters of a word except the last 2. ### Xify front
Words of length 1 and 2 are unmasked. If the attribute value is not a
string the result will be `xxxx`. This masking type replaces the front characters with `x` and
blanks. Alphanumeric characters, `_` and `-` are replaced by `x`,
everything else is replaced by a blank.
```json
{
"path": ".name",
"type": "xifyFront",
"unmaskedLength": 2
}
```
This will mask all alphanumeric characters of a word except the last
two characters. Words of length 1 and 2 are unmasked. If the
attribute value is not a string the result will be `xxxx`.
"This is a test!Do you agree?" "This is a test!Do you agree?"
will become … will become
"xxis is a xxst Do xou xxxee " "xxis is a xxst Do xou xxxee "
@ -113,30 +349,193 @@ There is a catch. If you have an index on the attribute the masking
might distort the index efficiency or even cause errors in case of a might distort the index efficiency or even cause errors in case of a
unique index. unique index.
{ ```json
"path": ".name", {
"unmaskedLength": 2, "type": "xifyFront",
"hash": true "path": ".name",
} "unmaskedLength": 2,
"hash": true
}
```
This will add a hash at the end of the string. This will add a hash at the end of the string.
"This is a test!Do you agree?" "This is a test!Do you agree?"
will become … will become
"xxis is a xxst Do xou xxxee NAATm8c9hVQ=" "xxis is a xxst Do xou xxxee NAATm8c9hVQ="
Note that the hash is based on a random secrect that is different for Note that the hash is based on a random secrect that is different for
each run. This avoids dictionary attacks. each run. This avoids dictionary attacks which can be used to guess
values based pre-computations on dictionaries.
If you need reproducable results, i.e. hash that do not change between If you need reproducible results, i.e. hashes that do not change between
different runs of *arangodump*, you need to specify a seed, which must different runs of *arangodump*, you need to specify a secret as seed,
not be `0`. a number which must not be `0`.
{ ```json
"path": ".name", {
"unmaskedLength": 2, "type": "xifyFront",
"hash": true, "path": ".name",
"seed": 246781478647 "unmaskedLength": 2,
} "hash": true,
"seed": 246781478647
}
```
### Zip
This masking type replaces a zip code with a random one. If the
attribute value is not a string then the default value of `"12345"` is
used as no zip is known. You can change the default value, see below.
```json
{
"path": ".code",
"type": "zip",
}
```
This will replace a real zip code with a random one. It uses the following
rule: If a character of the original zip code is a digit it will be replaced
by a random digit. If a character of the original zip code is a letter it
will be replaced by a random letter keeping the case.
```json
{
"path": ".code",
"type": "zip",
"default": "abcdef"
}
```
**Example**
If the original zip code is:
50674
… it will be replaced by e.g.:
98146
If the original zip code is:
SA34-EA
… it will be replaced by e.g.:
OW91-JI
Note that this will generate random zip code. Therefore there is a
chance generate the same zip code value multiple times, which can
cause unique constraint violations if a unique index is or will be
used on the zip code attribute.
### Datetime
This masking type replaces the value of the attribute with a random
date.
```json
{
"type": "datetime",
"begin" : "2019-01-01",
"end": "2019-12-31",
"format": "%yyyy-%mm-%dd",
}
```
`begin` and `end` are in ISO8601 format.
The format is described in
[DATE_FORMAT](../../../AQL/Functions/Date.html#dateformat).
### Integral number
This masking type replaces the value of the attribute with a random
integral number. It will replace the value even if it is a string,
boolean, or false.
```json
{
"type": "integer",
"lower" : -100,
"upper": 100
}
```
### Decimal number
This masking type replaces the value of the attribute with a random
decimal. It will replace the value even if it is a string, boolean,
or false.
```json
{
"type": "float",
"lower" : -0.3,
"upper": 0.3
}
```
By default, the decimal has a scale of 2. I.e. it has at most 2
decimal digits. The definition:
```json
{
"type": "float",
"lower" : -0.3,
"upper": 0.3,
"scale": 3
}
```
… will generate numbers with at most 3 decimal digits.
### Credit card number
This masking type replaces the value of the attribute with a random
credit card number.
```json
{
"type": "creditCard",
}
```
See [Luhn](https://en.wikipedia.org/wiki/Luhn_algorithm) for details.
### Phone number
This masking type replaces a phone number with a random one. If the
attribute value is not a string it is replaced by the string
`"+1234567890"`.
```json
{
"type": "phone",
"default": "+4912345123456789"
}
```
This will replace an existing phone number with a random one. It uses
the following rule: If a character of the original number is a digit
it will be replaced by a random digit. If it is a letter it is replaced
by a letter. All other characters are unchanged.
```json
{ "type": "zip",
"default": "+4912345123456789"
}
```
If the attribute value is not a string use the value of default
`"+4912345123456789"`.
### Email address
This masking type takes an email address, computes a hash value and
split it into three equal parts `AAAA`, `BBBB`, and `CCCC`. The
resulting email address is `AAAA.BBBB@CCCC.invalid`.

9
LICENSES-OTHER-COMPONENTS.md
View File

@ -29,6 +29,15 @@
* License Name: Boost Software License 1.0 * License Name: Boost Software License 1.0
* License Id: BSL-1.0 * License Id: BSL-1.0
### CreditCardGenerator 2016
* Name: CreditCardGenerator
* Version: 1.8.1
* Project Home: https://github.com/stormdark/CreditCardGenerator
* License: https://raw.githubusercontent.com/stormdark/CreditCardGenerator/master/LICENSE
* License Name: MIT License
* License Id: MIT
### Curl 7.50.3 ### Curl 7.50.3
* Name: Curl * Name: Curl

8
arangosh/Dump/arangodump.cpp
View File

@ -35,6 +35,7 @@
#include "Dump/DumpFeature.h" #include "Dump/DumpFeature.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "Logger/LoggerFeature.h" #include "Logger/LoggerFeature.h"
#include "Maskings/AttributeMasking.h"
#include "ProgramOptions/ProgramOptions.h" #include "ProgramOptions/ProgramOptions.h"
#include "Random/RandomFeature.h" #include "Random/RandomFeature.h"
#include "Shell/ClientFeature.h" #include "Shell/ClientFeature.h"
@ -42,6 +43,7 @@
#ifdef USE_ENTERPRISE #ifdef USE_ENTERPRISE
#include "Enterprise/Encryption/EncryptionFeature.h" #include "Enterprise/Encryption/EncryptionFeature.h"
#include "Enterprise/Maskings/AttributeMaskingEE.h"
#endif #endif
using namespace arangodb; using namespace arangodb;
@ -53,6 +55,12 @@ int main(int argc, char* argv[]) {
ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
context.installHup(); context.installHup();
maskings::InstallMaskings();
#ifdef USE_ENTERPRISE
maskings::InstallMaskingsEE();
#endif
std::shared_ptr<options::ProgramOptions> options( std::shared_ptr<options::ProgramOptions> options(
new options::ProgramOptions(argv[0], "Usage: arangodump [<options>]", new options::ProgramOptions(argv[0], "Usage: arangodump [<options>]",
"For more information use:", BIN_DIRECTORY)); "For more information use:", BIN_DIRECTORY));

6
js/client/modules/@arangodb/process-utils.js
View File

@ -107,6 +107,12 @@ class ConfigBuilder {
this.config['create-database'] = 'false'; this.config['create-database'] = 'false';
} }
} }
setMaskings(dir) {
if (this.type !== 'dump') {
throw '"maskings" is not supported for binary: ' + this.type;
}
this.config['maskings'] = fs.join(TOP_DIR, "tests/js/common/test-data/maskings", dir);
}
activateEncryption() { this.config['encription.keyfile'] = fs.join(this.rootDir, 'secret-key'); } activateEncryption() { this.config['encription.keyfile'] = fs.join(this.rootDir, 'secret-key'); }
setRootDir(dir) { this.rootDir = dir; } setRootDir(dir) { this.rootDir = dir; }
restrictToCollection(collection) { restrictToCollection(collection) {

140
js/client/modules/@arangodb/testsuites/dump.js
View File

@ -2,34 +2,36 @@
/* global print */ /* global print */
'use strict'; 'use strict';
// ////////////////////////////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER // DISCLAIMER
// / //
// / Copyright 2016 ArangoDB GmbH, Cologne, Germany // Copyright 2016-2019 ArangoDB GmbH, Cologne, Germany
// / Copyright 2014 triagens GmbH, Cologne, Germany // Copyright 2014 triagens GmbH, Cologne, Germany
// / //
// / Licensed under the Apache License, Version 2.0 (the "License") // Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at // You may obtain a copy of the License at
// / //
// / http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// / //
// / Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// / limitations under the License. // limitations under the License.
// / //
// / Copyright holder is ArangoDB GmbH, Cologne, Germany // Copyright holder is ArangoDB GmbH, Cologne, Germany
// / //
// / @author Max Neunhoeffer // @author Max Neunhoeffer
// ////////////////////////////////////////////////////////////////////////////// // //////////////////////////////////////////////////////////////////////////////
const functionsDocumentation = { const functionsDocumentation = {
'dump': 'dump tests', 'dump': 'dump tests',
'dump_authentication': 'dump tests with authentication',
'dump_encrypted': 'encrypted dump tests', 'dump_encrypted': 'encrypted dump tests',
'dump_authentication': 'dump tests with authentication' 'dump_maskings': 'masked dump tests'
}; };
const optionsDocumentation = [ const optionsDocumentation = [
' - `skipEncrypted` : if set to true the encryption tests are skipped' ' - `skipEncrypted` : if set to true the encryption tests are skipped'
]; ];
@ -48,16 +50,18 @@ const RESET = require('internal').COLORS.COLOR_RESET;
const testPaths = { const testPaths = {
'dump': [tu.pathForTesting('server/dump')], 'dump': [tu.pathForTesting('server/dump')],
'dump_authentication': [tu.pathForTesting('server/dump')],
'dump_encrypted': [tu.pathForTesting('server/dump')], 'dump_encrypted': [tu.pathForTesting('server/dump')],
'dump_authentication': [tu.pathForTesting('server/dump')] 'dump_maskings': [tu.pathForTesting('server/dump')]
}; };
class DumpRestoreHelper { class DumpRestoreHelper {
constructor(instanceInfo, options, clientAuth, dumpOptions, which, afterServerStart) { constructor(instanceInfo, options, clientAuth, dumpOptions, restoreOptions, which, afterServerStart) {
this.instanceInfo = instanceInfo; this.instanceInfo = instanceInfo;
this.options = options; this.options = options;
this.clientAuth = clientAuth; this.clientAuth = clientAuth;
this.dumpOptions = dumpOptions; this.dumpOptions = dumpOptions;
this.restoreOptions = restoreOptions;
this.which = which; this.which = which;
this.fn = afterServerStart(instanceInfo); this.fn = afterServerStart(instanceInfo);
this.results = {failed: 1}; this.results = {failed: 1};
@ -66,11 +70,15 @@ class DumpRestoreHelper {
this.dumpConfig.setOutputDirectory('dump'); this.dumpConfig.setOutputDirectory('dump');
this.dumpConfig.setIncludeSystem(true); this.dumpConfig.setIncludeSystem(true);
this.restoreConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo); if (dumpOptions.hasOwnProperty("maskings")) {
this.dumpConfig.setMaskings(dumpOptions.maskings);
}
this.restoreConfig = pu.createBaseConfig('restore', this.restoreOptions, this.instanceInfo);
this.restoreConfig.setInputDirectory('dump', true); this.restoreConfig.setInputDirectory('dump', true);
this.restoreConfig.setIncludeSystem(true); this.restoreConfig.setIncludeSystem(true);
this.restoreOldConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo); this.restoreOldConfig = pu.createBaseConfig('restore', this.restoreOptions, this.instanceInfo);
this.restoreOldConfig.setInputDirectory('dump', true); this.restoreOldConfig.setInputDirectory('dump', true);
this.restoreOldConfig.setIncludeSystem(true); this.restoreOldConfig.setIncludeSystem(true);
this.restoreOldConfig.setDatabase('_system'); this.restoreOldConfig.setDatabase('_system');
@ -81,8 +89,8 @@ class DumpRestoreHelper {
this.restoreOldConfig.activateEncryption(); this.restoreOldConfig.activateEncryption();
} }
this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.dumpOptions, this.instanceInfo.rootDir); this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.restoreOptions, this.instanceInfo.rootDir);
this.arangorestoreOld = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreOldConfig, this.dumpOptions, this.instanceInfo.rootDir); this.arangorestoreOld = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreOldConfig, this.restoreOptions, this.instanceInfo.rootDir);
this.arangodump = pu.run.arangoDumpRestoreWithConfig.bind(this, this.dumpConfig, this.dumpOptions, this.instanceInfo.rootDir); this.arangodump = pu.run.arangoDumpRestoreWithConfig.bind(this, this.dumpConfig, this.dumpOptions, this.instanceInfo.rootDir);
} }
@ -225,10 +233,7 @@ function getClusterStrings(options)
} }
} }
// ////////////////////////////////////////////////////////////////////////////// function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restoreOptions, which, tstFiles, afterServerStart) {
// / @brief TEST: dump
// //////////////////////////////////////////////////////////////////////////////
function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which, tstFiles, afterServerStart) {
print(CYAN + which + ' tests...' + RESET); print(CYAN + which + ' tests...' + RESET);
let instanceInfo = pu.startInstance('tcp', options, serverAuthInfo, which); let instanceInfo = pu.startInstance('tcp', options, serverAuthInfo, which);
@ -243,7 +248,7 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which,
}; };
return rc; return rc;
} }
const helper = new DumpRestoreHelper(instanceInfo, options, clientAuth, dumpOptions, which, afterServerStart); const helper = new DumpRestoreHelper(instanceInfo, options, clientAuth, dumpOptions, restoreOptions, which, afterServerStart);
const setupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpSetup)); const setupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpSetup));
const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain)); const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain));
@ -267,21 +272,21 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which,
} }
} }
const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest)); if (tstFiles.hasOwnProperty("foxxTest")) {
if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') || const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest));
!helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') || if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') ||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') || !helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') ||
!helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxAppsBundle') || !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') ||
!helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxBundleApps') || !helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxAppsBundle') ||
!helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxBundleApps')) { !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxBundleApps') ||
return helper.extractResults(); !helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxBundleApps')) {
return helper.extractResults();
}
} }
return helper.extractResults(); return helper.extractResults();
} }
// /////////////////////////////////////////////////////////////////////////////
function dump (options) { function dump (options) {
let c = getClusterStrings(options); let c = getClusterStrings(options);
let tstFiles = { let tstFiles = {
@ -292,7 +297,7 @@ function dump (options) {
foxxTest: 'check-foxx.js' foxxTest: 'check-foxx.js'
}; };
return dump_backend(options, {}, {}, options, 'dump', tstFiles, function(){}); return dump_backend(options, {}, {}, options, options, 'dump', tstFiles, function(){});
} }
function dumpAuthentication (options) { function dumpAuthentication (options) {
@ -332,7 +337,7 @@ function dumpAuthentication (options) {
foxxTest: 'check-foxx.js' foxxTest: 'check-foxx.js'
}; };
return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, 'dump_authentication', tstFiles, function(){}); return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, dumpAuthOpts, 'dump_authentication', tstFiles, function(){});
} }
function dumpEncrypted (options) { function dumpEncrypted (options) {
@ -373,20 +378,57 @@ function dumpEncrypted (options) {
foxxTest: 'check-foxx.js' foxxTest: 'check-foxx.js'
}; };
return dump_backend(options, {}, {}, dumpOptions, 'dump_encrypted', tstFiles, afterServerStart); return dump_backend(options, {}, {}, dumpOptions, dumpOptions, 'dump_encrypted', tstFiles, afterServerStart);
}
function dumpMaskings (options) {
// test is only meaningful in the enterprise version
let skip = true;
if (global.ARANGODB_CLIENT_VERSION) {
let version = global.ARANGODB_CLIENT_VERSION(true);
if (version.hasOwnProperty('enterprise-version')) {
skip = false;
}
}
if (skip) {
print('skipping dump_maskings test');
return {
dump_maskings: {
status: true,
skipped: true
}
};
}
let tstFiles = {
dumpSetup: 'dump-maskings-setup.js',
dumpAgain: 'dump-maskings.js',
dumpTearDown: 'dump-teardown.js'
};
let dumpMaskingsOpts = {
maskings: 'maskings1.json'
};
_.defaults(dumpMaskingsOpts, options);
return dump_backend(options, {}, {}, dumpMaskingsOpts, options, 'dump_maskings', tstFiles, function(){});
} }
// /////////////////////////////////////////////////////////////////////////////
exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) { exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {
Object.assign(allTestPaths, testPaths); Object.assign(allTestPaths, testPaths);
testFns['dump'] = dump; testFns['dump'] = dump;
defaultFns.push('dump'); defaultFns.push('dump');
testFns['dump_authentication'] = dumpAuthentication;
defaultFns.push('dump_authentication');
testFns['dump_encrypted'] = dumpEncrypted; testFns['dump_encrypted'] = dumpEncrypted;
defaultFns.push('dump_encrypted'); defaultFns.push('dump_encrypted');
testFns['dump_authentication'] = dumpAuthentication; testFns['dump_maskings'] = dumpMaskings;
defaultFns.push('dump_authentication'); defaultFns.push('dump_maskings');
for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; } for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; }
for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); } for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); }

29
lib/Basics/Utf8Helper.h
View File

@ -1,7 +1,7 @@
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER /// DISCLAIMER
/// ///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany /// Copyright 2014-2019 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
/// ///
/// Licensed under the Apache License, Version 2.0 (the "License"); /// Licensed under the Apache License, Version 2.0 (the "License");
@ -25,9 +25,10 @@
#ifndef ARANGODB_BASICS_UTF8HELPER_H #ifndef ARANGODB_BASICS_UTF8HELPER_H
#define ARANGODB_BASICS_UTF8HELPER_H 1 #define ARANGODB_BASICS_UTF8HELPER_H 1
#include <velocypack/StringRef.h>
#include "Basics/Common.h" #include "Basics/Common.h"
#include <velocypack/StringRef.h>
#include <unicode/coll.h> #include <unicode/coll.h>
#include <unicode/regex.h> #include <unicode/regex.h>
#include <unicode/ustring.h> #include <unicode/ustring.h>
@ -40,10 +41,6 @@ class Utf8Helper {
Utf8Helper& operator=(Utf8Helper const&) = delete; Utf8Helper& operator=(Utf8Helper const&) = delete;
public: public:
//////////////////////////////////////////////////////////////////////////////
/// @brief a default helper
//////////////////////////////////////////////////////////////////////////////
static Utf8Helper DefaultUtf8Helper; static Utf8Helper DefaultUtf8Helper;
public: public:
@ -153,6 +150,26 @@ class Utf8Helper {
char const* replacement, size_t replacementLength, char const* replacement, size_t replacementLength,
bool partial, bool& error); bool partial, bool& error);
// append an UTF8 to a string. This will append 1 to 4 bytes.
static void appendUtf8Character(std::string& result, uint32_t ch) {
if (ch <= 0x7f) {
result.push_back((uint8_t)ch);
} else {
if (ch <= 0x7ff) {
result.push_back((uint8_t)((ch >> 6) | 0xc0));
} else {
if (ch <= 0xffff) {
result.push_back((uint8_t)((ch >> 12) | 0xe0));
} else {
result.push_back((uint8_t)((ch >> 18) | 0xf0));
result.push_back((uint8_t)(((ch >> 12) & 0x3f) | 0x80));
}
result.push_back((uint8_t)(((ch >> 6) & 0x3f) | 0x80));
}
result.push_back((uint8_t)((ch & 0x3f) | 0x80));
}
}
private: private:
Collator* _coll; Collator* _coll;
}; };

2
lib/Basics/datetime.h
View File

@ -23,6 +23,8 @@
#ifndef ARANGODB_BASICS_DATETIME_H #ifndef ARANGODB_BASICS_DATETIME_H
#define ARANGODB_BASICS_DATETIME_H 1 #define ARANGODB_BASICS_DATETIME_H 1
#include "Basics/Common.h"
#include <chrono> #include <chrono>
#include <regex> #include <regex>

2
lib/CMakeLists.txt
View File

@ -237,7 +237,7 @@ add_library(${LIB_ARANGO} STATIC
Maskings/Collection.cpp Maskings/Collection.cpp
Maskings/Maskings.cpp Maskings/Maskings.cpp
Maskings/Path.cpp Maskings/Path.cpp
Maskings/XifyFront.cpp Maskings/RandomStringMask.cpp
ProgramOptions/Option.cpp ProgramOptions/Option.cpp
ProgramOptions/ProgramOptions.cpp ProgramOptions/ProgramOptions.cpp
ProgramOptions/Section.cpp ProgramOptions/Section.cpp

47
lib/Maskings/AttributeMasking.cpp
View File

@ -24,11 +24,17 @@
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "Maskings/XifyFront.h" #include "Maskings/RandomStringMask.h"
using namespace arangodb; using namespace arangodb;
using namespace arangodb::maskings; using namespace arangodb::maskings;
void arangodb::maskings::InstallMaskings() {
AttributeMasking::installMasking("randomString", RandomStringMask::create);
}
std::unordered_map<std::string, ParseResult<AttributeMasking> (*)(Path, Maskings*, VPackSlice const&)> AttributeMasking::_maskings;
ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings, ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
VPackSlice const& def) { VPackSlice const& def) {
if (!def.isObject()) { if (!def.isObject()) {
@ -39,9 +45,6 @@ ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
std::string path = ""; std::string path = "";
std::string type = ""; std::string type = "";
uint64_t length = 2;
uint64_t seed = 0;
bool hash = false;
for (auto const& entry : VPackObjectIterator(def, false)) { for (auto const& entry : VPackObjectIterator(def, false)) {
std::string key = entry.key.copyString(); std::string key = entry.key.copyString();
@ -60,27 +63,6 @@ ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
} }
path = entry.value.copyString(); path = entry.value.copyString();
} else if (key == "unmaskedLength") {
if (!entry.value.isInteger()) {
return ParseResult<AttributeMasking>(ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
"length must be an integer");
}
length = entry.value.getInt();
} else if (key == "hash") {
if (!entry.value.isBool()) {
return ParseResult<AttributeMasking>(ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
"hash must be an integer");
}
hash = entry.value.getBool();
} else if (key == "seed") {
if (!entry.value.isInteger()) {
return ParseResult<AttributeMasking>(ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
"seed must be an integer");
}
seed = entry.value.getInt();
} }
} }
@ -96,20 +78,15 @@ ParseResult<AttributeMasking> AttributeMasking::parse(Maskings* maskings,
(ParseResult<AttributeMasking>::StatusCode)(int)ap.status, ap.message); (ParseResult<AttributeMasking>::StatusCode)(int)ap.status, ap.message);
} }
if (type == "xify_front") { auto const& it = _maskings.find(type);
if (length < 1) {
return ParseResult<AttributeMasking>(
ParseResult<AttributeMasking>::ILLEGAL_PARAMETER,
"expecting length to be at least for xify_front");
}
if (it == _maskings.end()) {
return ParseResult<AttributeMasking>( return ParseResult<AttributeMasking>(
AttributeMasking(ap.result, new XifyFront(maskings, length, hash, seed))); ParseResult<AttributeMasking>::UNKNOWN_TYPE,
"unknown attribute masking type '" + type + "'");
} }
return ParseResult<AttributeMasking>( return it->second(ap.result, maskings, def);
ParseResult<AttributeMasking>::UNKNOWN_TYPE,
"expecting unknown attribute masking type '" + type + "'");
} }
bool AttributeMasking::match(std::vector<std::string> const& path) const { bool AttributeMasking::match(std::vector<std::string> const& path) const {

8
lib/Maskings/AttributeMasking.h
View File

@ -37,9 +37,14 @@
namespace arangodb { namespace arangodb {
namespace maskings { namespace maskings {
void InstallMaskings();
class AttributeMasking { class AttributeMasking {
public: public:
static ParseResult<AttributeMasking> parse(Maskings*, VPackSlice const&); static ParseResult<AttributeMasking> parse(Maskings*, VPackSlice const&);
static void installMasking(std::string const& name, ParseResult<AttributeMasking> (* func)(Path, Maskings*, VPackSlice const&)) {
_maskings[name] = func;
}
public: public:
AttributeMasking() = default; AttributeMasking() = default;
@ -52,6 +57,9 @@ class AttributeMasking {
MaskingFunction* func() const { return _func.get(); } MaskingFunction* func() const { return _func.get(); }
private:
static std::unordered_map<std::string, ParseResult<AttributeMasking> (*)(Path, Maskings*, VPackSlice const&)> _maskings;
private: private:
Path _path; Path _path;
std::shared_ptr<MaskingFunction> _func; std::shared_ptr<MaskingFunction> _func;

7
lib/Maskings/MaskingFunction.h
View File

@ -25,6 +25,8 @@
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Basics/Utf8Helper.h"
#include <velocypack/Builder.h> #include <velocypack/Builder.h>
#include <velocypack/Iterator.h> #include <velocypack/Iterator.h>
#include <velocypack/Parser.h> #include <velocypack/Parser.h>
@ -37,9 +39,8 @@ class Maskings;
class MaskingFunction { class MaskingFunction {
public: public:
static bool isNameChar(char c) { static bool isNameChar(UChar32 ch) {
return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || return u_isalpha(ch) || u_isdigit(ch) || ch == U'_' || ch == U'-';
('0' <= c && c <= '9') || c == '_' || c == '-';
} }
public: public:

58
lib/Maskings/Maskings.cpp
View File

@ -85,11 +85,21 @@ ParseResult<Maskings> Maskings::parse(VPackSlice const& def) {
for (auto const& entry : VPackObjectIterator(def, false)) { for (auto const& entry : VPackObjectIterator(def, false)) {
std::string key = entry.key.copyString(); std::string key = entry.key.copyString();
LOG_TOPIC(TRACE, Logger::CONFIG) << "masking collection '" << key << "'";
if (_collections.find(key) != _collections.end()) { if (key == "*") {
return ParseResult<Maskings>(ParseResult<Maskings>::DUPLICATE_COLLECTION, LOG_TOPIC(TRACE, Logger::CONFIG) << "default masking";
"duplicate collection entry '" + key + "'");
if (_hasDefaultCollection) {
return ParseResult<Maskings>(ParseResult<Maskings>::DUPLICATE_COLLECTION,
"duplicate default entry");
}
} else {
LOG_TOPIC(TRACE, Logger::CONFIG) << "masking collection '" << key << "'";
if (_collections.find(key) != _collections.end()) {
return ParseResult<Maskings>(ParseResult<Maskings>::DUPLICATE_COLLECTION,
"duplicate collection entry '" + key + "'");
}
} }
ParseResult<Collection> c = Collection::parse(this, entry.value); ParseResult<Collection> c = Collection::parse(this, entry.value);
@ -99,20 +109,30 @@ ParseResult<Maskings> Maskings::parse(VPackSlice const& def) {
c.message); c.message);
} }
_collections[key] = c.result; if (key == "*") {
_hasDefaultCollection = true;
_defaultCollection = c.result;
} else {
_collections[key] = c.result;
}
} }
return ParseResult<Maskings>(ParseResult<Maskings>::VALID); return ParseResult<Maskings>(ParseResult<Maskings>::VALID);
} }
bool Maskings::shouldDumpStructure(std::string const& name) { bool Maskings::shouldDumpStructure(std::string const& name) {
CollectionSelection select = CollectionSelection::EXCLUDE;
auto const itr = _collections.find(name); auto const itr = _collections.find(name);
if (itr == _collections.end()) { if (itr == _collections.end()) {
return false; if (_hasDefaultCollection) {
select = _defaultCollection.selection();
}
} else {
select = itr->second.selection();
} }
switch (itr->second.selection()) { switch (select) {
case CollectionSelection::FULL: case CollectionSelection::FULL:
return true; return true;
case CollectionSelection::MASKED: case CollectionSelection::MASKED:
@ -129,13 +149,18 @@ bool Maskings::shouldDumpStructure(std::string const& name) {
} }
bool Maskings::shouldDumpData(std::string const& name) { bool Maskings::shouldDumpData(std::string const& name) {
CollectionSelection select = CollectionSelection::EXCLUDE;
auto const itr = _collections.find(name); auto const itr = _collections.find(name);
if (itr == _collections.end()) { if (itr == _collections.end()) {
return false; if (_hasDefaultCollection) {
select = _defaultCollection.selection();
}
} else {
select = itr->second.selection();
} }
switch (itr->second.selection()) { switch (select) {
case CollectionSelection::FULL: case CollectionSelection::FULL:
return true; return true;
case CollectionSelection::MASKED: case CollectionSelection::MASKED:
@ -289,14 +314,21 @@ void Maskings::mask(std::string const& name, basics::StringBuffer const& data,
basics::StringBuffer& result) { basics::StringBuffer& result) {
result.clear(); result.clear();
Collection* collection;
auto const itr = _collections.find(name); auto const itr = _collections.find(name);
if (itr == _collections.end()) { if (itr == _collections.end()) {
result.copy(data); if (_hasDefaultCollection) {
return; collection = &_defaultCollection;
} else {
result.copy(data);
return;
}
} else {
collection = &(itr->second);
} }
if (itr->second.selection() == CollectionSelection::FULL) { if (collection->selection() == CollectionSelection::FULL) {
result.copy(data); result.copy(data);
return; return;
} }
@ -314,7 +346,7 @@ void Maskings::mask(std::string const& name, basics::StringBuffer const& data,
std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(q, p - q); std::shared_ptr<VPackBuilder> builder = VPackParser::fromJson(q, p - q);
addMasked(itr->second, result, builder->slice()); addMasked(*collection, result, builder->slice());
while (p < e && (*p == '\n' || *p == '\r')) { while (p < e && (*p == '\n' || *p == '\r')) {
++p; ++p;

2
lib/Maskings/Maskings.h
View File

@ -80,6 +80,8 @@ class Maskings {
private: private:
std::map<std::string, Collection> _collections; std::map<std::string, Collection> _collections;
bool _hasDefaultCollection = false;
Collection _defaultCollection;
uint64_t _randomSeed = 0; uint64_t _randomSeed = 0;
}; };

72
lib/Maskings/Path.cpp
View File

@ -23,6 +23,7 @@
#include "Collection.h" #include "Collection.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Basics/Utf8Helper.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
using namespace arangodb; using namespace arangodb;
@ -40,71 +41,66 @@ ParseResult<Path> Path::parse(std::string const& def) {
wildcard = true; wildcard = true;
} }
char const* p = def.c_str(); uint8_t const* p = reinterpret_cast<uint8_t const*>(def.c_str());
char const* e = p + def.size(); int32_t off = 0;
int32_t len = def.size();
UChar32 ch;
if (wildcard) { if (wildcard) {
++p; U8_NEXT(p, off, len, ch);
} }
std::vector<std::string> components; std::vector<std::string> components;
std::string buffer; std::string buffer;
while (p < e) { while (off < len) {
if (*p == '.') { U8_NEXT(p, off, len, ch);
if (ch < 0) {
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def + "' contains illegal UTF-8");
} else if (ch == 46) {
if (buffer.size() == 0) { if (buffer.size() == 0) {
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER, return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def + "path '" + def +
"' contains an empty component"); "' contains an empty component");
} }
++p;
components.push_back(buffer); components.push_back(buffer);
buffer.clear(); buffer.clear();
} else if (*p == 96) { // backtick ` } else if (ch == 96 || ch == 180) { // windows does not like U'`' and U'´'
++p; UChar32 quote = ch;
U8_NEXT(p, off, len, ch);
while (p < e && *p != 96) { if (ch < 0) {
buffer.push_back(*p++); return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def + "' contains illegal UTF-8");
} }
if (p == e) { while (off < len && ch != quote) {
basics::Utf8Helper::appendUtf8Character(buffer, ch);
U8_NEXT(p, off, len, ch);
if (ch < 0) {
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def + "' contains illegal UTF-8");
}
}
if (ch != quote) {
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER, return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def + "path '" + def +
"' contains an unbalanced quote"); "' contains an unbalanced quote");
} }
++p; U8_NEXT(p, off, len, ch);
} else if (p[0] == -62 && p[1] == -76) { // there is also a 0 at *e, so p[1] is ok
p += 2;
while (p < e - 1 && (p[0] != -62 || p[1] != -76)) { if (ch < 0) {
buffer.push_back(*p++);
}
if (p == e) {
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER, return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def + "path '" + def + "' contains illegal UTF-8");
"' contains an unbalanced quote");
} }
p += 2;
} else if (p[0] == -76 && p[1] == -62) { // there is also a 0 at *e, so p[1] is ok
p += 2;
while (p < e - 1 && (p[0] != -76 || p[1] != -62)) {
buffer.push_back(*p++);
}
if (p == e) {
return ParseResult<Path>(ParseResult<Path>::ILLEGAL_PARAMETER,
"path '" + def +
"' contains an unbalanced quote");
}
p += 2;
} else { } else {
buffer.push_back(*p++); basics::Utf8Helper::appendUtf8Character(buffer, ch);
} }
} }

73
lib/Maskings/RandomStringMask.cpp Normal file
View File

@ -0,0 +1,73 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Frank Celler
////////////////////////////////////////////////////////////////////////////////
#include "RandomStringMask.h"
#include "Basics/StringUtils.h"
#include "Basics/fasthash.h"
#include "Maskings/Maskings.h"
static std::string const xxxx("xxxx");
using namespace arangodb;
using namespace arangodb::maskings;
ParseResult<AttributeMasking> RandomStringMask::create(Path path, Maskings* maskings,
VPackSlice const&) {
return ParseResult<AttributeMasking>(AttributeMasking(path, new RandomStringMask(maskings)));
}
VPackValue RandomStringMask::mask(bool value) const {
return VPackValue(value);
}
VPackValue RandomStringMask::mask(std::string const& data, std::string& buffer) const {
uint64_t len = data.size();
uint64_t hash;
hash = fasthash64(data.c_str(), data.size(), _maskings->randomSeed());
std::string hash64 = basics::StringUtils::encodeBase64(
std::string((char const*)&hash, sizeof(decltype(hash))));
buffer.clear();
buffer.reserve(len);
buffer.append(hash64);
if (buffer.size() < len) {
while (buffer.size() < len) {
buffer.append(hash64);
}
buffer.resize(len);
}
return VPackValue(buffer);
}
VPackValue RandomStringMask::mask(int64_t value) const {
return VPackValue(value);
}
VPackValue RandomStringMask::mask(double value) const {
return VPackValue(value);
}

21
lib/Maskings/XifyFront.h → lib/Maskings/RandomStringMask.h
View File

@ -20,30 +20,27 @@
/// @author Frank Celler /// @author Frank Celler
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_MASKINGS_ATTRIBUTE_XIFY_FRONT_H #ifndef ARANGODB_MASKINGS_ATTRIBUTE_RANDOM_STRING_MASK_H
#define ARANGODB_MASKINGS_ATTRIBUTE_XIFY_FRONT_H 1 #define ARANGODB_MASKINGS_ATTRIBUTE_RANDOM_STRING_MASK_H 1
#include "Maskings/AttributeMasking.h"
#include "Maskings/MaskingFunction.h" #include "Maskings/MaskingFunction.h"
#include "Maskings/ParseResult.h"
namespace arangodb { namespace arangodb {
namespace maskings { namespace maskings {
class XifyFront : public MaskingFunction { class RandomStringMask : public MaskingFunction {
public: public:
XifyFront(Maskings* maskings, int64_t length, bool hash, uint64_t seed) static ParseResult<AttributeMasking> create(Path, Maskings*, VPackSlice const& def);
: MaskingFunction(maskings),
_length((uint64_t)length),
_randomSeed(seed),
_hash(hash) {}
public:
VPackValue mask(bool) const override; VPackValue mask(bool) const override;
VPackValue mask(std::string const&, std::string& buffer) const override; VPackValue mask(std::string const& data, std::string& buffer) const override;
VPackValue mask(int64_t) const override; VPackValue mask(int64_t) const override;
VPackValue mask(double) const override; VPackValue mask(double) const override;
private: private:
uint64_t _length; explicit RandomStringMask(Maskings* maskings) : MaskingFunction(maskings) {}
uint64_t _randomSeed;
bool _hash;
}; };
} // namespace maskings } // namespace maskings
} // namespace arangodb } // namespace arangodb

92
lib/Maskings/XifyFront.cpp
View File

@ -1,92 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Frank Celler
////////////////////////////////////////////////////////////////////////////////
#include "XifyFront.h"
#include "Basics/StringUtils.h"
#include "Basics/fasthash.h"
#include "Maskings/Maskings.h"
static std::string const xxxx("xxxx");
using namespace arangodb;
using namespace arangodb::maskings;
VPackValue XifyFront::mask(bool) const { return VPackValue(xxxx); }
VPackValue XifyFront::mask(std::string const& data, std::string& buffer) const {
char const* p = data.c_str();
char const* q = p;
char const* e = p + data.size();
buffer.clear();
buffer.reserve(data.size());
while (p < e) {
while (p < e && isNameChar(*p)) {
++p;
}
if (p != q) {
char const* w = p - _length;
while (q < w) {
buffer.push_back('x');
++q;
}
while (q < p) {
buffer.push_back(*q);
++q;
}
}
while (p < e && !isNameChar(*p)) {
buffer.push_back(' ');
++p;
}
q = p;
}
if (_hash) {
uint64_t hash;
if (_randomSeed == 0) {
hash = fasthash64(data.c_str(), data.size(), _maskings->randomSeed());
} else {
hash = fasthash64(data.c_str(), data.size(), _randomSeed);
}
std::string hash64 =
basics::StringUtils::encodeBase64(std::string((char const*)&hash, 8));
buffer.push_back(' ');
buffer.append(hash64);
}
return VPackValue(buffer);
}
VPackValue XifyFront::mask(int64_t) const { return VPackValue(xxxx); }
VPackValue XifyFront::mask(double) const { return VPackValue(xxxx); }

10
lib/Random/UniformCharacter.cpp
View File

@ -38,9 +38,15 @@ UniformCharacter::UniformCharacter(std::string const& characters)
UniformCharacter::UniformCharacter(size_t length, std::string const& characters) UniformCharacter::UniformCharacter(size_t length, std::string const& characters)
: _length(length), _characters(characters) {} : _length(length), _characters(characters) {}
std::string UniformCharacter::random() { return random(_length); } char UniformCharacter::randomChar() const {
size_t r = RandomGenerator::interval((uint32_t)(_characters.size() - 1));
std::string UniformCharacter::random(size_t length) { return _characters[r];
}
std::string UniformCharacter::random() const { return random(_length); }
std::string UniformCharacter::random(size_t length) const {
std::string buffer; std::string buffer;
buffer.reserve(length); buffer.reserve(length);

7
lib/Random/UniformCharacter.h
View File

@ -38,11 +38,12 @@ class UniformCharacter {
UniformCharacter(size_t length, std::string const& characters); UniformCharacter(size_t length, std::string const& characters);
public: public:
std::string random(); std::string random() const;
std::string random(size_t length); std::string random(size_t length) const;
char randomChar() const;
private: private:
size_t _length; size_t const _length;
std::string const _characters; std::string const _characters;
}; };
} // namespace arangodb } // namespace arangodb

66
tests/js/common/test-data/maskings/maskings1.json Normal file
View File

@ -0,0 +1,66 @@
{ "maskings1": {
"type": "masked",
"maskings": [
{
"path": "´name´",
"type": "xifyFront",
"unmaskedLength": 1
},
{
"path": ".`name`",
"type": "xifyFront",
"unmaskedLength": 2
},
{
"path": "email",
"type": "xifyFront",
"unmaskedLength": 3
}
]
},
"maskings2": {
"type": "masked",
"maskings": [
{
"path": "random",
"type": "randomString"
},
{
"path": "zip",
"type": "zip"
},
{
"path": "date",
"type": "date",
"begin": "1900-01-01",
"end": "2017-12-31",
"format": "%yyyy %mm %dd"
},
{
"path": "integer",
"type": "integer",
"lower": -10,
"upper": 10
},
{
"path": "decimal",
"type": "decimal",
"lower": -10,
"upper": 10,
"scale": 2
},
{
"path": "ccard",
"type": "creditCard"
},
{
"path": "phone",
"type": "phone"
},
{
"path": "email",
"type": "email"
}
]
}
}

88
tests/js/server/dump/dump-maskings-setup.js Normal file
View File

@ -0,0 +1,88 @@
/*jshint globalstrict:false, strict:false, maxlen:4000, unused:false */
/*global arango */
// /////////////////////////////////////////////////////////////////////////////
// @brief tests for dump/reload
//
// @file
//
// DISCLAIMER
//
// Copyright 2019 ArangoDB GmbH, Cologne, Germany
// Copyright 2010-2012 triagens GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Frank Celler
// /////////////////////////////////////////////////////////////////////////////
(function () {
'use strict';
var db = require("@arangodb").db;
var i, c;
try {
db._dropDatabase("UnitTestsDumpSrc");
} catch (err1) {
}
db._createDatabase("UnitTestsDumpSrc");
db._useDatabase("UnitTestsDumpSrc");
db._create("maskings1");
db.maskings1.save({
_key: "1",
name: "Hallo World! This is a t0st a top-level",
blub: {
name: "Hallo World! This is a t0st in a sub-object",
},
email: [
"testing arrays",
"this is another one",
{ something: "something else" },
{ email: "within a subject" },
{ name: [ "emails within a subject", "as list" ] }
],
sub: {
name: "this is a name leaf attribute",
email: [ "in this case as list", "with more than one entry" ]
}
});
db._create("maskings2");
db.maskings2.save({
_key: "2",
random: "a",
zip: "12345",
date: "2018-01-01",
integer: 100,
decimal: 100.12,
ccard: "1234 1234 1234 1234",
phone: "abcd 1234",
email: "me@you.here"
});
})();
return {
status: true
};

84
tests/js/server/dump/dump-maskings.js Normal file
View File

@ -0,0 +1,84 @@
/*jshint globalstrict:false, strict:false, maxlen:4000 */
/*global assertEqual, assertTrue, assertFalse, assertNotNull */
// /////////////////////////////////////////////////////////////////////////////
// @brief tests for dump/reload
//
// @file
//
// DISCLAIMER
//
// Copyright 2019 ArangoDB GmbH, Cologne, Germany
// Copyright 2010-2012 triagens GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// @author Frank Celler
// /////////////////////////////////////////////////////////////////////////////
var internal = require("internal");
var jsunity = require("jsunity");
let users = require("@arangodb/users");
function dumpMaskingSuite () {
'use strict';
var db = internal.db;
return {
setUp : function () {
},
tearDown : function () {
},
testGeneral : function () {
var c = db._collection("maskings1");
var d = c.document("1");
assertNotNull(d, "document '1' was restored");
assertEqual(d.name, "xxxxo xxxxd xxxs xs a xxxt a xxxxxxxxl");
assertEqual(d.blub.name, "xxxlo xxxld xxis is a xxst in a xxxxxxxxct");
assertEqual(d.email.length, 5);
assertEqual(d.email[0], "xxxxing xxxays");
assertEqual(d.email[1], "xhis is xxxxher one");
assertEqual(d.email[2].something, "something else");
assertEqual(d.email[3].email, "within a subject");
assertEqual(d.email[4].name.length, 2);
assertEqual(d.email[4].name[0], "xxxxls xxxxin a xxxxxct");
assertEqual(d.email[4].name[1], "as xxst");
assertEqual(d.sub.name, "xxis is a xxme xxaf xxxxxxxte");
assertEqual(d.sub.email.length, 2);
assertEqual(d.sub.email[0], "in this case as list");
assertEqual(d.sub.email[1], "with more than one entry");
},
testRandomString : function () {
var c = db._collection("maskings2");
var d = c.document("2");
assertFalse(d.random === "a");
assertFalse(d.zip === "12345");
assertFalse(d.date === "2018-01-01");
assertFalse(d.integer === 100);
assertFalse(d.ccard === "1234 1234 1234 1234");
assertFalse(d.phone === "abcd 1234");
assertFalse(d.emil === "me@you.here");
}
};
}
jsunity.run(dumpMaskingSuite);
return jsunity.done();