diff --git a/Documentation/Books/Manual/Programs/Arangodump/Maskings.md b/Documentation/Books/Manual/Programs/Arangodump/Maskings.md index f47f341673..97e7f25419 100644 --- a/Documentation/Books/Manual/Programs/Arangodump/Maskings.md +++ b/Documentation/Books/Manual/Programs/Arangodump/Maskings.md @@ -3,84 +3,111 @@ Arangodump Data Maskings *--maskings path-of-config* -It is possible to mask certain fields during dump. A JSON config file is +It is possible to mask certain fields for a dump. A JSON configuration file is used to define which fields should be masked and how. -The general structure of the config file is +The general structure of the config file looks like this: - { - "collection-name": { - "type": MASKING_TYPE - "maskings" : [ - MASKING1, - MASKING2, - ... - ] - }, +```json +{ + "collection-name": { + "type": MASKING_TYPE + "maskings" : [ + MASKING1, + MASKING2, ... - } + ] + }, + ... +} +``` + +Using `"*"` as collection name defines a default behavior for collections not +listed explicitly. Masking Types ------------- -This is a string describing how to mask this collection. Possible values are +`type` is a string describing how to mask the given collection. +Possible values are: -- "exclude": the collection is ignored completely and not even the structure data +- `"exclude"`: the collection is ignored completely and not even the structure data is dumped. -- "structure": only the collection structure is dumped, but no data at all +- `"structure"`: only the collection structure is dumped, but no data at all -- "masked": the collection structure and all data is dumped. However, the data - is subject to maskings defined in the attribute maskings. +- `"masked"`: the collection structure and all data is dumped. However, the data + is subject to obfuscation defined in the attribute `maskings`. -- "full": the collection structure and all data is dumped. No masking at all - is done for this collection. +- `"full"`: the collection structure and all data is dumped. No masking is + applied to this collection at all. -For example: +**Example** - { - "private": { - "type": "exclude" +```json +{ + "private": { + "type": "exclude" + }, + + "log": { + "type": "structure" + }, + + "person": { + "type": "masked", + "maskings": [ + { + "path": "name", + "type": "xifyFront", + "unmaskedLength": 2 }, - - "log": { - "type": "structure" - }, - - "person": { - "type": "masked", - "maskings": [ - { - "path": "name", - "type": "xify_front", - "unmaskedLength": 2 - }, - { - "path": ".security_id", - "type": "xify_front", - "unmaskedLength": 2 - } - ] + { + "path": ".security_id", + "type": "xifyFront", + "unmaskedLength": 2 } - } + ] + } +} +``` -In the example the collection "private" is completely ignored. Only the -structure of the collection "log" is dumped, but not the data itself. -The collection "person" is dumped completely but masking the "name" field if -it occurs on the top-level. It masks the field "security_id" anywhere in the -document. See below for a complete description of the parameters of -"xify_front". +In the example the collection _private_ is completely ignored. Only the +structure of the collection _log_ is dumped, but not the data itself. +The collection _person_ is dumped completely but with the _name_ field masked +if it occurs on the top-level. It also masks fields with the name "security_id" +anywhere in the document. See below for a complete description of the parameters +of [type "xifyFront"](#xify-front). + +### Masking vs. dump-data option + +*arangodump* also supports a very coarse masking with the option +`--dump-data false`. This basically removes all data from the dump. + +You can either use `--masking` or `--dump-data false`, but not both. + +### Masking vs. include-collection option + +*arangodump* also supports a very coarse masking with the option +`--include-collection`. This will restrict the collections that are +dumped to the ones explicitly listed. + +It is possible to combine `--masking` and `--include-collection`. +This will take the intersection of exportable collections. Path ---- -If the path starts with a `.` then it is considered to be a wildcard match. -For example, `.name` will match the attribute name `name` everywhere in the -document. `name` will only match at top level. `person.name` will match -the attribute `name` in the top-level object `person`. +If the path starts with a `.` then it is considered to match any path +ending in `name`. For example, `.name` will match the attribute name +`name` all leaf attributes in the document. Leaf attributes are +attributes whose value is `null` or of data type `string`, `number`, +`bool` and `array` (see below). `name` will only match leaf attributes +at top level. `person.name` will match the attribute `name` of a leaf +in the top-level object `person`. -If you have a attribute name that contains a dot, you need to quote the -name with either a tick or a backtick. For example +If you have an attribute name that contains a dot, you need to quote the +name with either a tick or a backtick. For example: "path": "´name.with.dots´" @@ -88,24 +115,233 @@ or "path": "`name.with.dots`" -xify_front ----------- +If the attribute value is an array the masking is applied to all the +array elements individually. -This masking replaces characters with `x` and ` `. Alphanumeric characters, -`_` and `-` are replaced by `x`, everything else is replaced by ` `. +**Example** +The following configuration will replace the value of the "name" +attribute with an "XXXX"-masked string: + +```json +{ + "type": "xifyFront", + "path": ".name", + "unmaskedLength": 2 +} +``` + +The document: + +```json +{ + "name": "top-level-name", + "age": 42, + "nicknames" : [ { "name": "hugo" }, "egon" ], + "other": { + "name": [ "emil", { "secret": "superman" } ] + } +} +``` + +… will be changed as follows: + +```json +{ + "name": "xxxxxxxxxxxxme", + "age": 42, + "nicknames" : [ { "name": "xxgo" }, "egon" ], + "other": { + "name": [ "xxil", { "secret": "superman" } ] + } +} +``` + +The values `"egon"` and `"superman"` are not replaced, because they +are not contained in an attribute value of which the attribute name is +`name`. + +### Nested objects and arrays + +If you specify a path and the attribute value is an array then the +masking decision is applied to each element of the array as if this +was the value of the attribute. + +If the attribute value is an object, then the attribute is not masked. +Instead the nested object is checked further for leaf attributes. + +**Example** + +Masking `email` will convert: + + +```json +{ + "email" : "email address" +} +``` + +… into: + +```json +{ + "email" : "xxil xxxxxxss" +} +``` + +because `email` is a leaf attribute. The document: + +```json +{ + "email" : [ + "address one", + "address two" + ] +} +``` + +… will be converted into: + +```json +{ + "email" : [ + "xxxxxss xne", + "xxxxxss xwo" + ] +} +``` + +… because the array is "unfolded". The document: + +```json +{ + "email" : { + "address" : "email address" + } +} +``` + +… will not be changed because `email` is not a leaf attribute. + + +Masking Functions +----------------- + +{% hint 'info' %} +The following masking functions are only available in the +[**Enterprise Edition**](https://www.arangodb.com/why-arangodb/arangodb-enterprise/) +{% endhint %} + +- xify front +- zip +- datetime +- integral number +- decimal number +- credit card number +- phone number +- email address + +The function: + +- random string + +… is available on Community Edition and in the Enterprise Edition. + + +### Random string + +```json +{ + "path": ".name", + "type": "randomString" +} +``` + +This masking type will replace all values of attributes with key +`name` with an anonymized string. It is not guaranteed that the string +will be of the same length. + +A hash of the original string is computed. If the original string is +shorter then the hash will be used. This will result in a longer +replacement string. If the string is longer than the hash then +characters will be repeated as many times as needed to reach the full +original string length. + +**Example** + +Masking name as above, the document: + +```json +{ + "_key" : "38937", + "_id" : "examplecollection/38937", + "_rev" : "_YFaGG1u--_", + "name" : [ + "My Name", + { + "other" : "Hallo Name" + }, + [ + "Name One", + "Name Two" + ], + true, + false, + null, + 1.0, + 1234, + "This is a very long name" + ] +} +``` + +… will be converted into + +```json +{ + "_key": "38937", + "_id": "examplecollection/38937", + "_rev": "_YFaGG1u--_", + "name": [ + "+y5OQiYmp/o=", { - "path": ".name", - "unmaskedLength": 2 - } + "other": "Hallo Name" + }, + [ + "ihCTrlsKKdk=", + "yo/55hfla0U=" + ], + true, + false, + null, + 1.0, + 1234, + "hwjAfNe5BGw=hwjAfNe5BGw=" + ] +} +``` -This will mask all alphanumeric characters of a word except the last 2. -Words of length 1 and 2 are unmasked. If the attribute value is not a -string the result will be `xxxx`. +### Xify front + +This masking type replaces the front characters with `x` and +blanks. Alphanumeric characters, `_` and `-` are replaced by `x`, +everything else is replaced by a blank. + +```json +{ + "path": ".name", + "type": "xifyFront", + "unmaskedLength": 2 +} +``` + +This will mask all alphanumeric characters of a word except the last +two characters. Words of length 1 and 2 are unmasked. If the +attribute value is not a string the result will be `xxxx`. "This is a test!Do you agree?" -will become +… will become "xxis is a xxst Do xou xxxee " @@ -113,30 +349,193 @@ There is a catch. If you have an index on the attribute the masking might distort the index efficiency or even cause errors in case of a unique index. - { - "path": ".name", - "unmaskedLength": 2, - "hash": true - } +```json +{ + "type": "xifyFront", + "path": ".name", + "unmaskedLength": 2, + "hash": true +} +``` This will add a hash at the end of the string. "This is a test!Do you agree?" -will become +… will become "xxis is a xxst Do xou xxxee NAATm8c9hVQ=" Note that the hash is based on a random secrect that is different for -each run. This avoids dictionary attacks. +each run. This avoids dictionary attacks which can be used to guess +values based pre-computations on dictionaries. -If you need reproducable results, i.e. hash that do not change between -different runs of *arangodump*, you need to specify a seed, which must -not be `0`. +If you need reproducible results, i.e. hashes that do not change between +different runs of *arangodump*, you need to specify a secret as seed, +a number which must not be `0`. - { - "path": ".name", - "unmaskedLength": 2, - "hash": true, - "seed": 246781478647 - } +```json +{ + "type": "xifyFront", + "path": ".name", + "unmaskedLength": 2, + "hash": true, + "seed": 246781478647 +} +``` + +### Zip + +This masking type replaces a zip code with a random one. If the +attribute value is not a string then the default value of `"12345"` is +used as no zip is known. You can change the default value, see below. + +```json +{ + "path": ".code", + "type": "zip", +} +``` + +This will replace a real zip code with a random one. It uses the following +rule: If a character of the original zip code is a digit it will be replaced +by a random digit. If a character of the original zip code is a letter it +will be replaced by a random letter keeping the case. + +```json +{ + "path": ".code", + "type": "zip", + "default": "abcdef" +} +``` + +**Example** + +If the original zip code is: + + 50674 + +… it will be replaced by e.g.: + + 98146 + +If the original zip code is: + + SA34-EA + +… it will be replaced by e.g.: + + OW91-JI + +Note that this will generate random zip code. Therefore there is a +chance generate the same zip code value multiple times, which can +cause unique constraint violations if a unique index is or will be +used on the zip code attribute. + +### Datetime + +This masking type replaces the value of the attribute with a random +date. + +```json +{ + "type": "datetime", + "begin" : "2019-01-01", + "end": "2019-12-31", + "format": "%yyyy-%mm-%dd", +} +``` + +`begin` and `end` are in ISO8601 format. + +The format is described in +[DATE_FORMAT](../../../AQL/Functions/Date.html#dateformat). + +### Integral number + +This masking type replaces the value of the attribute with a random +integral number. It will replace the value even if it is a string, +boolean, or false. + +```json +{ + "type": "integer", + "lower" : -100, + "upper": 100 +} +``` + +### Decimal number + +This masking type replaces the value of the attribute with a random +decimal. It will replace the value even if it is a string, boolean, +or false. + +```json +{ + "type": "float", + "lower" : -0.3, + "upper": 0.3 +} +``` + +By default, the decimal has a scale of 2. I.e. it has at most 2 +decimal digits. The definition: + +```json +{ + "type": "float", + "lower" : -0.3, + "upper": 0.3, + "scale": 3 +} +``` + +… will generate numbers with at most 3 decimal digits. + +### Credit card number + +This masking type replaces the value of the attribute with a random +credit card number. + +```json +{ + "type": "creditCard", +} +``` + +See [Luhn](https://en.wikipedia.org/wiki/Luhn_algorithm) for details. + +### Phone number + +This masking type replaces a phone number with a random one. If the +attribute value is not a string it is replaced by the string +`"+1234567890"`. + +```json +{ + "type": "phone", + "default": "+4912345123456789" +} +``` + +This will replace an existing phone number with a random one. It uses +the following rule: If a character of the original number is a digit +it will be replaced by a random digit. If it is a letter it is replaced +by a letter. All other characters are unchanged. + +```json +{ "type": "zip", + "default": "+4912345123456789" +} +``` + +If the attribute value is not a string use the value of default +`"+4912345123456789"`. + +### Email address + +This masking type takes an email address, computes a hash value and +split it into three equal parts `AAAA`, `BBBB`, and `CCCC`. The +resulting email address is `AAAA.BBBB@CCCC.invalid`. diff --git a/LICENSES-OTHER-COMPONENTS.md b/LICENSES-OTHER-COMPONENTS.md index 53656a5492..87e6778ca1 100644 --- a/LICENSES-OTHER-COMPONENTS.md +++ b/LICENSES-OTHER-COMPONENTS.md @@ -29,6 +29,15 @@ * License Name: Boost Software License 1.0 * License Id: BSL-1.0 +### CreditCardGenerator 2016 + +* Name: CreditCardGenerator +* Version: 1.8.1 +* Project Home: https://github.com/stormdark/CreditCardGenerator +* License: https://raw.githubusercontent.com/stormdark/CreditCardGenerator/master/LICENSE +* License Name: MIT License +* License Id: MIT + ### Curl 7.50.3 * Name: Curl diff --git a/arangosh/Dump/arangodump.cpp b/arangosh/Dump/arangodump.cpp index 75823b0268..2d646b6b1f 100644 --- a/arangosh/Dump/arangodump.cpp +++ b/arangosh/Dump/arangodump.cpp @@ -35,6 +35,7 @@ #include "Dump/DumpFeature.h" #include "Logger/Logger.h" #include "Logger/LoggerFeature.h" +#include "Maskings/AttributeMasking.h" #include "ProgramOptions/ProgramOptions.h" #include "Random/RandomFeature.h" #include "Shell/ClientFeature.h" @@ -42,6 +43,7 @@ #ifdef USE_ENTERPRISE #include "Enterprise/Encryption/EncryptionFeature.h" +#include "Enterprise/Maskings/AttributeMaskingEE.h" #endif using namespace arangodb; @@ -53,6 +55,12 @@ int main(int argc, char* argv[]) { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); + maskings::InstallMaskings(); + +#ifdef USE_ENTERPRISE + maskings::InstallMaskingsEE(); +#endif + std::shared_ptr options( new options::ProgramOptions(argv[0], "Usage: arangodump []", "For more information use:", BIN_DIRECTORY)); diff --git a/js/client/modules/@arangodb/process-utils.js b/js/client/modules/@arangodb/process-utils.js index 73e645f240..dbacc51d50 100755 --- a/js/client/modules/@arangodb/process-utils.js +++ b/js/client/modules/@arangodb/process-utils.js @@ -107,6 +107,12 @@ class ConfigBuilder { this.config['create-database'] = 'false'; } } + setMaskings(dir) { + if (this.type !== 'dump') { + throw '"maskings" is not supported for binary: ' + this.type; + } + this.config['maskings'] = fs.join(TOP_DIR, "tests/js/common/test-data/maskings", dir); + } activateEncryption() { this.config['encription.keyfile'] = fs.join(this.rootDir, 'secret-key'); } setRootDir(dir) { this.rootDir = dir; } restrictToCollection(collection) { diff --git a/js/client/modules/@arangodb/testsuites/dump.js b/js/client/modules/@arangodb/testsuites/dump.js index 4b3e61baa2..08f2e22a54 100644 --- a/js/client/modules/@arangodb/testsuites/dump.js +++ b/js/client/modules/@arangodb/testsuites/dump.js @@ -2,34 +2,36 @@ /* global print */ 'use strict'; -// ////////////////////////////////////////////////////////////////////////////// -// / DISCLAIMER -// / -// / Copyright 2016 ArangoDB GmbH, Cologne, Germany -// / Copyright 2014 triagens GmbH, Cologne, Germany -// / -// / Licensed under the Apache License, Version 2.0 (the "License") -// / you may not use this file except in compliance with the License. -// / You may obtain a copy of the License at -// / -// / http://www.apache.org/licenses/LICENSE-2.0 -// / -// / Unless required by applicable law or agreed to in writing, software -// / distributed under the License is distributed on an "AS IS" BASIS, -// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// / See the License for the specific language governing permissions and -// / limitations under the License. -// / -// / Copyright holder is ArangoDB GmbH, Cologne, Germany -// / -// / @author Max Neunhoeffer +// ///////////////////////////////////////////////////////////////////////////// +// DISCLAIMER +// +// Copyright 2016-2019 ArangoDB GmbH, Cologne, Germany +// Copyright 2014 triagens GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// @author Max Neunhoeffer // ////////////////////////////////////////////////////////////////////////////// const functionsDocumentation = { 'dump': 'dump tests', + 'dump_authentication': 'dump tests with authentication', 'dump_encrypted': 'encrypted dump tests', - 'dump_authentication': 'dump tests with authentication' + 'dump_maskings': 'masked dump tests' }; + const optionsDocumentation = [ ' - `skipEncrypted` : if set to true the encryption tests are skipped' ]; @@ -48,16 +50,18 @@ const RESET = require('internal').COLORS.COLOR_RESET; const testPaths = { 'dump': [tu.pathForTesting('server/dump')], + 'dump_authentication': [tu.pathForTesting('server/dump')], 'dump_encrypted': [tu.pathForTesting('server/dump')], - 'dump_authentication': [tu.pathForTesting('server/dump')] + 'dump_maskings': [tu.pathForTesting('server/dump')] }; class DumpRestoreHelper { - constructor(instanceInfo, options, clientAuth, dumpOptions, which, afterServerStart) { + constructor(instanceInfo, options, clientAuth, dumpOptions, restoreOptions, which, afterServerStart) { this.instanceInfo = instanceInfo; this.options = options; this.clientAuth = clientAuth; this.dumpOptions = dumpOptions; + this.restoreOptions = restoreOptions; this.which = which; this.fn = afterServerStart(instanceInfo); this.results = {failed: 1}; @@ -66,11 +70,15 @@ class DumpRestoreHelper { this.dumpConfig.setOutputDirectory('dump'); this.dumpConfig.setIncludeSystem(true); - this.restoreConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo); + if (dumpOptions.hasOwnProperty("maskings")) { + this.dumpConfig.setMaskings(dumpOptions.maskings); + } + + this.restoreConfig = pu.createBaseConfig('restore', this.restoreOptions, this.instanceInfo); this.restoreConfig.setInputDirectory('dump', true); this.restoreConfig.setIncludeSystem(true); - this.restoreOldConfig = pu.createBaseConfig('restore', this.dumpOptions, this.instanceInfo); + this.restoreOldConfig = pu.createBaseConfig('restore', this.restoreOptions, this.instanceInfo); this.restoreOldConfig.setInputDirectory('dump', true); this.restoreOldConfig.setIncludeSystem(true); this.restoreOldConfig.setDatabase('_system'); @@ -81,8 +89,8 @@ class DumpRestoreHelper { this.restoreOldConfig.activateEncryption(); } - this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.dumpOptions, this.instanceInfo.rootDir); - this.arangorestoreOld = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreOldConfig, this.dumpOptions, this.instanceInfo.rootDir); + this.arangorestore = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreConfig, this.restoreOptions, this.instanceInfo.rootDir); + this.arangorestoreOld = pu.run.arangoDumpRestoreWithConfig.bind(this, this.restoreOldConfig, this.restoreOptions, this.instanceInfo.rootDir); this.arangodump = pu.run.arangoDumpRestoreWithConfig.bind(this, this.dumpConfig, this.dumpOptions, this.instanceInfo.rootDir); } @@ -225,10 +233,7 @@ function getClusterStrings(options) } } -// ////////////////////////////////////////////////////////////////////////////// -// / @brief TEST: dump -// ////////////////////////////////////////////////////////////////////////////// -function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which, tstFiles, afterServerStart) { +function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, restoreOptions, which, tstFiles, afterServerStart) { print(CYAN + which + ' tests...' + RESET); let instanceInfo = pu.startInstance('tcp', options, serverAuthInfo, which); @@ -243,7 +248,7 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which, }; return rc; } - const helper = new DumpRestoreHelper(instanceInfo, options, clientAuth, dumpOptions, which, afterServerStart); + const helper = new DumpRestoreHelper(instanceInfo, options, clientAuth, dumpOptions, restoreOptions, which, afterServerStart); const setupFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpSetup)); const testFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.dumpAgain)); @@ -267,21 +272,21 @@ function dump_backend (options, serverAuthInfo, clientAuth, dumpOptions, which, } } - const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest)); - if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') || - !helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') || - !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') || - !helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxAppsBundle') || - !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxBundleApps') || - !helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxBundleApps')) { - return helper.extractResults(); + if (tstFiles.hasOwnProperty("foxxTest")) { + const foxxTestFile = tu.makePathUnix(fs.join(testPaths[which][0], tstFiles.foxxTest)); + if (!helper.restoreFoxxComplete('UnitTestsDumpFoxxComplete') || + !helper.testFoxxComplete(foxxTestFile, 'UnitTestsDumpFoxxComplete') || + !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxAppsBundle') || + !helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxAppsBundle') || + !helper.restoreFoxxAppsBundle('UnitTestsDumpFoxxBundleApps') || + !helper.testFoxxAppsBundle(foxxTestFile, 'UnitTestsDumpFoxxBundleApps')) { + return helper.extractResults(); + } } return helper.extractResults(); } -// ///////////////////////////////////////////////////////////////////////////// - function dump (options) { let c = getClusterStrings(options); let tstFiles = { @@ -292,7 +297,7 @@ function dump (options) { foxxTest: 'check-foxx.js' }; - return dump_backend(options, {}, {}, options, 'dump', tstFiles, function(){}); + return dump_backend(options, {}, {}, options, options, 'dump', tstFiles, function(){}); } function dumpAuthentication (options) { @@ -332,7 +337,7 @@ function dumpAuthentication (options) { foxxTest: 'check-foxx.js' }; - return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, 'dump_authentication', tstFiles, function(){}); + return dump_backend(options, serverAuthInfo, clientAuth, dumpAuthOpts, dumpAuthOpts, 'dump_authentication', tstFiles, function(){}); } function dumpEncrypted (options) { @@ -373,20 +378,57 @@ function dumpEncrypted (options) { foxxTest: 'check-foxx.js' }; - return dump_backend(options, {}, {}, dumpOptions, 'dump_encrypted', tstFiles, afterServerStart); + return dump_backend(options, {}, {}, dumpOptions, dumpOptions, 'dump_encrypted', tstFiles, afterServerStart); +} + +function dumpMaskings (options) { + // test is only meaningful in the enterprise version + let skip = true; + if (global.ARANGODB_CLIENT_VERSION) { + let version = global.ARANGODB_CLIENT_VERSION(true); + if (version.hasOwnProperty('enterprise-version')) { + skip = false; + } + } + + if (skip) { + print('skipping dump_maskings test'); + return { + dump_maskings: { + status: true, + skipped: true + } + }; + } + + let tstFiles = { + dumpSetup: 'dump-maskings-setup.js', + dumpAgain: 'dump-maskings.js', + dumpTearDown: 'dump-teardown.js' + }; + + let dumpMaskingsOpts = { + maskings: 'maskings1.json' + }; + + _.defaults(dumpMaskingsOpts, options); + + return dump_backend(options, {}, {}, dumpMaskingsOpts, options, 'dump_maskings', tstFiles, function(){}); } -// ///////////////////////////////////////////////////////////////////////////// exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) { Object.assign(allTestPaths, testPaths); testFns['dump'] = dump; defaultFns.push('dump'); + testFns['dump_authentication'] = dumpAuthentication; + defaultFns.push('dump_authentication'); + testFns['dump_encrypted'] = dumpEncrypted; defaultFns.push('dump_encrypted'); - testFns['dump_authentication'] = dumpAuthentication; - defaultFns.push('dump_authentication'); + testFns['dump_maskings'] = dumpMaskings; + defaultFns.push('dump_maskings'); for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; } for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); } diff --git a/lib/Basics/Utf8Helper.h b/lib/Basics/Utf8Helper.h index 40fe934050..868d98b161 100644 --- a/lib/Basics/Utf8Helper.h +++ b/lib/Basics/Utf8Helper.h @@ -1,7 +1,7 @@ //////////////////////////////////////////////////////////////////////////////// /// DISCLAIMER /// -/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2014-2019 ArangoDB GmbH, Cologne, Germany /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany /// /// Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,9 +25,10 @@ #ifndef ARANGODB_BASICS_UTF8HELPER_H #define ARANGODB_BASICS_UTF8HELPER_H 1 -#include #include "Basics/Common.h" +#include + #include #include #include @@ -40,10 +41,6 @@ class Utf8Helper { Utf8Helper& operator=(Utf8Helper const&) = delete; public: - ////////////////////////////////////////////////////////////////////////////// - /// @brief a default helper - ////////////////////////////////////////////////////////////////////////////// - static Utf8Helper DefaultUtf8Helper; public: @@ -153,6 +150,26 @@ class Utf8Helper { char const* replacement, size_t replacementLength, bool partial, bool& error); + // append an UTF8 to a string. This will append 1 to 4 bytes. + static void appendUtf8Character(std::string& result, uint32_t ch) { + if (ch <= 0x7f) { + result.push_back((uint8_t)ch); + } else { + if (ch <= 0x7ff) { + result.push_back((uint8_t)((ch >> 6) | 0xc0)); + } else { + if (ch <= 0xffff) { + result.push_back((uint8_t)((ch >> 12) | 0xe0)); + } else { + result.push_back((uint8_t)((ch >> 18) | 0xf0)); + result.push_back((uint8_t)(((ch >> 12) & 0x3f) | 0x80)); + } + result.push_back((uint8_t)(((ch >> 6) & 0x3f) | 0x80)); + } + result.push_back((uint8_t)((ch & 0x3f) | 0x80)); + } + } + private: Collator* _coll; }; diff --git a/lib/Basics/datetime.h b/lib/Basics/datetime.h index e2b4f08dcf..bdd509fc05 100644 --- a/lib/Basics/datetime.h +++ b/lib/Basics/datetime.h @@ -23,6 +23,8 @@ #ifndef ARANGODB_BASICS_DATETIME_H #define ARANGODB_BASICS_DATETIME_H 1 +#include "Basics/Common.h" + #include #include diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index fe60be1202..a6d24ce803 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -237,7 +237,7 @@ add_library(${LIB_ARANGO} STATIC Maskings/Collection.cpp Maskings/Maskings.cpp Maskings/Path.cpp - Maskings/XifyFront.cpp + Maskings/RandomStringMask.cpp ProgramOptions/Option.cpp ProgramOptions/ProgramOptions.cpp ProgramOptions/Section.cpp diff --git a/lib/Maskings/AttributeMasking.cpp b/lib/Maskings/AttributeMasking.cpp index 5477cacb32..c95e843b92 100644 --- a/lib/Maskings/AttributeMasking.cpp +++ b/lib/Maskings/AttributeMasking.cpp @@ -24,11 +24,17 @@ #include "Basics/StringUtils.h" #include "Logger/Logger.h" -#include "Maskings/XifyFront.h" +#include "Maskings/RandomStringMask.h" using namespace arangodb; using namespace arangodb::maskings; +void arangodb::maskings::InstallMaskings() { + AttributeMasking::installMasking("randomString", RandomStringMask::create); +} + +std::unordered_map (*)(Path, Maskings*, VPackSlice const&)> AttributeMasking::_maskings; + ParseResult AttributeMasking::parse(Maskings* maskings, VPackSlice const& def) { if (!def.isObject()) { @@ -39,9 +45,6 @@ ParseResult AttributeMasking::parse(Maskings* maskings, std::string path = ""; std::string type = ""; - uint64_t length = 2; - uint64_t seed = 0; - bool hash = false; for (auto const& entry : VPackObjectIterator(def, false)) { std::string key = entry.key.copyString(); @@ -60,27 +63,6 @@ ParseResult AttributeMasking::parse(Maskings* maskings, } path = entry.value.copyString(); - } else if (key == "unmaskedLength") { - if (!entry.value.isInteger()) { - return ParseResult(ParseResult::ILLEGAL_PARAMETER, - "length must be an integer"); - } - - length = entry.value.getInt(); - } else if (key == "hash") { - if (!entry.value.isBool()) { - return ParseResult(ParseResult::ILLEGAL_PARAMETER, - "hash must be an integer"); - } - - hash = entry.value.getBool(); - } else if (key == "seed") { - if (!entry.value.isInteger()) { - return ParseResult(ParseResult::ILLEGAL_PARAMETER, - "seed must be an integer"); - } - - seed = entry.value.getInt(); } } @@ -96,20 +78,15 @@ ParseResult AttributeMasking::parse(Maskings* maskings, (ParseResult::StatusCode)(int)ap.status, ap.message); } - if (type == "xify_front") { - if (length < 1) { - return ParseResult( - ParseResult::ILLEGAL_PARAMETER, - "expecting length to be at least for xify_front"); - } + auto const& it = _maskings.find(type); + if (it == _maskings.end()) { return ParseResult( - AttributeMasking(ap.result, new XifyFront(maskings, length, hash, seed))); + ParseResult::UNKNOWN_TYPE, + "unknown attribute masking type '" + type + "'"); } - return ParseResult( - ParseResult::UNKNOWN_TYPE, - "expecting unknown attribute masking type '" + type + "'"); + return it->second(ap.result, maskings, def); } bool AttributeMasking::match(std::vector const& path) const { diff --git a/lib/Maskings/AttributeMasking.h b/lib/Maskings/AttributeMasking.h index c0aa79b95a..6a7db88b65 100644 --- a/lib/Maskings/AttributeMasking.h +++ b/lib/Maskings/AttributeMasking.h @@ -37,9 +37,14 @@ namespace arangodb { namespace maskings { +void InstallMaskings(); + class AttributeMasking { public: static ParseResult parse(Maskings*, VPackSlice const&); + static void installMasking(std::string const& name, ParseResult (* func)(Path, Maskings*, VPackSlice const&)) { + _maskings[name] = func; + } public: AttributeMasking() = default; @@ -52,6 +57,9 @@ class AttributeMasking { MaskingFunction* func() const { return _func.get(); } + private: + static std::unordered_map (*)(Path, Maskings*, VPackSlice const&)> _maskings; + private: Path _path; std::shared_ptr _func; diff --git a/lib/Maskings/MaskingFunction.h b/lib/Maskings/MaskingFunction.h index 83f52c7da4..6ebe8c6c39 100644 --- a/lib/Maskings/MaskingFunction.h +++ b/lib/Maskings/MaskingFunction.h @@ -25,6 +25,8 @@ #include "Basics/Common.h" +#include "Basics/Utf8Helper.h" + #include #include #include @@ -37,9 +39,8 @@ class Maskings; class MaskingFunction { public: - static bool isNameChar(char c) { - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || - ('0' <= c && c <= '9') || c == '_' || c == '-'; + static bool isNameChar(UChar32 ch) { + return u_isalpha(ch) || u_isdigit(ch) || ch == U'_' || ch == U'-'; } public: diff --git a/lib/Maskings/Maskings.cpp b/lib/Maskings/Maskings.cpp index f36197891c..eec71809b6 100644 --- a/lib/Maskings/Maskings.cpp +++ b/lib/Maskings/Maskings.cpp @@ -85,11 +85,21 @@ ParseResult Maskings::parse(VPackSlice const& def) { for (auto const& entry : VPackObjectIterator(def, false)) { std::string key = entry.key.copyString(); - LOG_TOPIC(TRACE, Logger::CONFIG) << "masking collection '" << key << "'"; - if (_collections.find(key) != _collections.end()) { - return ParseResult(ParseResult::DUPLICATE_COLLECTION, - "duplicate collection entry '" + key + "'"); + if (key == "*") { + LOG_TOPIC(TRACE, Logger::CONFIG) << "default masking"; + + if (_hasDefaultCollection) { + return ParseResult(ParseResult::DUPLICATE_COLLECTION, + "duplicate default entry"); + } + } else { + LOG_TOPIC(TRACE, Logger::CONFIG) << "masking collection '" << key << "'"; + + if (_collections.find(key) != _collections.end()) { + return ParseResult(ParseResult::DUPLICATE_COLLECTION, + "duplicate collection entry '" + key + "'"); + } } ParseResult c = Collection::parse(this, entry.value); @@ -99,20 +109,30 @@ ParseResult Maskings::parse(VPackSlice const& def) { c.message); } - _collections[key] = c.result; + if (key == "*") { + _hasDefaultCollection = true; + _defaultCollection = c.result; + } else { + _collections[key] = c.result; + } } return ParseResult(ParseResult::VALID); } bool Maskings::shouldDumpStructure(std::string const& name) { + CollectionSelection select = CollectionSelection::EXCLUDE; auto const itr = _collections.find(name); if (itr == _collections.end()) { - return false; + if (_hasDefaultCollection) { + select = _defaultCollection.selection(); + } + } else { + select = itr->second.selection(); } - switch (itr->second.selection()) { + switch (select) { case CollectionSelection::FULL: return true; case CollectionSelection::MASKED: @@ -129,13 +149,18 @@ bool Maskings::shouldDumpStructure(std::string const& name) { } bool Maskings::shouldDumpData(std::string const& name) { + CollectionSelection select = CollectionSelection::EXCLUDE; auto const itr = _collections.find(name); if (itr == _collections.end()) { - return false; + if (_hasDefaultCollection) { + select = _defaultCollection.selection(); + } + } else { + select = itr->second.selection(); } - switch (itr->second.selection()) { + switch (select) { case CollectionSelection::FULL: return true; case CollectionSelection::MASKED: @@ -289,14 +314,21 @@ void Maskings::mask(std::string const& name, basics::StringBuffer const& data, basics::StringBuffer& result) { result.clear(); + Collection* collection; auto const itr = _collections.find(name); if (itr == _collections.end()) { - result.copy(data); - return; + if (_hasDefaultCollection) { + collection = &_defaultCollection; + } else { + result.copy(data); + return; + } + } else { + collection = &(itr->second); } - if (itr->second.selection() == CollectionSelection::FULL) { + if (collection->selection() == CollectionSelection::FULL) { result.copy(data); return; } @@ -314,7 +346,7 @@ void Maskings::mask(std::string const& name, basics::StringBuffer const& data, std::shared_ptr builder = VPackParser::fromJson(q, p - q); - addMasked(itr->second, result, builder->slice()); + addMasked(*collection, result, builder->slice()); while (p < e && (*p == '\n' || *p == '\r')) { ++p; diff --git a/lib/Maskings/Maskings.h b/lib/Maskings/Maskings.h index 074cad9bfc..d2248064a5 100644 --- a/lib/Maskings/Maskings.h +++ b/lib/Maskings/Maskings.h @@ -80,6 +80,8 @@ class Maskings { private: std::map _collections; + bool _hasDefaultCollection = false; + Collection _defaultCollection; uint64_t _randomSeed = 0; }; diff --git a/lib/Maskings/Path.cpp b/lib/Maskings/Path.cpp index fc177cc92a..bd51c91e25 100644 --- a/lib/Maskings/Path.cpp +++ b/lib/Maskings/Path.cpp @@ -23,6 +23,7 @@ #include "Collection.h" #include "Basics/StringUtils.h" +#include "Basics/Utf8Helper.h" #include "Logger/Logger.h" using namespace arangodb; @@ -40,71 +41,66 @@ ParseResult Path::parse(std::string const& def) { wildcard = true; } - char const* p = def.c_str(); - char const* e = p + def.size(); + uint8_t const* p = reinterpret_cast(def.c_str()); + int32_t off = 0; + int32_t len = def.size(); + UChar32 ch; if (wildcard) { - ++p; + U8_NEXT(p, off, len, ch); } std::vector components; std::string buffer; - while (p < e) { - if (*p == '.') { + while (off < len) { + U8_NEXT(p, off, len, ch); + + if (ch < 0) { + return ParseResult(ParseResult::ILLEGAL_PARAMETER, + "path '" + def + "' contains illegal UTF-8"); + } else if (ch == 46) { if (buffer.size() == 0) { return ParseResult(ParseResult::ILLEGAL_PARAMETER, "path '" + def + "' contains an empty component"); } - ++p; components.push_back(buffer); buffer.clear(); - } else if (*p == 96) { // backtick ` - ++p; + } else if (ch == 96 || ch == 180) { // windows does not like U'`' and U'´' + UChar32 quote = ch; + U8_NEXT(p, off, len, ch); - while (p < e && *p != 96) { - buffer.push_back(*p++); + if (ch < 0) { + return ParseResult(ParseResult::ILLEGAL_PARAMETER, + "path '" + def + "' contains illegal UTF-8"); } - if (p == e) { + while (off < len && ch != quote) { + basics::Utf8Helper::appendUtf8Character(buffer, ch); + U8_NEXT(p, off, len, ch); + + if (ch < 0) { + return ParseResult(ParseResult::ILLEGAL_PARAMETER, + "path '" + def + "' contains illegal UTF-8"); + } + } + + if (ch != quote) { return ParseResult(ParseResult::ILLEGAL_PARAMETER, "path '" + def + "' contains an unbalanced quote"); } - ++p; - } else if (p[0] == -62 && p[1] == -76) { // there is also a 0 at *e, so p[1] is ok - p += 2; + U8_NEXT(p, off, len, ch); - while (p < e - 1 && (p[0] != -62 || p[1] != -76)) { - buffer.push_back(*p++); - } - - if (p == e) { + if (ch < 0) { return ParseResult(ParseResult::ILLEGAL_PARAMETER, - "path '" + def + - "' contains an unbalanced quote"); + "path '" + def + "' contains illegal UTF-8"); } - - p += 2; - } else if (p[0] == -76 && p[1] == -62) { // there is also a 0 at *e, so p[1] is ok - p += 2; - - while (p < e - 1 && (p[0] != -76 || p[1] != -62)) { - buffer.push_back(*p++); - } - - if (p == e) { - return ParseResult(ParseResult::ILLEGAL_PARAMETER, - "path '" + def + - "' contains an unbalanced quote"); - } - - p += 2; } else { - buffer.push_back(*p++); + basics::Utf8Helper::appendUtf8Character(buffer, ch); } } diff --git a/lib/Maskings/RandomStringMask.cpp b/lib/Maskings/RandomStringMask.cpp new file mode 100644 index 0000000000..5cdd2dff9c --- /dev/null +++ b/lib/Maskings/RandomStringMask.cpp @@ -0,0 +1,73 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Frank Celler +//////////////////////////////////////////////////////////////////////////////// + +#include "RandomStringMask.h" + +#include "Basics/StringUtils.h" +#include "Basics/fasthash.h" +#include "Maskings/Maskings.h" + +static std::string const xxxx("xxxx"); + +using namespace arangodb; +using namespace arangodb::maskings; + +ParseResult RandomStringMask::create(Path path, Maskings* maskings, + VPackSlice const&) { + return ParseResult(AttributeMasking(path, new RandomStringMask(maskings))); +} + +VPackValue RandomStringMask::mask(bool value) const { + return VPackValue(value); +} + +VPackValue RandomStringMask::mask(std::string const& data, std::string& buffer) const { + uint64_t len = data.size(); + uint64_t hash; + + hash = fasthash64(data.c_str(), data.size(), _maskings->randomSeed()); + + std::string hash64 = basics::StringUtils::encodeBase64( + std::string((char const*)&hash, sizeof(decltype(hash)))); + + buffer.clear(); + buffer.reserve(len); + buffer.append(hash64); + + if (buffer.size() < len) { + while (buffer.size() < len) { + buffer.append(hash64); + } + + buffer.resize(len); + } + + return VPackValue(buffer); +} + +VPackValue RandomStringMask::mask(int64_t value) const { + return VPackValue(value); +} + +VPackValue RandomStringMask::mask(double value) const { + return VPackValue(value); +} diff --git a/lib/Maskings/XifyFront.h b/lib/Maskings/RandomStringMask.h similarity index 70% rename from lib/Maskings/XifyFront.h rename to lib/Maskings/RandomStringMask.h index 1c43d10483..2bed882af8 100644 --- a/lib/Maskings/XifyFront.h +++ b/lib/Maskings/RandomStringMask.h @@ -20,30 +20,27 @@ /// @author Frank Celler //////////////////////////////////////////////////////////////////////////////// -#ifndef ARANGODB_MASKINGS_ATTRIBUTE_XIFY_FRONT_H -#define ARANGODB_MASKINGS_ATTRIBUTE_XIFY_FRONT_H 1 +#ifndef ARANGODB_MASKINGS_ATTRIBUTE_RANDOM_STRING_MASK_H +#define ARANGODB_MASKINGS_ATTRIBUTE_RANDOM_STRING_MASK_H 1 +#include "Maskings/AttributeMasking.h" #include "Maskings/MaskingFunction.h" +#include "Maskings/ParseResult.h" namespace arangodb { namespace maskings { -class XifyFront : public MaskingFunction { +class RandomStringMask : public MaskingFunction { public: - XifyFront(Maskings* maskings, int64_t length, bool hash, uint64_t seed) - : MaskingFunction(maskings), - _length((uint64_t)length), - _randomSeed(seed), - _hash(hash) {} + static ParseResult create(Path, Maskings*, VPackSlice const& def); + public: VPackValue mask(bool) const override; - VPackValue mask(std::string const&, std::string& buffer) const override; + VPackValue mask(std::string const& data, std::string& buffer) const override; VPackValue mask(int64_t) const override; VPackValue mask(double) const override; private: - uint64_t _length; - uint64_t _randomSeed; - bool _hash; + explicit RandomStringMask(Maskings* maskings) : MaskingFunction(maskings) {} }; } // namespace maskings } // namespace arangodb diff --git a/lib/Maskings/XifyFront.cpp b/lib/Maskings/XifyFront.cpp deleted file mode 100644 index b22eac1da6..0000000000 --- a/lib/Maskings/XifyFront.cpp +++ /dev/null @@ -1,92 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2018 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Frank Celler -//////////////////////////////////////////////////////////////////////////////// - -#include "XifyFront.h" - -#include "Basics/StringUtils.h" -#include "Basics/fasthash.h" -#include "Maskings/Maskings.h" - -static std::string const xxxx("xxxx"); - -using namespace arangodb; -using namespace arangodb::maskings; - -VPackValue XifyFront::mask(bool) const { return VPackValue(xxxx); } - -VPackValue XifyFront::mask(std::string const& data, std::string& buffer) const { - char const* p = data.c_str(); - char const* q = p; - char const* e = p + data.size(); - - buffer.clear(); - buffer.reserve(data.size()); - - while (p < e) { - while (p < e && isNameChar(*p)) { - ++p; - } - - if (p != q) { - char const* w = p - _length; - - while (q < w) { - buffer.push_back('x'); - ++q; - } - - while (q < p) { - buffer.push_back(*q); - ++q; - } - } - - while (p < e && !isNameChar(*p)) { - buffer.push_back(' '); - ++p; - } - - q = p; - } - - if (_hash) { - uint64_t hash; - - if (_randomSeed == 0) { - hash = fasthash64(data.c_str(), data.size(), _maskings->randomSeed()); - } else { - hash = fasthash64(data.c_str(), data.size(), _randomSeed); - } - - std::string hash64 = - basics::StringUtils::encodeBase64(std::string((char const*)&hash, 8)); - - buffer.push_back(' '); - buffer.append(hash64); - } - - return VPackValue(buffer); -} - -VPackValue XifyFront::mask(int64_t) const { return VPackValue(xxxx); } - -VPackValue XifyFront::mask(double) const { return VPackValue(xxxx); } diff --git a/lib/Random/UniformCharacter.cpp b/lib/Random/UniformCharacter.cpp index 8051fd670d..215d516aa0 100644 --- a/lib/Random/UniformCharacter.cpp +++ b/lib/Random/UniformCharacter.cpp @@ -38,9 +38,15 @@ UniformCharacter::UniformCharacter(std::string const& characters) UniformCharacter::UniformCharacter(size_t length, std::string const& characters) : _length(length), _characters(characters) {} -std::string UniformCharacter::random() { return random(_length); } +char UniformCharacter::randomChar() const { + size_t r = RandomGenerator::interval((uint32_t)(_characters.size() - 1)); -std::string UniformCharacter::random(size_t length) { + return _characters[r]; +} + +std::string UniformCharacter::random() const { return random(_length); } + +std::string UniformCharacter::random(size_t length) const { std::string buffer; buffer.reserve(length); diff --git a/lib/Random/UniformCharacter.h b/lib/Random/UniformCharacter.h index 82c8fdb8c3..220c01903f 100644 --- a/lib/Random/UniformCharacter.h +++ b/lib/Random/UniformCharacter.h @@ -38,11 +38,12 @@ class UniformCharacter { UniformCharacter(size_t length, std::string const& characters); public: - std::string random(); - std::string random(size_t length); + std::string random() const; + std::string random(size_t length) const; + char randomChar() const; private: - size_t _length; + size_t const _length; std::string const _characters; }; } // namespace arangodb diff --git a/tests/js/common/test-data/maskings/maskings1.json b/tests/js/common/test-data/maskings/maskings1.json new file mode 100644 index 0000000000..116b363319 --- /dev/null +++ b/tests/js/common/test-data/maskings/maskings1.json @@ -0,0 +1,66 @@ +{ "maskings1": { + "type": "masked", + "maskings": [ + { + "path": "´name´", + "type": "xifyFront", + "unmaskedLength": 1 + }, + { + "path": ".`name`", + "type": "xifyFront", + "unmaskedLength": 2 + }, + { + "path": "email", + "type": "xifyFront", + "unmaskedLength": 3 + } + ] + }, + "maskings2": { + "type": "masked", + "maskings": [ + { + "path": "random", + "type": "randomString" + }, + { + "path": "zip", + "type": "zip" + }, + { + "path": "date", + "type": "date", + "begin": "1900-01-01", + "end": "2017-12-31", + "format": "%yyyy %mm %dd" + }, + { + "path": "integer", + "type": "integer", + "lower": -10, + "upper": 10 + }, + { + "path": "decimal", + "type": "decimal", + "lower": -10, + "upper": 10, + "scale": 2 + }, + { + "path": "ccard", + "type": "creditCard" + }, + { + "path": "phone", + "type": "phone" + }, + { + "path": "email", + "type": "email" + } + ] + } +} diff --git a/tests/js/server/dump/dump-maskings-setup.js b/tests/js/server/dump/dump-maskings-setup.js new file mode 100644 index 0000000000..37017a8647 --- /dev/null +++ b/tests/js/server/dump/dump-maskings-setup.js @@ -0,0 +1,88 @@ +/*jshint globalstrict:false, strict:false, maxlen:4000, unused:false */ +/*global arango */ + +// ///////////////////////////////////////////////////////////////////////////// +// @brief tests for dump/reload +// +// @file +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// Copyright 2010-2012 triagens GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// @author Frank Celler +// ///////////////////////////////////////////////////////////////////////////// + +(function () { + 'use strict'; + var db = require("@arangodb").db; + var i, c; + + try { + db._dropDatabase("UnitTestsDumpSrc"); + } catch (err1) { + } + + db._createDatabase("UnitTestsDumpSrc"); + db._useDatabase("UnitTestsDumpSrc"); + + db._create("maskings1"); + + db.maskings1.save({ + _key: "1", + + name: "Hallo World! This is a t0st a top-level", + + blub: { + name: "Hallo World! This is a t0st in a sub-object", + }, + + email: [ + "testing arrays", + "this is another one", + { something: "something else" }, + { email: "within a subject" }, + { name: [ "emails within a subject", "as list" ] } + ], + + sub: { + name: "this is a name leaf attribute", + email: [ "in this case as list", "with more than one entry" ] + } + }); + + db._create("maskings2"); + + db.maskings2.save({ + _key: "2", + + random: "a", + zip: "12345", + date: "2018-01-01", + integer: 100, + decimal: 100.12, + ccard: "1234 1234 1234 1234", + phone: "abcd 1234", + email: "me@you.here" + }); +})(); + +return { + status: true +}; + diff --git a/tests/js/server/dump/dump-maskings.js b/tests/js/server/dump/dump-maskings.js new file mode 100644 index 0000000000..ebad921d31 --- /dev/null +++ b/tests/js/server/dump/dump-maskings.js @@ -0,0 +1,84 @@ +/*jshint globalstrict:false, strict:false, maxlen:4000 */ +/*global assertEqual, assertTrue, assertFalse, assertNotNull */ + +// ///////////////////////////////////////////////////////////////////////////// +// @brief tests for dump/reload +// +// @file +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// Copyright 2010-2012 triagens GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// @author Frank Celler +// ///////////////////////////////////////////////////////////////////////////// + +var internal = require("internal"); +var jsunity = require("jsunity"); +let users = require("@arangodb/users"); + +function dumpMaskingSuite () { + 'use strict'; + var db = internal.db; + + return { + setUp : function () { + }, + + tearDown : function () { + }, + + testGeneral : function () { + var c = db._collection("maskings1"); + var d = c.document("1"); + + assertNotNull(d, "document '1' was restored"); + assertEqual(d.name, "xxxxo xxxxd xxxs xs a xxxt a xxxxxxxxl"); + assertEqual(d.blub.name, "xxxlo xxxld xxis is a xxst in a xxxxxxxxct"); + assertEqual(d.email.length, 5); + assertEqual(d.email[0], "xxxxing xxxays"); + assertEqual(d.email[1], "xhis is xxxxher one"); + assertEqual(d.email[2].something, "something else"); + assertEqual(d.email[3].email, "within a subject"); + assertEqual(d.email[4].name.length, 2); + assertEqual(d.email[4].name[0], "xxxxls xxxxin a xxxxxct"); + assertEqual(d.email[4].name[1], "as xxst"); + assertEqual(d.sub.name, "xxis is a xxme xxaf xxxxxxxte"); + assertEqual(d.sub.email.length, 2); + assertEqual(d.sub.email[0], "in this case as list"); + assertEqual(d.sub.email[1], "with more than one entry"); + }, + + testRandomString : function () { + var c = db._collection("maskings2"); + var d = c.document("2"); + + assertFalse(d.random === "a"); + assertFalse(d.zip === "12345"); + assertFalse(d.date === "2018-01-01"); + assertFalse(d.integer === 100); + assertFalse(d.ccard === "1234 1234 1234 1234"); + assertFalse(d.phone === "abcd 1234"); + assertFalse(d.emil === "me@you.here"); + } +}; +} + +jsunity.run(dumpMaskingSuite); + +return jsunity.done();