Feature/debugdump ui (#5187) (#5188)

* Adeded a route to aardvark to export a debug dump, it can optinally contain anonymized example data

* Added the Download Debug Button to UI

* fixed zip file download of query dump file

* removed not needed vars

* Added JWT token authentication and error display on debug dump button

* Fixed Error reporting on download debug dump.

* Anonymization does not anonymize system attributes any more.

* Anonymize in explainer debugDump will now anonymize bindVars and will not anonymize system document attributes.

js/apps/system/_admin/aardvark/APP/aardvark.js

@@ -38,6 +38,8 @@ const createRouter = require('@arangodb/foxx/router');
 const users = require('@arangodb/users');
 const cluster = require('@arangodb/cluster');
 const isEnterprise = require('internal').isEnterprise();
+const explainer = require('@arangodb/aql/explainer');
+const fs = require('fs');
 
 const ERROR_USER_NOT_FOUND = errors.ERROR_USER_NOT_FOUND.code;
 const API_DOCS = require(module.context.fileName('api-docs.json'));
@@ -143,14 +145,14 @@ authRouter.post('/query/explain', function (req, res) {
 
   try {
     if (bindVars) {
-      msg = require('@arangodb/aql/explainer').explain({
+      msg = explainer.explain({
         query: query,
         bindVars: bindVars,
         batchSize: batchSize,
         id: id
       }, {colors: false}, false, bindVars);
     } else {
-      msg = require('@arangodb/aql/explainer').explain(query, {colors: false}, false);
+      msg = explainer.explain(query, {colors: false}, false);
     }
   } catch (e) {
     res.throw('bad request', e.message, {cause: e});
@@ -169,6 +171,54 @@ authRouter.post('/query/explain', function (req, res) {
   Explains a query in a more user-friendly way than the query_api/explain
 `);
 
+authRouter.post('/query/debugDump', function (req, res) {
+  const bindVars = req.body.bindVars || {};
+  const query = req.body.query;
+  const tmpDebugFolder = fs.getTempFile();
+  const tmpDebugFileName = fs.join(tmpDebugFolder, 'debugDump.json');
+  const tmpDebugZipFileName = fs.join(tmpDebugFolder, 'debugDump.zip');
+
+  try {
+    fs.makeDirectory(tmpDebugFolder);
+  } catch (e) {
+    require('console').error(e);
+    res.throw('Server error, failed to create temp directory', e.message, {cause: e});
+  }
+  let options = {};
+  if (req.body.examples) {
+    options.anonymize = true;
+    options.examples = true;
+  }
+
+  try {
+    explainer.debugDump(tmpDebugFileName, query, bindVars, options);
+  } catch (e) {
+    res.throw('bad request', e.message, {cause: e});
+  }
+  try {
+    fs.zipFile(tmpDebugZipFileName, tmpDebugFolder, ['debugDump.json']);
+  } catch (e) {
+    require('console').error(e);
+    res.throw('Server error, failed to create zip file', e.message, {cause: e});
+  }
+
+  res.download(tmpDebugZipFileName, 'debugDump.zip');
+})
+.body(joi.object({
+  query: joi.string().required(),
+  bindVars: joi.object().optional(),
+  examples: joi.bool().optional()
+}).required(), 'Query and bindVars to generate debug dump output')
+.summary('Generate Debug Output for Query')
+.description(dd`
+  Creates a debug output for the query in a zip file.
+  This file includes the query plan and anonymized test data as
+  well es collection information required for this query.
+  It is extremely helpful for the ArangoDB team to get this archive
+  and to reproduce your case. Whenever you submit a query based issue
+  please attach this file and the Team can help you much faster with it.
+`);
+
 authRouter.post('/query/upload/:user', function (req, res) {
   let user = req.pathParams.user;
 

js/apps/system/_admin/aardvark/APP/frontend/js/arango/arango.js

@@ -1,5 +1,5 @@
 /* jshint unused: false */
-/* global Blob, window, Joi, sigma, $, Tippy, document, _, arangoHelper, frontendConfig, arangoHelper, sessionStorage, localStorage */
+/* global Blob, window, Joi, sigma, $, Tippy, document, _, arangoHelper, frontendConfig, arangoHelper, sessionStorage, localStorage, XMLHttpRequest */
 
 (function () {
   'use strict';
@@ -1029,26 +1029,60 @@
 
     download: function (url, callback) {
       $.ajax(url)
-        .success(function (result, dummy, request) {
-          if (callback) {
-            callback(result);
-            return;
-          }
+      .success(function (result, dummy, request) {
+        if (callback) {
+          callback(result);
+          return;
+        }
 
-          var blob = new Blob([JSON.stringify(result)], {type: request.getResponseHeader('Content-Type') || 'application/octet-stream'});
-          var blobUrl = window.URL.createObjectURL(blob);
+        var blob = new Blob([JSON.stringify(result)], {type: request.getResponseHeader('Content-Type') || 'application/octet-stream'});
+        var blobUrl = window.URL.createObjectURL(blob);
+        var a = document.createElement('a');
+        document.body.appendChild(a);
+        a.style = 'display: none';
+        a.href = blobUrl;
+        a.download = request.getResponseHeader('Content-Disposition').replace(/.* filename="([^")]*)"/, '$1');
+        a.click();
+
+        window.setTimeout(function () {
+          window.URL.revokeObjectURL(blobUrl);
+          document.body.removeChild(a);
+        }, 500);
+      });
+    },
+
+    downloadPost: function (url, body, callback, errorCB) {
+      var xhr = new XMLHttpRequest();
+      xhr.onreadystatechange = function () {
+        if (this.readyState === 4 && this.status === 200) {
+          if (callback) {
+            callback();
+          }
           var a = document.createElement('a');
+          a.download = this.getResponseHeader('Content-Disposition').replace(/.* filename="([^")]*)"/, '$1');
           document.body.appendChild(a);
-          a.style = 'display: none';
+          var blobUrl = window.URL.createObjectURL(this.response);
           a.href = blobUrl;
-          a.download = request.getResponseHeader('Content-Disposition').replace(/.* filename="([^")]*)"/, '$1');
           a.click();
 
           window.setTimeout(function () {
             window.URL.revokeObjectURL(blobUrl);
             document.body.removeChild(a);
           }, 500);
-        });
+        } else {
+          if (this.readyState === 4) {
+            if (errorCB !== undefined) {
+              errorCB(this.status, this.statusText);
+            }
+          }
+        }
+      };
+      xhr.open('POST', url);
+      if (window.arangoHelper.getCurrentJwt()) {
+        xhr.setRequestHeader('Authorization', 'bearer ' + window.arangoHelper.getCurrentJwt());
+      }
+      xhr.responseType = 'blob';
+      xhr.send(body);
     },
 
     checkCollectionPermissions: function (collectionID, roCallback) {

js/apps/system/_admin/aardvark/APP/frontend/js/templates/queryView.ejs

@@ -47,6 +47,7 @@
         <button id="importQuery" class="button-success query-button" style="display:none">Import Queries</button>
         <button id="executeQuery" class="button-success query-button">Execute</button>
         <button id="explainQuery" class="button-info query-button">Explain</button>
+        <button id="debugQuery" class="button-warning query-button">Create Debug Package</button>
         <button id="removeResults" class="button-close query-button" style="display: none">Remove all results</button>
       </div>
     </div>

js/apps/system/_admin/aardvark/APP/frontend/js/views/queryView.js

@@ -72,6 +72,7 @@
     events: {
       'click #executeQuery': 'executeQuery',
       'click #explainQuery': 'explainQuery',
+      'click #debugQuery': 'debugDownloadDialog',
       'click #clearQuery': 'clearQuery',
       'click .outputEditorWrapper #downloadQueryResult': 'downloadQueryResult',
       'click .outputEditorWrapper #downloadCsvResult': 'downloadCsvResult',
@@ -525,6 +526,62 @@
       this.outputCounter++;
     },
 
+    debugDownloadDialog: function () {
+      var buttons = [];
+      var tableContent = [];
+
+      tableContent.push(
+        window.modalView.createReadOnlyEntry(
+          'debug-download-package-disclaimer',
+          'Disclaimer',
+          'This will generate a package containing a lot of commonly required information about your query and environment that helps the ArangoDB Team to reproduce your issue. This debug package will include collection names and created indexes, including AttributeNames and bindParameters. All String-Values will be obfuscated in a not-reversable way. If the below check box is not checked this package will not include any data. If the below check box is checked it will include a sample data-set again obfuscating all string values, all number values are not obfuscated. In order to check if any sensitive data is shared open the package locally and check if it contains anything that you are not allowed/willing to share and obfuscate it before. Including this package in bug reports will lower the amout of questioning back and forth until the issue is reproduced.',
+          undefined,
+          false,
+          false
+        )
+      );
+      tableContent.push(
+        window.modalView.createCheckboxEntry(
+          'debug-download-package-examples',
+          'Include obfuscated examples',
+          'includeExamples',
+          'Includes an example set of documents, obfuscating all String values inside the data. This helps the Team in many ways as many issues are related to the document structure / format and the indexes defined on them.',
+          true
+        )
+      );
+      buttons.push(
+        window.modalView.createSuccessButton('Download Package', this.downloadDebugZip.bind(this))
+      );
+      window.modalView.show('modalTable.ejs', 'Download Query Debug Package', buttons, tableContent, undefined, undefined);
+    },
+
+    downloadDebugZip: function () {
+      if (this.verifyQueryAndParams()) {
+        return;
+      }
+
+      var cbFunction = function () {
+        window.modalView.hide();
+      };
+      var errorFunction = function (errorCode, response) {
+        window.arangoHelper.arangoError('Debug Dump', errorCode + ': ' + response);
+        window.modalView.hide();
+      };
+
+      var query = this.aqlEditor.getValue();
+      if (query !== '' && query !== undefined && query !== null) {
+        var url = 'query/debugDump';
+        var body = {
+          query: query,
+          bindVars: this.bindParamTableObj || {},
+          examples: $('#debug-download-package-examples').is(':checked')
+        };
+        arangoHelper.downloadPost(url, JSON.stringify(body), cbFunction, errorFunction);
+      } else {
+        arangoHelper.arangoError('Query error', 'Could not create a debug package.');
+      }
+    },
+
     fillExplain: function (outputEditor, counter) {
       var self = this;
       var queryData = this.readQueryData();
@@ -1578,10 +1635,6 @@
         }
       }
 
-      if (quit === true) {
-        return quit;
-      }
-
       return quit;
     },
 

js/common/modules/@arangodb/aql/explainer.js

@@ -11,6 +11,32 @@ if (typeof internal.printBrowser === 'function') {
   print = internal.printBrowser;
 }
 
+const anonymize = function(doc) {
+  if (Array.isArray(doc)) {
+    return doc.map(anonymize);
+  }
+  if (typeof doc === 'string') {
+    return Array(doc.length).join("X");
+  }
+  if (doc === null || typeof doc === 'number' || typeof doc === 'boolean') {
+    return doc;
+  } 
+  if (typeof doc === 'object') {
+    let result = {};
+    Object.keys(doc).forEach(function(key) {
+      if (key.startsWith("_") || key.startsWith("@")) {
+        // This excludes system attributes in examples
+        // and collections in bindVars
+        result[key] = doc[key];
+      } else {
+        result[key] = anonymize(doc[key]);
+      }
+    });
+    return result;
+  }
+  return doc;
+};
+
 var stringBuilder = {
   output: '',
 
@@ -1287,6 +1313,7 @@ function explain(data, options, shouldPrint) {
 function debug(query, bindVars, options) {
   'use strict';
   let input = {};
+
   if (query instanceof Object) {
     if (typeof query.toAQL === 'function') {
       query = query.toAQL();
@@ -1295,7 +1322,7 @@ function debug(query, bindVars, options) {
   } else {
     input.query = query;
     if (bindVars !== undefined) {
-      input.bindVars = bindVars;
+      input.bindVars = anonymize(bindVars);
     }
     if (options !== undefined) {
       input.options = options;
@@ -1304,27 +1331,6 @@ function debug(query, bindVars, options) {
   if (!input.options) {
     input.options = {};
   }
-
-  let anonymize = function(doc) {
-    if (Array.isArray(doc)) {
-      return doc.map(anonymize);
-    }
-    if (typeof doc === 'string') {
-      return Array(doc.length).join("X");
-    }
-    if (doc === null || typeof doc === 'number' || typeof doc === 'boolean') {
-      return doc;
-    } 
-    if (typeof doc === 'object') {
-      let result = {};
-      Object.keys(doc).forEach(function(key) {
-        result[key] = anonymize(doc[key]);
-      });
-      return result;
-    }
-    return doc;
-  };
-
   let result = {
     engine: db._engine(),
     version: db._version(true),