diff --git a/js/apps/system/_admin/aardvark/APP/foxxes.js b/js/apps/system/_admin/aardvark/APP/foxxes.js index a69fbca2bb..54053e4e01 100644 --- a/js/apps/system/_admin/aardvark/APP/foxxes.js +++ b/js/apps/system/_admin/aardvark/APP/foxxes.js @@ -1,28 +1,28 @@ 'use strict'; -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2010-2013 triAGENS GmbH, Cologne, Germany -/// Copyright 2016 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Michael Hackstein -/// @author Alan Plum -//////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////// +// / DISCLAIMER +// / +// / Copyright 2010-2013 triAGENS GmbH, Cologne, Germany +// / Copyright 2016 ArangoDB GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License"); +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is ArangoDB GmbH, Cologne, Germany +// / +// / @author Michael Hackstein +// / @author Alan Plum +// ////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); const joi = require('joi'); @@ -37,7 +37,6 @@ const FoxxGenerator = require('./generator'); const fmu = require('@arangodb/foxx/manager-utils'); const createRouter = require('@arangodb/foxx/router'); const joinPath = require('path').join; -const posix = require('path').posix; const DEFAULT_THUMBNAIL = module.context.fileName('default-thumbnail.png'); @@ -77,168 +76,143 @@ foxxRouter.use(installer) Triggers teardown and setup. `); -if (FoxxManager.isFoxxmaster()) { - installer.use(function (req, res, next) { - const mount = decodeURIComponent(req.queryParams.mount); - const upgrade = req.queryParams.upgrade; - const replace = req.queryParams.replace; - next(); - const options = {}; - const appInfo = req.body; - options.legacy = req.queryParams.legacy; - let service; - try { - if (upgrade) { - service = FoxxManager.upgrade(appInfo, mount, options); - } else if (replace) { - service = FoxxManager.replace(appInfo, mount, options); - } else { - service = FoxxManager.install(appInfo, mount, options); - } - } catch (e) { - if (e.isArangoError && [ - errors.ERROR_MODULE_FAILURE.code, - errors.ERROR_MALFORMED_MANIFEST_FILE.code, - errors.ERROR_INVALID_SERVICE_MANIFEST.code - ].indexOf(e.errorNum) !== -1) { - res.throw('bad request', e); - } - if ( - e.isArangoError && - e.errorNum === errors.ERROR_SERVICE_NOT_FOUND.code - ) { - res.throw('not found', e); - } - if ( - e.isArangoError && - e.errorNum === errors.ERROR_SERVICE_MOUNTPOINT_CONFLICT.code - ) { - res.throw('conflict', e); - } - throw e; +installer.use(function (req, res, next) { + const mount = decodeURIComponent(req.queryParams.mount); + const upgrade = req.queryParams.upgrade; + const replace = req.queryParams.replace; + next(); + const options = {}; + const appInfo = req.body; + options.legacy = req.queryParams.legacy; + let service; + try { + if (upgrade) { + service = FoxxManager.upgrade(appInfo, mount, options); + } else if (replace) { + service = FoxxManager.replace(appInfo, mount, options); + } else { + service = FoxxManager.install(appInfo, mount, options); } - const configuration = service.getConfiguration(); - res.json(Object.assign( - {error: false, configuration}, - service.simpleJSON() - )); + } catch (e) { + if (e.isArangoError && [ + errors.ERROR_MODULE_FAILURE.code, + errors.ERROR_MALFORMED_MANIFEST_FILE.code, + errors.ERROR_INVALID_SERVICE_MANIFEST.code + ].indexOf(e.errorNum) !== -1) { + res.throw('bad request', e); + } + if ( + e.isArangoError && + e.errorNum === errors.ERROR_SERVICE_NOT_FOUND.code + ) { + res.throw('not found', e); + } + if ( + e.isArangoError && + e.errorNum === errors.ERROR_SERVICE_MOUNTPOINT_CONFLICT.code + ) { + res.throw('conflict', e); + } + throw e; + } + const configuration = service.getConfiguration(); + res.json(Object.assign({ + error: false, + configuration + }, service.simpleJSON())); +}); + +installer.put('/store', function (req) { + req.body = `${req.body.name}:${req.body.version}`; +}) +.body(joi.object({ + name: joi.string().required(), + version: joi.string().required() +}).required(), 'A Foxx service from the store.') +.summary('Install a Foxx from the store') +.description(dd` + Downloads a Foxx from the store and installs it at the given mount. +`); + +installer.put('/git', function (req) { + const baseUrl = process.env.FOXX_BASE_URL || 'https://github.com'; + req.body = `${baseUrl}${req.body.url}/archive/${req.body.version || 'master'}.zip`; +}) +.body(joi.object({ + url: joi.string().required(), + version: joi.string().default('master') +}).required(), 'A GitHub reference.') +.summary('Install a Foxx from Github') +.description(dd` + Install a Foxx with user/repository and version. +`); + +installer.put('/generate', (req, res) => { + const tempDir = fs.getTempFile('aardvark', false); + const generated = FoxxGenerator.generate(req.body); + FoxxGenerator.write(tempDir, generated.files, generated.folders); + const tempFile = fmu.zipDirectory(tempDir); + req.body = fs.readFileSync(tempFile); + try { + fs.removeDirectoryRecursive(tempDir, true); + } catch (e) { + console.warn(`Failed to remove temporary Foxx generator folder: ${tempDir}`); + } + try { + fs.remove(tempFile); + } catch (e) { + console.warn(`Failed to remove temporary Foxx generator bundle: ${tempFile}`); + } +}) +.body(joi.object().required(), 'A Foxx generator configuration.') +.summary('Generate a new foxx') +.description(dd` + Generate a new empty foxx on the given mount point. +`); + +installer.put('/zip', function (req) { + const tempFile = joinPath(fs.getTempPath(), req.body.zipFile); + req.body = fs.readFileSync(tempFile); + try { + fs.remove(tempFile); + } catch (e) { + console.warn(`Failed to remove uploaded file: ${tempFile}`); + } +}) +.body(joi.object({ + zipFile: joi.string().required() +}).required(), 'A zip file path.') +.summary('Install a Foxx from temporary zip file') +.description(dd` + Install a Foxx from the given zip path. + This path has to be created via _api/upload. +`); + +installer.put('/raw', function (req) { + req.body = req.rawBody; +}) +.summary('Install a Foxx from a direct upload') +.description(dd` + Install a Foxx from raw request body. +`); + +foxxRouter.delete('/', function (req, res) { + const mount = decodeURIComponent(req.queryParams.mount); + const runTeardown = req.queryParams.teardown; + const service = FoxxManager.uninstall(mount, { + teardown: runTeardown, + force: true }); - - installer.put('/store', function (req) { - req.body = `${req.body.name}:${req.body.version}`; - }) - .body(joi.object({ - name: joi.string().required(), - version: joi.string().required() - }).required(), 'A Foxx service from the store.') - .summary('Install a Foxx from the store') - .description(dd` - Downloads a Foxx from the store and installs it at the given mount. - `); - - installer.put('/git', function (req) { - const baseUrl = process.env.FOXX_BASE_URL || 'https://github.com'; - req.body = `${baseUrl}${req.body.url}/archive/${req.body.version || 'master'}.zip`; - }) - .body(joi.object({ - url: joi.string().required(), - version: joi.string().default('master') - }).required(), 'A GitHub reference.') - .summary('Install a Foxx from Github') - .description(dd` - Install a Foxx with user/repository and version. - `); - - installer.put('/generate', (req, res) => { - const tempDir = fs.getTempFile('aardvark', false); - const generated = FoxxGenerator.generate(req.body); - FoxxGenerator.write(tempDir, generated.files, generated.folders); - const tempFile = fmu.zipDirectory(tempDir); - req.body = fs.readFileSync(tempFile); - try { - fs.removeDirectoryRecursive(tempDir, true); - } catch (e) { - console.warn(`Failed to remove temporary Foxx generator folder: ${tempDir}`); - } - try { - fs.remove(tempFile); - } catch (e) { - console.warn(`Failed to remove temporary Foxx generator bundle: ${tempFile}`); - } - }) - .body(joi.object().required(), 'A Foxx generator configuration.') - .summary('Generate a new foxx') - .description(dd` - Generate a new empty foxx on the given mount point. - `); - - installer.put('/zip', function (req) { - const tempFile = joinPath(fs.getTempPath(), req.body.zipFile); - req.body = fs.readFileSync(tempFile); - try { - fs.remove(tempFile); - } catch (e) { - console.warn(`Failed to remove uploaded file: ${tempFile}`); - } - }) - .body(joi.object({ - zipFile: joi.string().required() - }).required(), 'A zip file path.') - .summary('Install a Foxx from temporary zip file') - .description(dd` - Install a Foxx from the given zip path. - This path has to be created via _api/upload. - `); - - installer.put('/raw', function (req) { - req.body = req.rawBody; - }) - .summary('Install a Foxx from a direct upload') - .description(dd` - Install a Foxx from raw request body. - `); - - foxxRouter.delete('/', function (req, res) { - const mount = decodeURIComponent(req.queryParams.mount); - const runTeardown = req.queryParams.teardown; - const service = FoxxManager.uninstall(mount, { - teardown: runTeardown, - force: true - }); - res.json(Object.assign( - {error: false}, - service.simpleJSON() - )); - }) - .queryParam('teardown', joi.boolean().default(true)) - .summary('Uninstall a Foxx') - .description(dd` - Uninstall the Foxx at the given mount-point. - `); -} else { - installer.put('/store', FoxxManager.proxyToFoxxmaster); - installer.put('/git', FoxxManager.proxyToFoxxmaster); - installer.put('/generate', FoxxManager.proxyToFoxxmaster); - installer.put('/zip', (req, res) => { - const zipData = fs.readFileSync(joinPath(fs.getTempPath(), req.body.zipFile)); - FoxxManager.proxyToFoxxmaster({ - method: req.method, - rawBody: zipData, - headers: Object.assign({}, req.headers, { - 'content-length': zipData.length - }), - _url: { - pathname: posix.resolve(req._url.pathname, '../raw'), - search: req._url.search - } - }, res); - }) - .body(joi.object({ - zipFile: joi.string().required() - }).required(), 'A zip file path.'); - installer.put('/raw', FoxxManager.proxyToFoxxmaster); - foxxRouter.delete('/', FoxxManager.proxyToFoxxmaster); -} + res.json(Object.assign( + {error: false}, + service.simpleJSON() + )); +}) +.queryParam('teardown', joi.boolean().default(true)) +.summary('Uninstall a Foxx') +.description(dd` + Uninstall the Foxx at the given mount-point. +`); router.get('/', function (req, res) { const foxxes = FoxxManager.listJson(); @@ -299,36 +273,37 @@ foxxRouter.get('/deps', function (req, res) { Used to request the dependencies options for services. `); -if (FoxxManager.isFoxxmaster()) { - foxxRouter.patch('/config', function (req, res) { - const mount = decodeURIComponent(req.queryParams.mount); - const configuration = req.body; - const service = FoxxManager.lookupService(mount); - FoxxManager.setConfiguration(mount, {configuration, replace: !service.isDevelopment}); - res.json(service.getConfiguration()); - }) - .body(joi.object().optional(), 'Configuration to apply.') - .summary('Set the configuration for a service') - .description(dd` - Used to overwrite the configuration options for services. - `); +foxxRouter.patch('/config', function (req, res) { + const mount = decodeURIComponent(req.queryParams.mount); + const configuration = req.body; + const service = FoxxManager.lookupService(mount); + FoxxManager.setConfiguration(mount, { + configuration, + replace: !service.isDevelopment + }); + res.json(service.getConfiguration()); +}) +.body(joi.object().optional(), 'Configuration to apply.') +.summary('Set the configuration for a service') +.description(dd` + Used to overwrite the configuration options for services. +`); - foxxRouter.patch('/deps', function (req, res) { - const mount = decodeURIComponent(req.queryParams.mount); - const dependencies = req.body; - const service = FoxxManager.lookupService(mount); - FoxxManager.setDependencies(mount, {dependencies, replace: !service.isDevelopment}); - res.json(service.getDependencies()); - }) - .body(joi.object().optional(), 'Dependency options to apply.') - .summary('Set the dependencies for a service') - .description(dd` - Used to overwrite the dependencies options for services. - `); -} else { - foxxRouter.patch('/config', FoxxManager.proxyToFoxxmaster); - foxxRouter.patch('/deps', FoxxManager.proxyToFoxxmaster); -} +foxxRouter.patch('/deps', function (req, res) { + const mount = decodeURIComponent(req.queryParams.mount); + const dependencies = req.body; + const service = FoxxManager.lookupService(mount); + FoxxManager.setDependencies(mount, { + dependencies, + replace: !service.isDevelopment + }); + res.json(service.getDependencies()); +}) +.body(joi.object().optional(), 'Dependency options to apply.') +.summary('Set the dependencies for a service') +.description(dd` + Used to overwrite the dependencies options for services. +`); foxxRouter.post('/tests', function (req, res) { const mount = decodeURIComponent(req.queryParams.mount); diff --git a/js/apps/system/_api/foxx/APP/index.js b/js/apps/system/_api/foxx/APP/index.js index 224afb6669..26b0491486 100644 --- a/js/apps/system/_api/foxx/APP/index.js +++ b/js/apps/system/_api/foxx/APP/index.js @@ -1,4 +1,29 @@ 'use strict'; + +// ////////////////////////////////////////////////////////////////////////////// +// / DISCLAIMER +// / +// / Copyright 2010-2013 triAGENS GmbH, Cologne, Germany +// / Copyright 2016 ArangoDB GmbH, Cologne, Germany +// / +// / Licensed under the Apache License, Version 2.0 (the "License") +// / you may not use this file except in compliance with the License. +// / You may obtain a copy of the License at +// / +// / http://www.apache.org/licenses/LICENSE-2.0 +// / +// / Unless required by applicable law or agreed to in writing, software +// / distributed under the License is distributed on an "AS IS" BASIS, +// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// / See the License for the specific language governing permissions and +// / limitations under the License. +// / +// / Copyright holder is ArangoDB GmbH, Cologne, Germany +// / +// / @author Alan Plum +// ////////////////////////////////////////////////////////////////////////////// + + const _ = require('lodash'); const dd = require('dedent'); const fs = require('fs'); @@ -10,7 +35,6 @@ const errors = require('@arangodb').errors; const jsonml2xml = require('@arangodb/util').jsonml2xml; const swaggerJson = require('@arangodb/foxx/legacy/swagger').swaggerJson; const FoxxManager = require('@arangodb/foxx/manager'); -const FoxxService = require('@arangodb/foxx/service'); const createRouter = require('@arangodb/foxx/router'); const reporters = Object.keys(require('@arangodb/mocha').reporters); const schemas = require('./schemas'); @@ -64,7 +88,10 @@ router.use((req, res, next) => { } catch (e) { if (e.isArangoError) { const status = actions.arangoErrorToHttpCode(e.errorNum); - res.throw(status, e.errorMessage, {errorNum: e.errorNum, cause: e}); + res.throw(status, e.errorMessage, { + errorNum: e.errorNum, + cause: e + }); } throw e; } @@ -87,31 +114,33 @@ router.get((req, res) => { }) .response(200, joi.array().items(schemas.shortInfo).required()); -if (FoxxManager.isFoxxmaster()) { - router.post(prepareServiceRequestBody, (req, res) => { - const mount = req.queryParams.mount; - FoxxManager.install(req.body.source, mount, _.omit(req.queryParams, ['mount', 'development'])); - if (req.body.configuration) { - FoxxManager.setConfiguration(mount, {configuration: req.body.configuration, replace: true}); - } - if (req.body.dependencies) { - FoxxManager.setDependencies(mount, {dependencies: req.body.dependencies, replace: true}); - } - if (req.queryParams.development) { - FoxxManager.development(mount); - } - const service = FoxxManager.lookupService(mount); - res.json(serviceToJson(service)); - }) - .body(schemas.service, ['application/javascript', 'application/zip', 'multipart/form-data', 'application/json']) - .queryParam('mount', schemas.mount) - .queryParam('development', schemas.flag.default(false)) - .queryParam('setup', schemas.flag.default(true)) - .queryParam('legacy', schemas.flag.default(false)) - .response(201, schemas.fullInfo); -} else { - router.post(FoxxManager.proxyToFoxxmaster); -} +router.post(prepareServiceRequestBody, (req, res) => { + const mount = req.queryParams.mount; + FoxxManager.install(req.body.source, mount, _.omit(req.queryParams, ['mount', 'development'])); + if (req.body.configuration) { + FoxxManager.setConfiguration(mount, { + configuration: req.body.configuration, + replace: true + }); + } + if (req.body.dependencies) { + FoxxManager.setDependencies(mount, { + dependencies: req.body.dependencies, + replace: true + }); + } + if (req.queryParams.development) { + FoxxManager.development(mount); + } + const service = FoxxManager.lookupService(mount); + res.json(serviceToJson(service)); +}) +.body(schemas.service, ['application/javascript', 'application/zip', 'multipart/form-data', 'application/json']) +.queryParam('mount', schemas.mount) +.queryParam('development', schemas.flag.default(false)) +.queryParam('setup', schemas.flag.default(true)) +.queryParam('legacy', schemas.flag.default(false)) +.response(201, schemas.fullInfo); const instanceRouter = createRouter(); instanceRouter.use((req, res, next) => { @@ -133,62 +162,68 @@ serviceRouter.get((req, res) => { res.json(serviceToJson(req.service)); }) .response(200, schemas.fullInfo) -.summary(`Service description`) +.summary('Service description') .description(dd` Fetches detailed information for the service at the given mount path. `); -if (FoxxManager.isFoxxmaster()) { - serviceRouter.patch(prepareServiceRequestBody, (req, res) => { - const mount = req.queryParams.mount; - FoxxManager.upgrade(req.body.source, mount, _.omit(req.queryParams, ['mount'])); - if (req.body.configuration) { - FoxxManager.setConfiguration(mount, {configuration: req.body.configuration, replace: false}); - } - if (req.body.dependencies) { - FoxxManager.setDependencies(mount, {dependencies: req.body.dependencies, replace: false}); - } - const service = FoxxManager.lookupService(mount); - res.json(serviceToJson(service)); - }) - .body(schemas.service, ['application/javascript', 'application/zip', 'multipart/form-data', 'application/json']) - .queryParam('teardown', schemas.flag.default(false)) - .queryParam('setup', schemas.flag.default(true)) - .queryParam('legacy', schemas.flag.default(false)) - .response(200, schemas.fullInfo); +serviceRouter.patch(prepareServiceRequestBody, (req, res) => { + const mount = req.queryParams.mount; + FoxxManager.upgrade(req.body.source, mount, _.omit(req.queryParams, ['mount'])); + if (req.body.configuration) { + FoxxManager.setConfiguration(mount, { + configuration: req.body.configuration, + replace: false + }); + } + if (req.body.dependencies) { + FoxxManager.setDependencies(mount, { + dependencies: req.body.dependencies, + replace: false + }); + } + const service = FoxxManager.lookupService(mount); + res.json(serviceToJson(service)); +}) +.body(schemas.service, ['application/javascript', 'application/zip', 'multipart/form-data', 'application/json']) +.queryParam('teardown', schemas.flag.default(false)) +.queryParam('setup', schemas.flag.default(true)) +.queryParam('legacy', schemas.flag.default(false)) +.response(200, schemas.fullInfo); - serviceRouter.put(prepareServiceRequestBody, (req, res) => { - const mount = req.queryParams.mount; - FoxxManager.replace(req.body.source, mount, _.omit(req.queryParams, ['mount'])); - if (req.body.configuration) { - FoxxManager.setConfiguration(mount, {configuration: req.body.configuration, replace: true}); - } - if (req.body.dependencies) { - FoxxManager.setDependencies(mount, {dependencies: req.body.dependencies, replace: true}); - } - const service = FoxxManager.lookupService(mount); - res.json(serviceToJson(service)); - }) - .body(schemas.service, ['application/javascript', 'application/zip', 'multipart/form-data', 'application/json']) - .queryParam('teardown', schemas.flag.default(true)) - .queryParam('setup', schemas.flag.default(true)) - .queryParam('legacy', schemas.flag.default(false)) - .response(200, schemas.fullInfo); +serviceRouter.put(prepareServiceRequestBody, (req, res) => { + const mount = req.queryParams.mount; + FoxxManager.replace(req.body.source, mount, _.omit(req.queryParams, ['mount'])); + if (req.body.configuration) { + FoxxManager.setConfiguration(mount, { + configuration: req.body.configuration, + replace: true + }); + } + if (req.body.dependencies) { + FoxxManager.setDependencies(mount, { + dependencies: req.body.dependencies, + replace: true + }); + } + const service = FoxxManager.lookupService(mount); + res.json(serviceToJson(service)); +}) +.body(schemas.service, ['application/javascript', 'application/zip', 'multipart/form-data', 'application/json']) +.queryParam('teardown', schemas.flag.default(true)) +.queryParam('setup', schemas.flag.default(true)) +.queryParam('legacy', schemas.flag.default(false)) +.response(200, schemas.fullInfo); - serviceRouter.delete((req, res) => { - FoxxManager.uninstall( - req.queryParams.mount, - _.omit(req.queryParams, ['mount']) - ); - res.status(204); - }) - .queryParam('teardown', schemas.flag.default(true)) - .response(204, null); -} else { - serviceRouter.patch(FoxxManager.proxyToFoxxmaster); - serviceRouter.put(FoxxManager.proxyToFoxxmaster); - serviceRouter.delete(FoxxManager.proxyToFoxxmaster); -} +serviceRouter.delete((req, res) => { + FoxxManager.uninstall( + req.queryParams.mount, + _.omit(req.queryParams, ['mount']) + ); + res.status(204); +}) +.queryParam('teardown', schemas.flag.default(true)) +.response(204, null); const configRouter = createRouter(); instanceRouter.use('/configuration', configRouter) @@ -198,30 +233,31 @@ configRouter.get((req, res) => { res.json(req.service.getConfiguration()); }); -if (FoxxManager.isFoxxmaster()) { - configRouter.patch((req, res) => { - const warnings = FoxxManager.setConfiguration(req.service.mount, { - configuration: req.body, - replace: false - }); - const values = req.service.getConfiguration(true); - res.json({values, warnings}); - }) - .body(joi.object().required()); +configRouter.patch((req, res) => { + const warnings = FoxxManager.setConfiguration(req.service.mount, { + configuration: req.body, + replace: false + }); + const values = req.service.getConfiguration(true); + res.json({ + values, + warnings + }); +}) +.body(joi.object().required()); - configRouter.put((req, res) => { - const warnings = FoxxManager.setConfiguration(req.service.mount, { - configuration: req.body, - replace: true - }); - const values = req.service.getConfiguration(true); - res.json({values, warnings}); - }) - .body(joi.object().required()); -} else { - configRouter.patch(FoxxManager.proxyToFoxxmaster); - configRouter.put(FoxxManager.proxyToFoxxmaster); -} +configRouter.put((req, res) => { + const warnings = FoxxManager.setConfiguration(req.service.mount, { + configuration: req.body, + replace: true + }); + const values = req.service.getConfiguration(true); + res.json({ + values, + warnings + }); +}) +.body(joi.object().required()); const depsRouter = createRouter(); instanceRouter.use('/dependencies', depsRouter) @@ -231,30 +267,31 @@ depsRouter.get((req, res) => { res.json(req.service.getDependencies()); }); -if (FoxxManager.isFoxxmaster()) { - depsRouter.patch((req, res) => { - const warnings = FoxxManager.setDependencies(req.service.mount, { - dependencies: req.body, - replace: true - }); - const values = req.service.getDependencies(true); - res.json({values, warnings}); - }) - .body(joi.object().required()); +depsRouter.patch((req, res) => { + const warnings = FoxxManager.setDependencies(req.service.mount, { + dependencies: req.body, + replace: true + }); + const values = req.service.getDependencies(true); + res.json({ + values, + warnings + }); +}) +.body(joi.object().required()); - depsRouter.put((req, res) => { - const warnings = FoxxManager.setDependencies(req.service.mount, { - dependencies: req.body, - replace: true - }); - const values = req.service.getDependencies(true); - res.json({values, warnings}); - }) - .body(joi.object().required()); -} else { - depsRouter.patch(FoxxManager.proxyToFoxxmaster); - depsRouter.put(FoxxManager.proxyToFoxxmaster); -} +depsRouter.put((req, res) => { + const warnings = FoxxManager.setDependencies(req.service.mount, { + dependencies: req.body, + replace: true + }); + const values = req.service.getDependencies(true); + res.json({ + values, + warnings + }); +}) +.body(joi.object().required()); const devRouter = createRouter(); instanceRouter.use('/development', devRouter) @@ -356,86 +393,6 @@ instanceRouter.get('/swagger', (req, res) => { const localRouter = createRouter(); router.use('/_local', localRouter); -localRouter.post((req, res) => { - const result = {}; - for (const mount of Object.keys(req.body)) { - const coordIds = req.body[mount]; - result[mount] = FoxxManager._installLocal(mount, coordIds); - } - FoxxManager._reloadRouting(); - res.json(result); -}) -.body(joi.object()); - -localRouter.post('/service', (req, res) => { - FoxxManager._reloadRouting(); -}) -.queryParam('mount', schemas.mount); - -localRouter.delete('/service', (req, res) => { - FoxxManager._uninstallLocal(req.queryParams.mount); - FoxxManager._reloadRouting(); -}) -.queryParam('mount', schemas.mount); - -localRouter.get('/bundle', (req, res) => { - const mount = req.queryParams.mount; - const bundlePath = FoxxService.bundlePath(mount); - if (!fs.isFile(bundlePath)) { - res.throw(404, 'Bundle not available'); - } - const etag = `"${FoxxService.checksum(mount)}"`; - if (req.get('if-none-match') === etag) { - res.status(304); - return; - } - if (req.get('if-match') && req.get('if-match') !== etag) { - res.throw(404, 'No matching bundle available'); - } - res.set('etag', etag); - res.download(bundlePath); -}) -.response(200, ['application/zip']) -.header('if-match', joi.string().optional()) -.header('if-none-match', joi.string().optional()) -.queryParam('mount', schemas.mount); - -localRouter.get('/status', (req, res) => { - const ready = global.KEY_GET('foxx', 'ready'); - if (ready || FoxxManager.isFoxxmaster()) { - res.json({ready}); - return; - } - FoxxManager.proxyToFoxxmaster(req, res); - if (res.statusCode < 400) { - const result = JSON.parse(res.body.toString('utf-8')); - if (result.ready) { - global.KEY_SET('foxx', 'ready', result.ready); - } - } +localRouter.post('/heal', (req, res) => { + FoxxManager.heal(); }); - -localRouter.get('/checksums', (req, res) => { - const mountParam = req.queryParams.mount || []; - const mounts = Array.isArray(mountParam) ? mountParam : [mountParam]; - const checksums = {}; - for (const mount of mounts) { - try { - checksums[mount] = FoxxService.checksum(mount); - } catch (e) { - } - } - res.json(checksums); -}) -.queryParam('mount', joi.alternatives( - joi.array().items(schemas.mount), - schemas.mount -)); - -if (FoxxManager.isFoxxmaster()) { - localRouter.post('/heal', (req, res) => { - FoxxManager.heal(); - }); -} else { - localRouter.post('/heal', FoxxManager.proxyToFoxxmaster); -} diff --git a/js/client/tests/shell/shell-foxx-manager-spec.js b/js/client/tests/shell/shell-foxx-manager-spec.js index b14d24300f..7768cb3d8f 100644 --- a/js/client/tests/shell/shell-foxx-manager-spec.js +++ b/js/client/tests/shell/shell-foxx-manager-spec.js @@ -55,19 +55,6 @@ describe('Foxx Manager', function () { expect(checksum).not.to.be.empty; }); - it('should provide a bundle', function () { - FoxxManager.install(setupTeardownApp, mount); - const url = `${arango.getEndpoint().replace('tcp://', 'http://')}/_api/foxx/_local/bundle?mount=${encodeURIComponent(mount)}`; - const res = download(url); - expect(res.code).to.equal(200); - const checksum = db._query(aql` - FOR service IN _apps - FILTER service.mount == ${mount} - RETURN service.checksum - `).next(); - expect(res.headers.etag).to.equal(`"${checksum}"`); - }); - it('should run the setup script', function () { FoxxManager.install(setupTeardownApp, mount); expect(db._collection(setupTeardownCol)).to.be.an.instanceOf(ArangoCollection); diff --git a/js/common/bootstrap/errors.js b/js/common/bootstrap/errors.js index 58354fa56a..f53c839f6e 100644 --- a/js/common/bootstrap/errors.js +++ b/js/common/bootstrap/errors.js @@ -47,6 +47,7 @@ "ERROR_HTTP_METHOD_NOT_ALLOWED" : { "code" : 405, "message" : "method not supported" }, "ERROR_HTTP_PRECONDITION_FAILED" : { "code" : 412, "message" : "precondition failed" }, "ERROR_HTTP_SERVER_ERROR" : { "code" : 500, "message" : "internal server error" }, + "ERROR_HTTP_SERVICE_UNAVAILABLE" : { "code" : 503, "message" : "service unavailable" }, "ERROR_HTTP_CORRUPTED_JSON" : { "code" : 600, "message" : "invalid JSON object" }, "ERROR_HTTP_SUPERFLUOUS_SUFFICES" : { "code" : 601, "message" : "superfluous URL suffices" }, "ERROR_ARANGO_ILLEGAL_STATE" : { "code" : 1000, "message" : "illegal state" }, @@ -274,6 +275,8 @@ "COMMUNICATOR_REQUEST_ABORTED" : { "code" : 2100, "message" : "Request aborted" }, "ERROR_MALFORMED_MANIFEST_FILE" : { "code" : 3000, "message" : "failed to parse manifest file" }, "ERROR_INVALID_SERVICE_MANIFEST" : { "code" : 3001, "message" : "manifest file is invalid" }, + "ERROR_SERVICE_FILES_MISSING" : { "code" : 3002, "message" : "service files missing" }, + "ERROR_SERVICE_FILES_OUTDATED" : { "code" : 3003, "message" : "service files outdated" }, "ERROR_INVALID_FOXX_OPTIONS" : { "code" : 3004, "message" : "service options are invalid" }, "ERROR_INVALID_MOUNTPOINT" : { "code" : 3007, "message" : "invalid mountpath" }, "ERROR_SERVICE_NOT_FOUND" : { "code" : 3009, "message" : "service not found" }, diff --git a/js/common/modules/@arangodb/foxx/manager-utils.js b/js/common/modules/@arangodb/foxx/manager-utils.js index bc6c5d65f8..3bd2fe4503 100644 --- a/js/common/modules/@arangodb/foxx/manager-utils.js +++ b/js/common/modules/@arangodb/foxx/manager-utils.js @@ -54,11 +54,46 @@ function getReadableName (name) { } function getStorage () { - var c = db._collection('_apps'); + let c = db._collection('_apps'); if (c === null) { - c = db._create('_apps', {isSystem: true, replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM, - distributeShardsLike: '_graphs', journalSize: 4 * 1024 * 1024}); - c.ensureIndex({ type: 'hash', fields: [ 'mount' ], unique: true }); + try { + c = db._create('_apps', { + isSystem: true, + replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM, + distributeShardsLike: '_graphs', + journalSize: 4 * 1024 * 1024 + }); + c.ensureIndex({ + type: 'hash', + fields: ['mount'], + unique: true + }); + } catch (e) { + c = db._collection('_apps'); + if (!c) { + throw e; + } + } + } + return c; +} + +function getBundleStorage () { + let c = db._collection('_appbundles'); + if (c === null) { + try { + c = db._create('_appbundles', { + isSystem: true, + replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM, + distributeShardsLike: '_graphs', + journalSize: 4 * 1024 * 1024 + }); + } catch (e) { + c = db._collection('_appbundles'); + if (!c) { + throw e; + } + } } return c; } @@ -231,4 +266,5 @@ exports.buildGithubUrl = buildGithubUrl; exports.validateMount = validateMount; exports.zipDirectory = zipDirectory; exports.getStorage = getStorage; +exports.getBundleStorage = getBundleStorage; exports.pathRegex = pathRegex; diff --git a/js/common/modules/@arangodb/request.js b/js/common/modules/@arangodb/request.js index 5b7f9439df..afbb7b1c37 100644 --- a/js/common/modules/@arangodb/request.js +++ b/js/common/modules/@arangodb/request.js @@ -61,6 +61,29 @@ class Response { } } } + _PRINT (ctx) { + const MAX_BYTES = 100; + ctx.output += `[IncomingResponse ${this.status} ${this.message} `; + if (!this.body || !this.body.length) { + ctx.output += 'empty'; + } else { + ctx.output += `${this.body.length} bytes `; + if (typeof this.body !== 'string') { + ctx.output += ''; + } else if (this.body.length <= MAX_BYTES) { + ctx.output += `"${this.body}"`; + } else { + const offset = (this.body.length - MAX_BYTES) / 2; + ctx.output += `"…${ + this.body.slice(offset, offset + MAX_BYTES) + .replace('\n', '\\n') + .replace('\r', '\\r') + .replace('\t', '\\t') + }…"`; + } + } + ctx.output += ']'; + } } function querystringify (query, useQuerystring) { diff --git a/js/server/bootstrap/coordinator.js b/js/server/bootstrap/coordinator.js index b498ee22a1..2d49928abc 100644 --- a/js/server/bootstrap/coordinator.js +++ b/js/server/bootstrap/coordinator.js @@ -44,26 +44,14 @@ internal.loadStartup('server/bootstrap/routing.js').startup(); if (internal.threadNumber === 0) { - global.KEYSPACE_CREATE('foxx', 1, true); - global.KEY_SET('foxx', 'ready', false); - const isFoxxmaster = global.ArangoServerState.isFoxxmaster(); - require('@arangodb/foxx/manager')._startup(isFoxxmaster); + require('@arangodb/foxx/manager')._startup(); require('@arangodb/tasks').register({ id: 'self-heal', isSystem: true, - period: 0.1, // secs + period: 5 * 60, // secs command: function () { const FoxxManager = require('@arangodb/foxx/manager'); - const isReady = FoxxManager._isClusterReady(); - if (!isReady) { - return; - } - require('@arangodb/tasks').unregister('self-heal'); - const isFoxxmaster = global.ArangoServerState.isFoxxmaster(); - const foxxmasterIsReady = FoxxManager._selfHeal(isFoxxmaster); - if (foxxmasterIsReady) { - global.KEY_SET('foxx', 'ready', true); - } + FoxxManager.healAll(); } }); // start the queue manager once diff --git a/js/server/modules/@arangodb/foxx/manager.js b/js/server/modules/@arangodb/foxx/manager.js index 9b562af5a4..d1a078f374 100644 --- a/js/server/modules/@arangodb/foxx/manager.js +++ b/js/server/modules/@arangodb/foxx/manager.js @@ -30,7 +30,6 @@ const fs = require('fs'); const path = require('path'); -const querystringify = require('querystring').encode; const dd = require('dedent'); const utils = require('@arangodb/foxx/manager-utils'); const store = require('@arangodb/foxx/store'); @@ -44,8 +43,6 @@ const db = arangodb.db; const ArangoClusterControl = require('@arangodb/cluster'); const request = require('@arangodb/request'); const actions = require('@arangodb/actions'); -const shuffle = require('lodash/shuffle'); -const zip = require('lodash/zip'); const isZipBuffer = require('@arangodb/util').isZipBuffer; const SYSTEM_SERVICE_MOUNTS = [ @@ -72,13 +69,6 @@ function getMyCoordinatorId () { return global.ArangoServerState.id(); } -function getFoxmasterCoordinatorId () { - if (!ArangoClusterControl.isCluster()) { - return null; - } - return global.ArangoServerState.getFoxxmaster(); -} - function getPeerCoordinatorIds () { const myId = getMyCoordinatorId(); return getAllCoordinatorIds().filter((id) => id !== myId); @@ -91,20 +81,6 @@ function isFoxxmaster () { return global.ArangoServerState.isFoxxmaster(); } -function proxyToFoxxmaster (req, res) { - const coordId = getFoxmasterCoordinatorId(); - const response = parallelClusterRequests([[ - coordId, - req.method, - req._url.pathname + (req._url.search || ''), - req.rawBody, - req.headers - ]])[0]; - res.statusCode = response.statusCode; - res.headers = response.headers; - res.body = response.rawBody; -} - function isClusterReadyForBusiness () { const coordIds = getPeerCoordinatorIds(); return parallelClusterRequests(function * () { @@ -157,70 +133,166 @@ function parallelClusterRequests (requests) { ); } -function isFoxxmasterReady () { - if (!ArangoClusterControl.isCluster()) { - return true; - } - const coordId = getFoxmasterCoordinatorId(); - const response = parallelClusterRequests([[ - coordId, - 'GET', - '/_api/foxx/_local/status' - ]])[0]; - if (response.statusCode >= 400) { - return false; - } - return JSON.parse(response.body).ready; -} - -function getChecksumsFromPeers (mounts) { - const coordinatorIds = getPeerCoordinatorIds(); - const responses = parallelClusterRequests(function * () { - for (const coordId of coordinatorIds) { - yield [ - coordId, - 'GET', - `/_api/foxx/_local/checksums?${querystringify({mount: mounts})}` - ]; - } - }()); - const peerChecksums = new Map(); - for (const [coordId, response] of zip(coordinatorIds, responses)) { - const body = JSON.parse(response.body); - const coordChecksums = new Map(); - for (const mount of mounts) { - coordChecksums.set(mount, body[mount] || null); - } - peerChecksums.set(coordId, coordChecksums); - } - return peerChecksums; -} - // Startup and self-heal -function startup () { +function selfHealAll (skipReloadRouting) { const db = require('internal').db; const dbName = db._name(); + let modified; try { db._useDatabase('_system'); - const writeToDatabase = isFoxxmaster(); const databases = db._databases(); for (const name of databases) { try { db._useDatabase(name); - rebuildAllServiceBundles(writeToDatabase); - if (writeToDatabase) { - upsertSystemServices(); - } + modified = selfHeal() || modified; } catch (e) { console.warnStack(e); } } } finally { db._useDatabase(dbName); + if (modified && !skipReloadRouting) { + reloadRouting(); + } } } +function triggerSelfHeal () { + const modified = selfHeal(); + if (modified) { + reloadRouting(); + } +} + +function selfHeal () { + const dirname = FoxxService.rootBundlePath(); + if (!fs.exists(dirname)) { + fs.makeDirectoryRecursive(dirname); + } + + const serviceCollection = utils.getStorage(); + const bundleCollection = utils.getBundleStorage(); + const serviceDefinitions = db._query(aql` + FOR doc IN ${serviceCollection} + FILTER LEFT(doc.mount, 2) != "/_" + LET bundleExists = DOCUMENT(${bundleCollection}, doc.checksum) != null + RETURN [doc.mount, doc.checksum, doc._rev, bundleExists] + `).toArray(); + + let modified = false; + const knownBundlePaths = new Array(serviceDefinitions.length); + const knownServicePaths = new Array(serviceDefinitions.length); + const localServiceMap = GLOBAL_SERVICE_MAP.get(db._name()); + for (const [mount, checksum, rev, bundleExists] of serviceDefinitions) { + const bundlePath = FoxxService.bundlePath(mount); + const basePath = FoxxService.basePath(mount); + knownBundlePaths.push(bundlePath); + knownServicePaths.push(basePath); + + if (localServiceMap) { + if (!localServiceMap.has(mount) || localServiceMap.get(mount)._rev !== rev) { + modified = true; + } + } + + const hasBundle = fs.exists(bundlePath); + const hasFolder = fs.exists(basePath); + if (!hasBundle && hasFolder) { + createServiceBundle(mount); + } else if (hasBundle && !hasFolder) { + extractServiceBundle(bundlePath, basePath); + modified = true; + } + const isInstalled = hasBundle || hasFolder; + const localChecksum = isInstalled ? safeChecksum(mount) : null; + + if (!checksum) { + // pre-3.2 service, can't self-heal this + continue; + } + + if (checksum === localChecksum) { + if (!bundleExists) { + try { + bundleCollection._binaryInsert({_key: checksum}, bundlePath); + } catch (e) { + console.warnStack(e, `Failed to store missing service bundle for service at "${mount}"`); + } + } + } else if (bundleExists) { + try { + bundleCollection._binaryDocument(checksum, bundlePath); + extractServiceBundle(bundlePath, basePath); + modified = true; + } catch (e) { + console.errorStack(e, `Failed to load service bundle for service at "${mount}"`); + } + } + } + + const rootPath = FoxxService.rootPath(); + for (const relPath of fs.listTree(rootPath)) { + if (!relPath) { + continue; + } + const basename = path.basename(relPath); + if (basename.toUpperCase() !== 'APP') { + continue; + } + const basePath = path.resolve(rootPath, relPath); + if (!knownServicePaths.includes(basePath)) { + modified = true; + try { + fs.removeDirectoryRecursive(basePath, true); + console.debug(`Deleted orphaned service folder ${basePath}`); + } catch (e) { + console.warnStack(e, `Failed to delete orphaned service folder ${basePath}`); + } + } + } + + const bundlesPath = FoxxService.rootBundlePath(); + for (const relPath of fs.listTree(bundlesPath)) { + if (!relPath) { + continue; + } + const bundlePath = path.resolve(bundlesPath, relPath); + if (!knownBundlePaths.includes(bundlePath)) { + try { + fs.remove(bundlePath); + console.debug(`Deleted orphaned service bundle ${bundlePath}`); + } catch (e) { + console.warnStack(e, `Failed to delete orphaned service bundle ${bundlePath}`); + } + } + } + + return modified; +} + +function startup () { + if (isFoxxmaster()) { + const db = require('internal').db; + const dbName = db._name(); + try { + db._useDatabase('_system'); + const databases = db._databases(); + for (const name of databases) { + try { + db._useDatabase(name); + upsertSystemServices(); + } catch (e) { + console.warnStack(e); + } + } + } finally { + db._useDatabase(dbName); + } + } + selfHealAll(true); +} + function upsertSystemServices () { const serviceDefinitions = new Map(); for (const mount of SYSTEM_SERVICE_MOUNTS) { @@ -237,238 +309,6 @@ function upsertSystemServices () { `); } -function rebuildAllServiceBundles (fixMissingChecksums) { - const servicesMissingChecksums = []; - const collection = utils.getStorage(); - for (const serviceDefinition of collection.all()) { - const mount = serviceDefinition.mount; - if (mount.startsWith('/_')) { - continue; - } - const bundlePath = FoxxService.bundlePath(mount); - const basePath = FoxxService.basePath(mount); - - const hasBundle = fs.exists(bundlePath); - const hasFolder = fs.exists(basePath); - if (!hasBundle && hasFolder) { - createServiceBundle(mount); - } else if (hasBundle && !hasFolder) { - extractServiceBundle(bundlePath, basePath); - } else if (!hasBundle && !hasFolder) { - continue; - } - if (fixMissingChecksums && !serviceDefinition.checksum) { - servicesMissingChecksums.push({ - checksum: FoxxService.checksum(mount), - _key: serviceDefinition._key - }); - } - } - if (!servicesMissingChecksums.length) { - return; - } - db._query(aql` - FOR service IN ${servicesMissingChecksums} - UPDATE service._key - WITH {checksum: service.checksum} - IN ${collection} - `); -} - -function selfHeal () { - const db = require('internal').db; - const dbName = db._name(); - try { - db._useDatabase('_system'); - const databases = db._databases(); - const weAreFoxxmaster = isFoxxmaster(); - const foxxmasterIsReady = weAreFoxxmaster || isFoxxmasterReady(); - for (const name of databases) { - try { - db._useDatabase(name); - if (weAreFoxxmaster) { - healMyselfAndCoords(); - } else if (foxxmasterIsReady) { - healMyself(); - } - } catch (e) { - console.warnStack(e); - } - } - return foxxmasterIsReady; - } finally { - db._useDatabase(dbName); - } -} - -function healMyself () { - const servicesINeedToFix = new Map(); - - const collection = utils.getStorage(); - for (const serviceDefinition of collection.all()) { - const mount = serviceDefinition.mount; - const checksum = serviceDefinition.checksum; - if (mount.startsWith('/_')) { - continue; - } - if (!checksum || checksum !== safeChecksum(mount)) { - servicesINeedToFix.set(mount, checksum); - } - } - - const coordinatorIds = getPeerCoordinatorIds(); - let clusterIsInconsistent = false; - for (const [mount, checksum] of servicesINeedToFix) { - const coordIdsToTry = shuffle(coordinatorIds); - let found = false; - for (const coordId of coordIdsToTry) { - const bundle = downloadServiceBundleFromCoordinator(coordId, mount, checksum); - if (bundle) { - replaceLocalServiceFromTempBundle(mount, bundle); - found = true; - break; - } - } - if (!found) { - clusterIsInconsistent = true; - break; - } - } - - if (clusterIsInconsistent) { - const coordId = getFoxmasterCoordinatorId(); - parallelClusterRequests([[ - coordId, - 'POST', - '/_api/foxx/_local/heal' - ]]); - } -} - -function healMyselfAndCoords () { - const checksumsINeedToFixLocally = []; - const actualChecksums = new Map(); - const coordsKnownToBeGoodSources = new Map(); - const coordsKnownToBeBadSources = new Map(); - const allKnownMounts = []; - - const collection = utils.getStorage(); - for (const serviceDefinition of collection.all()) { - const mount = serviceDefinition.mount; - const checksum = serviceDefinition.checksum; - if (mount.startsWith('/_')) { - continue; - } - allKnownMounts.push(mount); - actualChecksums.set(mount, checksum); - coordsKnownToBeGoodSources.set(mount, []); - coordsKnownToBeBadSources.set(mount, new Map()); - if (!checksum || checksum !== safeChecksum(mount)) { - checksumsINeedToFixLocally.push(mount); - } - } - - const serviceChecksumsByCoordinator = getChecksumsFromPeers(allKnownMounts); - for (const [coordId, serviceChecksums] of serviceChecksumsByCoordinator) { - for (const [mount, checksum] of serviceChecksums) { - if (!checksum) { - coordsKnownToBeBadSources.get(mount).set(coordId, null); - } else if (!actualChecksums.get(mount)) { - actualChecksums.set(mount, checksum); - coordsKnownToBeGoodSources.get(mount).push(coordId); - } else if (actualChecksums.get(mount) === checksum) { - coordsKnownToBeGoodSources.get(mount).push(coordId); - } else { - coordsKnownToBeBadSources.get(mount).set(coordId, checksum); - } - } - } - - const myId = getMyCoordinatorId(); - const serviceMountsToDeleteInCollection = []; - const serviceChecksumsToUpdateInCollection = new Map(); - for (const mount of checksumsINeedToFixLocally) { - const possibleSources = coordsKnownToBeGoodSources.get(mount); - if (!possibleSources.length) { - const myChecksum = safeChecksum(mount); - if (myChecksum) { - serviceChecksumsToUpdateInCollection.set(mount, myChecksum); - } else { - let found = false; - for (const [coordId, coordChecksum] of coordsKnownToBeBadSources.get(mount)) { - if (!coordChecksum) { - continue; - } - const bundle = downloadServiceBundleFromCoordinator(coordId, mount, coordChecksum); - if (bundle) { - serviceChecksumsToUpdateInCollection.set(mount, coordChecksum); - possibleSources.push(coordId); - replaceLocalServiceFromTempBundle(mount, bundle); - found = true; - break; - } - } - if (!found) { - serviceMountsToDeleteInCollection.push(mount); - coordsKnownToBeBadSources.delete(mount); - } - } - } else { - const checksum = actualChecksums.get(mount); - for (const coordId of possibleSources) { - const bundle = downloadServiceBundleFromCoordinator(coordId, mount, checksum); - if (bundle) { - replaceLocalServiceFromTempBundle(mount, bundle); - break; - } - } - } - } - - for (const ids of coordsKnownToBeGoodSources.values()) { - ids.push(myId); - } - - db._query(aql` - FOR service IN ${collection} - FILTER service.mount IN ${serviceMountsToDeleteInCollection} - REMOVE service - IN ${collection} - `); - - db._query(aql` - FOR service IN ${collection} - FOR item IN ${Array.from(serviceChecksumsToUpdateInCollection)} - FILTER service.mount == item[0] - UPDATE service - WITH {checksum: item[1]} - IN ${collection} - `); - - parallelClusterRequests(function * () { - for (const coordId of getPeerCoordinatorIds()) { - const servicesYouNeedToUpdate = {}; - for (const [mount, badCoordinatorIds] of coordsKnownToBeBadSources) { - if (!badCoordinatorIds.has(coordId)) { - continue; - } - const goodCoordIds = coordsKnownToBeGoodSources.get(mount); - servicesYouNeedToUpdate[mount] = shuffle(goodCoordIds); - } - if (!Object.keys(servicesYouNeedToUpdate).length) { - continue; - } - yield [ - coordId, - 'POST', - '/_api/foxx/_local', - JSON.stringify(servicesYouNeedToUpdate), - {'content-type': 'application/json'} - ]; - } - }()); -} - // Change propagation function reloadRouting () { @@ -476,45 +316,10 @@ function reloadRouting () { actions.reloadRouting(); } -function propagateServiceDestroyed (service) { +function propagateSelfHeal () { parallelClusterRequests(function * () { for (const coordId of getPeerCoordinatorIds()) { - yield [coordId, 'DELETE', `/_api/foxx/_local/service?${querystringify({ - mount: service.mount - })}`]; - } - }()); - reloadRouting(); -} - -function propagateServiceReplaced (service) { - const myId = getMyCoordinatorId(); - const coordIds = getPeerCoordinatorIds(); - const results = parallelClusterRequests(function * () { - for (const coordId of coordIds) { - yield [ - coordId, - 'POST', - '/_api/foxx/_local', - JSON.stringify({[service.mount]: [myId]}), - {'content-type': 'application/json'} - ]; - } - }()); - for (const [coordId, result] of zip(coordIds, results)) { - if (result.statusCode >= 400) { - console.error(`Failed to propagate service ${service.mount} to coord ${coordId}`); - } - } - reloadRouting(); -} - -function propagateServiceReconfigured (service) { - parallelClusterRequests(function * () { - for (const coordId of getPeerCoordinatorIds()) { - yield [coordId, 'POST', `/_api/foxx/_local/service?${querystringify({ - mount: service.mount - })}`]; + yield [coordId, 'POST', '/_api/foxx/_local/heal']; } }()); reloadRouting(); @@ -715,13 +520,19 @@ function _install (mount, options = {}) { service.executeScript('setup'); } service.updateChecksum(); + const bundleCollection = utils.getBundleStorage(); + if (!bundleCollection.exists(service.checksum)) { + bundleCollection._binaryInsert({_key: service.checksum}, service.bundlePath); + } const serviceDefinition = service.toJSON(); - db._query(aql` + const meta = db._query(aql` UPSERT {mount: ${mount}} INSERT ${serviceDefinition} REPLACE ${serviceDefinition} IN ${collection} - `); + RETURN NEW + `).next(); + service._rev = meta._rev; ensureServiceExecuted(service, true); return service; } @@ -746,11 +557,22 @@ function _uninstall (mount, options = {}) { } } const collection = utils.getStorage(); - db._query(aql` + const serviceDefinition = db._query(aql` FOR service IN ${collection} FILTER service.mount == ${mount} REMOVE service IN ${collection} - `); + RETURN OLD + `).next(); + if (serviceDefinition) { + const checksumRefs = db._query(aql` + FOR service IN ${collection} + FILTER service.checksum == ${serviceDefinition.checksum} + RETURN 1 + `).toArray(); + if (!checksumRefs.length) { + utils.getBundleStorage().remove(serviceDefinition.checksum); + } + } GLOBAL_SERVICE_MAP.get(db._name()).delete(mount); const servicePath = FoxxService.basePath(mount); if (fs.exists(servicePath)) { @@ -812,22 +634,6 @@ function downloadServiceBundleFromRemote (url) { } } -function downloadServiceBundleFromCoordinator (coordId, mount, checksum) { - const response = parallelClusterRequests([[ - coordId, - 'GET', - `/_api/foxx/_local/bundle?${querystringify({mount})}`, - null, - checksum ? {'if-match': `"${checksum}"`} : undefined - ]])[0]; - if (response.headers['x-arango-response-code'].startsWith('404')) { - return null; - } - const filename = fs.getTempFile('bundles', true); - fs.writeFileSync(filename, response.rawBody); - return filename; -} - function extractServiceBundle (archive, targetPath) { const tempFolder = fs.getTempFile('services', false); fs.makeDirectory(tempFolder); @@ -866,12 +672,6 @@ function extractServiceBundle (archive, targetPath) { } } -function replaceLocalServiceFromTempBundle (mount, tempBundlePath) { - const tempServicePath = fs.getTempFile('services', false); - extractServiceBundle(tempBundlePath, tempServicePath); - _buildServiceInPath(mount, tempServicePath, tempBundlePath); -} - // Exported functions for manipulating services function install (serviceInfo, mount, options = {}) { @@ -889,44 +689,14 @@ function install (serviceInfo, mount, options = {}) { const tempPaths = _prepareService(serviceInfo, options); _buildServiceInPath(mount, tempPaths.tempServicePath, tempPaths.tempBundlePath); const service = _install(mount, options); - propagateServiceReplaced(service); + propagateSelfHeal(); return service; } -function installLocal (mount, coordIds) { - for (const coordId of coordIds) { - const filename = downloadServiceBundleFromCoordinator(coordId, mount); - if (filename) { - replaceLocalServiceFromTempBundle(mount, filename); - return true; - } - } - return false; -} - -function uninstallLocal (mount) { - const servicePath = FoxxService.basePath(mount); - if (fs.exists(servicePath)) { - try { - fs.removeDirectoryRecursive(servicePath, true); - } catch (e) { - console.warnStack(e); - } - } - const bundlePath = FoxxService.bundlePath(mount); - if (fs.exists(bundlePath)) { - try { - fs.remove(bundlePath); - } catch (e) { - console.warnStack(e); - } - } -} - function uninstall (mount, options = {}) { ensureFoxxInitialized(); const service = _uninstall(mount, options); - propagateServiceDestroyed(service); + propagateSelfHeal(); return service; } @@ -942,7 +712,7 @@ function replace (serviceInfo, mount, options = {}) { _uninstall(mount, Object.assign({teardown: true}, options, {force: true})); _buildServiceInPath(mount, tempPaths.tempServicePath, tempPaths.tempBundlePath); const service = _install(mount, Object.assign({}, options, {force: true})); - propagateServiceReplaced(service); + propagateSelfHeal(); return service; } @@ -962,7 +732,7 @@ function upgrade (serviceInfo, mount, options = {}) { _uninstall(mount, Object.assign({teardown: false}, options, {force: true})); _buildServiceInPath(mount, tempPaths.tempServicePath, tempPaths.tempBundlePath); const service = _install(mount, Object.assign({}, options, serviceOptions, {force: true})); - propagateServiceReplaced(service); + propagateSelfHeal(); return service; } @@ -990,7 +760,7 @@ function enableDevelopmentMode (mount) { const service = getServiceInstance(mount); service.development(true); utils.updateService(mount, service.toJSON()); - propagateServiceReconfigured(service); + propagateSelfHeal(); return service; } @@ -1002,7 +772,7 @@ function disableDevelopmentMode (mount) { utils.updateService(mount, service.toJSON()); // Make sure setup changes from devmode are respected service.executeScript('setup'); - propagateServiceReplaced(service); + propagateSelfHeal(); return service; } @@ -1010,7 +780,7 @@ function setConfiguration (mount, options = {}) { const service = getServiceInstance(mount); const warnings = service.applyConfiguration(options.configuration, options.replace); utils.updateService(mount, service.toJSON()); - propagateServiceReconfigured(service); + propagateSelfHeal(); return warnings; } @@ -1018,7 +788,7 @@ function setDependencies (mount, options = {}) { const service = getServiceInstance(mount); const warnings = service.applyDependencies(options.dependencies, options.replace); utils.updateService(mount, service.toJSON()); - propagateServiceReconfigured(service); + propagateSelfHeal(); return warnings; } @@ -1073,8 +843,6 @@ function safeChecksum (mount) { // Exports -exports._installLocal = installLocal; -exports._uninstallLocal = uninstallLocal; exports.install = install; exports.uninstall = uninstall; exports.replace = replace; @@ -1094,15 +862,14 @@ exports.installedServices = installedServices; // ------------------------------------------------- exports.isFoxxmaster = isFoxxmaster; -exports.proxyToFoxxmaster = proxyToFoxxmaster; exports._reloadRouting = reloadRouting; exports.reloadInstalledService = reloadInstalledService; exports.ensureRouted = ensureServiceLoaded; exports.initializeFoxx = initLocalServiceMap; exports.ensureFoxxInitialized = ensureFoxxInitialized; -exports.heal = healMyselfAndCoords; exports._startup = startup; -exports._selfHeal = selfHeal; +exports.heal = triggerSelfHeal; +exports.healAll = selfHealAll; exports._createServiceBundle = createServiceBundle; exports._resetCache = () => GLOBAL_SERVICE_MAP.clear(); exports._mountPoints = getMountPoints; diff --git a/js/server/modules/@arangodb/foxx/service.js b/js/server/modules/@arangodb/foxx/service.js index 64bd06dba4..75e78917e5 100644 --- a/js/server/modules/@arangodb/foxx/service.js +++ b/js/server/modules/@arangodb/foxx/service.js @@ -107,7 +107,7 @@ module.exports = return manifest; } - static validateServiceFiles (mount, manifest) { + static validateServiceFiles (mount, manifest, rev) { const servicePath = FoxxService.basePath(mount); if (manifest.main) { parseFile(servicePath, manifest.main); @@ -140,6 +140,7 @@ module.exports = } constructor (definition, manifest) { + this._rev = definition._rev; this.mount = definition.mount; this.checksum = definition.checksum; this.basePath = definition.basePath || FoxxService.basePath(this.mount); @@ -642,12 +643,20 @@ module.exports = } static rootPath (mount) { - if (mount.charAt(1) === '_') { + if (mount && mount.charAt(1) === '_') { return FoxxService._systemAppPath; } return FoxxService._appPath; } + static rootBundlePath (mount) { + return path.resolve( + FoxxService.rootPath(mount), + '_appbundles' + ); + } + + static basePath (mount) { return path.resolve( FoxxService.rootPath(mount), @@ -661,7 +670,7 @@ module.exports = mount = '/' + mount; } const bundleName = mount.substr(1).replace(/[-.:/]/g, '_'); - return path.join(FoxxService.rootPath(mount), bundleName + '.zip'); + return path.join(FoxxService.rootBundlePath(mount), bundleName + '.zip'); } static get _startupPath () { diff --git a/js/server/server.js b/js/server/server.js index 345fc59e43..d84cba19f5 100644 --- a/js/server/server.js +++ b/js/server/server.js @@ -67,8 +67,7 @@ if (internal.threadNumber === 0 && global.ArangoServerState.role() === 'SINGLE') { if (restServer) { // startup the foxx manager once - require('@arangodb/foxx/manager')._startup(true); - require('@arangodb/foxx/manager')._selfHeal(true); + require('@arangodb/foxx/manager')._startup(); } // start the queue manager once diff --git a/lib/Basics/errors.dat b/lib/Basics/errors.dat index 9048e39800..93bfec0508 100755 --- a/lib/Basics/errors.dat +++ b/lib/Basics/errors.dat @@ -44,6 +44,7 @@ ERROR_HTTP_NOT_FOUND,404,"not found","Will be raised when an URI is unknown." ERROR_HTTP_METHOD_NOT_ALLOWED,405,"method not supported","Will be raised when an unsupported HTTP method is used for an operation." ERROR_HTTP_PRECONDITION_FAILED,412,"precondition failed","Will be raised when a precondition for an HTTP request is not met." ERROR_HTTP_SERVER_ERROR,500,"internal server error","Will be raised when an internal server is encountered." +ERROR_HTTP_SERVICE_UNAVAILABLE,503,"service unavailable","Will be raised when a service is temporarily unavailable." ################################################################################ ## HTTP processing errors @@ -382,6 +383,8 @@ COMMUNICATOR_REQUEST_ABORTED,2100,"Request aborted","Request was aborted." ERROR_MALFORMED_MANIFEST_FILE,3000,"failed to parse manifest file","The service manifest file is not well-formed JSON." ERROR_INVALID_SERVICE_MANIFEST,3001,"manifest file is invalid","The service manifest contains invalid values." +ERROR_SERVICE_FILES_MISSING,3002,"service files missing","The service folder or bundle does not exist on this server." +ERROR_SERVICE_FILES_OUTDATED,3003,"service files outdated","The local service bundle does not match the checksum in the database." ERROR_INVALID_FOXX_OPTIONS,3004,"service options are invalid","The service options contain invalid values." ERROR_INVALID_MOUNTPOINT,3007,"invalid mountpath","The service mountpath contains invalid characters." ERROR_SERVICE_NOT_FOUND,3009,"service not found","No service found at the given mountpath." diff --git a/lib/Basics/voc-errors.cpp b/lib/Basics/voc-errors.cpp index fb1965d956..fbd7f08df7 100644 --- a/lib/Basics/voc-errors.cpp +++ b/lib/Basics/voc-errors.cpp @@ -43,6 +43,7 @@ void TRI_InitializeErrorMessages () { REG_ERROR(ERROR_HTTP_METHOD_NOT_ALLOWED, "method not supported"); REG_ERROR(ERROR_HTTP_PRECONDITION_FAILED, "precondition failed"); REG_ERROR(ERROR_HTTP_SERVER_ERROR, "internal server error"); + REG_ERROR(ERROR_HTTP_SERVICE_UNAVAILABLE, "service unavailable"); REG_ERROR(ERROR_HTTP_CORRUPTED_JSON, "invalid JSON object"); REG_ERROR(ERROR_HTTP_SUPERFLUOUS_SUFFICES, "superfluous URL suffices"); REG_ERROR(ERROR_ARANGO_ILLEGAL_STATE, "illegal state"); @@ -270,6 +271,8 @@ void TRI_InitializeErrorMessages () { REG_ERROR(COMMUNICATOR_REQUEST_ABORTED, "Request aborted"); REG_ERROR(ERROR_MALFORMED_MANIFEST_FILE, "failed to parse manifest file"); REG_ERROR(ERROR_INVALID_SERVICE_MANIFEST, "manifest file is invalid"); + REG_ERROR(ERROR_SERVICE_FILES_MISSING, "service files missing"); + REG_ERROR(ERROR_SERVICE_FILES_OUTDATED, "service files outdated"); REG_ERROR(ERROR_INVALID_FOXX_OPTIONS, "service options are invalid"); REG_ERROR(ERROR_INVALID_MOUNTPOINT, "invalid mountpath"); REG_ERROR(ERROR_SERVICE_NOT_FOUND, "service not found"); diff --git a/lib/Basics/voc-errors.h b/lib/Basics/voc-errors.h index eedfdd92cf..c46e134d07 100644 --- a/lib/Basics/voc-errors.h +++ b/lib/Basics/voc-errors.h @@ -85,6 +85,8 @@ /// Will be raised when a precondition for an HTTP request is not met. /// - 500: @LIT{internal server error} /// Will be raised when an internal server is encountered. +/// - 503: @LIT{service unavailable} +/// Will be raised when a service is temporarily unavailable. /// - 600: @LIT{invalid JSON object} /// Will be raised when a string representation of a JSON object is corrupt. /// - 601: @LIT{superfluous URL suffices} @@ -646,6 +648,10 @@ /// The service manifest file is not well-formed JSON. /// - 3001: @LIT{manifest file is invalid} /// The service manifest contains invalid values. +/// - 3002: @LIT{service files missing} +/// The service folder or bundle does not exist on this server. +/// - 3003: @LIT{service files outdated} +/// The local service bundle does not match the checksum in the database. /// - 3004: @LIT{service options are invalid} /// The service options contain invalid values. /// - 3007: @LIT{invalid mountpath} @@ -1093,6 +1099,16 @@ void TRI_InitializeErrorMessages (); #define TRI_ERROR_HTTP_SERVER_ERROR (500) +//////////////////////////////////////////////////////////////////////////////// +/// @brief 503: ERROR_HTTP_SERVICE_UNAVAILABLE +/// +/// service unavailable +/// +/// Will be raised when a service is temporarily unavailable. +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_ERROR_HTTP_SERVICE_UNAVAILABLE (503) + //////////////////////////////////////////////////////////////////////////////// /// @brief 600: ERROR_HTTP_CORRUPTED_JSON /// @@ -3461,6 +3477,26 @@ void TRI_InitializeErrorMessages (); #define TRI_ERROR_INVALID_SERVICE_MANIFEST (3001) +//////////////////////////////////////////////////////////////////////////////// +/// @brief 3002: ERROR_SERVICE_FILES_MISSING +/// +/// service files missing +/// +/// The service folder or bundle does not exist on this server. +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_ERROR_SERVICE_FILES_MISSING (3002) + +//////////////////////////////////////////////////////////////////////////////// +/// @brief 3003: ERROR_SERVICE_FILES_OUTDATED +/// +/// service files outdated +/// +/// The local service bundle does not match the checksum in the database. +//////////////////////////////////////////////////////////////////////////////// + +#define TRI_ERROR_SERVICE_FILES_OUTDATED (3003) + //////////////////////////////////////////////////////////////////////////////// /// @brief 3004: ERROR_INVALID_FOXX_OPTIONS ///