Simplified Foxx self healing (#2511)

* Implement new self-heal
* Add error codes for 503, service missing/outdated
* Detect changes to service via rev
* Pretty print incoming response object in log

js/apps/system/_admin/aardvark/APP/foxxes.js

@@ -37,7 +37,6 @@ const FoxxGenerator = require('./generator');
 const fmu = require('@arangodb/foxx/manager-utils');
 const createRouter = require('@arangodb/foxx/router');
 const joinPath = require('path').join;
-const posix = require('path').posix;
 
 const DEFAULT_THUMBNAIL = module.context.fileName('default-thumbnail.png');
 
@@ -77,7 +76,6 @@ foxxRouter.use(installer)
   Triggers teardown and setup.
 `);
 
-if (FoxxManager.isFoxxmaster()) {
 installer.use(function (req, res, next) {
   const mount = decodeURIComponent(req.queryParams.mount);
   const upgrade = req.queryParams.upgrade;
@@ -118,10 +116,10 @@ if (FoxxManager.isFoxxmaster()) {
     throw e;
   }
   const configuration = service.getConfiguration();
-    res.json(Object.assign(
+  res.json(Object.assign({
-      {error: false, configuration},
+    error: false,
-      service.simpleJSON()
+    configuration
-    ));
+  }, service.simpleJSON()));
 });
 
 installer.put('/store', function (req) {
@@ -215,30 +213,6 @@ if (FoxxManager.isFoxxmaster()) {
 .description(dd`
   Uninstall the Foxx at the given mount-point.
 `);
-} else {
-  installer.put('/store', FoxxManager.proxyToFoxxmaster);
-  installer.put('/git', FoxxManager.proxyToFoxxmaster);
-  installer.put('/generate', FoxxManager.proxyToFoxxmaster);
-  installer.put('/zip', (req, res) => {
-    const zipData = fs.readFileSync(joinPath(fs.getTempPath(), req.body.zipFile));
-    FoxxManager.proxyToFoxxmaster({
-      method: req.method,
-      rawBody: zipData,
-      headers: Object.assign({}, req.headers, {
-        'content-length': zipData.length
-      }),
-      _url: {
-        pathname: posix.resolve(req._url.pathname, '../raw'),
-        search: req._url.search
-      }
-    }, res);
-  })
-  .body(joi.object({
-    zipFile: joi.string().required()
-  }).required(), 'A zip file path.');
-  installer.put('/raw', FoxxManager.proxyToFoxxmaster);
-  foxxRouter.delete('/', FoxxManager.proxyToFoxxmaster);
-}
 
 router.get('/', function (req, res) {
   const foxxes = FoxxManager.listJson();
@@ -299,12 +273,14 @@ foxxRouter.get('/deps', function (req, res) {
   Used to request the dependencies options for services.
 `);
 
-if (FoxxManager.isFoxxmaster()) {
 foxxRouter.patch('/config', function (req, res) {
   const mount = decodeURIComponent(req.queryParams.mount);
   const configuration = req.body;
   const service = FoxxManager.lookupService(mount);
-    FoxxManager.setConfiguration(mount, {configuration, replace: !service.isDevelopment});
+  FoxxManager.setConfiguration(mount, {
+    configuration,
+    replace: !service.isDevelopment
+  });
   res.json(service.getConfiguration());
 })
 .body(joi.object().optional(), 'Configuration to apply.')
@@ -317,7 +293,10 @@ if (FoxxManager.isFoxxmaster()) {
   const mount = decodeURIComponent(req.queryParams.mount);
   const dependencies = req.body;
   const service = FoxxManager.lookupService(mount);
-    FoxxManager.setDependencies(mount, {dependencies, replace: !service.isDevelopment});
+  FoxxManager.setDependencies(mount, {
+    dependencies,
+    replace: !service.isDevelopment
+  });
   res.json(service.getDependencies());
 })
 .body(joi.object().optional(), 'Dependency options to apply.')
@@ -325,10 +304,6 @@ if (FoxxManager.isFoxxmaster()) {
 .description(dd`
   Used to overwrite the dependencies options for services.
 `);
-} else {
-  foxxRouter.patch('/config', FoxxManager.proxyToFoxxmaster);
-  foxxRouter.patch('/deps', FoxxManager.proxyToFoxxmaster);
-}
 
 foxxRouter.post('/tests', function (req, res) {
   const mount = decodeURIComponent(req.queryParams.mount);

js/apps/system/_api/foxx/APP/index.js

@@ -1,4 +1,29 @@
 'use strict';
+
+// //////////////////////////////////////////////////////////////////////////////
+// / DISCLAIMER
+// /
+// / Copyright 2010-2013 triAGENS GmbH, Cologne, Germany
+// / Copyright 2016 ArangoDB GmbH, Cologne, Germany
+// /
+// / Licensed under the Apache License, Version 2.0 (the "License")
+// / you may not use this file except in compliance with the License.
+// / You may obtain a copy of the License at
+// /
+// /     http://www.apache.org/licenses/LICENSE-2.0
+// /
+// / Unless required by applicable law or agreed to in writing, software
+// / distributed under the License is distributed on an "AS IS" BASIS,
+// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// / See the License for the specific language governing permissions and
+// / limitations under the License.
+// /
+// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// /
+// / @author Alan Plum
+// //////////////////////////////////////////////////////////////////////////////
+
+
 const _ = require('lodash');
 const dd = require('dedent');
 const fs = require('fs');
@@ -10,7 +35,6 @@ const errors = require('@arangodb').errors;
 const jsonml2xml = require('@arangodb/util').jsonml2xml;
 const swaggerJson = require('@arangodb/foxx/legacy/swagger').swaggerJson;
 const FoxxManager = require('@arangodb/foxx/manager');
-const FoxxService = require('@arangodb/foxx/service');
 const createRouter = require('@arangodb/foxx/router');
 const reporters = Object.keys(require('@arangodb/mocha').reporters);
 const schemas = require('./schemas');
@@ -64,7 +88,10 @@ router.use((req, res, next) => {
   } catch (e) {
     if (e.isArangoError) {
       const status = actions.arangoErrorToHttpCode(e.errorNum);
-      res.throw(status, e.errorMessage, {errorNum: e.errorNum, cause: e});
+      res.throw(status, e.errorMessage, {
+        errorNum: e.errorNum,
+        cause: e
+      });
     }
     throw e;
   }
@@ -87,15 +114,20 @@ router.get((req, res) => {
 })
 .response(200, joi.array().items(schemas.shortInfo).required());
 
-if (FoxxManager.isFoxxmaster()) {
 router.post(prepareServiceRequestBody, (req, res) => {
   const mount = req.queryParams.mount;
   FoxxManager.install(req.body.source, mount, _.omit(req.queryParams, ['mount', 'development']));
   if (req.body.configuration) {
-      FoxxManager.setConfiguration(mount, {configuration: req.body.configuration, replace: true});
+    FoxxManager.setConfiguration(mount, {
+      configuration: req.body.configuration,
+      replace: true
+    });
   }
   if (req.body.dependencies) {
-      FoxxManager.setDependencies(mount, {dependencies: req.body.dependencies, replace: true});
+    FoxxManager.setDependencies(mount, {
+      dependencies: req.body.dependencies,
+      replace: true
+    });
   }
   if (req.queryParams.development) {
     FoxxManager.development(mount);
@@ -109,9 +141,6 @@ if (FoxxManager.isFoxxmaster()) {
 .queryParam('setup', schemas.flag.default(true))
 .queryParam('legacy', schemas.flag.default(false))
 .response(201, schemas.fullInfo);
-} else {
-  router.post(FoxxManager.proxyToFoxxmaster);
-}
 
 const instanceRouter = createRouter();
 instanceRouter.use((req, res, next) => {
@@ -133,20 +162,25 @@ serviceRouter.get((req, res) => {
   res.json(serviceToJson(req.service));
 })
 .response(200, schemas.fullInfo)
-.summary(`Service description`)
+.summary('Service description')
 .description(dd`
   Fetches detailed information for the service at the given mount path.
 `);
 
-if (FoxxManager.isFoxxmaster()) {
 serviceRouter.patch(prepareServiceRequestBody, (req, res) => {
   const mount = req.queryParams.mount;
   FoxxManager.upgrade(req.body.source, mount, _.omit(req.queryParams, ['mount']));
   if (req.body.configuration) {
-      FoxxManager.setConfiguration(mount, {configuration: req.body.configuration, replace: false});
+    FoxxManager.setConfiguration(mount, {
+      configuration: req.body.configuration,
+      replace: false
+    });
   }
   if (req.body.dependencies) {
-      FoxxManager.setDependencies(mount, {dependencies: req.body.dependencies, replace: false});
+    FoxxManager.setDependencies(mount, {
+      dependencies: req.body.dependencies,
+      replace: false
+    });
   }
   const service = FoxxManager.lookupService(mount);
   res.json(serviceToJson(service));
@@ -161,10 +195,16 @@ if (FoxxManager.isFoxxmaster()) {
   const mount = req.queryParams.mount;
   FoxxManager.replace(req.body.source, mount, _.omit(req.queryParams, ['mount']));
   if (req.body.configuration) {
-      FoxxManager.setConfiguration(mount, {configuration: req.body.configuration, replace: true});
+    FoxxManager.setConfiguration(mount, {
+      configuration: req.body.configuration,
+      replace: true
+    });
   }
   if (req.body.dependencies) {
-      FoxxManager.setDependencies(mount, {dependencies: req.body.dependencies, replace: true});
+    FoxxManager.setDependencies(mount, {
+      dependencies: req.body.dependencies,
+      replace: true
+    });
   }
   const service = FoxxManager.lookupService(mount);
   res.json(serviceToJson(service));
@@ -184,11 +224,6 @@ if (FoxxManager.isFoxxmaster()) {
 })
 .queryParam('teardown', schemas.flag.default(true))
 .response(204, null);
-} else {
-  serviceRouter.patch(FoxxManager.proxyToFoxxmaster);
-  serviceRouter.put(FoxxManager.proxyToFoxxmaster);
-  serviceRouter.delete(FoxxManager.proxyToFoxxmaster);
-}
 
 const configRouter = createRouter();
 instanceRouter.use('/configuration', configRouter)
@@ -198,14 +233,16 @@ configRouter.get((req, res) => {
   res.json(req.service.getConfiguration());
 });
 
-if (FoxxManager.isFoxxmaster()) {
 configRouter.patch((req, res) => {
   const warnings = FoxxManager.setConfiguration(req.service.mount, {
     configuration: req.body,
     replace: false
   });
   const values = req.service.getConfiguration(true);
-    res.json({values, warnings});
+  res.json({
+    values,
+    warnings
+  });
 })
 .body(joi.object().required());
 
@@ -215,13 +252,12 @@ if (FoxxManager.isFoxxmaster()) {
     replace: true
   });
   const values = req.service.getConfiguration(true);
-    res.json({values, warnings});
+  res.json({
+    values,
+    warnings
+  });
 })
 .body(joi.object().required());
-} else {
-  configRouter.patch(FoxxManager.proxyToFoxxmaster);
-  configRouter.put(FoxxManager.proxyToFoxxmaster);
-}
 
 const depsRouter = createRouter();
 instanceRouter.use('/dependencies', depsRouter)
@@ -231,14 +267,16 @@ depsRouter.get((req, res) => {
   res.json(req.service.getDependencies());
 });
 
-if (FoxxManager.isFoxxmaster()) {
 depsRouter.patch((req, res) => {
   const warnings = FoxxManager.setDependencies(req.service.mount, {
     dependencies: req.body,
     replace: true
   });
   const values = req.service.getDependencies(true);
-    res.json({values, warnings});
+  res.json({
+    values,
+    warnings
+  });
 })
 .body(joi.object().required());
 
@@ -248,13 +286,12 @@ if (FoxxManager.isFoxxmaster()) {
     replace: true
   });
   const values = req.service.getDependencies(true);
-    res.json({values, warnings});
+  res.json({
+    values,
+    warnings
+  });
 })
 .body(joi.object().required());
-} else {
-  depsRouter.patch(FoxxManager.proxyToFoxxmaster);
-  depsRouter.put(FoxxManager.proxyToFoxxmaster);
-}
 
 const devRouter = createRouter();
 instanceRouter.use('/development', devRouter)
@@ -356,86 +393,6 @@ instanceRouter.get('/swagger', (req, res) => {
 const localRouter = createRouter();
 router.use('/_local', localRouter);
 
-localRouter.post((req, res) => {
-  const result = {};
-  for (const mount of Object.keys(req.body)) {
-    const coordIds = req.body[mount];
-    result[mount] = FoxxManager._installLocal(mount, coordIds);
-  }
-  FoxxManager._reloadRouting();
-  res.json(result);
-})
-.body(joi.object());
-
-localRouter.post('/service', (req, res) => {
-  FoxxManager._reloadRouting();
-})
-.queryParam('mount', schemas.mount);
-
-localRouter.delete('/service', (req, res) => {
-  FoxxManager._uninstallLocal(req.queryParams.mount);
-  FoxxManager._reloadRouting();
-})
-.queryParam('mount', schemas.mount);
-
-localRouter.get('/bundle', (req, res) => {
-  const mount = req.queryParams.mount;
-  const bundlePath = FoxxService.bundlePath(mount);
-  if (!fs.isFile(bundlePath)) {
-    res.throw(404, 'Bundle not available');
-  }
-  const etag = `"${FoxxService.checksum(mount)}"`;
-  if (req.get('if-none-match') === etag) {
-    res.status(304);
-    return;
-  }
-  if (req.get('if-match') && req.get('if-match') !== etag) {
-    res.throw(404, 'No matching bundle available');
-  }
-  res.set('etag', etag);
-  res.download(bundlePath);
-})
-.response(200, ['application/zip'])
-.header('if-match', joi.string().optional())
-.header('if-none-match', joi.string().optional())
-.queryParam('mount', schemas.mount);
-
-localRouter.get('/status', (req, res) => {
-  const ready = global.KEY_GET('foxx', 'ready');
-  if (ready || FoxxManager.isFoxxmaster()) {
-    res.json({ready});
-    return;
-  }
-  FoxxManager.proxyToFoxxmaster(req, res);
-  if (res.statusCode < 400) {
-    const result = JSON.parse(res.body.toString('utf-8'));
-    if (result.ready) {
-      global.KEY_SET('foxx', 'ready', result.ready);
-    }
-  }
-});
-
-localRouter.get('/checksums', (req, res) => {
-  const mountParam = req.queryParams.mount || [];
-  const mounts = Array.isArray(mountParam) ? mountParam : [mountParam];
-  const checksums = {};
-  for (const mount of mounts) {
-    try {
-      checksums[mount] = FoxxService.checksum(mount);
-    } catch (e) {
-    }
-  }
-  res.json(checksums);
-})
-.queryParam('mount', joi.alternatives(
-  joi.array().items(schemas.mount),
-  schemas.mount
-));
-
-if (FoxxManager.isFoxxmaster()) {
 localRouter.post('/heal', (req, res) => {
   FoxxManager.heal();
 });
-} else {
-  localRouter.post('/heal', FoxxManager.proxyToFoxxmaster);
-}

js/client/tests/shell/shell-foxx-manager-spec.js

@@ -55,19 +55,6 @@ describe('Foxx Manager', function () {
         expect(checksum).not.to.be.empty;
       });
 
-      it('should provide a bundle', function () {
-        FoxxManager.install(setupTeardownApp, mount);
-        const url = `${arango.getEndpoint().replace('tcp://', 'http://')}/_api/foxx/_local/bundle?mount=${encodeURIComponent(mount)}`;
-        const res = download(url);
-        expect(res.code).to.equal(200);
-        const checksum = db._query(aql`
-          FOR service IN _apps
-          FILTER service.mount == ${mount}
-          RETURN service.checksum
-        `).next();
-        expect(res.headers.etag).to.equal(`"${checksum}"`);
-      });
-
       it('should run the setup script', function () {
         FoxxManager.install(setupTeardownApp, mount);
         expect(db._collection(setupTeardownCol)).to.be.an.instanceOf(ArangoCollection);

js/common/bootstrap/errors.js

@@ -47,6 +47,7 @@
     "ERROR_HTTP_METHOD_NOT_ALLOWED" : { "code" : 405, "message" : "method not supported" },
     "ERROR_HTTP_PRECONDITION_FAILED" : { "code" : 412, "message" : "precondition failed" },
     "ERROR_HTTP_SERVER_ERROR"      : { "code" : 500, "message" : "internal server error" },
+    "ERROR_HTTP_SERVICE_UNAVAILABLE" : { "code" : 503, "message" : "service unavailable" },
     "ERROR_HTTP_CORRUPTED_JSON"    : { "code" : 600, "message" : "invalid JSON object" },
     "ERROR_HTTP_SUPERFLUOUS_SUFFICES" : { "code" : 601, "message" : "superfluous URL suffices" },
     "ERROR_ARANGO_ILLEGAL_STATE"   : { "code" : 1000, "message" : "illegal state" },
@@ -274,6 +275,8 @@
     "COMMUNICATOR_REQUEST_ABORTED" : { "code" : 2100, "message" : "Request aborted" },
     "ERROR_MALFORMED_MANIFEST_FILE" : { "code" : 3000, "message" : "failed to parse manifest file" },
     "ERROR_INVALID_SERVICE_MANIFEST" : { "code" : 3001, "message" : "manifest file is invalid" },
+    "ERROR_SERVICE_FILES_MISSING"  : { "code" : 3002, "message" : "service files missing" },
+    "ERROR_SERVICE_FILES_OUTDATED" : { "code" : 3003, "message" : "service files outdated" },
     "ERROR_INVALID_FOXX_OPTIONS"   : { "code" : 3004, "message" : "service options are invalid" },
     "ERROR_INVALID_MOUNTPOINT"     : { "code" : 3007, "message" : "invalid mountpath" },
     "ERROR_SERVICE_NOT_FOUND"      : { "code" : 3009, "message" : "service not found" },

js/common/modules/@arangodb/foxx/manager-utils.js

@@ -54,11 +54,46 @@ function getReadableName (name) {
 }
 
 function getStorage () {
-  var c = db._collection('_apps');
+  let c = db._collection('_apps');
   if (c === null) {
-    c = db._create('_apps', {isSystem: true, replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM,
+    try {
-    distributeShardsLike: '_graphs', journalSize: 4 * 1024 * 1024});
+      c = db._create('_apps', {
-    c.ensureIndex({ type: 'hash', fields: [ 'mount' ], unique: true });
+        isSystem: true,
+        replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM,
+        distributeShardsLike: '_graphs',
+        journalSize: 4 * 1024 * 1024
+      });
+      c.ensureIndex({
+        type: 'hash',
+        fields: ['mount'],
+        unique: true
+      });
+    } catch (e) {
+      c = db._collection('_apps');
+      if (!c) {
+        throw e;
+      }
+    }
+  }
+  return c;
+}
+
+function getBundleStorage () {
+  let c = db._collection('_appbundles');
+  if (c === null) {
+    try {
+      c = db._create('_appbundles', {
+        isSystem: true,
+        replicationFactor: DEFAULT_REPLICATION_FACTOR_SYSTEM,
+        distributeShardsLike: '_graphs',
+        journalSize: 4 * 1024 * 1024
+      });
+    } catch (e) {
+      c = db._collection('_appbundles');
+      if (!c) {
+        throw e;
+      }
+    }
   }
   return c;
 }
@@ -231,4 +266,5 @@ exports.buildGithubUrl = buildGithubUrl;
 exports.validateMount = validateMount;
 exports.zipDirectory = zipDirectory;
 exports.getStorage = getStorage;
+exports.getBundleStorage = getBundleStorage;
 exports.pathRegex = pathRegex;

js/common/modules/@arangodb/request.js

@@ -61,6 +61,29 @@ class Response {
       }
     }
   }
+  _PRINT (ctx) {
+    const MAX_BYTES = 100;
+    ctx.output += `[IncomingResponse ${this.status} ${this.message} `;
+    if (!this.body || !this.body.length) {
+      ctx.output += 'empty';
+    } else {
+      ctx.output += `${this.body.length} bytes `;
+      if (typeof this.body !== 'string') {
+        ctx.output += '<binary>';
+      } else if (this.body.length <= MAX_BYTES) {
+        ctx.output += `"${this.body}"`;
+      } else {
+        const offset = (this.body.length - MAX_BYTES) / 2;
+        ctx.output += `"…${
+          this.body.slice(offset, offset + MAX_BYTES)
+          .replace('\n', '\\n')
+          .replace('\r', '\\r')
+          .replace('\t', '\\t')
+        }…"`;
+      }
+    }
+    ctx.output += ']';
+  }
 }
 
 function querystringify (query, useQuerystring) {

js/server/bootstrap/coordinator.js

@@ -44,26 +44,14 @@
   internal.loadStartup('server/bootstrap/routing.js').startup();
 
   if (internal.threadNumber === 0) {
-    global.KEYSPACE_CREATE('foxx', 1, true);
+    require('@arangodb/foxx/manager')._startup();
-    global.KEY_SET('foxx', 'ready', false);
-    const isFoxxmaster = global.ArangoServerState.isFoxxmaster();
-    require('@arangodb/foxx/manager')._startup(isFoxxmaster);
     require('@arangodb/tasks').register({
       id: 'self-heal',
       isSystem: true,
-      period: 0.1, // secs
+      period: 5 * 60, // secs
       command: function () {
         const FoxxManager = require('@arangodb/foxx/manager');
-        const isReady = FoxxManager._isClusterReady();
+        FoxxManager.healAll();
-        if (!isReady) {
-          return;
-        }
-        require('@arangodb/tasks').unregister('self-heal');
-        const isFoxxmaster = global.ArangoServerState.isFoxxmaster();
-        const foxxmasterIsReady = FoxxManager._selfHeal(isFoxxmaster);
-        if (foxxmasterIsReady) {
-          global.KEY_SET('foxx', 'ready', true);
-        }
       }
     });
     // start the queue manager once

js/server/modules/@arangodb/foxx/manager.js

@@ -30,7 +30,6 @@
 
 const fs = require('fs');
 const path = require('path');
-const querystringify = require('querystring').encode;
 const dd = require('dedent');
 const utils = require('@arangodb/foxx/manager-utils');
 const store = require('@arangodb/foxx/store');
@@ -44,8 +43,6 @@ const db = arangodb.db;
 const ArangoClusterControl = require('@arangodb/cluster');
 const request = require('@arangodb/request');
 const actions = require('@arangodb/actions');
-const shuffle = require('lodash/shuffle');
-const zip = require('lodash/zip');
 const isZipBuffer = require('@arangodb/util').isZipBuffer;
 
 const SYSTEM_SERVICE_MOUNTS = [
@@ -72,13 +69,6 @@ function getMyCoordinatorId () {
   return global.ArangoServerState.id();
 }
 
-function getFoxmasterCoordinatorId () {
-  if (!ArangoClusterControl.isCluster()) {
-    return null;
-  }
-  return global.ArangoServerState.getFoxxmaster();
-}
-
 function getPeerCoordinatorIds () {
   const myId = getMyCoordinatorId();
   return getAllCoordinatorIds().filter((id) => id !== myId);
@@ -91,20 +81,6 @@ function isFoxxmaster () {
   return global.ArangoServerState.isFoxxmaster();
 }
 
-function proxyToFoxxmaster (req, res) {
-  const coordId = getFoxmasterCoordinatorId();
-  const response = parallelClusterRequests([[
-    coordId,
-    req.method,
-    req._url.pathname + (req._url.search || ''),
-    req.rawBody,
-    req.headers
-  ]])[0];
-  res.statusCode = response.statusCode;
-  res.headers = response.headers;
-  res.body = response.rawBody;
-}
-
 function isClusterReadyForBusiness () {
   const coordIds = getPeerCoordinatorIds();
   return parallelClusterRequests(function * () {
@@ -157,61 +133,155 @@ function parallelClusterRequests (requests) {
   );
 }
 
-function isFoxxmasterReady () {
-  if (!ArangoClusterControl.isCluster()) {
-    return true;
-  }
-  const coordId = getFoxmasterCoordinatorId();
-  const response = parallelClusterRequests([[
-    coordId,
-    'GET',
-    '/_api/foxx/_local/status'
-  ]])[0];
-  if (response.statusCode >= 400) {
-    return false;
-  }
-  return JSON.parse(response.body).ready;
-}
-
-function getChecksumsFromPeers (mounts) {
-  const coordinatorIds = getPeerCoordinatorIds();
-  const responses = parallelClusterRequests(function * () {
-    for (const coordId of coordinatorIds) {
-      yield [
-        coordId,
-        'GET',
-        `/_api/foxx/_local/checksums?${querystringify({mount: mounts})}`
-      ];
-    }
-  }());
-  const peerChecksums = new Map();
-  for (const [coordId, response] of zip(coordinatorIds, responses)) {
-    const body = JSON.parse(response.body);
-    const coordChecksums = new Map();
-    for (const mount of mounts) {
-      coordChecksums.set(mount, body[mount] || null);
-    }
-    peerChecksums.set(coordId, coordChecksums);
-  }
-  return peerChecksums;
-}
-
 // Startup and self-heal
 
-function startup () {
+function selfHealAll (skipReloadRouting) {
   const db = require('internal').db;
   const dbName = db._name();
+  let modified;
   try {
     db._useDatabase('_system');
-    const writeToDatabase = isFoxxmaster();
     const databases = db._databases();
     for (const name of databases) {
       try {
         db._useDatabase(name);
-        rebuildAllServiceBundles(writeToDatabase);
+        modified = selfHeal() || modified;
-        if (writeToDatabase) {
+      } catch (e) {
-          upsertSystemServices();
+        console.warnStack(e);
       }
+    }
+  } finally {
+    db._useDatabase(dbName);
+    if (modified && !skipReloadRouting) {
+      reloadRouting();
+    }
+  }
+}
+
+function triggerSelfHeal () {
+  const modified = selfHeal();
+  if (modified) {
+    reloadRouting();
+  }
+}
+
+function selfHeal () {
+  const dirname = FoxxService.rootBundlePath();
+  if (!fs.exists(dirname)) {
+    fs.makeDirectoryRecursive(dirname);
+  }
+
+  const serviceCollection = utils.getStorage();
+  const bundleCollection = utils.getBundleStorage();
+  const serviceDefinitions = db._query(aql`
+    FOR doc IN ${serviceCollection}
+    FILTER LEFT(doc.mount, 2) != "/_"
+    LET bundleExists = DOCUMENT(${bundleCollection}, doc.checksum) != null
+    RETURN [doc.mount, doc.checksum, doc._rev, bundleExists]
+  `).toArray();
+
+  let modified = false;
+  const knownBundlePaths = new Array(serviceDefinitions.length);
+  const knownServicePaths = new Array(serviceDefinitions.length);
+  const localServiceMap = GLOBAL_SERVICE_MAP.get(db._name());
+  for (const [mount, checksum, rev, bundleExists] of serviceDefinitions) {
+    const bundlePath = FoxxService.bundlePath(mount);
+    const basePath = FoxxService.basePath(mount);
+    knownBundlePaths.push(bundlePath);
+    knownServicePaths.push(basePath);
+
+    if (localServiceMap) {
+      if (!localServiceMap.has(mount) || localServiceMap.get(mount)._rev !== rev) {
+        modified = true;
+      }
+    }
+
+    const hasBundle = fs.exists(bundlePath);
+    const hasFolder = fs.exists(basePath);
+    if (!hasBundle && hasFolder) {
+      createServiceBundle(mount);
+    } else if (hasBundle && !hasFolder) {
+      extractServiceBundle(bundlePath, basePath);
+      modified = true;
+    }
+    const isInstalled = hasBundle || hasFolder;
+    const localChecksum = isInstalled ? safeChecksum(mount) : null;
+
+    if (!checksum) {
+      // pre-3.2 service, can't self-heal this
+      continue;
+    }
+
+    if (checksum === localChecksum) {
+      if (!bundleExists) {
+        try {
+          bundleCollection._binaryInsert({_key: checksum}, bundlePath);
+        } catch (e) {
+          console.warnStack(e, `Failed to store missing service bundle for service at "${mount}"`);
+        }
+      }
+    } else if (bundleExists) {
+      try {
+        bundleCollection._binaryDocument(checksum, bundlePath);
+        extractServiceBundle(bundlePath, basePath);
+        modified = true;
+      } catch (e) {
+        console.errorStack(e, `Failed to load service bundle for service at "${mount}"`);
+      }
+    }
+  }
+
+  const rootPath = FoxxService.rootPath();
+  for (const relPath of fs.listTree(rootPath)) {
+    if (!relPath) {
+      continue;
+    }
+    const basename = path.basename(relPath);
+    if (basename.toUpperCase() !== 'APP') {
+      continue;
+    }
+    const basePath = path.resolve(rootPath, relPath);
+    if (!knownServicePaths.includes(basePath)) {
+      modified = true;
+      try {
+        fs.removeDirectoryRecursive(basePath, true);
+        console.debug(`Deleted orphaned service folder ${basePath}`);
+      } catch (e) {
+        console.warnStack(e, `Failed to delete orphaned service folder ${basePath}`);
+      }
+    }
+  }
+
+  const bundlesPath = FoxxService.rootBundlePath();
+  for (const relPath of fs.listTree(bundlesPath)) {
+    if (!relPath) {
+      continue;
+    }
+    const bundlePath = path.resolve(bundlesPath, relPath);
+    if (!knownBundlePaths.includes(bundlePath)) {
+      try {
+        fs.remove(bundlePath);
+        console.debug(`Deleted orphaned service bundle ${bundlePath}`);
+      } catch (e) {
+        console.warnStack(e, `Failed to delete orphaned service bundle ${bundlePath}`);
+      }
+    }
+  }
+
+  return modified;
+}
+
+function startup () {
+  if (isFoxxmaster()) {
+    const db = require('internal').db;
+    const dbName = db._name();
+    try {
+      db._useDatabase('_system');
+      const databases = db._databases();
+      for (const name of databases) {
+        try {
+          db._useDatabase(name);
+          upsertSystemServices();
         } catch (e) {
           console.warnStack(e);
         }
@@ -220,6 +290,8 @@ function startup () {
       db._useDatabase(dbName);
     }
   }
+  selfHealAll(true);
+}
 
 function upsertSystemServices () {
   const serviceDefinitions = new Map();
@@ -237,238 +309,6 @@ function upsertSystemServices () {
   `);
 }
 
-function rebuildAllServiceBundles (fixMissingChecksums) {
-  const servicesMissingChecksums = [];
-  const collection = utils.getStorage();
-  for (const serviceDefinition of collection.all()) {
-    const mount = serviceDefinition.mount;
-    if (mount.startsWith('/_')) {
-      continue;
-    }
-    const bundlePath = FoxxService.bundlePath(mount);
-    const basePath = FoxxService.basePath(mount);
-
-    const hasBundle = fs.exists(bundlePath);
-    const hasFolder = fs.exists(basePath);
-    if (!hasBundle && hasFolder) {
-      createServiceBundle(mount);
-    } else if (hasBundle && !hasFolder) {
-      extractServiceBundle(bundlePath, basePath);
-    } else if (!hasBundle && !hasFolder) {
-      continue;
-    }
-    if (fixMissingChecksums && !serviceDefinition.checksum) {
-      servicesMissingChecksums.push({
-        checksum: FoxxService.checksum(mount),
-        _key: serviceDefinition._key
-      });
-    }
-  }
-  if (!servicesMissingChecksums.length) {
-    return;
-  }
-  db._query(aql`
-    FOR service IN ${servicesMissingChecksums}
-    UPDATE service._key
-    WITH {checksum: service.checksum}
-    IN ${collection}
-  `);
-}
-
-function selfHeal () {
-  const db = require('internal').db;
-  const dbName = db._name();
-  try {
-    db._useDatabase('_system');
-    const databases = db._databases();
-    const weAreFoxxmaster = isFoxxmaster();
-    const foxxmasterIsReady = weAreFoxxmaster || isFoxxmasterReady();
-    for (const name of databases) {
-      try {
-        db._useDatabase(name);
-        if (weAreFoxxmaster) {
-          healMyselfAndCoords();
-        } else if (foxxmasterIsReady) {
-          healMyself();
-        }
-      } catch (e) {
-        console.warnStack(e);
-      }
-    }
-    return foxxmasterIsReady;
-  } finally {
-    db._useDatabase(dbName);
-  }
-}
-
-function healMyself () {
-  const servicesINeedToFix = new Map();
-
-  const collection = utils.getStorage();
-  for (const serviceDefinition of collection.all()) {
-    const mount = serviceDefinition.mount;
-    const checksum = serviceDefinition.checksum;
-    if (mount.startsWith('/_')) {
-      continue;
-    }
-    if (!checksum || checksum !== safeChecksum(mount)) {
-      servicesINeedToFix.set(mount, checksum);
-    }
-  }
-
-  const coordinatorIds = getPeerCoordinatorIds();
-  let clusterIsInconsistent = false;
-  for (const [mount, checksum] of servicesINeedToFix) {
-    const coordIdsToTry = shuffle(coordinatorIds);
-    let found = false;
-    for (const coordId of coordIdsToTry) {
-      const bundle = downloadServiceBundleFromCoordinator(coordId, mount, checksum);
-      if (bundle) {
-        replaceLocalServiceFromTempBundle(mount, bundle);
-        found = true;
-        break;
-      }
-    }
-    if (!found) {
-      clusterIsInconsistent = true;
-      break;
-    }
-  }
-
-  if (clusterIsInconsistent) {
-    const coordId = getFoxmasterCoordinatorId();
-    parallelClusterRequests([[
-      coordId,
-      'POST',
-      '/_api/foxx/_local/heal'
-    ]]);
-  }
-}
-
-function healMyselfAndCoords () {
-  const checksumsINeedToFixLocally = [];
-  const actualChecksums = new Map();
-  const coordsKnownToBeGoodSources = new Map();
-  const coordsKnownToBeBadSources = new Map();
-  const allKnownMounts = [];
-
-  const collection = utils.getStorage();
-  for (const serviceDefinition of collection.all()) {
-    const mount = serviceDefinition.mount;
-    const checksum = serviceDefinition.checksum;
-    if (mount.startsWith('/_')) {
-      continue;
-    }
-    allKnownMounts.push(mount);
-    actualChecksums.set(mount, checksum);
-    coordsKnownToBeGoodSources.set(mount, []);
-    coordsKnownToBeBadSources.set(mount, new Map());
-    if (!checksum || checksum !== safeChecksum(mount)) {
-      checksumsINeedToFixLocally.push(mount);
-    }
-  }
-
-  const serviceChecksumsByCoordinator = getChecksumsFromPeers(allKnownMounts);
-  for (const [coordId, serviceChecksums] of serviceChecksumsByCoordinator) {
-    for (const [mount, checksum] of serviceChecksums) {
-      if (!checksum) {
-        coordsKnownToBeBadSources.get(mount).set(coordId, null);
-      } else if (!actualChecksums.get(mount)) {
-        actualChecksums.set(mount, checksum);
-        coordsKnownToBeGoodSources.get(mount).push(coordId);
-      } else if (actualChecksums.get(mount) === checksum) {
-        coordsKnownToBeGoodSources.get(mount).push(coordId);
-      } else {
-        coordsKnownToBeBadSources.get(mount).set(coordId, checksum);
-      }
-    }
-  }
-
-  const myId = getMyCoordinatorId();
-  const serviceMountsToDeleteInCollection = [];
-  const serviceChecksumsToUpdateInCollection = new Map();
-  for (const mount of checksumsINeedToFixLocally) {
-    const possibleSources = coordsKnownToBeGoodSources.get(mount);
-    if (!possibleSources.length) {
-      const myChecksum = safeChecksum(mount);
-      if (myChecksum) {
-        serviceChecksumsToUpdateInCollection.set(mount, myChecksum);
-      } else {
-        let found = false;
-        for (const [coordId, coordChecksum] of coordsKnownToBeBadSources.get(mount)) {
-          if (!coordChecksum) {
-            continue;
-          }
-          const bundle = downloadServiceBundleFromCoordinator(coordId, mount, coordChecksum);
-          if (bundle) {
-            serviceChecksumsToUpdateInCollection.set(mount, coordChecksum);
-            possibleSources.push(coordId);
-            replaceLocalServiceFromTempBundle(mount, bundle);
-            found = true;
-            break;
-          }
-        }
-        if (!found) {
-          serviceMountsToDeleteInCollection.push(mount);
-          coordsKnownToBeBadSources.delete(mount);
-        }
-      }
-    } else {
-      const checksum = actualChecksums.get(mount);
-      for (const coordId of possibleSources) {
-        const bundle = downloadServiceBundleFromCoordinator(coordId, mount, checksum);
-        if (bundle) {
-          replaceLocalServiceFromTempBundle(mount, bundle);
-          break;
-        }
-      }
-    }
-  }
-
-  for (const ids of coordsKnownToBeGoodSources.values()) {
-    ids.push(myId);
-  }
-
-  db._query(aql`
-    FOR service IN ${collection}
-    FILTER service.mount IN ${serviceMountsToDeleteInCollection}
-    REMOVE service
-    IN ${collection}
-  `);
-
-  db._query(aql`
-    FOR service IN ${collection}
-    FOR item IN ${Array.from(serviceChecksumsToUpdateInCollection)}
-    FILTER service.mount == item[0]
-    UPDATE service
-    WITH {checksum: item[1]}
-    IN ${collection}
-  `);
-
-  parallelClusterRequests(function * () {
-    for (const coordId of getPeerCoordinatorIds()) {
-      const servicesYouNeedToUpdate = {};
-      for (const [mount, badCoordinatorIds] of coordsKnownToBeBadSources) {
-        if (!badCoordinatorIds.has(coordId)) {
-          continue;
-        }
-        const goodCoordIds = coordsKnownToBeGoodSources.get(mount);
-        servicesYouNeedToUpdate[mount] = shuffle(goodCoordIds);
-      }
-      if (!Object.keys(servicesYouNeedToUpdate).length) {
-        continue;
-      }
-      yield [
-        coordId,
-        'POST',
-        '/_api/foxx/_local',
-        JSON.stringify(servicesYouNeedToUpdate),
-        {'content-type': 'application/json'}
-      ];
-    }
-  }());
-}
-
 // Change propagation
 
 function reloadRouting () {
@@ -476,45 +316,10 @@ function reloadRouting () {
   actions.reloadRouting();
 }
 
-function propagateServiceDestroyed (service) {
+function propagateSelfHeal () {
   parallelClusterRequests(function * () {
     for (const coordId of getPeerCoordinatorIds()) {
-      yield [coordId, 'DELETE', `/_api/foxx/_local/service?${querystringify({
+      yield [coordId, 'POST', '/_api/foxx/_local/heal'];
-        mount: service.mount
-      })}`];
-    }
-  }());
-  reloadRouting();
-}
-
-function propagateServiceReplaced (service) {
-  const myId = getMyCoordinatorId();
-  const coordIds = getPeerCoordinatorIds();
-  const results = parallelClusterRequests(function * () {
-    for (const coordId of coordIds) {
-      yield [
-        coordId,
-        'POST',
-        '/_api/foxx/_local',
-        JSON.stringify({[service.mount]: [myId]}),
-        {'content-type': 'application/json'}
-      ];
-    }
-  }());
-  for (const [coordId, result] of zip(coordIds, results)) {
-    if (result.statusCode >= 400) {
-      console.error(`Failed to propagate service ${service.mount} to coord ${coordId}`);
-    }
-  }
-  reloadRouting();
-}
-
-function propagateServiceReconfigured (service) {
-  parallelClusterRequests(function * () {
-    for (const coordId of getPeerCoordinatorIds()) {
-      yield [coordId, 'POST', `/_api/foxx/_local/service?${querystringify({
-        mount: service.mount
-      })}`];
     }
   }());
   reloadRouting();
@@ -715,13 +520,19 @@ function _install (mount, options = {}) {
     service.executeScript('setup');
   }
   service.updateChecksum();
+  const bundleCollection = utils.getBundleStorage();
+  if (!bundleCollection.exists(service.checksum)) {
+    bundleCollection._binaryInsert({_key: service.checksum}, service.bundlePath);
+  }
   const serviceDefinition = service.toJSON();
-  db._query(aql`
+  const meta = db._query(aql`
     UPSERT {mount: ${mount}}
     INSERT ${serviceDefinition}
     REPLACE ${serviceDefinition}
     IN ${collection}
-  `);
+    RETURN NEW
+  `).next();
+  service._rev = meta._rev;
   ensureServiceExecuted(service, true);
   return service;
 }
@@ -746,11 +557,22 @@ function _uninstall (mount, options = {}) {
     }
   }
   const collection = utils.getStorage();
-  db._query(aql`
+  const serviceDefinition = db._query(aql`
     FOR service IN ${collection}
     FILTER service.mount == ${mount}
     REMOVE service IN ${collection}
-  `);
+    RETURN OLD
+  `).next();
+  if (serviceDefinition) {
+    const checksumRefs = db._query(aql`
+      FOR service IN ${collection}
+      FILTER service.checksum == ${serviceDefinition.checksum}
+      RETURN 1
+    `).toArray();
+    if (!checksumRefs.length) {
+      utils.getBundleStorage().remove(serviceDefinition.checksum);
+    }
+  }
   GLOBAL_SERVICE_MAP.get(db._name()).delete(mount);
   const servicePath = FoxxService.basePath(mount);
   if (fs.exists(servicePath)) {
@@ -812,22 +634,6 @@ function downloadServiceBundleFromRemote (url) {
   }
 }
 
-function downloadServiceBundleFromCoordinator (coordId, mount, checksum) {
-  const response = parallelClusterRequests([[
-    coordId,
-    'GET',
-    `/_api/foxx/_local/bundle?${querystringify({mount})}`,
-    null,
-    checksum ? {'if-match': `"${checksum}"`} : undefined
-  ]])[0];
-  if (response.headers['x-arango-response-code'].startsWith('404')) {
-    return null;
-  }
-  const filename = fs.getTempFile('bundles', true);
-  fs.writeFileSync(filename, response.rawBody);
-  return filename;
-}
-
 function extractServiceBundle (archive, targetPath) {
   const tempFolder = fs.getTempFile('services', false);
   fs.makeDirectory(tempFolder);
@@ -866,12 +672,6 @@ function extractServiceBundle (archive, targetPath) {
   }
 }
 
-function replaceLocalServiceFromTempBundle (mount, tempBundlePath) {
-  const tempServicePath = fs.getTempFile('services', false);
-  extractServiceBundle(tempBundlePath, tempServicePath);
-  _buildServiceInPath(mount, tempServicePath, tempBundlePath);
-}
-
 // Exported functions for manipulating services
 
 function install (serviceInfo, mount, options = {}) {
@@ -889,44 +689,14 @@ function install (serviceInfo, mount, options = {}) {
   const tempPaths = _prepareService(serviceInfo, options);
   _buildServiceInPath(mount, tempPaths.tempServicePath, tempPaths.tempBundlePath);
   const service = _install(mount, options);
-  propagateServiceReplaced(service);
+  propagateSelfHeal();
   return service;
 }
 
-function installLocal (mount, coordIds) {
-  for (const coordId of coordIds) {
-    const filename = downloadServiceBundleFromCoordinator(coordId, mount);
-    if (filename) {
-      replaceLocalServiceFromTempBundle(mount, filename);
-      return true;
-    }
-  }
-  return false;
-}
-
-function uninstallLocal (mount) {
-  const servicePath = FoxxService.basePath(mount);
-  if (fs.exists(servicePath)) {
-    try {
-      fs.removeDirectoryRecursive(servicePath, true);
-    } catch (e) {
-      console.warnStack(e);
-    }
-  }
-  const bundlePath = FoxxService.bundlePath(mount);
-  if (fs.exists(bundlePath)) {
-    try {
-      fs.remove(bundlePath);
-    } catch (e) {
-      console.warnStack(e);
-    }
-  }
-}
-
 function uninstall (mount, options = {}) {
   ensureFoxxInitialized();
   const service = _uninstall(mount, options);
-  propagateServiceDestroyed(service);
+  propagateSelfHeal();
   return service;
 }
 
@@ -942,7 +712,7 @@ function replace (serviceInfo, mount, options = {}) {
   _uninstall(mount, Object.assign({teardown: true}, options, {force: true}));
   _buildServiceInPath(mount, tempPaths.tempServicePath, tempPaths.tempBundlePath);
   const service = _install(mount, Object.assign({}, options, {force: true}));
-  propagateServiceReplaced(service);
+  propagateSelfHeal();
   return service;
 }
 
@@ -962,7 +732,7 @@ function upgrade (serviceInfo, mount, options = {}) {
   _uninstall(mount, Object.assign({teardown: false}, options, {force: true}));
   _buildServiceInPath(mount, tempPaths.tempServicePath, tempPaths.tempBundlePath);
   const service = _install(mount, Object.assign({}, options, serviceOptions, {force: true}));
-  propagateServiceReplaced(service);
+  propagateSelfHeal();
   return service;
 }
 
@@ -990,7 +760,7 @@ function enableDevelopmentMode (mount) {
   const service = getServiceInstance(mount);
   service.development(true);
   utils.updateService(mount, service.toJSON());
-  propagateServiceReconfigured(service);
+  propagateSelfHeal();
   return service;
 }
 
@@ -1002,7 +772,7 @@ function disableDevelopmentMode (mount) {
   utils.updateService(mount, service.toJSON());
   // Make sure setup changes from devmode are respected
   service.executeScript('setup');
-  propagateServiceReplaced(service);
+  propagateSelfHeal();
   return service;
 }
 
@@ -1010,7 +780,7 @@ function setConfiguration (mount, options = {}) {
   const service = getServiceInstance(mount);
   const warnings = service.applyConfiguration(options.configuration, options.replace);
   utils.updateService(mount, service.toJSON());
-  propagateServiceReconfigured(service);
+  propagateSelfHeal();
   return warnings;
 }
 
@@ -1018,7 +788,7 @@ function setDependencies (mount, options = {}) {
   const service = getServiceInstance(mount);
   const warnings = service.applyDependencies(options.dependencies, options.replace);
   utils.updateService(mount, service.toJSON());
-  propagateServiceReconfigured(service);
+  propagateSelfHeal();
   return warnings;
 }
 
@@ -1073,8 +843,6 @@ function safeChecksum (mount) {
 
 // Exports
 
-exports._installLocal = installLocal;
-exports._uninstallLocal = uninstallLocal;
 exports.install = install;
 exports.uninstall = uninstall;
 exports.replace = replace;
@@ -1094,15 +862,14 @@ exports.installedServices = installedServices;
 // -------------------------------------------------
 
 exports.isFoxxmaster = isFoxxmaster;
-exports.proxyToFoxxmaster = proxyToFoxxmaster;
 exports._reloadRouting = reloadRouting;
 exports.reloadInstalledService = reloadInstalledService;
 exports.ensureRouted = ensureServiceLoaded;
 exports.initializeFoxx = initLocalServiceMap;
 exports.ensureFoxxInitialized = ensureFoxxInitialized;
-exports.heal = healMyselfAndCoords;
 exports._startup = startup;
-exports._selfHeal = selfHeal;
+exports.heal = triggerSelfHeal;
+exports.healAll = selfHealAll;
 exports._createServiceBundle = createServiceBundle;
 exports._resetCache = () => GLOBAL_SERVICE_MAP.clear();
 exports._mountPoints = getMountPoints;

js/server/modules/@arangodb/foxx/service.js

@@ -107,7 +107,7 @@ module.exports =
       return manifest;
     }
 
-    static validateServiceFiles (mount, manifest) {
+    static validateServiceFiles (mount, manifest, rev) {
       const servicePath = FoxxService.basePath(mount);
       if (manifest.main) {
         parseFile(servicePath, manifest.main);
@@ -140,6 +140,7 @@ module.exports =
     }
 
     constructor (definition, manifest) {
+      this._rev = definition._rev;
       this.mount = definition.mount;
       this.checksum = definition.checksum;
       this.basePath = definition.basePath || FoxxService.basePath(this.mount);
@@ -642,12 +643,20 @@ module.exports =
     }
 
     static rootPath (mount) {
-      if (mount.charAt(1) === '_') {
+      if (mount && mount.charAt(1) === '_') {
         return FoxxService._systemAppPath;
       }
       return FoxxService._appPath;
     }
 
+    static rootBundlePath (mount) {
+      return path.resolve(
+        FoxxService.rootPath(mount),
+        '_appbundles'
+      );
+    }
+
+
     static basePath (mount) {
       return path.resolve(
         FoxxService.rootPath(mount),
@@ -661,7 +670,7 @@ module.exports =
         mount = '/' + mount;
       }
       const bundleName = mount.substr(1).replace(/[-.:/]/g, '_');
-      return path.join(FoxxService.rootPath(mount), bundleName + '.zip');
+      return path.join(FoxxService.rootBundlePath(mount), bundleName + '.zip');
     }
 
     static get _startupPath () {

js/server/server.js

@@ -67,8 +67,7 @@
   if (internal.threadNumber === 0 && global.ArangoServerState.role() === 'SINGLE') {
     if (restServer) {
       // startup the foxx manager once
-      require('@arangodb/foxx/manager')._startup(true);
+      require('@arangodb/foxx/manager')._startup();
-      require('@arangodb/foxx/manager')._selfHeal(true);
     }
 
     // start the queue manager once

lib/Basics/errors.dat

@@ -44,6 +44,7 @@ ERROR_HTTP_NOT_FOUND,404,"not found","Will be raised when an URI is unknown."
 ERROR_HTTP_METHOD_NOT_ALLOWED,405,"method not supported","Will be raised when an unsupported HTTP method is used for an operation."
 ERROR_HTTP_PRECONDITION_FAILED,412,"precondition failed","Will be raised when a precondition for an HTTP request is not met."
 ERROR_HTTP_SERVER_ERROR,500,"internal server error","Will be raised when an internal server is encountered."
+ERROR_HTTP_SERVICE_UNAVAILABLE,503,"service unavailable","Will be raised when a service is temporarily unavailable."
 
 ################################################################################
 ## HTTP processing errors
@@ -382,6 +383,8 @@ COMMUNICATOR_REQUEST_ABORTED,2100,"Request aborted","Request was aborted."
 
 ERROR_MALFORMED_MANIFEST_FILE,3000,"failed to parse manifest file","The service manifest file is not well-formed JSON."
 ERROR_INVALID_SERVICE_MANIFEST,3001,"manifest file is invalid","The service manifest contains invalid values."
+ERROR_SERVICE_FILES_MISSING,3002,"service files missing","The service folder or bundle does not exist on this server."
+ERROR_SERVICE_FILES_OUTDATED,3003,"service files outdated","The local service bundle does not match the checksum in the database."
 ERROR_INVALID_FOXX_OPTIONS,3004,"service options are invalid","The service options contain invalid values."
 ERROR_INVALID_MOUNTPOINT,3007,"invalid mountpath","The service mountpath contains invalid characters."
 ERROR_SERVICE_NOT_FOUND,3009,"service not found","No service found at the given mountpath."

lib/Basics/voc-errors.cpp

@@ -43,6 +43,7 @@ void TRI_InitializeErrorMessages () {
   REG_ERROR(ERROR_HTTP_METHOD_NOT_ALLOWED, "method not supported");
   REG_ERROR(ERROR_HTTP_PRECONDITION_FAILED, "precondition failed");
   REG_ERROR(ERROR_HTTP_SERVER_ERROR, "internal server error");
+  REG_ERROR(ERROR_HTTP_SERVICE_UNAVAILABLE, "service unavailable");
   REG_ERROR(ERROR_HTTP_CORRUPTED_JSON, "invalid JSON object");
   REG_ERROR(ERROR_HTTP_SUPERFLUOUS_SUFFICES, "superfluous URL suffices");
   REG_ERROR(ERROR_ARANGO_ILLEGAL_STATE, "illegal state");
@@ -270,6 +271,8 @@ void TRI_InitializeErrorMessages () {
   REG_ERROR(COMMUNICATOR_REQUEST_ABORTED, "Request aborted");
   REG_ERROR(ERROR_MALFORMED_MANIFEST_FILE, "failed to parse manifest file");
   REG_ERROR(ERROR_INVALID_SERVICE_MANIFEST, "manifest file is invalid");
+  REG_ERROR(ERROR_SERVICE_FILES_MISSING, "service files missing");
+  REG_ERROR(ERROR_SERVICE_FILES_OUTDATED, "service files outdated");
   REG_ERROR(ERROR_INVALID_FOXX_OPTIONS, "service options are invalid");
   REG_ERROR(ERROR_INVALID_MOUNTPOINT, "invalid mountpath");
   REG_ERROR(ERROR_SERVICE_NOT_FOUND, "service not found");

lib/Basics/voc-errors.h

@@ -85,6 +85,8 @@
 ///   Will be raised when a precondition for an HTTP request is not met.
 /// - 500: @LIT{internal server error}
 ///   Will be raised when an internal server is encountered.
+/// - 503: @LIT{service unavailable}
+///   Will be raised when a service is temporarily unavailable.
 /// - 600: @LIT{invalid JSON object}
 ///   Will be raised when a string representation of a JSON object is corrupt.
 /// - 601: @LIT{superfluous URL suffices}
@@ -646,6 +648,10 @@
 ///   The service manifest file is not well-formed JSON.
 /// - 3001: @LIT{manifest file is invalid}
 ///   The service manifest contains invalid values.
+/// - 3002: @LIT{service files missing}
+///   The service folder or bundle does not exist on this server.
+/// - 3003: @LIT{service files outdated}
+///   The local service bundle does not match the checksum in the database.
 /// - 3004: @LIT{service options are invalid}
 ///   The service options contain invalid values.
 /// - 3007: @LIT{invalid mountpath}
@@ -1093,6 +1099,16 @@ void TRI_InitializeErrorMessages ();
 
 #define TRI_ERROR_HTTP_SERVER_ERROR                                       (500)
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief 503: ERROR_HTTP_SERVICE_UNAVAILABLE
+///
+/// service unavailable
+///
+/// Will be raised when a service is temporarily unavailable.
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_ERROR_HTTP_SERVICE_UNAVAILABLE                                (503)
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief 600: ERROR_HTTP_CORRUPTED_JSON
 ///
@@ -3461,6 +3477,26 @@ void TRI_InitializeErrorMessages ();
 
 #define TRI_ERROR_INVALID_SERVICE_MANIFEST                                (3001)
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief 3002: ERROR_SERVICE_FILES_MISSING
+///
+/// service files missing
+///
+/// The service folder or bundle does not exist on this server.
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_ERROR_SERVICE_FILES_MISSING                                   (3002)
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief 3003: ERROR_SERVICE_FILES_OUTDATED
+///
+/// service files outdated
+///
+/// The local service bundle does not match the checksum in the database.
+////////////////////////////////////////////////////////////////////////////////
+
+#define TRI_ERROR_SERVICE_FILES_OUTDATED                                  (3003)
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief 3004: ERROR_INVALID_FOXX_OPTIONS
 ///