From 920cf4b7c37b68f617653f5b8145c44cdae4443a Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 6 Jun 2014 15:00:26 +0200 Subject: [PATCH] Added fetchFromGithub to server-side foxx manager. Makes it a lot easier to update an already installed foxx from aardvark --- .../modules/org/arangodb/foxx/manager.js | 210 +-------------- .../org/arangodb/foxx/manager-utils.js | 243 ++++++++++++++++++ .../modules/org/arangodb/foxx/manager.js | 48 ++++ 3 files changed, 301 insertions(+), 200 deletions(-) create mode 100644 js/common/modules/org/arangodb/foxx/manager-utils.js diff --git a/js/client/modules/org/arangodb/foxx/manager.js b/js/client/modules/org/arangodb/foxx/manager.js index f56717b35e..618e388efb 100644 --- a/js/client/modules/org/arangodb/foxx/manager.js +++ b/js/client/modules/org/arangodb/foxx/manager.js @@ -1,4 +1,4 @@ -/*jslint indent: 2, nomen: true, maxlen: 130, vars: true, white: true, plusplus: true, nonpropdel: true, continue: true, regexp: true */ +/*jslint indent: 2, nomen: true, maxlen: 130, vars: true, white: true, plusplus: true, continue: true, regexp: true */ /*global require, exports, module */ //////////////////////////////////////////////////////////////////////////////// @@ -45,6 +45,7 @@ var checkedFishBowl = false; var arango = require("internal").arango; var download = require("internal").download; +var utils = require("org/arangodb/foxx/manager-utils"); // ----------------------------------------------------------------------------- // --SECTION-- private functions @@ -89,25 +90,12 @@ function getFishbowlStorage () { /// @brief returns the fishbow repository //////////////////////////////////////////////////////////////////////////////// -function getFishbowlUrl (version) { +function getFishbowlUrl () { 'use strict'; return "arangodb/foxx-apps"; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief builds a github repository URL -//////////////////////////////////////////////////////////////////////////////// - -function buildGithubUrl (repository, version) { - 'use strict'; - - if (typeof version === "undefined") { - version = "master"; - } - - return 'https://github.com/' + repository + '/archive/' + version + '.zip'; -} //////////////////////////////////////////////////////////////////////////////// /// @brief builds a github repository URL @@ -153,152 +141,6 @@ function validateMount (mnt) { }); } -//////////////////////////////////////////////////////////////////////////////// -/// @brief extracts the name and version from a manifest file -//////////////////////////////////////////////////////////////////////////////// - -function extractNameAndVersionManifest (source, filename) { - 'use strict'; - - var manifest = JSON.parse(fs.read(filename)); - - source.name = manifest.name; - source.version = manifest.version; -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief processes files in a directory -//////////////////////////////////////////////////////////////////////////////// - -function processDirectory (source) { - 'use strict'; - - var location = source.location; - - if (! fs.exists(location) || ! fs.isDirectory(location)) { - throwFileNotFound("'" + String(location) + "' is not a directory"); - } - - // ............................................................................. - // extract name and version from manifest - // ............................................................................. - - extractNameAndVersionManifest(source, fs.join(location, "manifest.json")); - - // ............................................................................. - // extract name and version from manifest - // ............................................................................. - - var tree = fs.listTree(location); - var files = []; - var i; - - for (i = 0; i < tree.length; ++i) { - var filename = fs.join(location, tree[i]); - - if (fs.isFile(filename)) { - files.push(tree[i]); - } - } - - if (files.length === 0) { - throwFileNotFound("Directory '" + String(location) + "' is empty"); - } - - var tempFile = fs.getTempFile("downloads", false); - source.filename = tempFile; - source.removeFile = true; - - fs.zipFile(tempFile, location, files); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief extracts the name and version from a zip -//////////////////////////////////////////////////////////////////////////////// - -function repackZipFile (source) { - 'use strict'; - - var i; - - var filename = source.filename; - var path = fs.getTempFile("zip", false); - - fs.unzipFile(filename, path, false, true); - - // ............................................................................. - // locate the manifest file - // ............................................................................. - - var tree = fs.listTree(path).sort(function(a,b) { - return a.length - b.length; - }); - var found; - var mf = "manifest.json"; - var re = /[\/\\\\]manifest\.json$/; // Windows! - - for (i = 0; i < tree.length && found === undefined; ++i) { - var tf = tree[i]; - - if (re.test(tf) || tf === mf) { - found = tf; - } - } - - if (typeof found === "undefined") { - throwFileNotFound("Cannot find manifest file in zip file '" + filename + "'"); - } - - var mp; - - if (found === mf) { - mp = "."; - } - else { - mp = found.substr(0, found.length - mf.length - 1); - } - - // ............................................................................. - // throw away source file if necessary - // ............................................................................. - - if (source.removeFile && source.filename !== '') { - try { - fs.remove(source.filename); - } - catch (err1) { - arangodb.printf("Cannot remove temporary file '%s'\n", source.filename); - } - } - - delete source.filename; - delete source.removeFile; - - // ............................................................................. - // repack the zip file - // ............................................................................. - - var newSource = { location: fs.join(path, mp) }; - - processDirectory(newSource); - - source.name = newSource.name; - source.version = newSource.version; - source.filename = newSource.filename; - source.removeFile = newSource.removeFile; - - // ............................................................................. - // cleanup temporary paths - // ............................................................................. - - try { - fs.removeDirectoryRecursive(path); - } - catch (err2) { - arangodb.printf("Cannot remove temporary directory '%s'\n", path); - } -} - //////////////////////////////////////////////////////////////////////////////// /// @brief processes files in a zip file //////////////////////////////////////////////////////////////////////////////// @@ -315,39 +157,7 @@ function processZip (source) { source.filename = source.location; source.removeFile = false; - repackZipFile(source); -} - -//////////////////////////////////////////////////////////////////////////////// -/// @brief processes files from a github repository -//////////////////////////////////////////////////////////////////////////////// - -function processGithubRepository (source) { - 'use strict'; - - var url = buildGithubUrl(source.location, source.version); - var tempFile = fs.getTempFile("downloads", false); - - try { - var result = download(url, "", { - method: "get", - followRedirects: true, - timeout: 30 - }, tempFile); - - if (result.code >= 200 && result.code <= 299) { - source.filename = tempFile; - source.removeFile = true; - } - else { - throwDownloadError("Could not download from repository '" + url + "'"); - } - } - catch (err) { - throwDownloadError("Could not download from repository '" + url + "': " + String(err)); - } - - repackZipFile(source); + utils.repackZipFile(source); } //////////////////////////////////////////////////////////////////////////////// @@ -361,10 +171,10 @@ function processSource (src) { processZip(src); } else if (src.type === "directory") { - processDirectory(src); + utils.processDirectory(src); } else if (src.type === "github") { - processGithubRepository(src); + utils.processGithubRepository(src); } else { throwBadParameter("Unknown application type '" + src.type + "'. " + @@ -415,7 +225,7 @@ function updateFishbowlFromZip (filename) { try { fs.remove(fs.join(root, file)); } - catch (err3) { + catch (ignore) { } } }); @@ -487,7 +297,7 @@ function updateFishbowlFromZip (filename) { try { fs.removeDirectoryRecursive(tempPath); } - catch (err2) { + catch (ignore) { } } @@ -502,7 +312,7 @@ function updateFishbowlFromZip (filename) { function updateFishbowl () { 'use strict'; - var url = buildGithubUrl(getFishbowlUrl()); + var url = utils.buildGithubUrl(getFishbowlUrl()); var filename = fs.getTempFile("downloads", false); var path = fs.getTempFile("zip", false); @@ -529,7 +339,7 @@ function updateFishbowl () { try { fs.removeDirectoryRecursive(path); } - catch (err2) { + catch (ignore) { } throw err; diff --git a/js/common/modules/org/arangodb/foxx/manager-utils.js b/js/common/modules/org/arangodb/foxx/manager-utils.js new file mode 100644 index 0000000000..9281bff7d8 --- /dev/null +++ b/js/common/modules/org/arangodb/foxx/manager-utils.js @@ -0,0 +1,243 @@ +/*jslint indent: 2, nomen: true, maxlen: 130, vars: true, white: true, plusplus: true, continue: true, regexp: true */ +/*global require, exports, module */ + +//////////////////////////////////////////////////////////////////////////////// +/// @brief ArangoDB Application Launcher +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2013 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is triAGENS GmbH, Cologne, Germany +/// +/// @author Jan Steemann +/// @author Dr. Frank Celler +/// @author Michael Hackstein +/// @author Copyright 2014, triAGENS GmbH, Cologne, Germany +//////////////////////////////////////////////////////////////////////////////// + +var fs = require("fs"); +var arangodb = require("org/arangodb"); +var download = require("internal").download; + +var throwFileNotFound = arangodb.throwFileNotFound; +var throwDownloadError = arangodb.throwDownloadError; + +//////////////////////////////////////////////////////////////////////////////// +/// @brief builds a github repository URL +//////////////////////////////////////////////////////////////////////////////// + +function buildGithubUrl (repository, version) { + 'use strict'; + + if (version === undefined) { + version = "master"; + } + + return 'https://github.com/' + repository + '/archive/' + version + '.zip'; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief extracts the name and version from a manifest file +//////////////////////////////////////////////////////////////////////////////// + +function extractNameAndVersionManifest (source, filename) { + 'use strict'; + + var manifest = JSON.parse(fs.read(filename)); + + source.name = manifest.name; + source.version = manifest.version; +} + + +//////////////////////////////////////////////////////////////////////////////// +/// @brief processes files in a directory +//////////////////////////////////////////////////////////////////////////////// + +function processDirectory (source) { + 'use strict'; + + var location = source.location; + + if (! fs.exists(location) || ! fs.isDirectory(location)) { + throwFileNotFound("'" + String(location) + "' is not a directory"); + } + + // ............................................................................. + // extract name and version from manifest + // ............................................................................. + + extractNameAndVersionManifest(source, fs.join(location, "manifest.json")); + + // ............................................................................. + // extract name and version from manifest + // ............................................................................. + + var tree = fs.listTree(location); + var files = []; + var i; + var filename; + + for (i = 0; i < tree.length; ++i) { + filename = fs.join(location, tree[i]); + + if (fs.isFile(filename)) { + files.push(tree[i]); + } + } + + if (files.length === 0) { + throwFileNotFound("Directory '" + String(location) + "' is empty"); + } + + var tempFile = fs.getTempFile("downloads", false); + source.filename = tempFile; + source.removeFile = true; + + fs.zipFile(tempFile, location, files); +} + + +//////////////////////////////////////////////////////////////////////////////// +/// @brief extracts the name and version from a zip +//////////////////////////////////////////////////////////////////////////////// + +function repackZipFile (source) { + 'use strict'; + + var i; + + var filename = source.filename; + var path = fs.getTempFile("zip", false); + + fs.unzipFile(filename, path, false, true); + + // ............................................................................. + // locate the manifest file + // ............................................................................. + + var tree = fs.listTree(path).sort(function(a,b) { + return a.length - b.length; + }); + var found; + var mf = "manifest.json"; + var re = /[\/\\\\]manifest\.json$/; // Windows! + var tf; + + for (i = 0; i < tree.length && found === undefined; ++i) { + tf = tree[i]; + + if (re.test(tf) || tf === mf) { + found = tf; + } + } + + if (found === undefined) { + throwFileNotFound("Cannot find manifest file in zip file '" + filename + "'"); + } + + var mp; + + if (found === mf) { + mp = "."; + } + else { + mp = found.substr(0, found.length - mf.length - 1); + } + + // ............................................................................. + // throw away source file if necessary + // ............................................................................. + + if (source.removeFile && source.filename !== '') { + try { + fs.remove(source.filename); + } + catch (err1) { + arangodb.printf("Cannot remove temporary file '%s'\n", source.filename); + } + } + + delete source.filename; + delete source.removeFile; + + // ............................................................................. + // repack the zip file + // ............................................................................. + + var newSource = { location: fs.join(path, mp) }; + + processDirectory(newSource); + + source.name = newSource.name; + source.version = newSource.version; + source.filename = newSource.filename; + source.removeFile = newSource.removeFile; + + // ............................................................................. + // cleanup temporary paths + // ............................................................................. + + try { + fs.removeDirectoryRecursive(path); + } + catch (err2) { + arangodb.printf("Cannot remove temporary directory '%s'\n", path); + } +} +//////////////////////////////////////////////////////////////////////////////// +/// @brief processes files from a github repository +//////////////////////////////////////////////////////////////////////////////// + +function processGithubRepository (source) { + 'use strict'; + + var url = buildGithubUrl(source.location, source.version); + var tempFile = fs.getTempFile("downloads", false); + + try { + var result = download(url, "", { + method: "get", + followRedirects: true, + timeout: 30 + }, tempFile); + + if (result.code >= 200 && result.code <= 299) { + source.filename = tempFile; + source.removeFile = true; + } + else { + throwDownloadError("Could not download from repository '" + url + "'"); + } + } + catch (err) { + throwDownloadError("Could not download from repository '" + url + "': " + String(err)); + } + + repackZipFile(source); +} + + +//////////////////////////////////////////////////////////////////////////////// +/// @brief Exports +//////////////////////////////////////////////////////////////////////////////// + +exports.buildGithubUrl = buildGithubUrl; +exports.repackZipFile = repackZipFile; +exports.processDirectory = processDirectory; +exports.processGithubRepository = processGithubRepository; diff --git a/js/server/modules/org/arangodb/foxx/manager.js b/js/server/modules/org/arangodb/foxx/manager.js index 5769c72299..e1fe0e8063 100644 --- a/js/server/modules/org/arangodb/foxx/manager.js +++ b/js/server/modules/org/arangodb/foxx/manager.js @@ -32,6 +32,7 @@ var arangodb = require("org/arangodb"); var console = require("console"); var fs = require("fs"); +var utils = require("org/arangodb/foxx/manager-utils"); var _ = require("underscore"); @@ -1416,6 +1417,53 @@ exports.mountedApp = function (path) { return {}; }; +//////////////////////////////////////////////////////////////////////////////// +/// @brief builds a github repository URL +//////////////////////////////////////////////////////////////////////////////// + +function buildGithubUrl (repository, version) { + 'use strict'; + + if (typeof version === "undefined") { + version = "master"; + } + + return 'https://github.com/' + repository + '/archive/' + version + '.zip'; +} + +//////////////////////////////////////////////////////////////////////////////// +/// @brief fetches a foxx app from a remote repository +//////////////////////////////////////////////////////////////////////////////// + +exports.fetchFromGithub = function (url, name, version) { + + var source = { + location: url, + name: name, + version: version + }; + utils.processGithubRepository(source); + var realFile = source.filename; + + var appPath = module.appPath(); + if (appPath === undefined) { + fs.remove(realFile); + throw "javascript.app-path not set, rejecting app loading"; + } + var path = fs.join(appPath, source.name + "-" + source.version); + + if (fs.exists(path)) { + fs.remove(realFile); + throw "destination path '" + path + "' already exists"; + } + + fs.makeDirectoryRecursive(path); + fs.unzipFile(realFile, path, false, true); + + this.scanAppDirectory(); + return "app:" + source.name + ":" + source.version; +}; + // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE // -----------------------------------------------------------------------------