Copy installation files on startup

arangod/Cluster/v8-cluster.cpp

@@ -1883,7 +1883,6 @@ static void JS_Drop(v8::FunctionCallbackInfo<v8::Value> const& args) {
 
 static void JS_GetId(v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_BEGIN(isolate);
-  ONLY_IN_CLUSTER
 
   if (args.Length() != 0) {
     TRI_V8_THROW_EXCEPTION_USAGE("getId()");

arangod/RestServer/DatabasePathFeature.cpp

@@ -37,7 +37,7 @@ using namespace arangodb::basics;
 using namespace arangodb::options;
 
 DatabasePathFeature::DatabasePathFeature(ApplicationServer* server)
-    : ApplicationFeature(server, "DatabasePath"),
+    : ApplicationFeature(server, DatabasePathFeature::name()),
       _requiredDirectoryState("any") {
   setOptional(false);
   requiresElevatedPrivileges(false);

arangod/RestServer/DatabasePathFeature.h

@@ -32,6 +32,10 @@ class DatabasePathFeature final
  public:
   explicit DatabasePathFeature(
       application_features::ApplicationServer* server);
+      
+  static constexpr const char* name() {
+    return "DatabasePath";
+  }
 
  public:
   void collectOptions(std::shared_ptr<options::ProgramOptions>) override final;

arangod/V8Server/V8DealerFeature.cpp

@@ -40,6 +40,7 @@
 #include "ProgramOptions/Section.h"
 #include "Random/RandomGenerator.h"
 #include "RestServer/DatabaseFeature.h"
+#include "RestServer/DatabasePathFeature.h"
 #include "Scheduler/JobGuard.h"
 #include "Scheduler/SchedulerFeature.h"
 #include "Transaction/V8Context.h"
@@ -95,6 +96,7 @@ V8DealerFeature::V8DealerFeature(
       _gcFrequency(30.0),
       _gcInterval(1000),
       _maxContextAge(60.0),
+      _copyInstallation(false),
       _nrMaxContexts(0),
       _nrMinContexts(0),
       _nrInflightContexts(0),
@@ -141,6 +143,11 @@ void V8DealerFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
       "--javascript.module-directory",
       "additional paths containing JavaScript modules",
       new VectorParameter<StringParameter>(&_moduleDirectory));
+  
+  options->addOption(
+       "--javascript.copy-installation",
+       "copy contents of 'javascript.startup-directory' on first start",
+       new BooleanParameter(&_copyInstallation));
 
   options->addOption(
       "--javascript.v8-contexts",
@@ -187,9 +194,6 @@ void V8DealerFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
   ctx->normalizePath(_startupDirectory, "javascript.startup-directory", true);
   ctx->normalizePath(_moduleDirectory, "javascript.module-directory", false);
 
-  _startupLoader.setDirectory(_startupDirectory);
-  ServerState::instance()->setJavaScriptPath(_startupDirectory);
-
   // check whether app-path was specified
   if (_appPath.empty()) {
     LOG_TOPIC(FATAL, arangodb::Logger::V8) << "no value has been specified for --javascript.app-path";
@@ -207,6 +211,15 @@ void V8DealerFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
 }
 
 void V8DealerFeature::start() {
+  if (_copyInstallation) {
+    copyInstallationFiles(); // will exit process if it fails
+  }
+  LOG_TOPIC(DEBUG, Logger::V8) << "effective startup-directory is '" << _startupDirectory <<
+  "', effective module-directory is " << _moduleDirectory;
+  
+  _startupLoader.setDirectory(_startupDirectory);
+  ServerState::instance()->setJavaScriptPath(_startupDirectory);
+  
   // dump paths
   {
     std::vector<std::string> paths;
@@ -307,6 +320,69 @@ void V8DealerFeature::start() {
   startGarbageCollection();
 }
 
+void V8DealerFeature::copyInstallationFiles() {
+  // get base path from DatabasePathFeature
+  auto dbPathFeature = application_features::ApplicationServer::getFeature<DatabasePathFeature>(DatabasePathFeature::name());
+  const std::string copyJSPath = FileUtils::buildFilename(dbPathFeature->directory(), "js");
+  if (copyJSPath == _startupDirectory) {
+    LOG_TOPIC(FATAL, arangodb::Logger::V8)
+    << "'javascript.startup-directory' cannot be inside 'database.directory'";
+    FATAL_ERROR_EXIT();
+  }
+  TRI_ASSERT(!copyJSPath.empty());
+  
+  const std::string checksumFile = FileUtils::buildFilename(_startupDirectory, StaticStrings::checksumFileJs);
+  const std::string copyChecksumFile = FileUtils::buildFilename(copyJSPath, StaticStrings::checksumFileJs);
+  
+  bool overwriteCopy = false;
+  if (!FileUtils::exists(copyJSPath) ||
+      !FileUtils::exists(checksumFile) ||
+      !FileUtils::exists(copyChecksumFile)) {
+    overwriteCopy = true;
+  } else {
+    try {
+      overwriteCopy = (FileUtils::slurp(copyChecksumFile) != FileUtils::slurp(checksumFile));
+    } catch(basics::Exception const& e) {
+      LOG_TOPIC(ERR, Logger::V8) << "Error reading '" << StaticStrings::checksumFileJs <<
+      "' from disk: " << e.what();
+      overwriteCopy = true;
+    }
+  }
+  
+  if (overwriteCopy) {
+    // sanity check before removing an existing directory:
+    // check if for some reason we will be trying to remove the entire database directory...
+    if (FileUtils::exists(FileUtils::buildFilename(copyJSPath, "ENGINE"))) {
+      LOG_TOPIC(FATAL, Logger::V8) << "JS installation path '" << copyJSPath
+      << "' seems to be invalid";
+      FATAL_ERROR_EXIT();
+    }
+    
+    LOG_TOPIC(DEBUG, Logger::V8) << "Copying JS installation files to '" << copyJSPath << "'";
+    int res = TRI_ERROR_NO_ERROR;
+    if (FileUtils::exists(copyJSPath)) {
+      res = TRI_RemoveDirectory(copyJSPath.c_str());
+      if (res != TRI_ERROR_NO_ERROR) {
+        LOG_TOPIC(FATAL, Logger::V8) << "Error cleaning JS installation path '" << copyJSPath
+        << "': " << TRI_errno_string(res);
+        FATAL_ERROR_EXIT();
+      }
+    }
+    if (!FileUtils::createDirectory(copyJSPath, &res)) {
+      LOG_TOPIC(FATAL, Logger::V8) << "Error creating JS installation path '" << copyJSPath
+      << "': " << TRI_errno_string(res);
+      FATAL_ERROR_EXIT();
+    }
+    std::string error;
+    if (!FileUtils::copyRecursive(_startupDirectory, copyJSPath, error)) {
+      LOG_TOPIC(FATAL, Logger::V8) << "Error copying JS installation files to '" << copyJSPath
+      << "': " << error;
+      FATAL_ERROR_EXIT();
+    }
+  }
+  _startupDirectory = copyJSPath;
+}
+
 V8Context* V8DealerFeature::addContext() {
   V8Context* context = buildContext(nextId());
 

arangod/V8Server/V8DealerFeature.h

@@ -67,6 +67,7 @@ class V8DealerFeature final : public application_features::ApplicationFeature {
   std::string _appPath;
   std::string _startupDirectory;
   std::vector<std::string> _moduleDirectory;
+  bool _copyInstallation;
   uint64_t _nrMaxContexts;  // maximum number of contexts to create
   uint64_t _nrMinContexts; // minimum number of contexts to keep
   uint64_t _nrInflightContexts; // number of contexts currently in creation
@@ -127,6 +128,7 @@ class V8DealerFeature final : public application_features::ApplicationFeature {
 
  private:
   uint64_t nextId() { return _nextId++; }
+  void copyInstallationFiles();
   V8Context* addContext();
   V8Context* buildContext(size_t id);
   V8Context* pickFreeContextForGc();

arangosh/Shell/V8ShellFeature.cpp

@@ -32,6 +32,7 @@
 #include "Logger/Logger.h"
 #include "ProgramOptions/ProgramOptions.h"
 #include "ProgramOptions/Section.h"
+#include "Random/RandomGenerator.h"
 #include "Rest/HttpResponse.h"
 #include "Rest/Version.h"
 #include "Shell/ClientFeature.h"
@@ -62,6 +63,8 @@ V8ShellFeature::V8ShellFeature(application_features::ApplicationServer* server,
       _startupDirectory("js"),
       _clientModule(DEFAULT_CLIENT_MODULE),
       _currentModuleDirectory(true),
+      _copyInstallation(false),
+      _removeCopyInstallation(false),
       _gcInterval(50),
       _name(name),
       _isolate(nullptr),
@@ -72,6 +75,7 @@ V8ShellFeature::V8ShellFeature(application_features::ApplicationServer* server,
   startsAfter("Logger");
   startsAfter("Console");
   startsAfter("V8Platform");
+  startsAfter("Random");
 }
 
 void V8ShellFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
@@ -84,6 +88,11 @@ void V8ShellFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
   options->addHiddenOption("--javascript.client-module",
                            "client module to use at startup",
                            new StringParameter(&_clientModule));
+  
+  options->addOption("--javascript.copy-directory",
+                     "target directory to copy files from 'javascript.startup-directory' into "
+                     "(only used when `--javascript.copy-installation` is enabled)",
+                     new StringParameter(&_copyDirectory));
 
   options->addHiddenOption(
       "--javascript.module-directory",
@@ -93,10 +102,14 @@ void V8ShellFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
   options->addOption("--javascript.current-module-directory",
                      "add current directory to module path",
                      new BooleanParameter(&_currentModuleDirectory));
+  
+  options->addOption("--javascript.copy-installation",
+                     "copy contents of 'javascript.startup-directory'",
+                     new BooleanParameter(&_copyInstallation));
 
   options->addOption(
       "--javascript.gc-interval",
-      "request-based garbage collection interval (each n.th commands)",
+      "request-based garbage collection interval (each n.th command)",
       new UInt64Parameter(&_gcInterval));
 }
 
@@ -107,10 +120,7 @@ void V8ShellFeature::validateOptions(
         << "no 'javascript.startup-directory' has been supplied, giving up";
     FATAL_ERROR_EXIT();
   }
-
-  LOG_TOPIC(DEBUG, Logger::V8)
-      << "using Javascript startup files at '" << _startupDirectory << "'";
-
+  
   if (!_moduleDirectory.empty()) {
     LOG_TOPIC(DEBUG, Logger::V8)
         << "using Javascript modules at '"
@@ -125,6 +135,13 @@ void V8ShellFeature::start() {
   auto platform =
       application_features::ApplicationServer::getFeature<V8PlatformFeature>(
           "V8Platform");
+  
+  if (_copyInstallation) {
+    copyInstallationFiles(); // will exit process on error
+  }
+  
+  LOG_TOPIC(DEBUG, Logger::V8)
+  << "using Javascript startup files at '" << _startupDirectory << "'";
 
   _isolate = platform->createIsolate();
 
@@ -202,6 +219,50 @@ void V8ShellFeature::unprepare() {
   TRI_AllowMemoryFailures();
 }
 
+void V8ShellFeature::stop() {
+  if (_removeCopyInstallation && !_copyDirectory.empty()) {
+    int res = TRI_RemoveDirectory(_copyDirectory.c_str());
+    if (res != TRI_ERROR_NO_ERROR) {
+      LOG_TOPIC(DEBUG, Logger::V8) << "could not cleanup installation file copy in path '" << _copyDirectory << "': " << TRI_errno_string(res);
+    }
+  }
+}
+
+void V8ShellFeature::copyInstallationFiles() {
+  if (_copyDirectory.empty()) {
+    uint64_t r = RandomGenerator::interval(UINT64_MAX);
+    char buf[sizeof(uint64_t) * 2 + 1];
+    auto len = TRI_StringUInt64HexInPlace(r, buf);
+    std::string name("arangosh-js-");
+    name.append(buf, len);
+    _copyDirectory = FileUtils::buildFilename(TRI_GetTempPath(), name);
+    _removeCopyInstallation = true;
+  }
+  
+  LOG_TOPIC(DEBUG, Logger::V8) << "Copying JS installation files to '" << _copyDirectory << "'";
+  int res = TRI_ERROR_NO_ERROR;
+  if (FileUtils::exists(_copyDirectory)) {
+    res = TRI_RemoveDirectory(_copyDirectory.c_str());
+    if (res != TRI_ERROR_NO_ERROR) {
+      LOG_TOPIC(FATAL, Logger::V8) << "Error cleaning JS installation path '" << _copyDirectory
+      << "': " << TRI_errno_string(res);
+      FATAL_ERROR_EXIT();
+    }
+  }
+  if (!FileUtils::createDirectory(_copyDirectory, &res)) {
+    LOG_TOPIC(FATAL, Logger::V8) << "Error creating JS installation path '" << _copyDirectory
+    << "': " << TRI_errno_string(res);
+    FATAL_ERROR_EXIT();
+  }
+  std::string error;
+  if (!FileUtils::copyRecursive(_startupDirectory, _copyDirectory, error)) {
+    LOG_TOPIC(FATAL, Logger::V8) << "Error copying JS installation files to '" << _copyDirectory
+    << "': " << error;
+    FATAL_ERROR_EXIT();
+  }
+  _startupDirectory = _copyDirectory;
+}
+
 bool V8ShellFeature::printHello(V8ClientConnection* v8connection) {
   bool promptError = false;
 

arangosh/Shell/V8ShellFeature.h

@@ -46,6 +46,7 @@ class V8ShellFeature final : public application_features::ApplicationFeature {
       std::shared_ptr<options::ProgramOptions> options) override;
   void start() override final;
   void unprepare() override final;
+  void stop() override final;
 
   std::string const& startupDirectory() {
     return _startupDirectory;
@@ -54,8 +55,11 @@ class V8ShellFeature final : public application_features::ApplicationFeature {
  private:
   std::string _startupDirectory;
   std::string _clientModule;
+  std::string _copyDirectory;
   std::vector<std::string> _moduleDirectory;
   bool _currentModuleDirectory;
+  bool _copyInstallation;
+  bool _removeCopyInstallation;
   uint64_t _gcInterval;
 
  public:
@@ -69,6 +73,7 @@ class V8ShellFeature final : public application_features::ApplicationFeature {
   bool jslint(std::vector<std::string> const& files);
 
  private:
+  void copyInstallationFiles();
   bool printHello(V8ClientConnection*);
   void initGlobals();
   void initMode(ShellFeature::RunMode, std::vector<std::string> const&);

js/client/modules/@arangodb/process-utils.js

@@ -363,6 +363,7 @@ function makeArgsArangod (options, appDir, role, tmpDir) {
     'define': 'TOP_DIR=' + TOP_DIR,
     'wal.flush-timeout': options.walFlushTimeout,
     'javascript.app-path': appDir,
+    'javascript.copy-installation': false,
     'http.trusted-origin': options.httpTrustedOrigin || 'all',
     'cluster.create-waits-for-sync-replication': false,
     'temp.path': tmpDir

lib/Basics/StaticStrings.cpp

@@ -124,3 +124,6 @@ std::string const StaticStrings::MimeTypeJson(
     "application/json; charset=utf-8");
 std::string const StaticStrings::MimeTypeText("text/plain; charset=utf-8");
 std::string const StaticStrings::MimeTypeVPack("application/x-velocypack");
+
+// misc strings
+std::string const StaticStrings::checksumFileJs("JS_SHA1SUM.txt");

lib/Basics/StaticStrings.h

@@ -117,6 +117,9 @@ class StaticStrings {
   static std::string const MimeTypeJson;
   static std::string const MimeTypeText;
   static std::string const MimeTypeVPack;
+  
+  // misc strings
+  static std::string const checksumFileJs;
 };
 }