Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

do not copy node_modules directory on startup (#6945)

This commit is contained in:
Jan 2018-10-22 16:43:13 +02:00 committed by GitHub
parent 274c53444b
commit 360888c3ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 142 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
arangod/V8Server/V8DealerFeature.cpp
View File

@ -139,7 +139,7 @@ void V8DealerFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
options->addHiddenOption(
"--javascript.module-directory",
"additional paths containing JavaScript modules",
new VectorParameter<StringParameter>(&_moduleDirectory));
new VectorParameter<StringParameter>(&_moduleDirectories));
options->addOption(
"--javascript.copy-installation",
@ -211,7 +211,7 @@ void V8DealerFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
}
ctx->normalizePath(_startupDirectory, "javascript.startup-directory", true);
ctx->normalizePath(_moduleDirectory, "javascript.module-directory", false);
ctx->normalizePath(_moduleDirectories, "javascript.module-directory", false);
// try to append the current version name to the startup directory,
@ -225,7 +225,7 @@ void V8DealerFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
_startupDirectory = versionedPath;
}
for (auto& it : _moduleDirectory) {
for (auto& it : _moduleDirectories) {
versionedPath = basics::FileUtils::buildFilename(it, versionAppendix);
LOG_TOPIC(DEBUG, Logger::V8) << "checking for existence of version-specific module-directory '" << versionedPath << "'";
@ -263,7 +263,7 @@ void V8DealerFeature::start() {
}
LOG_TOPIC(DEBUG, Logger::V8) << "effective startup-directory is '" << _startupDirectory <<
"', effective module-directory is " << _moduleDirectory;
"', effective module-directories are " << _moduleDirectories;
_startupLoader.setDirectory(_startupDirectory);
ServerState::instance()->setJavaScriptPath(_startupDirectory);
@ -274,9 +274,9 @@ void V8DealerFeature::start() {
paths.push_back(std::string("startup '" + _startupDirectory + "'"));
if (!_moduleDirectory.empty()) {
if (!_moduleDirectories.empty()) {
paths.push_back(std::string(
"module '" + StringUtils::join(_moduleDirectory, ";") + "'"));
"module '" + StringUtils::join(_moduleDirectories, ";") + "'"));
}
if (!_appPath.empty()) {
@ -377,6 +377,12 @@ void V8DealerFeature::start() {
}
void V8DealerFeature::copyInstallationFiles() {
if (!_enableJS &&
(ServerState::instance()->isAgent() || ServerState::instance()->isDBServer())) {
// skip expensive file-copying in case we are an agency or db server
// these do not need JavaScript support
return;
}
// get base path from DatabasePathFeature
auto dbPathFeature = application_features::ApplicationServer::getFeature<DatabasePathFeature>();
const std::string copyJSPath = FileUtils::buildFilename(dbPathFeature->directory(), "js");
@ -414,7 +420,7 @@ void V8DealerFeature::copyInstallationFiles() {
FATAL_ERROR_EXIT();
}
LOG_TOPIC(DEBUG, Logger::V8) << "Copying JS installation files to '" << copyJSPath << "'";
LOG_TOPIC(DEBUG, Logger::V8) << "Copying JS installation files from '" << _startupDirectory << "' to '" << copyJSPath << "'";
int res = TRI_ERROR_NO_ERROR;
if (FileUtils::exists(copyJSPath)) {
res = TRI_RemoveDirectory(copyJSPath.c_str());
@ -429,12 +435,37 @@ void V8DealerFeature::copyInstallationFiles() {
<< "': " << TRI_errno_string(res);
FATAL_ERROR_EXIT();
}
// intentionally do not copy js/node/node_modules...
// we avoid copying this directory because it contains 5000+ files at the moment,
// and copying them one by one is darn slow at least on Windows...
std::string const versionAppendix = std::regex_replace(rest::Version::getServerVersion(), std::regex("-.*$"), "");
std::string const nodeModulesPath = FileUtils::buildFilename("js", "node", "node_modules");
std::string const nodeModulesPathVersioned = basics::FileUtils::buildFilename("js", versionAppendix, "node", "node_modules");
auto filter = [&nodeModulesPath, &nodeModulesPathVersioned, this](std::string const& filename) -> bool{
if (filename.size() >= nodeModulesPath.size()) {
std::string normalized = filename;
FileUtils::normalizePath(normalized);
TRI_ASSERT(filename.size() == normalized.size());
if (normalized.substr(normalized.size() - nodeModulesPath.size(), nodeModulesPath.size()) == nodeModulesPath ||
normalized.substr(normalized.size() - nodeModulesPathVersioned.size(), nodeModulesPathVersioned.size()) == nodeModulesPathVersioned) {
// filter it out!
_nodeModulesDirectory = _startupDirectory;
return true;
}
}
// let the file/directory pass through
return false;
};
std::string error;
if (!FileUtils::copyRecursive(_startupDirectory, copyJSPath, error)) {
if (!FileUtils::copyRecursive(_startupDirectory, copyJSPath, filter, error)) {
LOG_TOPIC(FATAL, Logger::V8) << "Error copying JS installation files to '" << copyJSPath
<< "': " << error;
FATAL_ERROR_EXIT();
}
// need to keep it because of js/node/node_modules, which we intentionally do not copy!
_moduleDirectories.push_back(_startupDirectory);
}
_startupDirectory = copyJSPath;
}
@ -1364,11 +1395,14 @@ V8Context* V8DealerFeature::buildContext(size_t id) {
std::string sep = "";
std::vector<std::string> directories;
directories.insert(directories.end(), _moduleDirectory.begin(),
_moduleDirectory.end());
directories.insert(directories.end(), _moduleDirectories.begin(),
_moduleDirectories.end());
directories.emplace_back(_startupDirectory);
if (!_nodeModulesDirectory.empty()) {
directories.emplace_back(_nodeModulesDirectory);
}
for (auto directory : directories) {
for (auto const& directory : directories) {
modules += sep;
sep = ";";

3
arangod/V8Server/V8DealerFeature.h
View File

@ -65,7 +65,8 @@ class V8DealerFeature final : public application_features::ApplicationFeature {
double _maxContextAge;
std::string _appPath;
std::string _startupDirectory;
std::vector<std::string> _moduleDirectory;
std::string _nodeModulesDirectory;
std::vector<std::string> _moduleDirectories;
bool _copyInstallation;
uint64_t _nrMaxContexts; // maximum number of contexts to create
uint64_t _nrMinContexts; // minimum number of contexts to keep

47
arangosh/Shell/V8ShellFeature.cpp
View File

@ -102,7 +102,7 @@ void V8ShellFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
options->addHiddenOption(
"--javascript.module-directory",
"additional paths containing JavaScript modules",
new VectorParameter<StringParameter>(&_moduleDirectory));
new VectorParameter<StringParameter>(&_moduleDirectories));
options->addOption("--javascript.current-module-directory",
"add current directory to module path",
@ -126,10 +126,10 @@ void V8ShellFeature::validateOptions(
FATAL_ERROR_EXIT();
}
if (!_moduleDirectory.empty()) {
if (!_moduleDirectories.empty()) {
LOG_TOPIC(DEBUG, Logger::V8)
<< "using Javascript modules at '"
<< StringUtils::join(_moduleDirectory, ";") << "'";
<< StringUtils::join(_moduleDirectories, ";") << "'";
}
}
@ -239,7 +239,7 @@ void V8ShellFeature::copyInstallationFiles() {
_removeCopyInstallation = true;
}
LOG_TOPIC(DEBUG, Logger::V8) << "Copying JS installation files to '" << _copyDirectory << "'";
LOG_TOPIC(DEBUG, Logger::V8) << "Copying JS installation files from '" << _startupDirectory << "' to '" << _copyDirectory << "'";
int res = TRI_ERROR_NO_ERROR;
if (FileUtils::exists(_copyDirectory)) {
@ -255,8 +255,30 @@ void V8ShellFeature::copyInstallationFiles() {
<< "': " << TRI_errno_string(res);
FATAL_ERROR_EXIT();
}
// intentionally do not copy js/node/node_modules...
// we avoid copying this directory because it contains 5000+ files at the moment,
// and copying them one by one is darn slow at least on Windows...
std::string const versionAppendix = std::regex_replace(rest::Version::getServerVersion(), std::regex("-.*$"), "");
std::string const nodeModulesPath = FileUtils::buildFilename("js", "node", "node_modules");
std::string const nodeModulesPathVersioned = basics::FileUtils::buildFilename("js", versionAppendix, "node", "node_modules");
auto filter = [&nodeModulesPath, &nodeModulesPathVersioned, this](std::string const& filename) -> bool{
if (filename.size() >= nodeModulesPath.size()) {
std::string normalized = filename;
FileUtils::normalizePath(normalized);
TRI_ASSERT(filename.size() == normalized.size());
if (normalized.substr(normalized.size() - nodeModulesPath.size(), nodeModulesPath.size()) == nodeModulesPath ||
normalized.substr(normalized.size() - nodeModulesPathVersioned.size(), nodeModulesPathVersioned.size()) == nodeModulesPathVersioned) {
// filter it out!
_nodeModulesDirectory = _startupDirectory;
return true;
}
}
// let the file/directory pass through
return false;
};
std::string error;
if (!FileUtils::copyRecursive(_startupDirectory, _copyDirectory, error)) {
if (!FileUtils::copyRecursive(_startupDirectory, _copyDirectory, filter, error)) {
LOG_TOPIC(FATAL, Logger::V8) << "Error copying JS installation files to '" << _copyDirectory
<< "': " << error;
FATAL_ERROR_EXIT();
@ -968,7 +990,7 @@ void V8ShellFeature::initGlobals() {
}
ctx->normalizePath(_startupDirectory, "javascript.startup-directory", true);
ctx->normalizePath(_moduleDirectory, "javascript.module-directory", false);
ctx->normalizePath(_moduleDirectories, "javascript.module-directory", false);
// try to append the current version name to the startup directory,
// so instead of "/path/to/js" we will get "/path/to/js/3.4.0"
@ -981,7 +1003,7 @@ void V8ShellFeature::initGlobals() {
_startupDirectory = versionedPath;
}
for (auto& it : _moduleDirectory) {
for (auto& it : _moduleDirectories) {
versionedPath = basics::FileUtils::buildFilename(it, versionAppendix);
LOG_TOPIC(DEBUG, Logger::V8) << "checking for existence of version-specific module-directory '" << versionedPath << "'";
@ -991,18 +1013,21 @@ void V8ShellFeature::initGlobals() {
}
}
LOG_TOPIC(DEBUG, Logger::V8) << "effective startup-directory is '" << _startupDirectory << "', effective module-directory is " << _moduleDirectory;
LOG_TOPIC(DEBUG, Logger::V8) << "effective startup-directory is '" << _startupDirectory << "', effective module-directory is " << _moduleDirectories;
// initialize standard modules
std::vector<std::string> directories;
directories.insert(directories.end(), _moduleDirectory.begin(),
_moduleDirectory.end());
directories.insert(directories.end(), _moduleDirectories.begin(),
_moduleDirectories.end());
directories.emplace_back(_startupDirectory);
if (!_nodeModulesDirectory.empty()) {
directories.emplace_back(_nodeModulesDirectory);
}
std::string modules = "";
std::string sep = "";
for (auto directory : directories) {
for (auto const& directory : directories) {
modules += sep;
sep = ";";

5
arangosh/Shell/V8ShellFeature.h
View File

@ -50,15 +50,16 @@ class V8ShellFeature final : public application_features::ApplicationFeature {
void unprepare() override final;
void stop() override final;
std::string const& startupDirectory() {
std::string const& startupDirectory() const {
return _startupDirectory;
}
private:
std::string _startupDirectory;
std::string _nodeModulesDirectory;
std::string _clientModule;
std::string _copyDirectory;
std::vector<std::string> _moduleDirectory;
std::vector<std::string> _moduleDirectories;
bool _currentModuleDirectory;
bool _copyInstallation;
bool _removeCopyInstallation;

7
cmake/ArangoDBInstall.cmake
View File

@ -83,6 +83,13 @@ install(
REGEX "^.*/.bin" EXCLUDE
)
install(
FILES
${ARANGODB_SOURCE_DIR}/js/JS_SHA1SUM.txt
DESTINATION
${CMAKE_INSTALL_DATAROOTDIR_ARANGO}/${ARANGODB_JS_VERSION}/JS_SHA1SUM.txt
)
if (USE_ENTERPRISE)
install(
DIRECTORY ${PROJECT_SOURCE_DIR}/enterprise/js/server

7
cmake/InstallArangoDBJSClient.cmake
View File

@ -12,6 +12,13 @@ install(
PATTERN "*.js"
)
install(
FILES
${ARANGODB_SOURCE_DIR}/js/JS_SHA1SUM.txt
DESTINATION
${CMAKE_INSTALL_DATAROOTDIR_ARANGO}/${ARANGODB_JS_VERSION}/JS_SHA1SUM.txt
)
if (USE_ENTERPRISE)
install(
DIRECTORY

36
lib/Basics/FileUtils.cpp
View File

@ -32,6 +32,8 @@
#endif
#include <unicode/unistr.h>
#include <functional>
#include "Basics/Exceptions.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringBuffer.h"
@ -39,6 +41,9 @@
#include "Basics/tri-strings.h"
#include "Logger/Logger.h"
namespace {
std::function<bool(std::string const&)> const passAllFilter = [](std::string const&) { return false; };
}
namespace arangodb {
namespace basics {
@ -265,17 +270,28 @@ bool createDirectory(std::string const& name, int mask, int* errorNumber) {
return (result != 0) ? false : true;
}
bool copyRecursive(std::string const& source, std::string const& target,
/// @brief will not copy files/directories for which the filter function
/// returns true
bool copyRecursive(std::string const& source,
std::string const& target,
std::function<bool(std::string const&)> const& filter,
std::string& error) {
if (isDirectory(source)) {
return copyDirectoryRecursive(source, target, error);
return copyDirectoryRecursive(source, target, filter, error);
}
if (filter(source)) {
return TRI_ERROR_NO_ERROR;
}
return TRI_CopyFile(source, target, error);
}
/// @brief will not copy files/directories for which the filter function
/// returns true
bool copyDirectoryRecursive(std::string const& source,
std::string const& target, std::string& error) {
std::string const& target,
std::function<bool(std::string const&)> const& filter,
std::string& error) {
char* fn = nullptr;
bool rc = true;
@ -287,14 +303,14 @@ bool copyDirectoryRecursive(std::string const& source,
intptr_t handle;
std::string rcs;
std::string filter = source + "\\*";
std::string flt = source + "\\*";
UnicodeString f(filter.c_str());
UnicodeString f(flt.c_str());
handle = _wfindfirst(f.getTerminatedBuffer(), &oneItem);
if (handle == -1) {
error = "directory " + source + "not found";
error = "directory " + source + " not found";
return false;
}
@ -308,7 +324,7 @@ bool copyDirectoryRecursive(std::string const& source,
DIR* filedir = opendir(source.c_str());
if (filedir == nullptr) {
error = "directory " + source + "not found";
error = "directory " + source + " not found";
return false;
}
@ -333,6 +349,10 @@ bool copyDirectoryRecursive(std::string const& source,
std::string dst = target + TRI_DIR_SEPARATOR_STR + fn;
std::string src = source + TRI_DIR_SEPARATOR_STR + fn;
if (filter(src)) {
continue;
}
// Handle subdirectories:
if (isSubDirectory(src)) {
long systemError;
@ -340,7 +360,7 @@ bool copyDirectoryRecursive(std::string const& source,
if (rc != TRI_ERROR_NO_ERROR) {
break;
}
if (!copyDirectoryRecursive(src, dst, error)) {
if (!copyDirectoryRecursive(src, dst, filter, error)) {
break;
}
if (!TRI_CopyAttributes(src, dst, error)) {

15
lib/Basics/FileUtils.h
View File

@ -76,11 +76,20 @@ bool remove(std::string const& fileName, int* errorNumber = nullptr);
bool createDirectory(std::string const& name, int* errorNumber = nullptr);
bool createDirectory(std::string const& name, int mask, int* errorNumber = nullptr);
// copies directories / files recursive
bool copyRecursive(std::string const& source, std::string const& target,
/// @brief copies directories / files recursive
/// will not copy files/directories for which the filter function
/// returns true
bool copyRecursive(std::string const& source,
std::string const& target,
std::function<bool(std::string const&)> const& filter,
std::string& error);
/// @brief will not copy files/directories for which the filter function
/// returns true
bool copyDirectoryRecursive(std::string const& source,
std::string const& target, std::string& error);
std::string const& target,
std::function<bool(std::string const&)> const& filter,
std::string& error);
// returns list of files
std::vector<std::string> listFiles(std::string const& directory);

3
lib/V8/v8-utils.cpp
View File

@ -2269,7 +2269,8 @@ static void JS_CopyRecursive(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_SYS_ERROR, errMsg);
}
if (!FileUtils::copyRecursive(source, destination, systemErrorStr)) {
std::function<bool(std::string const&)> const passAllFilter = [](std::string const&) { return false; };
if (!FileUtils::copyRecursive(source, destination, passAllFilter, systemErrorStr)) {
std::string errMsg = "cannot copy directory [" + source + "] to [" +
destination + " ] : " + std::to_string(errorNo) +
": " + systemErrorStr;