Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove OpenFilesTracker (#7189)

This commit is contained in:
Jan 2018-11-02 11:38:38 +01:00 committed by GitHub
parent f088733420
commit 976cc38e7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 103 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
arangod/MMFiles/MMFilesDatafile.cpp
View File

@ -24,7 +24,6 @@
#include "MMFilesDatafile.h" #include "MMFilesDatafile.h"
#include "ApplicationFeatures/PageSizeFeature.h" #include "ApplicationFeatures/PageSizeFeature.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StaticStrings.h" #include "Basics/StaticStrings.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Basics/encoding.h" #include "Basics/encoding.h"
@ -188,7 +187,7 @@ static MMFilesDatafile* CreateAnonymousDatafile(TRI_voc_fid_t fid,
int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED; int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED;
#else #else
// ugly workaround if MAP_ANONYMOUS is not available // ugly workaround if MAP_ANONYMOUS is not available
int fd = TRI_TRACKED_OPEN_FILE("/dev/zero", O_RDWR | TRI_O_CLOEXEC); int fd = TRI_OPEN("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
if (fd == -1) { if (fd == -1) {
return nullptr; return nullptr;
@ -207,7 +206,7 @@ static MMFilesDatafile* CreateAnonymousDatafile(TRI_voc_fid_t fid,
// nothing to do // nothing to do
#else #else
// close auxilliary file // close auxilliary file
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
fd = -1; fd = -1;
#endif #endif
@ -255,7 +254,7 @@ static MMFilesDatafile* CreatePhysicalDatafile(std::string const& filename,
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res); TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
// remove empty file // remove empty file
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());
@ -270,7 +269,7 @@ static MMFilesDatafile* CreatePhysicalDatafile(std::string const& filename,
try { try {
return new MMFilesDatafile(filename, fd, mmHandle, maximalSize, 0, fid, static_cast<char*>(data)); return new MMFilesDatafile(filename, fd, mmHandle, maximalSize, 0, fid, static_cast<char*>(data));
} catch (...) { } catch (...) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return nullptr; return nullptr;
} }
} }
@ -289,7 +288,7 @@ int MMFilesDatafile::judge(std::string const& filename) {
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE; return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
} }
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE; return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
@ -298,7 +297,7 @@ int MMFilesDatafile::judge(std::string const& filename) {
uint64_t buffer[256]; uint64_t buffer[256];
if (!TRI_ReadPointer(fd, &buffer, 256 * sizeof(uint64_t))) { if (!TRI_ReadPointer(fd, &buffer, 256 * sizeof(uint64_t))) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE; return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
} }
@ -307,13 +306,13 @@ int MMFilesDatafile::judge(std::string const& filename) {
while (ptr < end) { while (ptr < end) {
if (*ptr != 0) { if (*ptr != 0) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return TRI_ERROR_NO_ERROR; return TRI_ERROR_NO_ERROR;
} }
++ptr; ++ptr;
} }
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return TRI_ERROR_ARANGO_DATAFILE_EMPTY; return TRI_ERROR_ARANGO_DATAFILE_EMPTY;
} }
@ -1023,7 +1022,7 @@ int MMFilesDatafile::close() {
if (isPhysical()) { if (isPhysical()) {
TRI_ASSERT(_fd >= 0); TRI_ASSERT(_fd >= 0);
int res = TRI_TRACKED_CLOSE_FILE(_fd); int res = TRI_CLOSE(_fd);
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "unable to close datafile '" << getName() << "': " << res; LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "unable to close datafile '" << getName() << "': " << res;
@ -1111,7 +1110,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
std::string filename = getName() + ".new"; std::string filename = getName() + ".new";
int fd = int fd =
TRI_TRACKED_CREATE_FILE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR); S_IRUSR | S_IWUSR);
if (fd < 0) { if (fd < 0) {
@ -1127,7 +1126,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
if (offset == (TRI_lseek_t)-1) { if (offset == (TRI_lseek_t)-1) {
TRI_SYSTEM_ERROR(); TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
// remove empty file // remove empty file
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());
@ -1142,7 +1141,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
if (written < 0) { if (written < 0) {
TRI_SYSTEM_ERROR(); TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
// remove empty file // remove empty file
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());
@ -1159,7 +1158,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
TRI_SYSTEM_ERROR(); TRI_SYSTEM_ERROR();
TRI_set_errno(res); TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
// remove empty file // remove empty file
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());
@ -1179,7 +1178,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
res = TRI_UNMMFile(_data, _initSize, _fd, &_mmHandle); res = TRI_UNMMFile(_data, _initSize, _fd, &_mmHandle);
if (res < 0) { if (res < 0) {
TRI_TRACKED_CLOSE_FILE(_fd); TRI_CLOSE(_fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "munmap failed with: " << res; LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "munmap failed with: " << res;
return res; return res;
} }
@ -1191,7 +1190,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
// associated file below // associated file below
// ............................................................................................. // .............................................................................................
TRI_TRACKED_CLOSE_FILE(_fd); TRI_CLOSE(_fd);
_data = static_cast<char*>(data); _data = static_cast<char*>(data);
_next = (char*)(data) + position; _next = (char*)(data) + position;
@ -1795,7 +1794,7 @@ MMFilesDatafile* MMFilesDatafile::open(std::string const& filename, bool ignoreF
if (!ok) { if (!ok) {
TRI_UNMMFile(const_cast<char*>(datafile->data()), datafile->initSize(), datafile->fd(), &datafile->_mmHandle); TRI_UNMMFile(const_cast<char*>(datafile->data()), datafile->initSize(), datafile->fd(), &datafile->_mmHandle);
TRI_TRACKED_CLOSE_FILE(datafile->fd()); TRI_CLOSE(datafile->fd());
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile '" << datafile->getName() << "' is corrupt"; LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile '" << datafile->getName() << "' is corrupt";
// must free datafile here // must free datafile here
@ -1828,7 +1827,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
TRI_voc_fid_t fid = GetNumericFilenamePart(filename.c_str()); TRI_voc_fid_t fid = GetNumericFilenamePart(filename.c_str());
// attempt to open a datafile file // attempt to open a datafile file
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDWR | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename.c_str(), O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
TRI_SYSTEM_ERROR(); TRI_SYSTEM_ERROR();
@ -1846,7 +1845,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (res < 0) { if (res < 0) {
TRI_SYSTEM_ERROR(); TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot get status of datafile '" << filename << "': " << TRI_GET_ERRORBUF; LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot get status of datafile '" << filename << "': " << TRI_GET_ERRORBUF;
@ -1858,7 +1857,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (size < sizeof(MMFilesDatafileHeaderMarker) + sizeof(MMFilesDatafileFooterMarker)) { if (size < sizeof(MMFilesDatafileHeaderMarker) + sizeof(MMFilesDatafileFooterMarker)) {
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE); TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile '" << filename << "' is corrupt, size is only " << size; LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile '" << filename << "' is corrupt, size is only " << size;
@ -1881,7 +1880,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (!ok) { if (!ok) {
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot read datafile header from '" << filename << "': " << TRI_last_error(); LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot read datafile header from '" << filename << "': " << TRI_last_error();
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return nullptr; return nullptr;
} }
@ -1894,7 +1893,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (!ok) { if (!ok) {
if (IsMarker28(ptr, len)) { if (IsMarker28(ptr, len)) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile found from older version of ArangoDB. " LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile found from older version of ArangoDB. "
<< "Please dump data from that version with arangodump " << "Please dump data from that version with arangodump "
<< "and reload it into this ArangoDB instance with arangorestore"; << "and reload it into this ArangoDB instance with arangorestore";
@ -1908,7 +1907,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
printMarker(reinterpret_cast<MMFilesMarker const*>(ptr), len, &buffer[0], end); printMarker(reinterpret_cast<MMFilesMarker const*>(ptr), len, &buffer[0], end);
if (!ignoreErrors) { if (!ignoreErrors) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return nullptr; return nullptr;
} }
} }
@ -1921,7 +1920,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "unknown datafile version '" << header->_version << "' in datafile '" << filename << "'"; LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "unknown datafile version '" << header->_version << "' in datafile '" << filename << "'";
if (!ignoreErrors) { if (!ignoreErrors) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return nullptr; return nullptr;
} }
} }
@ -1939,7 +1938,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res); TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot memory map datafile '" << filename << "': " << TRI_errno_string(res); LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot memory map datafile '" << filename << "': " << TRI_errno_string(res);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "The database directory might reside on a shared folder " LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "The database directory might reside on a shared folder "
@ -1952,7 +1951,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
return new MMFilesDatafile(filename, fd, mmHandle, size, size, fid, static_cast<char*>(data)); return new MMFilesDatafile(filename, fd, mmHandle, size, size, fid, static_cast<char*>(data));
} catch (...) { } catch (...) {
TRI_UNMMFile(data, size, fd, &mmHandle); TRI_UNMMFile(data, size, fd, &mmHandle);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return nullptr; return nullptr;
} }

9
arangod/Pregel/TypedBuffer.h
View File

@ -25,7 +25,6 @@
#include "Basics/Common.h" #include "Basics/Common.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/files.h" #include "Basics/files.h"
#include "Basics/memory-map.h" #include "Basics/memory-map.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
@ -119,7 +118,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED; int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED;
#else #else
// ugly workaround if MAP_ANONYMOUS is not available // ugly workaround if MAP_ANONYMOUS is not available
_fd = TRI_TRACKED_OPEN_FILE("/dev/zero", O_RDWR | TRI_O_CLOEXEC); _fd = TRI_OPEN("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
if (_fd == -1) { if (_fd == -1) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL); THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
} }
@ -136,7 +135,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
// nothing to do // nothing to do
#else #else
// close auxilliary file // close auxilliary file
TRI_TRACKED_CLOSE_FILE(_fd); TRI_CLOSE(_fd);
_fd = -1; _fd = -1;
#endif #endif
@ -179,7 +178,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res); TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
// remove empty file // remove empty file
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());
@ -235,7 +234,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
} }
if (isPhysical()) { if (isPhysical()) {
TRI_ASSERT(_fd >= 0); TRI_ASSERT(_fd >= 0);
int res = TRI_TRACKED_CLOSE_FILE(_fd); int res = TRI_CLOSE(_fd);
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "unable to close pregel mapped file '" << _filename << "unable to close pregel mapped file '" << _filename

6
arangod/RestServer/FileDescriptorsFeature.cpp
View File

@ -23,7 +23,6 @@
#include "FileDescriptorsFeature.h" #include "FileDescriptorsFeature.h"
#include "Basics/OpenFilesTracker.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "ProgramOptions/ProgramOptions.h" #include "ProgramOptions/ProgramOptions.h"
#include "ProgramOptions/Section.h" #include "ProgramOptions/Section.h"
@ -95,11 +94,6 @@ void FileDescriptorsFeature::start() {
<< StringifyLimitValue(rlim.rlim_cur) << ", please raise to at least " << StringifyLimitValue(rlim.rlim_cur) << ", please raise to at least "
<< RECOMMENDED << " (e.g. ulimit -n " << RECOMMENDED << ")"; << RECOMMENDED << " (e.g. ulimit -n " << RECOMMENDED << ")";
} }
if (rlim.rlim_cur >= 1024) {
// set file descriptor warning threshold value to 95% of max available descriptors
OpenFilesTracker::instance()->warnThreshold(uint64_t(rlim.rlim_cur * 0.95));
}
#endif #endif
} }

1
arangosh/Dump/DumpFeature.cpp
View File

@ -36,7 +36,6 @@
#include "ApplicationFeatures/ApplicationServer.h" #include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/MutexLocker.h" #include "Basics/MutexLocker.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/Result.h" #include "Basics/Result.h"
#include "Basics/StaticStrings.h" #include "Basics/StaticStrings.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"

15
arangosh/Export/ExportFeature.cpp
View File

@ -24,7 +24,6 @@
#include "ApplicationFeatures/ApplicationServer.h" #include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
#include "ProgramOptions/ProgramOptions.h" #include "ProgramOptions/ProgramOptions.h"
@ -336,7 +335,7 @@ void ExportFeature::collectionExport(SimpleHttpClient* httpClient) {
VPackSlice body = parsedBody->slice(); VPackSlice body = parsedBody->slice();
int fd = int fd =
TRI_TRACKED_CREATE_FILE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR); S_IRUSR | S_IWUSR);
if (fd < 0) { if (fd < 0) {
@ -344,7 +343,7 @@ void ExportFeature::collectionExport(SimpleHttpClient* httpClient) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg);
} }
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
writeFirstLine(fd, fileName, collection); writeFirstLine(fd, fileName, collection);
@ -397,7 +396,7 @@ void ExportFeature::queryExport(SimpleHttpClient* httpClient) {
VPackSlice body = parsedBody->slice(); VPackSlice body = parsedBody->slice();
int fd = int fd =
TRI_TRACKED_CREATE_FILE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR); S_IRUSR | S_IWUSR);
if (fd < 0) { if (fd < 0) {
@ -405,7 +404,7 @@ void ExportFeature::queryExport(SimpleHttpClient* httpClient) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg);
} }
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
writeFirstLine(fd, fileName, ""); writeFirstLine(fd, fileName, "");
@ -651,14 +650,14 @@ void ExportFeature::graphExport(SimpleHttpClient* httpClient) {
} }
int fd = int fd =
TRI_TRACKED_CREATE_FILE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR); S_IRUSR | S_IWUSR);
if (fd < 0) { if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'"; errorMsg = "cannot write to file '" + fileName + "'";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg);
} }
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
std::string xmlHeader = std::string xmlHeader =
R"(<?xml version="1.0" encoding="UTF-8" standalone="yes"?> R"(<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
@ -848,4 +847,4 @@ void ExportFeature::xgmmlWriteOneAtt(int fd, std::string const& fileName,
} }
} }
} // arangodb } // arangodb

19
arangosh/Import/ImportHelper.cpp
View File

@ -25,7 +25,6 @@
#include "ImportHelper.h" #include "ImportHelper.h"
#include "Basics/ConditionLocker.h" #include "Basics/ConditionLocker.h"
#include "Basics/MutexLocker.h" #include "Basics/MutexLocker.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Basics/VelocyPackHelper.h" #include "Basics/VelocyPackHelper.h"
#include "Basics/files.h" #include "Basics/files.h"
@ -246,7 +245,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
} else { } else {
// read filesize // read filesize
totalLength = TRI_SizeFile(fileName.c_str()); totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_TRACKED_OPEN_FILE(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC); fd = TRI_OPEN(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
_errorMessages.push_back(TRI_LAST_ERROR_STR); _errorMessages.push_back(TRI_LAST_ERROR_STR);
@ -265,7 +264,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
if (separator == nullptr) { if (separator == nullptr) {
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
_errorMessages.push_back("out of memory"); _errorMessages.push_back("out of memory");
return false; return false;
@ -299,7 +298,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
TRI_Free(separator); TRI_Free(separator);
TRI_DestroyCsvParser(&parser); TRI_DestroyCsvParser(&parser);
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
_errorMessages.push_back(TRI_LAST_ERROR_STR); _errorMessages.push_back(TRI_LAST_ERROR_STR);
return false; return false;
@ -326,7 +325,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
TRI_Free(separator); TRI_Free(separator);
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
waitForSenders(); waitForSenders();
@ -363,7 +362,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
} else { } else {
// read filesize // read filesize
totalLength = TRI_SizeFile(fileName.c_str()); totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_TRACKED_OPEN_FILE(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC); fd = TRI_OPEN(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
_errorMessages.push_back(TRI_LAST_ERROR_STR); _errorMessages.push_back(TRI_LAST_ERROR_STR);
@ -391,7 +390,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
_errorMessages.push_back(TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY)); _errorMessages.push_back(TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY));
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
return false; return false;
} }
@ -402,7 +401,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
if (n < 0) { if (n < 0) {
_errorMessages.push_back(TRI_LAST_ERROR_STR); _errorMessages.push_back(TRI_LAST_ERROR_STR);
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
return false; return false;
} else if (n == 0) { } else if (n == 0) {
@ -434,7 +433,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
if (_outputBuffer.length() > _maxUploadSize) { if (_outputBuffer.length() > _maxUploadSize) {
if (isObject) { if (isObject) {
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
_errorMessages.push_back("import file is too big. please increase the value of --batch-size " _errorMessages.push_back("import file is too big. please increase the value of --batch-size "
"(currently " + "(currently " +
@ -459,7 +458,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
} }
if (fd != STDIN_FILENO) { if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
waitForSenders(); waitForSenders();

7
arangosh/Utils/ManagedDirectory.cpp
View File

@ -24,7 +24,6 @@
#include "ManagedDirectory.h" #include "ManagedDirectory.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Basics/files.h" #include "Basics/files.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
@ -77,14 +76,14 @@ inline std::string filePath(std::string const& directory,
/// @brief Opens a file given a path and flags /// @brief Opens a file given a path and flags
inline int openFile(std::string const& path, int flags) { inline int openFile(std::string const& path, int flags) {
return (::flagIsSet(flags, O_CREAT) return (::flagIsSet(flags, O_CREAT)
? TRI_TRACKED_CREATE_FILE(path.c_str(), flags, S_IRUSR | S_IWUSR) ? TRI_CREATE(path.c_str(), flags, S_IRUSR | S_IWUSR)
: TRI_TRACKED_OPEN_FILE(path.c_str(), flags)); : TRI_OPEN(path.c_str(), flags));
} }
/// @brief Closes an open file and sets the status /// @brief Closes an open file and sets the status
inline void closeFile(int& fd, arangodb::Result& status) { inline void closeFile(int& fd, arangodb::Result& status) {
TRI_ASSERT(fd >= 0); TRI_ASSERT(fd >= 0);
status = arangodb::Result{TRI_TRACKED_CLOSE_FILE(fd)}; status = arangodb::Result{TRI_CLOSE(fd)};
fd = -1; fd = -1;
} }

13
lib/Basics/FileUtils.cpp
View File

@ -35,7 +35,6 @@
#include <functional> #include <functional>
#include "Basics/Exceptions.h" #include "Basics/Exceptions.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringBuffer.h" #include "Basics/StringBuffer.h"
#include "Basics/files.h" #include "Basics/files.h"
#include "Basics/tri-strings.h" #include "Basics/tri-strings.h"
@ -151,13 +150,13 @@ static void fillStringBuffer(int fd, std::string const& filename,
} }
std::string slurp(std::string const& filename) { std::string slurp(std::string const& filename) {
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd == -1) { if (fd == -1) {
throwFileReadError(filename); throwFileReadError(filename);
} }
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
constexpr size_t chunkSize = 8192; constexpr size_t chunkSize = 8192;
StringBuffer buffer(chunkSize, false); StringBuffer buffer(chunkSize, false);
@ -168,13 +167,13 @@ std::string slurp(std::string const& filename) {
} }
void slurp(std::string const& filename, StringBuffer& result) { void slurp(std::string const& filename, StringBuffer& result) {
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd == -1) { if (fd == -1) {
throwFileReadError(filename); throwFileReadError(filename);
} }
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
result.reset(); result.reset();
constexpr size_t chunkSize = 8192; constexpr size_t chunkSize = 8192;
@ -192,7 +191,7 @@ static void throwFileWriteError(std::string const& filename) {
} }
void spit(std::string const& filename, char const* ptr, size_t len, bool sync) { void spit(std::string const& filename, char const* ptr, size_t len, bool sync) {
int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(), int fd = TRI_CREATE(filename.c_str(),
O_WRONLY | O_CREAT | O_TRUNC | TRI_O_CLOEXEC, O_WRONLY | O_CREAT | O_TRUNC | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP); S_IRUSR | S_IWUSR | S_IRGRP);
@ -200,7 +199,7 @@ void spit(std::string const& filename, char const* ptr, size_t len, bool sync) {
throwFileWriteError(filename); throwFileWriteError(filename);
} }
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
while (0 < len) { while (0 < len) {
ssize_t n = TRI_WRITE(fd, ptr, static_cast<TRI_write_t>(len)); ssize_t n = TRI_WRITE(fd, ptr, static_cast<TRI_write_t>(len));

94
lib/Basics/OpenFilesTracker.cpp
View File

@ -1,94 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "OpenFilesTracker.h"
#include "Logger/Logger.h"
using namespace arangodb;
namespace {
// single global instance which keeps track of all opened/closed files
OpenFilesTracker Instance;
}
OpenFilesTracker::OpenFilesTracker()
: _warnThreshold(0), _lastWarning(0.0) {}
OpenFilesTracker::~OpenFilesTracker() {}
int OpenFilesTracker::create(char const* path, int oflag, int mode) {
// call underlying open() function from OS
int res = TRI_CREATE(path, oflag, mode);
if (res >= 0) {
LOG_TOPIC(TRACE, Logger::SYSCALL) << "created file '" << path << "' was assigned file descriptor " << res;
increase();
}
return res;
}
int OpenFilesTracker::open(char const* path, int oflag) {
// call underlying open() function from OS
int res = TRI_OPEN(path, oflag);
if (res >= 0) {
LOG_TOPIC(TRACE, Logger::SYSCALL) << "opened file '" << path << "' was assigned file descriptor " << res;
increase();
}
return res;
}
int OpenFilesTracker::close(int fd) noexcept {
// call underlying close() function from OS
int res = TRI_CLOSE(fd);
if (res == TRI_ERROR_NO_ERROR) {
LOG_TOPIC(TRACE, Logger::SYSCALL) << "closed file with file descriptor " << fd;
decrease();
}
return res;
}
void OpenFilesTracker::increase() {
uint64_t nowOpen = ++_numOpenFiles;
if (nowOpen > _warnThreshold && _warnThreshold > 0) {
double now = TRI_microtime();
if (_lastWarning <= 0.0 || now - _lastWarning >= 30.0) {
// warn only every x seconds at most
LOG_TOPIC(WARN, Logger::SYSCALL) << "number of currently open files is now " << nowOpen << " and exceeds the warning threshold value " << _warnThreshold;
_lastWarning = now;
}
}
}
void OpenFilesTracker::decrease() noexcept { --_numOpenFiles; }
OpenFilesTracker* OpenFilesTracker::instance() {
return &Instance;
}

65
lib/Basics/OpenFilesTracker.h
View File

@ -1,65 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_BASICS_OPEN_FILES_TRACKER_H
#define ARANGODB_BASICS_OPEN_FILES_TRACKER_H 1
#include "Basics/Common.h"
#define TRI_TRACKED_CREATE_FILE(a, b, c) arangodb::OpenFilesTracker::instance()->create((a), (b), (c))
#define TRI_TRACKED_OPEN_FILE(a, b) arangodb::OpenFilesTracker::instance()->open((a), (b))
#define TRI_TRACKED_CLOSE_FILE arangodb::OpenFilesTracker::instance()->close
namespace arangodb {
class OpenFilesTracker {
public:
OpenFilesTracker();
~OpenFilesTracker();
int create(char const* path, int oflag, int mode);
int open(char const* path, int oflag);
int close(int fd) noexcept;
uint64_t numOpenFiles() const { return _numOpenFiles.load(); }
void warnThreshold(uint64_t threshold) {
_warnThreshold = threshold;
_lastWarning = 0.0;
}
static OpenFilesTracker* instance();
private:
void increase();
void decrease() noexcept;
private:
std::atomic<uint64_t> _numOpenFiles;
uint64_t _warnThreshold; // configured threshold
double _lastWarning; // timestamp of last warning
};
}
#endif

9
lib/Basics/VelocyPackHelper.cpp
View File

@ -24,7 +24,6 @@
#include "VelocyPackHelper.h" #include "VelocyPackHelper.h"
#include "Basics/Exceptions.h" #include "Basics/Exceptions.h"
#include "Basics/NumberUtils.h" #include "Basics/NumberUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StaticStrings.h" #include "Basics/StaticStrings.h"
#include "Basics/StringBuffer.h" #include "Basics/StringBuffer.h"
#include "Basics/StringRef.h" #include "Basics/StringRef.h"
@ -696,7 +695,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
TRI_UnlinkFile(tmp.c_str()); TRI_UnlinkFile(tmp.c_str());
} }
int fd = TRI_TRACKED_CREATE_FILE(tmp.c_str(), int fd = TRI_CREATE(tmp.c_str(),
O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC, O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
@ -708,7 +707,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
} }
if (!PrintVelocyPack(fd, slice, true)) { if (!PrintVelocyPack(fd, slice, true)) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot write to json file '" << tmp LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot write to json file '" << tmp
<< "': " << TRI_LAST_ERROR_STR; << "': " << TRI_LAST_ERROR_STR;
@ -720,7 +719,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "syncing tmp file '" << tmp << "'"; LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "syncing tmp file '" << tmp << "'";
if (!TRI_fsync(fd)) { if (!TRI_fsync(fd)) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot sync saved json '" << tmp LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot sync saved json '" << tmp
<< "': " << TRI_LAST_ERROR_STR; << "': " << TRI_LAST_ERROR_STR;
@ -729,7 +728,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
} }
} }
int res = TRI_TRACKED_CLOSE_FILE(fd); int res = TRI_CLOSE(fd);
if (res < 0) { if (res < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);

39
lib/Basics/files.cpp
View File

@ -38,7 +38,6 @@
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/ReadLocker.h" #include "Basics/ReadLocker.h"
#include "Basics/ReadWriteLock.h" #include "Basics/ReadWriteLock.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringBuffer.h" #include "Basics/StringBuffer.h"
#include "Basics/StringUtils.h" #include "Basics/StringUtils.h"
#include "Basics/Thread.h" #include "Basics/Thread.h"
@ -77,7 +76,7 @@ struct LockfileRemover {
CloseHandle(fd); CloseHandle(fd);
#else #else
int fd = it.second; int fd = it.second;
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
#endif #endif
TRI_UnlinkFile(it.first.c_str()); TRI_UnlinkFile(it.first.c_str());
@ -853,7 +852,7 @@ int TRI_WriteFile(char const* filename, char const* data, size_t length) {
int fd; int fd;
bool result; bool result;
fd = TRI_TRACKED_CREATE_FILE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR); S_IRUSR | S_IWUSR);
if (fd == -1) { if (fd == -1) {
@ -862,7 +861,7 @@ int TRI_WriteFile(char const* filename, char const* data, size_t length) {
result = TRI_WritePointer(fd, data, length); result = TRI_WritePointer(fd, data, length);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
if (!result) { if (!result) {
return TRI_errno(); return TRI_errno();
@ -901,7 +900,7 @@ bool TRI_fsync(int fd) {
char* TRI_SlurpFile(char const* filename, char* TRI_SlurpFile(char const* filename,
size_t* length) { size_t* length) {
TRI_set_errno(TRI_ERROR_NO_ERROR); TRI_set_errno(TRI_ERROR_NO_ERROR);
int fd = TRI_TRACKED_OPEN_FILE(filename, O_RDONLY | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename, O_RDONLY | TRI_O_CLOEXEC);
if (fd == -1) { if (fd == -1) {
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
@ -915,7 +914,7 @@ char* TRI_SlurpFile(char const* filename,
int res = TRI_ReserveStringBuffer(&result, READBUFFER_SIZE); int res = TRI_ReserveStringBuffer(&result, READBUFFER_SIZE);
if (res != TRI_ERROR_NO_ERROR) { if (res != TRI_ERROR_NO_ERROR) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_AnnihilateStringBuffer(&result); TRI_AnnihilateStringBuffer(&result);
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY); TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
@ -929,7 +928,7 @@ char* TRI_SlurpFile(char const* filename,
} }
if (n < 0) { if (n < 0) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_AnnihilateStringBuffer(&result); TRI_AnnihilateStringBuffer(&result);
@ -944,7 +943,7 @@ char* TRI_SlurpFile(char const* filename,
*length = TRI_LengthStringBuffer(&result); *length = TRI_LengthStringBuffer(&result);
} }
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
return result._buffer; return result._buffer;
} }
@ -1032,7 +1031,7 @@ int TRI_CreateLockFile(char const* filename) {
} }
} }
int fd = TRI_TRACKED_CREATE_FILE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC, int fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR); S_IRUSR | S_IWUSR);
if (fd == -1) { if (fd == -1) {
@ -1047,7 +1046,7 @@ int TRI_CreateLockFile(char const* filename) {
if (rv == -1) { if (rv == -1) {
int res = TRI_set_errno(TRI_ERROR_SYS_ERROR); int res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_UNLINK(filename); TRI_UNLINK(filename);
return res; return res;
@ -1065,7 +1064,7 @@ int TRI_CreateLockFile(char const* filename) {
if (rv == -1) { if (rv == -1) {
int res = TRI_set_errno(TRI_ERROR_SYS_ERROR); int res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_UNLINK(filename); TRI_UNLINK(filename);
return res; return res;
@ -1115,7 +1114,7 @@ int TRI_VerifyLockFile(char const* filename) {
return TRI_ERROR_NO_ERROR; return TRI_ERROR_NO_ERROR;
} }
int fd = TRI_TRACKED_OPEN_FILE(filename, O_RDWR | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
@ -1133,7 +1132,7 @@ int TRI_VerifyLockFile(char const* filename) {
sizeof(buffer)); // not really necessary, but this shuts up valgrind sizeof(buffer)); // not really necessary, but this shuts up valgrind
ssize_t n = TRI_READ(fd, buffer, static_cast<TRI_read_t>(sizeof(buffer))); ssize_t n = TRI_READ(fd, buffer, static_cast<TRI_read_t>(sizeof(buffer)));
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd)); TRI_DEFER(TRI_CLOSE(fd));
if (n <= 0 || n == sizeof(buffer)) { if (n <= 0 || n == sizeof(buffer)) {
return TRI_ERROR_NO_ERROR; return TRI_ERROR_NO_ERROR;
@ -1230,7 +1229,7 @@ int TRI_DestroyLockFile(char const* filename) {
for (size_t i = 0; i < OpenedFiles.size(); ++i) { for (size_t i = 0; i < OpenedFiles.size(); ++i) {
if (OpenedFiles[i].first == filename) { if (OpenedFiles[i].first == filename) {
int fd = TRI_TRACKED_OPEN_FILE(filename, O_RDWR | TRI_O_CLOEXEC); int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
return TRI_ERROR_NO_ERROR; return TRI_ERROR_NO_ERROR;
@ -1244,7 +1243,7 @@ int TRI_DestroyLockFile(char const* filename) {
lock.l_whence = SEEK_SET; lock.l_whence = SEEK_SET;
// release the lock // release the lock
int res = fcntl(fd, F_SETLK, &lock); int res = fcntl(fd, F_SETLK, &lock);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
if (res == 0) { if (res == 0) {
TRI_UnlinkFile(filename); TRI_UnlinkFile(filename);
@ -1252,7 +1251,7 @@ int TRI_DestroyLockFile(char const* filename) {
// close lock file descriptor // close lock file descriptor
fd = OpenedFiles[i].second; fd = OpenedFiles[i].second;
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
OpenedFiles.erase(OpenedFiles.begin() + i); OpenedFiles.erase(OpenedFiles.begin() + i);
@ -2256,12 +2255,12 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
TRI_ERRORBUF; TRI_ERRORBUF;
// open the file // open the file
int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC | TRI_NOATIME, int fd = TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC | TRI_NOATIME,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
TRI_IF_FAILURE("CreateDatafile1") { TRI_IF_FAILURE("CreateDatafile1") {
// intentionally fail // intentionally fail
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
fd = -1; fd = -1;
errno = ENOSPC; errno = ENOSPC;
} }
@ -2319,7 +2318,7 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF; LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF;
} }
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());
return -1; return -1;
@ -2335,7 +2334,7 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
if (offset == (TRI_lseek_t)-1) { if (offset == (TRI_lseek_t)-1) {
TRI_SYSTEM_ERROR(); TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR); TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
// remove empty file // remove empty file
TRI_UnlinkFile(filename.c_str()); TRI_UnlinkFile(filename.c_str());

6
lib/Basics/voc-errors.h
View File

@ -1020,9 +1020,9 @@ constexpr int TRI_ERROR_QUERY_VARIABLE_NAME_UNKNOWN
constexpr int TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED = 1521; constexpr int TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED = 1521;
/// 1522: ERROR_QUERY_TOO_MANY_COLLECTIONS /// 1522: ERROR_QUERY_TOO_MANY_COLLECTIONS
/// "too many collections" /// "too many collections/shards"
/// Will be raised when the number of collections in a query is beyond the /// Will be raised when the number of collections or shards in a query is
/// allowed value. /// beyond the allowed value.
constexpr int TRI_ERROR_QUERY_TOO_MANY_COLLECTIONS = 1522; constexpr int TRI_ERROR_QUERY_TOO_MANY_COLLECTIONS = 1522;
/// 1530: ERROR_QUERY_DOCUMENT_ATTRIBUTE_REDECLARED /// 1530: ERROR_QUERY_DOCUMENT_ATTRIBUTE_REDECLARED

1
lib/CMakeLists.txt
View File

@ -174,7 +174,6 @@ add_library(${LIB_ARANGO} STATIC
Basics/LocalTaskQueue.cpp Basics/LocalTaskQueue.cpp
Basics/Mutex.cpp Basics/Mutex.cpp
Basics/Nonce.cpp Basics/Nonce.cpp
Basics/OpenFilesTracker.cpp
Basics/ReadWriteLock.cpp Basics/ReadWriteLock.cpp
Basics/Result.cpp Basics/Result.cpp
Basics/RocksDBLogger.cpp Basics/RocksDBLogger.cpp

9
lib/Logger/LogAppenderFile.cpp
View File

@ -26,7 +26,6 @@
#include "ApplicationFeatures/ShellColorsFeature.h" #include "ApplicationFeatures/ShellColorsFeature.h"
#include "Basics/Exceptions.h" #include "Basics/Exceptions.h"
#include "Basics/FileUtils.h" #include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/tri-strings.h" #include "Basics/tri-strings.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
@ -123,7 +122,7 @@ LogAppenderFile::LogAppenderFile(std::string const& filename, std::string const&
if (_fd == -1) { if (_fd == -1) {
// no existing appender found yet // no existing appender found yet
int fd = TRI_TRACKED_CREATE_FILE(_filename.c_str(), int fd = TRI_CREATE(_filename.c_str(),
O_APPEND | O_CREAT | O_WRONLY | TRI_O_CLOEXEC, O_APPEND | O_CREAT | O_WRONLY | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP); S_IRUSR | S_IWUSR | S_IRGRP);
@ -205,7 +204,7 @@ void LogAppenderFile::reopenAll() {
TRI_RenameFile(filename.c_str(), backup.c_str()); TRI_RenameFile(filename.c_str(), backup.c_str());
// open new log file // open new log file
int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(), int fd = TRI_CREATE(filename.c_str(),
O_APPEND | O_CREAT | O_WRONLY | TRI_O_CLOEXEC, O_APPEND | O_CREAT | O_WRONLY | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP); S_IRUSR | S_IWUSR | S_IRGRP);
@ -224,7 +223,7 @@ void LogAppenderFile::reopenAll() {
std::get<2>(it)->updateFd(fd); std::get<2>(it)->updateFd(fd);
if (old > STDERR_FILENO) { if (old > STDERR_FILENO) {
TRI_TRACKED_CLOSE_FILE(old); TRI_CLOSE(old);
} }
} }
} }
@ -239,7 +238,7 @@ void LogAppenderFile::closeAll() {
if (fd > STDERR_FILENO) { if (fd > STDERR_FILENO) {
fsync(fd); fsync(fd);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
} }
} }

9
lib/Random/RandomGenerator.cpp
View File

@ -32,7 +32,6 @@
#include "Basics/Exceptions.h" #include "Basics/Exceptions.h"
#include "Basics/HybridLogicalClock.h" #include "Basics/HybridLogicalClock.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/Thread.h" #include "Basics/Thread.h"
#include "Basics/hashes.h" #include "Basics/hashes.h"
#include "Logger/Logger.h" #include "Logger/Logger.h"
@ -194,7 +193,7 @@ template <int N>
class RandomDeviceDirect : public RandomDevice { class RandomDeviceDirect : public RandomDevice {
public: public:
explicit RandomDeviceDirect(std::string const& path) : fd(-1), pos(0) { explicit RandomDeviceDirect(std::string const& path) : fd(-1), pos(0) {
fd = TRI_TRACKED_OPEN_FILE(path.c_str(), O_RDONLY | TRI_O_CLOEXEC); fd = TRI_OPEN(path.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
std::string message("cannot open random source '" + path + "'"); std::string message("cannot open random source '" + path + "'");
@ -206,7 +205,7 @@ class RandomDeviceDirect : public RandomDevice {
~RandomDeviceDirect() { ~RandomDeviceDirect() {
if (fd >= 0) { if (fd >= 0) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
} }
@ -263,7 +262,7 @@ class RandomDeviceCombined : public RandomDevice {
public: public:
explicit RandomDeviceCombined(std::string const& path) explicit RandomDeviceCombined(std::string const& path)
: fd(-1), pos(0), rseed(0) { : fd(-1), pos(0), rseed(0) {
fd = TRI_TRACKED_OPEN_FILE(path.c_str(), O_RDONLY | TRI_O_CLOEXEC); fd = TRI_OPEN(path.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
std::string message("cannot open random source '" + path + "'"); std::string message("cannot open random source '" + path + "'");
@ -290,7 +289,7 @@ class RandomDeviceCombined : public RandomDevice {
~RandomDeviceCombined() { ~RandomDeviceCombined() {
if (fd >= 0) { if (fd >= 0) {
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
} }
} }

7
lib/V8/v8-shell.cpp
View File

@ -26,7 +26,6 @@
#include "ApplicationFeatures/ShellColorsFeature.h" #include "ApplicationFeatures/ShellColorsFeature.h"
#include "Basics/csv.h" #include "Basics/csv.h"
#include "Basics/Exceptions.h" #include "Basics/Exceptions.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/tri-strings.h" #include "Basics/tri-strings.h"
#include "V8/v8-globals.h" #include "V8/v8-globals.h"
#include "V8/v8-conv.h" #include "V8/v8-conv.h"
@ -163,7 +162,7 @@ static void JS_ProcessCsvFile(v8::FunctionCallbackInfo<v8::Value> const& args) {
} }
// read and convert // read and convert
int fd = TRI_TRACKED_OPEN_FILE(*filename, O_RDONLY | TRI_O_CLOEXEC); int fd = TRI_OPEN(*filename, O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) { if (fd < 0) {
TRI_V8_THROW_EXCEPTION_SYS("cannot open file"); TRI_V8_THROW_EXCEPTION_SYS("cannot open file");
@ -193,7 +192,7 @@ static void JS_ProcessCsvFile(v8::FunctionCallbackInfo<v8::Value> const& args) {
if (n < 0) { if (n < 0) {
TRI_DestroyCsvParser(&parser); TRI_DestroyCsvParser(&parser);
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_V8_THROW_EXCEPTION_SYS("cannot read file"); TRI_V8_THROW_EXCEPTION_SYS("cannot read file");
} else if (n == 0) { } else if (n == 0) {
TRI_DestroyCsvParser(&parser); TRI_DestroyCsvParser(&parser);
@ -203,7 +202,7 @@ static void JS_ProcessCsvFile(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_ParseCsvString(&parser, buffer, n); TRI_ParseCsvString(&parser, buffer, n);
} }
TRI_TRACKED_CLOSE_FILE(fd); TRI_CLOSE(fd);
TRI_V8_RETURN_UNDEFINED(); TRI_V8_RETURN_UNDEFINED();
TRI_V8_TRY_CATCH_END TRI_V8_TRY_CATCH_END

24
lib/V8/v8-utils.cpp
View File

@ -102,9 +102,7 @@ TRI_Utf8ValueNFC::TRI_Utf8ValueNFC(v8::Handle<v8::Value> const obj)
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
TRI_Utf8ValueNFC::~TRI_Utf8ValueNFC() { TRI_Utf8ValueNFC::~TRI_Utf8ValueNFC() {
if (_str != nullptr) { TRI_Free(_str);
TRI_Free(_str);
}
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
@ -173,13 +171,16 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
char* content = TRI_SlurpFile(filename, &length); char* content = TRI_SlurpFile(filename, &length);
if (content == nullptr) { if (content == nullptr) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load java script file '" LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load JavaScript file '"
<< filename << filename
<< "': " << TRI_last_error(); << "': " << TRI_last_error();
return false; return false;
} }
// detect shebang auto guard = scopeGuard([&content] {
TRI_FreeString(content);
});
size_t bangOffset = 0; size_t bangOffset = 0;
if (stripShebang) { if (stripShebang) {
if (strncmp(content, "#!", 2) == 0) { if (strncmp(content, "#!", 2) == 0) {
@ -194,13 +195,14 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
} }
if (useGlobalContext) { if (useGlobalContext) {
char const* prologue = "(function() { "; constexpr char const* prologue = "(function() { ";
char const* epilogue = "/* end-of-file */ })()"; constexpr char const* epilogue = "/* end-of-file */ })()";
char* contentWrapper = TRI_Concatenate3String( char* contentWrapper = TRI_Concatenate3String(
prologue, content + bangOffset, epilogue); prologue, content + bangOffset, epilogue);
TRI_FreeString(content); TRI_FreeString(content);
content = nullptr;
length += strlen(prologue) + strlen(epilogue); length += strlen(prologue) + strlen(epilogue);
content = contentWrapper; content = contentWrapper;
@ -211,7 +213,7 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
if (content == nullptr) { if (content == nullptr) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "cannot load java script file '" << filename << "cannot load JavaScript file '" << filename
<< "': " << TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY); << "': " << TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY);
return false; return false;
} }
@ -220,8 +222,6 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
v8::Handle<v8::String> source = v8::Handle<v8::String> source =
TRI_V8_PAIR_STRING(isolate, content + bangOffset, (int)length); TRI_V8_PAIR_STRING(isolate, content + bangOffset, (int)length);
TRI_FreeString(content);
v8::TryCatch tryCatch; v8::TryCatch tryCatch;
v8::Handle<v8::Script> script = v8::Script::Compile(source, name); v8::Handle<v8::Script> script = v8::Script::Compile(source, name);
@ -233,7 +233,7 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
// compilation failed, print errors that happened during compilation // compilation failed, print errors that happened during compilation
if (script.IsEmpty()) { if (script.IsEmpty()) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load java script file '" LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load JavaScript file '"
<< filename << filename
<< "': compilation failed."; << "': compilation failed.";
return false; return false;
@ -253,7 +253,7 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
} }
} }
LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "loaded java script file: '" LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "loaded JavaScript file: '"
<< filename << "'"; << filename << "'";
return true; return true;
} }