1
0
Fork 0

remove OpenFilesTracker (#7189)

This commit is contained in:
Jan 2018-11-02 11:38:38 +01:00 committed by GitHub
parent f088733420
commit 976cc38e7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 103 additions and 281 deletions

View File

@ -24,7 +24,6 @@
#include "MMFilesDatafile.h"
#include "ApplicationFeatures/PageSizeFeature.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringUtils.h"
#include "Basics/encoding.h"
@ -188,7 +187,7 @@ static MMFilesDatafile* CreateAnonymousDatafile(TRI_voc_fid_t fid,
int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED;
#else
// ugly workaround if MAP_ANONYMOUS is not available
int fd = TRI_TRACKED_OPEN_FILE("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
int fd = TRI_OPEN("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
if (fd == -1) {
return nullptr;
@ -207,7 +206,7 @@ static MMFilesDatafile* CreateAnonymousDatafile(TRI_voc_fid_t fid,
// nothing to do
#else
// close auxilliary file
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
fd = -1;
#endif
@ -255,7 +254,7 @@ static MMFilesDatafile* CreatePhysicalDatafile(std::string const& filename,
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
@ -270,7 +269,7 @@ static MMFilesDatafile* CreatePhysicalDatafile(std::string const& filename,
try {
return new MMFilesDatafile(filename, fd, mmHandle, maximalSize, 0, fid, static_cast<char*>(data));
} catch (...) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return nullptr;
}
}
@ -289,7 +288,7 @@ int MMFilesDatafile::judge(std::string const& filename) {
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
}
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
@ -298,7 +297,7 @@ int MMFilesDatafile::judge(std::string const& filename) {
uint64_t buffer[256];
if (!TRI_ReadPointer(fd, &buffer, 256 * sizeof(uint64_t))) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
}
@ -307,13 +306,13 @@ int MMFilesDatafile::judge(std::string const& filename) {
while (ptr < end) {
if (*ptr != 0) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return TRI_ERROR_NO_ERROR;
}
++ptr;
}
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return TRI_ERROR_ARANGO_DATAFILE_EMPTY;
}
@ -1023,7 +1022,7 @@ int MMFilesDatafile::close() {
if (isPhysical()) {
TRI_ASSERT(_fd >= 0);
int res = TRI_TRACKED_CLOSE_FILE(_fd);
int res = TRI_CLOSE(_fd);
if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "unable to close datafile '" << getName() << "': " << res;
@ -1111,7 +1110,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
std::string filename = getName() + ".new";
int fd =
TRI_TRACKED_CREATE_FILE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
@ -1127,7 +1126,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
if (offset == (TRI_lseek_t)-1) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
@ -1142,7 +1141,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
if (written < 0) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
@ -1159,7 +1158,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
if (res != TRI_ERROR_NO_ERROR) {
TRI_SYSTEM_ERROR();
TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
@ -1179,7 +1178,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
res = TRI_UNMMFile(_data, _initSize, _fd, &_mmHandle);
if (res < 0) {
TRI_TRACKED_CLOSE_FILE(_fd);
TRI_CLOSE(_fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "munmap failed with: " << res;
return res;
}
@ -1191,7 +1190,7 @@ int MMFilesDatafile::truncateAndSeal(uint32_t position) {
// associated file below
// .............................................................................................
TRI_TRACKED_CLOSE_FILE(_fd);
TRI_CLOSE(_fd);
_data = static_cast<char*>(data);
_next = (char*)(data) + position;
@ -1795,7 +1794,7 @@ MMFilesDatafile* MMFilesDatafile::open(std::string const& filename, bool ignoreF
if (!ok) {
TRI_UNMMFile(const_cast<char*>(datafile->data()), datafile->initSize(), datafile->fd(), &datafile->_mmHandle);
TRI_TRACKED_CLOSE_FILE(datafile->fd());
TRI_CLOSE(datafile->fd());
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile '" << datafile->getName() << "' is corrupt";
// must free datafile here
@ -1828,7 +1827,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
TRI_voc_fid_t fid = GetNumericFilenamePart(filename.c_str());
// attempt to open a datafile file
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDWR | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename.c_str(), O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) {
TRI_SYSTEM_ERROR();
@ -1846,7 +1845,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (res < 0) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot get status of datafile '" << filename << "': " << TRI_GET_ERRORBUF;
@ -1858,7 +1857,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (size < sizeof(MMFilesDatafileHeaderMarker) + sizeof(MMFilesDatafileFooterMarker)) {
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile '" << filename << "' is corrupt, size is only " << size;
@ -1881,7 +1880,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (!ok) {
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot read datafile header from '" << filename << "': " << TRI_last_error();
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return nullptr;
}
@ -1894,7 +1893,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (!ok) {
if (IsMarker28(ptr, len)) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "datafile found from older version of ArangoDB. "
<< "Please dump data from that version with arangodump "
<< "and reload it into this ArangoDB instance with arangorestore";
@ -1908,7 +1907,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
printMarker(reinterpret_cast<MMFilesMarker const*>(ptr), len, &buffer[0], end);
if (!ignoreErrors) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return nullptr;
}
}
@ -1921,7 +1920,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "unknown datafile version '" << header->_version << "' in datafile '" << filename << "'";
if (!ignoreErrors) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return nullptr;
}
}
@ -1939,7 +1938,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "cannot memory map datafile '" << filename << "': " << TRI_errno_string(res);
LOG_TOPIC(ERR, arangodb::Logger::DATAFILES) << "The database directory might reside on a shared folder "
@ -1952,7 +1951,7 @@ MMFilesDatafile* MMFilesDatafile::openHelper(std::string const& filename, bool i
return new MMFilesDatafile(filename, fd, mmHandle, size, size, fid, static_cast<char*>(data));
} catch (...) {
TRI_UNMMFile(data, size, fd, &mmHandle);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return nullptr;
}

View File

@ -25,7 +25,6 @@
#include "Basics/Common.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/files.h"
#include "Basics/memory-map.h"
#include "Logger/Logger.h"
@ -119,7 +118,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED;
#else
// ugly workaround if MAP_ANONYMOUS is not available
_fd = TRI_TRACKED_OPEN_FILE("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
_fd = TRI_OPEN("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
if (_fd == -1) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
}
@ -136,7 +135,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
// nothing to do
#else
// close auxilliary file
TRI_TRACKED_CLOSE_FILE(_fd);
TRI_CLOSE(_fd);
_fd = -1;
#endif
@ -179,7 +178,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
@ -235,7 +234,7 @@ class MappedFileBuffer : public TypedBuffer<T> {
}
if (isPhysical()) {
TRI_ASSERT(_fd >= 0);
int res = TRI_TRACKED_CLOSE_FILE(_fd);
int res = TRI_CLOSE(_fd);
if (res != TRI_ERROR_NO_ERROR) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "unable to close pregel mapped file '" << _filename

View File

@ -23,7 +23,6 @@
#include "FileDescriptorsFeature.h"
#include "Basics/OpenFilesTracker.h"
#include "Logger/Logger.h"
#include "ProgramOptions/ProgramOptions.h"
#include "ProgramOptions/Section.h"
@ -95,11 +94,6 @@ void FileDescriptorsFeature::start() {
<< StringifyLimitValue(rlim.rlim_cur) << ", please raise to at least "
<< RECOMMENDED << " (e.g. ulimit -n " << RECOMMENDED << ")";
}
if (rlim.rlim_cur >= 1024) {
// set file descriptor warning threshold value to 95% of max available descriptors
OpenFilesTracker::instance()->warnThreshold(uint64_t(rlim.rlim_cur * 0.95));
}
#endif
}

View File

@ -36,7 +36,6 @@
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h"
#include "Basics/MutexLocker.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/Result.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringUtils.h"

View File

@ -24,7 +24,6 @@
#include "ApplicationFeatures/ApplicationServer.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringUtils.h"
#include "Logger/Logger.h"
#include "ProgramOptions/ProgramOptions.h"
@ -336,7 +335,7 @@ void ExportFeature::collectionExport(SimpleHttpClient* httpClient) {
VPackSlice body = parsedBody->slice();
int fd =
TRI_TRACKED_CREATE_FILE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
@ -344,7 +343,7 @@ void ExportFeature::collectionExport(SimpleHttpClient* httpClient) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg);
}
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
writeFirstLine(fd, fileName, collection);
@ -397,7 +396,7 @@ void ExportFeature::queryExport(SimpleHttpClient* httpClient) {
VPackSlice body = parsedBody->slice();
int fd =
TRI_TRACKED_CREATE_FILE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
@ -405,7 +404,7 @@ void ExportFeature::queryExport(SimpleHttpClient* httpClient) {
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg);
}
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
writeFirstLine(fd, fileName, "");
@ -651,14 +650,14 @@ void ExportFeature::graphExport(SimpleHttpClient* httpClient) {
}
int fd =
TRI_TRACKED_CREATE_FILE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
TRI_CREATE(fileName.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
errorMsg = "cannot write to file '" + fileName + "'";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_CANNOT_WRITE_FILE, errorMsg);
}
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
std::string xmlHeader =
R"(<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
@ -848,4 +847,4 @@ void ExportFeature::xgmmlWriteOneAtt(int fd, std::string const& fileName,
}
}
} // arangodb
} // arangodb

View File

@ -25,7 +25,6 @@
#include "ImportHelper.h"
#include "Basics/ConditionLocker.h"
#include "Basics/MutexLocker.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringUtils.h"
#include "Basics/VelocyPackHelper.h"
#include "Basics/files.h"
@ -246,7 +245,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
} else {
// read filesize
totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_TRACKED_OPEN_FILE(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC);
fd = TRI_OPEN(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
_errorMessages.push_back(TRI_LAST_ERROR_STR);
@ -265,7 +264,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
if (separator == nullptr) {
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
_errorMessages.push_back("out of memory");
return false;
@ -299,7 +298,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
TRI_Free(separator);
TRI_DestroyCsvParser(&parser);
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
_errorMessages.push_back(TRI_LAST_ERROR_STR);
return false;
@ -326,7 +325,7 @@ bool ImportHelper::importDelimited(std::string const& collectionName,
TRI_Free(separator);
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
waitForSenders();
@ -363,7 +362,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
} else {
// read filesize
totalLength = TRI_SizeFile(fileName.c_str());
fd = TRI_TRACKED_OPEN_FILE(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC);
fd = TRI_OPEN(fileName.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
_errorMessages.push_back(TRI_LAST_ERROR_STR);
@ -391,7 +390,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
_errorMessages.push_back(TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY));
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
return false;
}
@ -402,7 +401,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
if (n < 0) {
_errorMessages.push_back(TRI_LAST_ERROR_STR);
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
return false;
} else if (n == 0) {
@ -434,7 +433,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
if (_outputBuffer.length() > _maxUploadSize) {
if (isObject) {
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
_errorMessages.push_back("import file is too big. please increase the value of --batch-size "
"(currently " +
@ -459,7 +458,7 @@ bool ImportHelper::importJson(std::string const& collectionName,
}
if (fd != STDIN_FILENO) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
waitForSenders();

View File

@ -24,7 +24,6 @@
#include "ManagedDirectory.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringUtils.h"
#include "Basics/files.h"
#include "Logger/Logger.h"
@ -77,14 +76,14 @@ inline std::string filePath(std::string const& directory,
/// @brief Opens a file given a path and flags
inline int openFile(std::string const& path, int flags) {
return (::flagIsSet(flags, O_CREAT)
? TRI_TRACKED_CREATE_FILE(path.c_str(), flags, S_IRUSR | S_IWUSR)
: TRI_TRACKED_OPEN_FILE(path.c_str(), flags));
? TRI_CREATE(path.c_str(), flags, S_IRUSR | S_IWUSR)
: TRI_OPEN(path.c_str(), flags));
}
/// @brief Closes an open file and sets the status
inline void closeFile(int& fd, arangodb::Result& status) {
TRI_ASSERT(fd >= 0);
status = arangodb::Result{TRI_TRACKED_CLOSE_FILE(fd)};
status = arangodb::Result{TRI_CLOSE(fd)};
fd = -1;
}

View File

@ -35,7 +35,6 @@
#include <functional>
#include "Basics/Exceptions.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringBuffer.h"
#include "Basics/files.h"
#include "Basics/tri-strings.h"
@ -151,13 +150,13 @@ static void fillStringBuffer(int fd, std::string const& filename,
}
std::string slurp(std::string const& filename) {
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd == -1) {
throwFileReadError(filename);
}
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
constexpr size_t chunkSize = 8192;
StringBuffer buffer(chunkSize, false);
@ -168,13 +167,13 @@ std::string slurp(std::string const& filename) {
}
void slurp(std::string const& filename, StringBuffer& result) {
int fd = TRI_TRACKED_OPEN_FILE(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd == -1) {
throwFileReadError(filename);
}
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
result.reset();
constexpr size_t chunkSize = 8192;
@ -192,7 +191,7 @@ static void throwFileWriteError(std::string const& filename) {
}
void spit(std::string const& filename, char const* ptr, size_t len, bool sync) {
int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(),
int fd = TRI_CREATE(filename.c_str(),
O_WRONLY | O_CREAT | O_TRUNC | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP);
@ -200,7 +199,7 @@ void spit(std::string const& filename, char const* ptr, size_t len, bool sync) {
throwFileWriteError(filename);
}
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
while (0 < len) {
ssize_t n = TRI_WRITE(fd, ptr, static_cast<TRI_write_t>(len));

View File

@ -1,94 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "OpenFilesTracker.h"
#include "Logger/Logger.h"
using namespace arangodb;
namespace {
// single global instance which keeps track of all opened/closed files
OpenFilesTracker Instance;
}
OpenFilesTracker::OpenFilesTracker()
: _warnThreshold(0), _lastWarning(0.0) {}
OpenFilesTracker::~OpenFilesTracker() {}
int OpenFilesTracker::create(char const* path, int oflag, int mode) {
// call underlying open() function from OS
int res = TRI_CREATE(path, oflag, mode);
if (res >= 0) {
LOG_TOPIC(TRACE, Logger::SYSCALL) << "created file '" << path << "' was assigned file descriptor " << res;
increase();
}
return res;
}
int OpenFilesTracker::open(char const* path, int oflag) {
// call underlying open() function from OS
int res = TRI_OPEN(path, oflag);
if (res >= 0) {
LOG_TOPIC(TRACE, Logger::SYSCALL) << "opened file '" << path << "' was assigned file descriptor " << res;
increase();
}
return res;
}
int OpenFilesTracker::close(int fd) noexcept {
// call underlying close() function from OS
int res = TRI_CLOSE(fd);
if (res == TRI_ERROR_NO_ERROR) {
LOG_TOPIC(TRACE, Logger::SYSCALL) << "closed file with file descriptor " << fd;
decrease();
}
return res;
}
void OpenFilesTracker::increase() {
uint64_t nowOpen = ++_numOpenFiles;
if (nowOpen > _warnThreshold && _warnThreshold > 0) {
double now = TRI_microtime();
if (_lastWarning <= 0.0 || now - _lastWarning >= 30.0) {
// warn only every x seconds at most
LOG_TOPIC(WARN, Logger::SYSCALL) << "number of currently open files is now " << nowOpen << " and exceeds the warning threshold value " << _warnThreshold;
_lastWarning = now;
}
}
}
void OpenFilesTracker::decrease() noexcept { --_numOpenFiles; }
OpenFilesTracker* OpenFilesTracker::instance() {
return &Instance;
}

View File

@ -1,65 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_BASICS_OPEN_FILES_TRACKER_H
#define ARANGODB_BASICS_OPEN_FILES_TRACKER_H 1
#include "Basics/Common.h"
#define TRI_TRACKED_CREATE_FILE(a, b, c) arangodb::OpenFilesTracker::instance()->create((a), (b), (c))
#define TRI_TRACKED_OPEN_FILE(a, b) arangodb::OpenFilesTracker::instance()->open((a), (b))
#define TRI_TRACKED_CLOSE_FILE arangodb::OpenFilesTracker::instance()->close
namespace arangodb {
class OpenFilesTracker {
public:
OpenFilesTracker();
~OpenFilesTracker();
int create(char const* path, int oflag, int mode);
int open(char const* path, int oflag);
int close(int fd) noexcept;
uint64_t numOpenFiles() const { return _numOpenFiles.load(); }
void warnThreshold(uint64_t threshold) {
_warnThreshold = threshold;
_lastWarning = 0.0;
}
static OpenFilesTracker* instance();
private:
void increase();
void decrease() noexcept;
private:
std::atomic<uint64_t> _numOpenFiles;
uint64_t _warnThreshold; // configured threshold
double _lastWarning; // timestamp of last warning
};
}
#endif

View File

@ -24,7 +24,6 @@
#include "VelocyPackHelper.h"
#include "Basics/Exceptions.h"
#include "Basics/NumberUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringBuffer.h"
#include "Basics/StringRef.h"
@ -696,7 +695,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
TRI_UnlinkFile(tmp.c_str());
}
int fd = TRI_TRACKED_CREATE_FILE(tmp.c_str(),
int fd = TRI_CREATE(tmp.c_str(),
O_CREAT | O_TRUNC | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
@ -708,7 +707,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
}
if (!PrintVelocyPack(fd, slice, true)) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot write to json file '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
@ -720,7 +719,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "syncing tmp file '" << tmp << "'";
if (!TRI_fsync(fd)) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot sync saved json '" << tmp
<< "': " << TRI_LAST_ERROR_STR;
@ -729,7 +728,7 @@ bool VelocyPackHelper::velocyPackToFile(std::string const& filename,
}
}
int res = TRI_TRACKED_CLOSE_FILE(fd);
int res = TRI_CLOSE(fd);
if (res < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);

View File

@ -38,7 +38,6 @@
#include "Basics/FileUtils.h"
#include "Basics/ReadLocker.h"
#include "Basics/ReadWriteLock.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/StringBuffer.h"
#include "Basics/StringUtils.h"
#include "Basics/Thread.h"
@ -77,7 +76,7 @@ struct LockfileRemover {
CloseHandle(fd);
#else
int fd = it.second;
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
#endif
TRI_UnlinkFile(it.first.c_str());
@ -853,7 +852,7 @@ int TRI_WriteFile(char const* filename, char const* data, size_t length) {
int fd;
bool result;
fd = TRI_TRACKED_CREATE_FILE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd == -1) {
@ -862,7 +861,7 @@ int TRI_WriteFile(char const* filename, char const* data, size_t length) {
result = TRI_WritePointer(fd, data, length);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
if (!result) {
return TRI_errno();
@ -901,7 +900,7 @@ bool TRI_fsync(int fd) {
char* TRI_SlurpFile(char const* filename,
size_t* length) {
TRI_set_errno(TRI_ERROR_NO_ERROR);
int fd = TRI_TRACKED_OPEN_FILE(filename, O_RDONLY | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename, O_RDONLY | TRI_O_CLOEXEC);
if (fd == -1) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
@ -915,7 +914,7 @@ char* TRI_SlurpFile(char const* filename,
int res = TRI_ReserveStringBuffer(&result, READBUFFER_SIZE);
if (res != TRI_ERROR_NO_ERROR) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_AnnihilateStringBuffer(&result);
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
@ -929,7 +928,7 @@ char* TRI_SlurpFile(char const* filename,
}
if (n < 0) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_AnnihilateStringBuffer(&result);
@ -944,7 +943,7 @@ char* TRI_SlurpFile(char const* filename,
*length = TRI_LengthStringBuffer(&result);
}
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
return result._buffer;
}
@ -1032,7 +1031,7 @@ int TRI_CreateLockFile(char const* filename) {
}
}
int fd = TRI_TRACKED_CREATE_FILE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
int fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd == -1) {
@ -1047,7 +1046,7 @@ int TRI_CreateLockFile(char const* filename) {
if (rv == -1) {
int res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_UNLINK(filename);
return res;
@ -1065,7 +1064,7 @@ int TRI_CreateLockFile(char const* filename) {
if (rv == -1) {
int res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_UNLINK(filename);
return res;
@ -1115,7 +1114,7 @@ int TRI_VerifyLockFile(char const* filename) {
return TRI_ERROR_NO_ERROR;
}
int fd = TRI_TRACKED_OPEN_FILE(filename, O_RDWR | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
@ -1133,7 +1132,7 @@ int TRI_VerifyLockFile(char const* filename) {
sizeof(buffer)); // not really necessary, but this shuts up valgrind
ssize_t n = TRI_READ(fd, buffer, static_cast<TRI_read_t>(sizeof(buffer)));
TRI_DEFER(TRI_TRACKED_CLOSE_FILE(fd));
TRI_DEFER(TRI_CLOSE(fd));
if (n <= 0 || n == sizeof(buffer)) {
return TRI_ERROR_NO_ERROR;
@ -1230,7 +1229,7 @@ int TRI_DestroyLockFile(char const* filename) {
for (size_t i = 0; i < OpenedFiles.size(); ++i) {
if (OpenedFiles[i].first == filename) {
int fd = TRI_TRACKED_OPEN_FILE(filename, O_RDWR | TRI_O_CLOEXEC);
int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) {
return TRI_ERROR_NO_ERROR;
@ -1244,7 +1243,7 @@ int TRI_DestroyLockFile(char const* filename) {
lock.l_whence = SEEK_SET;
// release the lock
int res = fcntl(fd, F_SETLK, &lock);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
if (res == 0) {
TRI_UnlinkFile(filename);
@ -1252,7 +1251,7 @@ int TRI_DestroyLockFile(char const* filename) {
// close lock file descriptor
fd = OpenedFiles[i].second;
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
OpenedFiles.erase(OpenedFiles.begin() + i);
@ -2256,12 +2255,12 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
TRI_ERRORBUF;
// open the file
int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC | TRI_NOATIME,
int fd = TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC | TRI_NOATIME,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
TRI_IF_FAILURE("CreateDatafile1") {
// intentionally fail
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
fd = -1;
errno = ENOSPC;
}
@ -2319,7 +2318,7 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF;
}
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_UnlinkFile(filename.c_str());
return -1;
@ -2335,7 +2334,7 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
if (offset == (TRI_lseek_t)-1) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());

View File

@ -1020,9 +1020,9 @@ constexpr int TRI_ERROR_QUERY_VARIABLE_NAME_UNKNOWN
constexpr int TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED = 1521;
/// 1522: ERROR_QUERY_TOO_MANY_COLLECTIONS
/// "too many collections"
/// Will be raised when the number of collections in a query is beyond the
/// allowed value.
/// "too many collections/shards"
/// Will be raised when the number of collections or shards in a query is
/// beyond the allowed value.
constexpr int TRI_ERROR_QUERY_TOO_MANY_COLLECTIONS = 1522;
/// 1530: ERROR_QUERY_DOCUMENT_ATTRIBUTE_REDECLARED

View File

@ -174,7 +174,6 @@ add_library(${LIB_ARANGO} STATIC
Basics/LocalTaskQueue.cpp
Basics/Mutex.cpp
Basics/Nonce.cpp
Basics/OpenFilesTracker.cpp
Basics/ReadWriteLock.cpp
Basics/Result.cpp
Basics/RocksDBLogger.cpp

View File

@ -26,7 +26,6 @@
#include "ApplicationFeatures/ShellColorsFeature.h"
#include "Basics/Exceptions.h"
#include "Basics/FileUtils.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/tri-strings.h"
#include "Logger/Logger.h"
@ -123,7 +122,7 @@ LogAppenderFile::LogAppenderFile(std::string const& filename, std::string const&
if (_fd == -1) {
// no existing appender found yet
int fd = TRI_TRACKED_CREATE_FILE(_filename.c_str(),
int fd = TRI_CREATE(_filename.c_str(),
O_APPEND | O_CREAT | O_WRONLY | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP);
@ -205,7 +204,7 @@ void LogAppenderFile::reopenAll() {
TRI_RenameFile(filename.c_str(), backup.c_str());
// open new log file
int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(),
int fd = TRI_CREATE(filename.c_str(),
O_APPEND | O_CREAT | O_WRONLY | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP);
@ -224,7 +223,7 @@ void LogAppenderFile::reopenAll() {
std::get<2>(it)->updateFd(fd);
if (old > STDERR_FILENO) {
TRI_TRACKED_CLOSE_FILE(old);
TRI_CLOSE(old);
}
}
}
@ -239,7 +238,7 @@ void LogAppenderFile::closeAll() {
if (fd > STDERR_FILENO) {
fsync(fd);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
}
}

View File

@ -32,7 +32,6 @@
#include "Basics/Exceptions.h"
#include "Basics/HybridLogicalClock.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/Thread.h"
#include "Basics/hashes.h"
#include "Logger/Logger.h"
@ -194,7 +193,7 @@ template <int N>
class RandomDeviceDirect : public RandomDevice {
public:
explicit RandomDeviceDirect(std::string const& path) : fd(-1), pos(0) {
fd = TRI_TRACKED_OPEN_FILE(path.c_str(), O_RDONLY | TRI_O_CLOEXEC);
fd = TRI_OPEN(path.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
std::string message("cannot open random source '" + path + "'");
@ -206,7 +205,7 @@ class RandomDeviceDirect : public RandomDevice {
~RandomDeviceDirect() {
if (fd >= 0) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
}
@ -263,7 +262,7 @@ class RandomDeviceCombined : public RandomDevice {
public:
explicit RandomDeviceCombined(std::string const& path)
: fd(-1), pos(0), rseed(0) {
fd = TRI_TRACKED_OPEN_FILE(path.c_str(), O_RDONLY | TRI_O_CLOEXEC);
fd = TRI_OPEN(path.c_str(), O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
std::string message("cannot open random source '" + path + "'");
@ -290,7 +289,7 @@ class RandomDeviceCombined : public RandomDevice {
~RandomDeviceCombined() {
if (fd >= 0) {
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
}
}

View File

@ -26,7 +26,6 @@
#include "ApplicationFeatures/ShellColorsFeature.h"
#include "Basics/csv.h"
#include "Basics/Exceptions.h"
#include "Basics/OpenFilesTracker.h"
#include "Basics/tri-strings.h"
#include "V8/v8-globals.h"
#include "V8/v8-conv.h"
@ -163,7 +162,7 @@ static void JS_ProcessCsvFile(v8::FunctionCallbackInfo<v8::Value> const& args) {
}
// read and convert
int fd = TRI_TRACKED_OPEN_FILE(*filename, O_RDONLY | TRI_O_CLOEXEC);
int fd = TRI_OPEN(*filename, O_RDONLY | TRI_O_CLOEXEC);
if (fd < 0) {
TRI_V8_THROW_EXCEPTION_SYS("cannot open file");
@ -193,7 +192,7 @@ static void JS_ProcessCsvFile(v8::FunctionCallbackInfo<v8::Value> const& args) {
if (n < 0) {
TRI_DestroyCsvParser(&parser);
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_V8_THROW_EXCEPTION_SYS("cannot read file");
} else if (n == 0) {
TRI_DestroyCsvParser(&parser);
@ -203,7 +202,7 @@ static void JS_ProcessCsvFile(v8::FunctionCallbackInfo<v8::Value> const& args) {
TRI_ParseCsvString(&parser, buffer, n);
}
TRI_TRACKED_CLOSE_FILE(fd);
TRI_CLOSE(fd);
TRI_V8_RETURN_UNDEFINED();
TRI_V8_TRY_CATCH_END

View File

@ -102,9 +102,7 @@ TRI_Utf8ValueNFC::TRI_Utf8ValueNFC(v8::Handle<v8::Value> const obj)
////////////////////////////////////////////////////////////////////////////////
TRI_Utf8ValueNFC::~TRI_Utf8ValueNFC() {
if (_str != nullptr) {
TRI_Free(_str);
}
TRI_Free(_str);
}
////////////////////////////////////////////////////////////////////////////////
@ -173,13 +171,16 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
char* content = TRI_SlurpFile(filename, &length);
if (content == nullptr) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load java script file '"
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load JavaScript file '"
<< filename
<< "': " << TRI_last_error();
return false;
}
// detect shebang
auto guard = scopeGuard([&content] {
TRI_FreeString(content);
});
size_t bangOffset = 0;
if (stripShebang) {
if (strncmp(content, "#!", 2) == 0) {
@ -194,13 +195,14 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
}
if (useGlobalContext) {
char const* prologue = "(function() { ";
char const* epilogue = "/* end-of-file */ })()";
constexpr char const* prologue = "(function() { ";
constexpr char const* epilogue = "/* end-of-file */ })()";
char* contentWrapper = TRI_Concatenate3String(
prologue, content + bangOffset, epilogue);
TRI_FreeString(content);
content = nullptr;
length += strlen(prologue) + strlen(epilogue);
content = contentWrapper;
@ -211,7 +213,7 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
if (content == nullptr) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "cannot load java script file '" << filename
<< "cannot load JavaScript file '" << filename
<< "': " << TRI_errno_string(TRI_ERROR_OUT_OF_MEMORY);
return false;
}
@ -220,8 +222,6 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
v8::Handle<v8::String> source =
TRI_V8_PAIR_STRING(isolate, content + bangOffset, (int)length);
TRI_FreeString(content);
v8::TryCatch tryCatch;
v8::Handle<v8::Script> script = v8::Script::Compile(source, name);
@ -233,7 +233,7 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
// compilation failed, print errors that happened during compilation
if (script.IsEmpty()) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load java script file '"
LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot load JavaScript file '"
<< filename
<< "': compilation failed.";
return false;
@ -253,7 +253,7 @@ static bool LoadJavaScriptFile(v8::Isolate* isolate, char const* filename,
}
}
LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "loaded java script file: '"
LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "loaded JavaScript file: '"
<< filename << "'";
return true;
}