Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
arangodb/arangod/VocBase/datafile.cpp

2061 lines
64 KiB
C++
Raw Blame History

////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
////////////////////////////////////////////////////////////////////////////////
#include "datafile.h"
#include "Basics/FileUtils.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringUtils.h"
#include "Basics/files.h"
#include "Basics/hashes.h"
#include "Basics/memory-map.h"
#include "Basics/tri-strings.h"
#include "Logger/Logger.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/server.h"
#include <sstream>
#include <iomanip>
// #define DEBUG_DATAFILE 1
using namespace arangodb;
using namespace arangodb::basics;
////////////////////////////////////////////////////////////////////////////////
/// @brief check if a marker appears to be created by ArangoDB 28
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_crc_t Crc28(TRI_voc_crc_t crc, void const* data, size_t length) {
static TRI_voc_crc_t const CrcPolynomial = 0xEDB88320;
unsigned char* current = (unsigned char*) data;
while (length--) {
crc ^= *current++;
for (unsigned int i = 0; i < 8; ++i) {
if (crc & 1) {
crc = (crc >> 1) ^ CrcPolynomial;
} else {
crc = crc >> 1;
}
}
}
return crc;
}
static bool IsMarker28 (void const* marker) {
struct Marker28 {
TRI_voc_size_t _size;
TRI_voc_crc_t _crc;
uint32_t _type;
#ifdef TRI_PADDING_32
char _padding_df_marker[4];
#endif
TRI_voc_tick_t _tick;
};
TRI_voc_size_t zero = 0;
off_t o = offsetof(Marker28, _crc);
size_t n = sizeof(TRI_voc_crc_t);
char const* ptr = static_cast<char const*>(marker);
Marker28 const* m = static_cast<Marker28 const*>(marker);
TRI_voc_crc_t crc = TRI_InitialCrc32();
crc = Crc28(crc, ptr, o);
crc = Crc28(crc, (char*) &zero, n);
crc = Crc28(crc, ptr + o + n, m->_size - o - n);
crc = TRI_FinalCrc32(crc);
return crc == m->_crc;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return whether the datafile is a physical file (true) or an
/// anonymous mapped region (false)
////////////////////////////////////////////////////////////////////////////////
static bool IsPhysicalDatafile(TRI_datafile_t const* datafile) {
return datafile->_filename != nullptr;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return the name of a datafile
////////////////////////////////////////////////////////////////////////////////
static char const* GetNameDatafile(TRI_datafile_t const* datafile) {
if (datafile->_filename == nullptr) {
// anonymous regions do not have a filename
return "anonymous region";
}
// return name of the physical file
return datafile->_filename;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief close a datafile
////////////////////////////////////////////////////////////////////////////////
static void CloseDatafile(TRI_datafile_t* datafile) {
TRI_ASSERT(datafile->_state != TRI_DF_STATE_CLOSED);
if (datafile->isPhysical(datafile)) {
int res = TRI_CLOSE(datafile->_fd);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "unable to close datafile '" << datafile->getName(datafile) << "': " << res;
}
}
datafile->_state = TRI_DF_STATE_CLOSED;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a datafile
////////////////////////////////////////////////////////////////////////////////
static void DestroyDatafile(TRI_datafile_t* datafile) {
if (datafile->_filename != nullptr) {
TRI_FreeString(TRI_CORE_MEM_ZONE, datafile->_filename);
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief sync the data of a datafile
////////////////////////////////////////////////////////////////////////////////
static bool SyncDatafile(TRI_datafile_t* datafile, char const* begin,
char const* end) {
if (datafile->_filename == nullptr) {
// anonymous regions do not need to be synced
return true;
}
TRI_ASSERT(datafile->_fd >= 0);
if (begin == end) {
// no need to sync
return true;
}
return TRI_MSync(datafile->_fd, begin, end);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief calculates the actual CRC of a marker, without bounds checks
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_crc_t CalculateCrcValue(TRI_df_marker_t const* marker) {
TRI_voc_size_t zero = 0;
off_t o = marker->offsetOfCrc();
size_t n = sizeof(TRI_voc_crc_t);
char const* ptr = reinterpret_cast<char const*>(marker);
TRI_voc_crc_t crc = TRI_InitialCrc32();
crc = TRI_BlockCrc32(crc, ptr, o);
crc = TRI_BlockCrc32(crc, (char*)&zero, n);
crc = TRI_BlockCrc32(crc, ptr + o + n, marker->getSize() - o - n);
crc = TRI_FinalCrc32(crc);
return crc;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks a CRC of a marker, with bounds checks
////////////////////////////////////////////////////////////////////////////////
static bool CheckCrcMarker(TRI_df_marker_t const* marker, char const* end) {
TRI_voc_size_t const size = marker->getSize();
if (size < sizeof(TRI_df_marker_t)) {
return false;
}
if (reinterpret_cast<char const*>(marker) + size > end) {
return false;
}
auto expected = CalculateCrcValue(marker);
return marker->getCrc() == expected;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief creates a new datafile
///
/// returns the file descriptor or -1 if the file cannot be created
////////////////////////////////////////////////////////////////////////////////
static int CreateDatafile(char const* filename, TRI_voc_size_t maximalSize) {
TRI_ERRORBUF;
// open the file
int fd = TRI_CREATE(filename, O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
TRI_IF_FAILURE("CreateDatafile1") {
// intentionally fail
TRI_CLOSE(fd);
fd = -1;
errno = ENOSPC;
}
if (fd < 0) {
if (errno == ENOSPC) {
TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL);
LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_last_error();
} else {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF;
}
return -1;
}
// fill file with zeros from FileNullBuffer
size_t writeSize = TRI_GetNullBufferSizeFiles();
size_t written = 0;
while (written < maximalSize) {
if (writeSize + written > maximalSize) {
writeSize = maximalSize - written;
}
ssize_t writeResult =
TRI_WRITE(fd, TRI_GetNullBufferFiles(), static_cast<TRI_write_t>(writeSize));
TRI_IF_FAILURE("CreateDatafile2") {
// intentionally fail
writeResult = -1;
errno = ENOSPC;
}
if (writeResult < 0) {
if (errno == ENOSPC) {
TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL);
LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_last_error();
} else {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF;
}
TRI_CLOSE(fd);
TRI_UnlinkFile(filename);
return -1;
}
written += static_cast<size_t>(writeResult);
}
// go back to offset 0
TRI_lseek_t offset = TRI_LSEEK(fd, (TRI_lseek_t)0, SEEK_SET);
if (offset == (TRI_lseek_t)-1) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename);
LOG(ERR) << "cannot seek in datafile '" << filename << "': '" << TRI_GET_ERRORBUF << "'";
return -1;
}
return fd;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initializes a datafile
////////////////////////////////////////////////////////////////////////////////
static void InitDatafile(TRI_datafile_t* datafile, char* filename, int fd,
void* mmHandle, TRI_voc_size_t maximalSize,
TRI_voc_size_t currentSize, TRI_voc_fid_t fid,
char* data) {
// filename is a string for physical datafiles, and NULL for anonymous regions
// fd is a positive value for physical datafiles, and -1 for anonymous regions
if (filename == nullptr) {
TRI_ASSERT(fd == -1);
} else {
TRI_ASSERT(fd >= 0);
}
datafile->_state = TRI_DF_STATE_READ;
datafile->_fid = fid;
datafile->_filename = filename;
datafile->_fd = fd;
datafile->_mmHandle = mmHandle;
datafile->_initSize = maximalSize;
datafile->_maximalSize = maximalSize;
datafile->_currentSize = currentSize;
datafile->_footerSize = sizeof(TRI_df_footer_marker_t);
datafile->_isSealed = false;
datafile->_lastError = TRI_ERROR_NO_ERROR;
datafile->_full = false;
datafile->_data = data;
datafile->_next = data + currentSize;
datafile->_synced = data;
datafile->_written = nullptr;
// reset tick aggregates
datafile->_tickMin = 0;
datafile->_tickMax = 0;
datafile->_dataMin = 0;
datafile->_dataMax = 0;
// initialize function pointers
datafile->isPhysical = &IsPhysicalDatafile;
datafile->getName = &GetNameDatafile;
datafile->close = &CloseDatafile;
datafile->destroy = &DestroyDatafile;
datafile->sync = &SyncDatafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief truncates a datafile
///
/// Create a truncated datafile, seal it and rename the old.
////////////////////////////////////////////////////////////////////////////////
static int TruncateAndSealDatafile(TRI_datafile_t* datafile,
TRI_voc_size_t vocSize) {
TRI_ERRORBUF;
void* data;
void* mmHandle;
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
// use multiples of page-size
size_t maximalSize =
((vocSize + sizeof(TRI_df_footer_marker_t) + PageSize - 1) / PageSize) *
PageSize;
// sanity check
if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) >
maximalSize) {
LOG(ERR) << "cannot create datafile '" << datafile->getName(datafile) << "', maximal size " << (unsigned int)maximalSize << " is too small";
return TRI_set_errno(TRI_ERROR_ARANGO_MAXIMAL_SIZE_TOO_SMALL);
}
// open the file
std::string filename = arangodb::basics::FileUtils::buildFilename(datafile->_filename, ".new");
int fd =
TRI_CREATE(filename.c_str(), O_CREAT | O_EXCL | O_RDWR | TRI_O_CLOEXEC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
TRI_SYSTEM_ERROR();
LOG(ERR) << "cannot create new datafile '" << filename << "': " << TRI_GET_ERRORBUF;
return TRI_set_errno(TRI_ERROR_SYS_ERROR);
}
// go back to the beginning of the file
TRI_lseek_t offset = TRI_LSEEK(fd, (TRI_lseek_t)(maximalSize - 1), SEEK_SET);
if (offset == (TRI_lseek_t)-1) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
LOG(ERR) << "cannot seek in new datafile '" << filename << "': " << TRI_GET_ERRORBUF;
return TRI_ERROR_SYS_ERROR;
}
char zero = 0;
int res = TRI_WRITE(fd, &zero, 1);
if (res < 0) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
LOG(ERR) << "cannot create datafile '" << filename << "': " << TRI_GET_ERRORBUF;
return TRI_ERROR_SYS_ERROR;
}
// memory map the data
res = TRI_MMFile(0, maximalSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd,
&mmHandle, 0, &data);
if (res != TRI_ERROR_NO_ERROR) {
TRI_SYSTEM_ERROR();
TRI_set_errno(res);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename.c_str());
LOG(ERR) << "cannot memory map file '" << filename << "': " << TRI_GET_ERRORBUF;
LOG(ERR) << "The database directory might reside on a shared folder "
"(VirtualBox, VMWare) or an NFS "
"mounted volume which does not allow memory mapped files.";
return TRI_errno();
}
// copy the data
memcpy(data, datafile->_data, vocSize);
// patch the datafile structure
res = TRI_UNMMFile(datafile->_data, datafile->_initSize, datafile->_fd,
&datafile->_mmHandle);
if (res < 0) {
TRI_CLOSE(datafile->_fd);
LOG(ERR) << "munmap failed with: " << res;
return res;
}
// .............................................................................................
// For windows: Mem mapped files use handles
// the datafile->_mmHandle handle object has been closed in the underlying
// TRI_UNMMFile(...) call above so we do not need to close it for the
// associated file below
// .............................................................................................
TRI_CLOSE(datafile->_fd);
datafile->_data = static_cast<char*>(data);
datafile->_next = (char*)(data) + vocSize;
datafile->_currentSize = vocSize;
// do not change _initSize!
TRI_ASSERT(datafile->_initSize == datafile->_maximalSize);
datafile->_maximalSize = static_cast<TRI_voc_size_t>(maximalSize);
datafile->_fd = fd;
datafile->_mmHandle = mmHandle;
datafile->_state = TRI_DF_STATE_CLOSED;
datafile->_full = false;
datafile->_isSealed = false;
datafile->_synced = static_cast<char*>(data);
datafile->_written = datafile->_next;
// rename files
std::string oldname = arangodb::basics::FileUtils::buildFilename(datafile->_filename, ".corrupted");
res = TRI_RenameFile(datafile->_filename, oldname.c_str());
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
res = TRI_RenameFile(filename.c_str(), datafile->_filename);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
// need to reset the datafile state here to write, otherwise the following
// call will return an error
datafile->_state = TRI_DF_STATE_WRITE;
return TRI_SealDatafile(datafile);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief tries to repair a datafile
////////////////////////////////////////////////////////////////////////////////
static bool TryRepairDatafile(TRI_datafile_t* datafile) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
char* ptr = datafile->_data;
char* end = datafile->_data + datafile->_currentSize;
if (datafile->_currentSize == 0) {
end = datafile->_data + datafile->_maximalSize;
}
TRI_voc_size_t currentSize = 0;
while (ptr < end) {
TRI_df_marker_t* marker = reinterpret_cast<TRI_df_marker_t*>(ptr);
TRI_voc_size_t const size = marker->getSize();
if (size == 0) {
// reached end
return true;
}
if (size < sizeof(TRI_df_marker_t) || ptr + size > end) {
// marker too small or too big
return false;
}
if (!TRI_IsValidMarkerDatafile(marker)) {
// unknown marker type
return false;
}
if (marker->getType() != 0) {
if (!CheckCrcMarker(marker, end)) {
// CRC mismatch!
auto next = reinterpret_cast<char const*>(marker) + size;
auto p = next;
if (p < end) {
// check if the rest of the datafile is only followed by NULL bytes
bool isFollowedByNullBytes = true;
while (p < end) {
if (*p != '\0') {
isFollowedByNullBytes = false;
break;
}
++p;
}
if (isFollowedByNullBytes) {
// only last marker in datafile was corrupt. fix the datafile in
// place
LOG(INFO) << "truncating datafile '" << datafile->getName(datafile) << "' at position " << currentSize;
int res = TruncateAndSealDatafile(datafile, currentSize);
return (res == TRI_ERROR_NO_ERROR);
}
// there is some other stuff following. now inspect it...
TRI_ASSERT(next <= end);
if (next < end) {
// there is a next marker
auto nextMarker = reinterpret_cast<TRI_df_marker_t const*>(next);
if (nextMarker->getType() != 0 &&
nextMarker->getSize() >= sizeof(TRI_df_marker_t) &&
next + nextMarker->getSize() <= end &&
TRI_IsValidMarkerDatafile(nextMarker) &&
CheckCrcMarker(nextMarker, end)) {
// next marker looks good.
// create a temporary buffer
auto buffer =
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, size, false);
if (buffer == nullptr) {
return false;
}
// create a new marker in the temporary buffer
auto temp = reinterpret_cast<TRI_df_marker_t*>(buffer);
DatafileHelper::InitMarker(
reinterpret_cast<TRI_df_marker_t*>(buffer), TRI_DF_MARKER_BLANK,
static_cast<uint32_t>(size));
temp->setCrc(CalculateCrcValue(temp));
// all done. now copy back the marker into the file
memcpy(static_cast<void*>(ptr), buffer,
static_cast<size_t>(size));
TRI_Free(TRI_UNKNOWN_MEM_ZONE, buffer);
bool ok = datafile->sync(datafile, ptr, (ptr + size));
if (ok) {
LOG(INFO) << "zeroed single invalid marker in datafile '" << datafile->getName(datafile) << "' at position " << currentSize;
} else {
LOG(ERR) << "could not zero single invalid marker in datafile '" << datafile->getName(datafile) << "' at position " << currentSize;
return false;
}
} else {
// next marker looks broken, too.
int res = TruncateAndSealDatafile(datafile, currentSize);
return (res == TRI_ERROR_NO_ERROR);
}
}
}
}
}
size_t alignedSize = DatafileHelper::AlignedMarkerSize<TRI_voc_size_t>(marker);
currentSize += static_cast<TRI_voc_size_t>(alignedSize);
if (marker->getType() == TRI_DF_MARKER_FOOTER) {
return true;
}
ptr += alignedSize;
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief fixes a corrupted datafile
////////////////////////////////////////////////////////////////////////////////
static bool FixDatafile(TRI_datafile_t* datafile, TRI_voc_size_t currentSize) {
LOG(WARN) << "datafile '" << datafile->getName(datafile) << "' is corrupted at position " << currentSize;
LOG(WARN) << "setting datafile '" << datafile->getName(datafile) << "' to read-only and ignoring all data from this file beyond this position";
datafile->_currentSize = currentSize;
TRI_ASSERT(datafile->_initSize == datafile->_maximalSize);
datafile->_maximalSize = static_cast<TRI_voc_size_t>(currentSize);
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_full = true;
datafile->_state = TRI_DF_STATE_READ;
datafile->_isSealed = true;
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks a datafile
////////////////////////////////////////////////////////////////////////////////
static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
char* ptr = datafile->_data;
char* end = datafile->_data + datafile->_currentSize;
TRI_voc_size_t currentSize = 0;
if (datafile->_currentSize == 0) {
LOG(WARN) << "current size is 0 in read-only datafile '" << datafile->getName(datafile) << "', trying to fix";
end = datafile->_data + datafile->_maximalSize;
}
TRI_voc_tick_t maxTick = 0;
auto updateTick =
[](TRI_voc_tick_t maxTick) -> void { TRI_UpdateTickServer(maxTick); };
while (ptr < end) {
TRI_df_marker_t* marker = reinterpret_cast<TRI_df_marker_t*>(ptr);
TRI_voc_size_t const size = marker->getSize();
TRI_voc_tick_t const tick = marker->getTick();
TRI_df_marker_type_t const type = marker->getType();
#ifdef DEBUG_DATAFILE
LOG(TRACE) << "MARKER: size " << size << ", tick " << tick << ", crc " << marker->getCrc() << ", type " << type;
#endif
if (size == 0) {
LOG(DEBUG) << "reached end of datafile '" << datafile->getName(datafile) << "' data, current size " << currentSize;
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
updateTick(maxTick);
return true;
}
if (size < sizeof(TRI_df_marker_t)) {
if (ignoreFailures) {
return FixDatafile(datafile, currentSize);
}
datafile->_lastError =
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' too small, size " << size << ", should be at least " << sizeof(TRI_df_marker_t);
updateTick(maxTick);
return false;
}
// prevent reading over the end of the file
if (ptr + size > end) {
if (ignoreFailures) {
return FixDatafile(datafile, currentSize);
}
datafile->_lastError =
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' points with size " << size << " beyond end of file";
updateTick(maxTick);
return false;
}
// the following sanity check offers some, but not 100% crash-protection
// when reading
// totally corrupted datafiles
if (!TRI_IsValidMarkerDatafile(marker)) {
if (type == 0 && size < 128) {
// ignore markers with type 0 and a small size
LOG(WARN) << "ignoring suspicious marker in datafile '" << datafile->getName(datafile) << "': type: " << type << ", size: " << size;
} else {
if (ignoreFailures) {
return FixDatafile(datafile, currentSize);
}
datafile->_lastError =
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "marker in datafile '" << datafile->getName(datafile) << "' is corrupt: type: " << type << ", size: " << size;
updateTick(maxTick);
return false;
}
}
if (type != 0) {
bool ok = CheckCrcMarker(marker, end);
if (!ok) {
// CRC mismatch!
bool nextMarkerOk = false;
if (size > 0) {
auto next = reinterpret_cast<char const*>(marker) + DatafileHelper::AlignedSize<size_t>(size);
auto p = next;
if (p < end) {
// check if the rest of the datafile is only followed by NULL bytes
bool isFollowedByNullBytes = true;
while (p < end) {
if (*p != '\0') {
isFollowedByNullBytes = false;
break;
}
++p;
}
if (isFollowedByNullBytes) {
// only last marker in datafile was corrupt. fix the datafile in
// place
LOG(WARN) << "datafile '" << datafile->getName(datafile) << "' automatically truncated at last marker";
ignoreFailures = true;
} else {
// there is some other stuff following. now inspect it...
TRI_ASSERT(next <= end);
if (next < end) {
// there is a next marker
auto nextMarker =
reinterpret_cast<TRI_df_marker_t const*>(next);
if (nextMarker->getType() != 0 &&
nextMarker->getSize() >= sizeof(TRI_df_marker_t) &&
next + nextMarker->getSize() <= end &&
TRI_IsValidMarkerDatafile(nextMarker) &&
CheckCrcMarker(nextMarker, end)) {
// next marker looks good.
nextMarkerOk = true;
}
} else {
// EOF
nextMarkerOk = true;
}
}
}
}
if (ignoreFailures) {
return FixDatafile(datafile, currentSize);
}
datafile->_lastError =
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' of size "
<< datafile->_maximalSize << ", at position " << currentSize;
LOG(WARN) << "crc mismatch found inside marker of type '" << TRI_NameMarkerDatafile(marker)
<< "' and size " << size
<< ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
{
LOG(INFO) << "raw marker data following:";
char const* p = reinterpret_cast<char const*>(marker);
char const* e = reinterpret_cast<char const*>(marker) + DatafileHelper::AlignedSize<size_t>(size);
std::string line;
std::string raw;
size_t printed = 0;
while (p < e) {
// print offset
line.append("0x");
uintptr_t offset = static_cast<uintptr_t>(p - datafile->_data);
for (size_t i = 0; i < 8; ++i) {
uint8_t c = static_cast<uint8_t>((offset & (0xFFULL << 8 * (7 - i))) >> 8 * (7 - i));
uint8_t n1 = c >> 4;
uint8_t n2 = c & 0x0F;
line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10);
line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10);
}
// print data
line.append(": ");
for (size_t i = 0; i < 16; ++i) {
if (p >= e) {
line.append(" ");
} else {
uint8_t c = static_cast<uint8_t>(*p++);
uint8_t n1 = c >> 4;
uint8_t n2 = c & 0x0F;
line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10);
line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10);
line.push_back(' ');
raw.push_back((c < 32 || c >= 127) ? '.' : static_cast<unsigned char>(c));
++printed;
}
}
LOG(INFO) << line << " " << raw;
line.clear();
raw.clear();
if (printed >= 2048) {
LOG(INFO) << "(output truncated due to excessive length)";
break;
}
}
}
if (nextMarkerOk) {
LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it";
LOG(INFO) << "please restart the server with the parameter '--wal.ignore-logfile-errors true' to repair the marker";
} else {
LOG(WARN) << "data directly following this marker cannot be analyzed";
}
updateTick(maxTick);
return false;
}
}
if (tick > maxTick) {
maxTick = tick;
}
size_t alignedSize = DatafileHelper::AlignedMarkerSize<size_t>(marker);
currentSize += static_cast<TRI_voc_size_t>(alignedSize);
if (marker->getType() == TRI_DF_MARKER_FOOTER) {
LOG(DEBUG) << "found footer, reached end of datafile '" << datafile->getName(datafile) << "', current size " << currentSize;
datafile->_isSealed = true;
datafile->_currentSize = currentSize;
datafile->_next = datafile->_data + datafile->_currentSize;
updateTick(maxTick);
return true;
}
ptr += alignedSize;
}
updateTick(maxTick);
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief extract the numeric part from a filename
/// the filename must look like this: /.*type-abc\.ending$/, where abc is
/// a number, and type and ending are arbitrary letters
////////////////////////////////////////////////////////////////////////////////
static uint64_t GetNumericFilenamePart(char const* filename) {
char const* pos1 = strrchr(filename, '.');
if (pos1 == nullptr) {
return 0;
}
char const* pos2 = strrchr(filename, '-');
if (pos2 == nullptr || pos2 > pos1) {
return 0;
}
return StringUtils::uint64(pos2 + 1, pos1 - pos2 - 1);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief create the initial datafile header marker
////////////////////////////////////////////////////////////////////////////////
static int WriteInitialHeaderMarker(TRI_datafile_t* datafile, TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize) {
// create the header
TRI_df_header_marker_t header = DatafileHelper::CreateHeaderMarker(
maximalSize, static_cast<TRI_voc_tick_t>(fid));
// reserve space and write header to file
TRI_df_marker_t* position;
int res =
TRI_ReserveElementDatafile(datafile, header.base.getSize(), &position, 0);
if (res == TRI_ERROR_NO_ERROR) {
TRI_ASSERT(position != nullptr);
res = TRI_WriteCrcElementDatafile(datafile, position, &header.base, false);
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief creates a new anonymous datafile
///
/// this is only supported on certain platforms (Linux, MacOS)
////////////////////////////////////////////////////////////////////////////////
#ifdef TRI_HAVE_ANONYMOUS_MMAP
static TRI_datafile_t* CreateAnonymousDatafile(TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize) {
#ifdef TRI_MMAP_ANONYMOUS
// fd -1 is required for "real" anonymous regions
int fd = -1;
int flags = TRI_MMAP_ANONYMOUS | MAP_SHARED;
#else
// ugly workaround if MAP_ANONYMOUS is not available
int fd = TRI_OPEN("/dev/zero", O_RDWR | TRI_O_CLOEXEC);
if (fd == -1) {
return nullptr;
}
int flags = MAP_PRIVATE;
#endif
// memory map the data
void* data;
void* mmHandle;
ssize_t res = TRI_MMFile(nullptr, maximalSize, PROT_WRITE | PROT_READ, flags,
fd, &mmHandle, 0, &data);
#ifdef MAP_ANONYMOUS
// nothing to do
#else
// close auxilliary file
TRI_CLOSE(fd);
fd = -1;
#endif
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
LOG(ERR) << "cannot memory map anonymous region: " << TRI_last_error();
LOG(ERR) << "The database directory might reside on a shared folder "
"(VirtualBox, VMWare) or an NFS "
"mounted volume which does not allow memory mapped files.";
return nullptr;
}
// create datafile structure
TRI_datafile_t* datafile = static_cast<TRI_datafile_t*>(
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false));
if (datafile == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
LOG(ERR) << "out of memory";
return nullptr;
}
InitDatafile(datafile, nullptr, fd, mmHandle, maximalSize, 0, fid,
static_cast<char*>(data));
return datafile;
}
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief creates a new physical datafile
////////////////////////////////////////////////////////////////////////////////
static TRI_datafile_t* CreatePhysicalDatafile(char const* filename,
TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize) {
TRI_ASSERT(filename != nullptr);
int fd = CreateDatafile(filename, maximalSize);
if (fd < 0) {
// an error occurred
return nullptr;
}
// memory map the data
void* data;
void* mmHandle;
int flags = MAP_SHARED;
#ifdef __linux__
// try populating the mapping already
flags |= MAP_POPULATE;
#endif
ssize_t res = TRI_MMFile(0, maximalSize, PROT_WRITE | PROT_READ, flags, fd,
&mmHandle, 0, &data);
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
TRI_CLOSE(fd);
// remove empty file
TRI_UnlinkFile(filename);
LOG(ERR) << "cannot memory map file '" << filename << "': '" << TRI_errno_string((int)res) << "'";
LOG(ERR) << "The database directory might reside on a shared folder "
"(VirtualBox, VMWare) or an NFS "
"mounted volume which does not allow memory mapped files.";
return nullptr;
}
// create datafile structure
auto datafile = static_cast<TRI_datafile_t*>(
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false));
if (datafile == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
TRI_CLOSE(fd);
LOG(ERR) << "out of memory";
return nullptr;
}
InitDatafile(datafile, TRI_DuplicateString(filename), fd, mmHandle,
maximalSize, 0, fid, static_cast<char*>(data));
// Advise OS that sequential access is going to happen:
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_SEQUENTIAL);
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief opens a datafile
////////////////////////////////////////////////////////////////////////////////
static TRI_datafile_t* OpenDatafile(char const* filename, bool ignoreErrors) {
TRI_ERRORBUF;
void* data;
TRI_stat_t status;
void* mmHandle;
// this function must not be called for non-physical datafiles
TRI_ASSERT(filename != nullptr);
TRI_voc_fid_t fid = GetNumericFilenamePart(filename);
// ..........................................................................
// attempt to open a datafile file
// ..........................................................................
int fd = TRI_OPEN(filename, O_RDWR | TRI_O_CLOEXEC);
if (fd < 0) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG(ERR) << "cannot open datafile '" << filename << "': '" << TRI_GET_ERRORBUF << "'";
return nullptr;
}
// compute the size of the file
int res = TRI_FSTAT(fd, &status);
if (res < 0) {
TRI_SYSTEM_ERROR();
TRI_set_errno(TRI_ERROR_SYS_ERROR);
TRI_CLOSE(fd);
LOG(ERR) << "cannot get status of datafile '" << filename << "': " << TRI_GET_ERRORBUF;
return nullptr;
}
// check that file is not too small
TRI_voc_size_t size = static_cast<TRI_voc_size_t>(status.st_size);
if (size < sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t)) {
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
TRI_CLOSE(fd);
LOG(ERR) << "datafile '" << filename << "' is corrupt, size is only " << (unsigned int)size;
return nullptr;
}
// read header from file
char buffer[128];
memset(&buffer[0], 0, sizeof(buffer));
ssize_t len = sizeof(TRI_df_header_marker_t);
ssize_t toRead = sizeof(buffer);
if (toRead > static_cast<ssize_t>(status.st_size)) {
toRead = static_cast<ssize_t>(status.st_size);
}
bool ok = TRI_ReadPointer(fd, &buffer[0], toRead);
if (!ok) {
LOG(ERR) << "cannot read datafile header from '" << filename << "': " << TRI_last_error();
TRI_CLOSE(fd);
return nullptr;
}
char const* ptr = reinterpret_cast<char*>(&buffer[0]);
char const* end = static_cast<char const*>(ptr) + len;
TRI_df_header_marker_t const* header = reinterpret_cast<TRI_df_header_marker_t const*>(&buffer[0]);
// check CRC
ok = CheckCrcMarker(reinterpret_cast<TRI_df_marker_t const*>(ptr), end);
if (!ok) {
if (IsMarker28(ptr)) {
LOG(ERR) << "datafile found from older version of ArangoDB. "
<< "Please dump data from that version with arangodump "
<< "and reload it into this ArangoDB instance with arangorestore";
TRI_CLOSE(fd);
TRI_set_errno(TRI_ERROR_NOT_IMPLEMENTED);
FATAL_ERROR_EXIT();
}
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
LOG(ERR) << "corrupted datafile header read from '" << filename << "'";
if (!ignoreErrors) {
TRI_CLOSE(fd);
return nullptr;
}
}
// check the datafile version
if (ok) {
if (header->_version != TRI_DF_VERSION) {
TRI_set_errno(TRI_ERROR_ARANGO_CORRUPTED_DATAFILE);
LOG(ERR) << "unknown datafile version '" << header->_version << "' in datafile '" << filename << "'";
if (!ignoreErrors) {
TRI_CLOSE(fd);
return nullptr;
}
}
}
// check the maximal size
if (size > header->_maximalSize) {
LOG(DEBUG) << "datafile '" << filename << "' has size '" << size << "', but maximal size is '" << header->_maximalSize << "'";
}
// map datafile into memory
res = TRI_MMFile(0, size, PROT_READ, MAP_SHARED, fd, &mmHandle, 0, &data);
if (res != TRI_ERROR_NO_ERROR) {
TRI_set_errno(res);
TRI_CLOSE(fd);
LOG(ERR) << "cannot memory map datafile '" << filename << "': " << TRI_errno_string(res);
LOG(ERR) << "The database directory might reside on a shared folder "
"(VirtualBox, VMWare) or an NFS "
"mounted volume which does not allow memory mapped files.";
return nullptr;
}
// create datafile structure
TRI_datafile_t* datafile = static_cast<TRI_datafile_t*>(
TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(TRI_datafile_t), false));
if (datafile == nullptr) {
TRI_UNMMFile(data, size, fd, &mmHandle);
TRI_CLOSE(fd);
return nullptr;
}
InitDatafile(datafile, TRI_DuplicateString(filename), fd, mmHandle, size,
size, fid, static_cast<char*>(data));
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief creates either an anonymous or a physical datafile
////////////////////////////////////////////////////////////////////////////////
TRI_datafile_t* TRI_CreateDatafile(char const* filename, TRI_voc_fid_t fid,
TRI_voc_size_t maximalSize,
bool withInitialMarkers) {
TRI_datafile_t* datafile;
TRI_ASSERT(PageSize >= 256);
// use multiples of page-size
maximalSize =
(TRI_voc_size_t)(((maximalSize + PageSize - 1) / PageSize) * PageSize);
// sanity check maximal size
if (sizeof(TRI_df_header_marker_t) + sizeof(TRI_df_footer_marker_t) >
maximalSize) {
LOG(ERR) << "cannot create datafile, maximal size '" << (unsigned int)maximalSize << "' is too small";
TRI_set_errno(TRI_ERROR_ARANGO_MAXIMAL_SIZE_TOO_SMALL);
return nullptr;
}
// create either an anonymous or a physical datafile
if (filename == nullptr) {
#ifdef TRI_HAVE_ANONYMOUS_MMAP
datafile = CreateAnonymousDatafile(fid, maximalSize);
#else
// system does not support anonymous mmap
return nullptr;
#endif
} else {
datafile = CreatePhysicalDatafile(filename, fid, maximalSize);
}
if (datafile == nullptr) {
// an error occurred during creation
return nullptr;
}
datafile->_state = TRI_DF_STATE_WRITE;
if (withInitialMarkers) {
int res = WriteInitialHeaderMarker(datafile, fid, maximalSize);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "cannot write header to datafile '" << datafile->getName(datafile) << "'";
TRI_UNMMFile(datafile->_data, datafile->_maximalSize, datafile->_fd,
&datafile->_mmHandle);
datafile->close(datafile);
datafile->destroy(datafile);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, datafile);
return nullptr;
}
}
LOG(DEBUG) << "created datafile '" << datafile->getName(datafile) << "' of size " << (unsigned int)maximalSize << " and page-size " << (unsigned int)PageSize;
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief frees the memory allocated, but does not free the pointer
////////////////////////////////////////////////////////////////////////////////
void TRI_DestroyDatafile(TRI_datafile_t* datafile) {
datafile->destroy(datafile);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief frees the memory allocated and but frees the pointer
////////////////////////////////////////////////////////////////////////////////
void TRI_FreeDatafile(TRI_datafile_t* datafile) {
TRI_DestroyDatafile(datafile);
TRI_Free(TRI_UNKNOWN_MEM_ZONE, datafile);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the name for a marker
////////////////////////////////////////////////////////////////////////////////
char const* TRI_NameMarkerDatafile(TRI_df_marker_t const* marker) {
switch (marker->getType()) {
// general markers
case TRI_DF_MARKER_HEADER:
return "datafile header";
case TRI_DF_MARKER_FOOTER:
return "footer";
case TRI_DF_MARKER_BLANK:
return "blank marker (used when repairing datafiles)";
case TRI_DF_MARKER_COL_HEADER:
return "collection header";
case TRI_DF_MARKER_PROLOGUE:
return "prologue";
case TRI_DF_MARKER_VPACK_DOCUMENT:
return "document";
case TRI_DF_MARKER_VPACK_REMOVE:
return "remove document";
case TRI_DF_MARKER_VPACK_CREATE_COLLECTION:
return "create collection";
case TRI_DF_MARKER_VPACK_DROP_COLLECTION:
return "drop collection";
case TRI_DF_MARKER_VPACK_RENAME_COLLECTION:
return "rename collection";
case TRI_DF_MARKER_VPACK_CHANGE_COLLECTION:
return "change collection";
case TRI_DF_MARKER_VPACK_CREATE_INDEX:
return "create index";
case TRI_DF_MARKER_VPACK_DROP_INDEX:
return "drop index";
case TRI_DF_MARKER_VPACK_CREATE_DATABASE:
return "create database";
case TRI_DF_MARKER_VPACK_DROP_DATABASE:
return "drop database";
case TRI_DF_MARKER_VPACK_BEGIN_TRANSACTION:
return "begin transaction";
case TRI_DF_MARKER_VPACK_COMMIT_TRANSACTION:
return "commit transaction";
case TRI_DF_MARKER_VPACK_ABORT_TRANSACTION:
return "abort transaction";
default:
return "unused/unknown";
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks whether a marker is valid
////////////////////////////////////////////////////////////////////////////////
bool TRI_IsValidMarkerDatafile(TRI_df_marker_t const* marker) {
if (marker == nullptr) {
return false;
}
// check marker type
TRI_df_marker_type_t const type = marker->getType();
if (type <= TRI_DF_MARKER_MIN) {
// marker type is less than minimum allowed type value
return false;
}
if (type >= TRI_DF_MARKER_MAX) {
// marker type is greater than maximum allowed type value
return false;
}
if (marker->getSize() >= DatafileHelper::MaximalMarkerSize()) {
// a single marker bigger than 256 MB seems unreasonable
// note: this is an arbitrary limit
return false;
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief reserves room for an element, advances the pointer
///
/// note: maximalJournalSize is the collection's maximalJournalSize property,
/// which may be different from the size of the current datafile
/// some callers do not set the value of maximalJournalSize
////////////////////////////////////////////////////////////////////////////////
int TRI_ReserveElementDatafile(TRI_datafile_t* datafile, TRI_voc_size_t size,
TRI_df_marker_t** position,
TRI_voc_size_t maximalJournalSize) {
*position = nullptr;
size = DatafileHelper::AlignedSize<TRI_voc_size_t>(size);
if (datafile->_state != TRI_DF_STATE_WRITE) {
if (datafile->_state == TRI_DF_STATE_READ) {
LOG(ERR) << "cannot reserve marker, datafile is read-only";
return TRI_set_errno(TRI_ERROR_ARANGO_READ_ONLY);
}
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
}
// check the maximal size
if (size + DatafileHelper::JournalOverhead() > datafile->_maximalSize) {
// marker is bigger than journal size.
// adding the marker to this datafile will not work
if (maximalJournalSize <= datafile->_maximalSize) {
// the collection property 'maximalJournalSize' is equal to
// or smaller than the size of this datafile
// creating a new file and writing the marker into it will not work either
return TRI_set_errno(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE);
}
// if we get here, the collection's 'maximalJournalSize' property is
// higher than the size of this datafile.
// maybe the marker will fit into a new datafile with the bigger size?
if (size + DatafileHelper::JournalOverhead() > maximalJournalSize) {
// marker still won't fit
return TRI_set_errno(TRI_ERROR_ARANGO_DOCUMENT_TOO_LARGE);
}
// fall-through intentional
}
// add the marker, leave enough room for the footer
if (datafile->_currentSize + size + datafile->_footerSize >
datafile->_maximalSize) {
datafile->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_DATAFILE_FULL);
datafile->_full = true;
LOG(TRACE) << "cannot write marker, not enough space";
return TRI_ERROR_ARANGO_DATAFILE_FULL;
}
*position = reinterpret_cast<TRI_df_marker_t*>(datafile->_next);
TRI_ASSERT(*position != nullptr);
datafile->_next += size;
datafile->_currentSize += size;
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief writes a marker to the datafile
/// this function will write the marker as-is, without any CRC or tick updates
////////////////////////////////////////////////////////////////////////////////
int TRI_WriteElementDatafile(TRI_datafile_t* datafile, void* position,
TRI_df_marker_t const* marker, bool forceSync) {
TRI_ASSERT(marker->getTick() > 0);
TRI_ASSERT(marker->getSize() > 0);
TRI_UpdateTicksDatafile(datafile, marker);
if (datafile->_state != TRI_DF_STATE_WRITE) {
if (datafile->_state == TRI_DF_STATE_READ) {
LOG(ERR) << "cannot write marker, datafile is read-only";
return TRI_set_errno(TRI_ERROR_ARANGO_READ_ONLY);
}
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
}
TRI_ASSERT(position != nullptr);
// out of bounds check for writing into a datafile
if (position == nullptr || position < (void*)datafile->_data ||
position >= (void*)(datafile->_data + datafile->_maximalSize)) {
LOG(ERR) << "logic error. writing out of bounds of datafile '" << datafile->getName(datafile) << "'";
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
}
memcpy(position, marker, static_cast<size_t>(marker->getSize()));
if (forceSync) {
bool ok = datafile->sync(datafile, static_cast<char const*>(position),
((char*)position) + marker->getSize());
if (!ok) {
datafile->_state = TRI_DF_STATE_WRITE_ERROR;
if (errno == ENOSPC) {
datafile->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL);
} else {
datafile->_lastError = TRI_set_errno(TRI_ERROR_SYS_ERROR);
}
LOG(ERR) << "msync failed with: " << TRI_last_error();
return datafile->_lastError;
} else {
LOG(TRACE) << "msync succeeded " << (void*) position << ", size " << marker->getSize();
}
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief update tick values for a datafile
////////////////////////////////////////////////////////////////////////////////
void TRI_UpdateTicksDatafile(TRI_datafile_t* datafile,
TRI_df_marker_t const* marker) {
TRI_df_marker_type_t const type = marker->getType();
if (type != TRI_DF_MARKER_HEADER && type != TRI_DF_MARKER_FOOTER &&
type != TRI_DF_MARKER_COL_HEADER) {
// every marker but headers / footers counts
TRI_voc_tick_t tick = marker->getTick();
if (datafile->_tickMin == 0) {
datafile->_tickMin = tick;
}
if (datafile->_tickMax < tick) {
datafile->_tickMax = tick;
}
if (datafile->_dataMin == 0) {
datafile->_dataMin = tick;
}
if (datafile->_dataMax < tick) {
datafile->_dataMax = tick;
}
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checksums and writes a marker to the datafile
////////////////////////////////////////////////////////////////////////////////
int TRI_WriteCrcElementDatafile(TRI_datafile_t* datafile, void* position,
TRI_df_marker_t* marker, bool forceSync) {
TRI_ASSERT(marker->getTick() != 0);
if (datafile->isPhysical(datafile)) {
TRI_voc_crc_t crc = TRI_InitialCrc32();
crc = TRI_BlockCrc32(crc, (char const*)marker, marker->getSize());
marker->setCrc(TRI_FinalCrc32(crc));
}
return TRI_WriteElementDatafile(datafile, position, marker, forceSync);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief iterates over a datafile
/// also may set datafile's min/max tick values
////////////////////////////////////////////////////////////////////////////////
bool TRI_IterateDatafile(TRI_datafile_t* datafile,
bool (*iterator)(TRI_df_marker_t const*, void*,
TRI_datafile_t*),
void* data) {
TRI_ASSERT(iterator != nullptr);
LOG(TRACE) << "iterating over datafile '" << datafile->getName(datafile) << "', fid: " << datafile->_fid;
char const* ptr = datafile->_data;
char const* end = datafile->_data + datafile->_currentSize;
if (datafile->_state != TRI_DF_STATE_READ &&
datafile->_state != TRI_DF_STATE_WRITE) {
TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
return false;
}
while (ptr < end) {
auto const* marker = reinterpret_cast<TRI_df_marker_t const*>(ptr);
if (marker->getSize() == 0) {
return true;
}
// update the tick statistics
TRI_UpdateTicksDatafile(datafile, marker);
if (!iterator(marker, data, datafile)) {
return false;
}
ptr += DatafileHelper::AlignedMarkerSize<size_t>(marker);
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief opens an existing datafile
///
/// The datafile will be opened read-only if a footer is found
////////////////////////////////////////////////////////////////////////////////
TRI_datafile_t* TRI_OpenDatafile(char const* filename, bool ignoreFailures) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(filename != nullptr);
TRI_datafile_t* datafile = OpenDatafile(filename, false);
if (datafile == nullptr) {
return nullptr;
}
// check the datafile by scanning markers
bool ok = CheckDatafile(datafile, ignoreFailures);
if (!ok) {
TRI_UNMMFile(datafile->_data, datafile->_maximalSize, datafile->_fd,
&datafile->_mmHandle);
TRI_CLOSE(datafile->_fd);
LOG(ERR) << "datafile '" << datafile->getName(datafile) << "' is corrupt";
// must free datafile here
TRI_FreeDatafile(datafile);
return nullptr;
}
// change to read-write if no footer has been found
if (!datafile->_isSealed) {
datafile->_state = TRI_DF_STATE_WRITE;
TRI_ProtectMMFile(datafile->_data, datafile->_maximalSize,
PROT_READ | PROT_WRITE, datafile->_fd,
&datafile->_mmHandle);
}
// Advise on sequential use:
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_SEQUENTIAL);
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_WILLNEED);
return datafile;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief closes a datafile and all memory regions
////////////////////////////////////////////////////////////////////////////////
bool TRI_CloseDatafile(TRI_datafile_t* datafile) {
if (datafile->_state == TRI_DF_STATE_READ ||
datafile->_state == TRI_DF_STATE_WRITE) {
int res = TRI_UNMMFile(datafile->_data, datafile->_initSize, datafile->_fd,
&datafile->_mmHandle);
if (res != TRI_ERROR_NO_ERROR) {
LOG(ERR) << "munmap failed with: " << res;
datafile->_state = TRI_DF_STATE_WRITE_ERROR;
datafile->_lastError = res;
return false;
}
datafile->close(datafile);
datafile->_data = nullptr;
datafile->_next = nullptr;
datafile->_fd = -1;
return true;
} else if (datafile->_state == TRI_DF_STATE_CLOSED) {
LOG(WARN) << "closing an already closed datafile '" << datafile->getName(datafile) << "'";
return true;
} else {
TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
return false;
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief renames a datafile
////////////////////////////////////////////////////////////////////////////////
bool TRI_RenameDatafile(TRI_datafile_t* datafile, char const* filename) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
TRI_ASSERT(filename != nullptr);
if (TRI_ExistsFile(filename)) {
LOG(ERR) << "cannot overwrite datafile '" << filename << "'";
datafile->_lastError =
TRI_set_errno(TRI_ERROR_ARANGO_DATAFILE_ALREADY_EXISTS);
return false;
}
int res = TRI_RenameFile(datafile->_filename, filename);
if (res != TRI_ERROR_NO_ERROR) {
datafile->_state = TRI_DF_STATE_RENAME_ERROR;
datafile->_lastError = TRI_set_errno(TRI_ERROR_SYS_ERROR);
return false;
}
TRI_FreeString(TRI_CORE_MEM_ZONE, datafile->_filename);
datafile->_filename = TRI_DuplicateString(filename);
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief seals a datafile, writes a footer, sets it to read-only
////////////////////////////////////////////////////////////////////////////////
int TRI_SealDatafile(TRI_datafile_t* datafile) {
if (datafile->_state == TRI_DF_STATE_READ) {
return TRI_set_errno(TRI_ERROR_ARANGO_READ_ONLY);
}
if (datafile->_state != TRI_DF_STATE_WRITE) {
return TRI_set_errno(TRI_ERROR_ARANGO_ILLEGAL_STATE);
}
if (datafile->_isSealed) {
return TRI_set_errno(TRI_ERROR_ARANGO_DATAFILE_SEALED);
}
// set a proper tick value
if (datafile->_tickMax == 0) {
datafile->_tickMax = TRI_NewTickServer();
}
// create the footer
TRI_df_footer_marker_t footer = DatafileHelper::CreateFooterMarker(datafile->_tickMax);
// reserve space and write footer to file
datafile->_footerSize = 0;
TRI_df_marker_t* position;
int res =
TRI_ReserveElementDatafile(datafile, footer.base.getSize(), &position, 0);
if (res == TRI_ERROR_NO_ERROR) {
TRI_ASSERT(position != nullptr);
res = TRI_WriteCrcElementDatafile(datafile, position, &footer.base, false);
}
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
// sync file
bool ok = datafile->sync(datafile, datafile->_synced,
((char*)datafile->_data) + datafile->_currentSize);
if (!ok) {
datafile->_state = TRI_DF_STATE_WRITE_ERROR;
if (errno == ENOSPC) {
datafile->_lastError = TRI_set_errno(TRI_ERROR_ARANGO_FILESYSTEM_FULL);
} else {
datafile->_lastError = TRI_errno();
}
LOG(ERR) << "msync failed with: " << TRI_last_error();
}
// everything is now synced
datafile->_synced = datafile->_written;
TRI_ProtectMMFile(datafile->_data, datafile->_maximalSize, PROT_READ,
datafile->_fd, &datafile->_mmHandle);
// seal datafile
if (ok) {
datafile->_isSealed = true;
datafile->_state = TRI_DF_STATE_READ;
// note: _initSize must remain constant
TRI_ASSERT(datafile->_initSize == datafile->_maximalSize);
datafile->_maximalSize = datafile->_currentSize;
}
if (!ok) {
return datafile->_lastError;
}
if (datafile->isPhysical(datafile)) {
// From now on we predict random access (until collection or compaction):
TRI_MMFileAdvise(datafile->_data, datafile->_maximalSize,
TRI_MADVISE_RANDOM);
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief truncates a datafile and seals it
/// this is called from the recovery procedure only
////////////////////////////////////////////////////////////////////////////////
int TRI_TruncateDatafile(char const* path, TRI_voc_size_t position) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(path != nullptr);
TRI_datafile_t* datafile = OpenDatafile(path, true);
if (datafile == nullptr) {
return TRI_ERROR_ARANGO_DATAFILE_UNREADABLE;
}
int res = TruncateAndSealDatafile(datafile, position);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief try to repair a datafile
////////////////////////////////////////////////////////////////////////////////
bool TRI_TryRepairDatafile(char const* path) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(path != nullptr);
TRI_datafile_t* datafile = OpenDatafile(path, true);
if (datafile == nullptr) {
return false;
}
// set to read/write access
TRI_ProtectMMFile(datafile->_data, datafile->_maximalSize,
PROT_READ | PROT_WRITE, datafile->_fd,
&datafile->_mmHandle);
bool result = TryRepairDatafile(datafile);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
return result;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief diagnoses a marker
////////////////////////////////////////////////////////////////////////////////
static std::string DiagnoseMarker(TRI_df_marker_t const* marker,
char const* end) {
std::ostringstream result;
if (marker == nullptr) {
return "marker is undefined. should not happen";
}
// check marker type
TRI_df_marker_type_t type = marker->getType();
if (type <= TRI_DF_MARKER_MIN) {
// marker type is less than minimum allowed type value
result << "marker type value (" << static_cast<int>(type)
<< ") is wrong. expecting value higher than " << TRI_DF_MARKER_MIN;
return result.str();
}
if (type >= TRI_DF_MARKER_MAX) {
// marker type is greater than maximum allowed type value
result << "marker type value (" << static_cast<int>(type)
<< ") is wrong. expecting value less than " << TRI_DF_MARKER_MAX;
return result.str();
}
TRI_voc_size_t size = marker->getSize();
if (size >= DatafileHelper::MaximalMarkerSize()) {
// a single marker bigger than this size seems unreasonable
// note: this is an arbitrary limit
result << "marker size value (" << size
<< ") is wrong. expecting value less than "
<< DatafileHelper::MaximalMarkerSize();
return result.str();
}
if (size < sizeof(TRI_df_marker_t)) {
result << "marker size is too small (" << size
<< "). expecting at least " << sizeof(TRI_df_marker_t) << " bytes";
return result.str();
}
if (reinterpret_cast<char const*>(marker) + size > end) {
return "marker size is beyond end of datafile";
}
TRI_voc_crc_t crc = CalculateCrcValue(marker);
if (marker->getCrc() == crc) {
return "crc checksum is correct";
}
result << "crc checksum (hex " << std::hex << marker->getCrc()
<< ") is wrong. expecting (hex " << std::hex << crc << ")";
return result.str();
}
////////////////////////////////////////////////////////////////////////////////
/// @brief scans a datafile
////////////////////////////////////////////////////////////////////////////////
static DatafileScan ScanDatafile(TRI_datafile_t const* datafile) {
// this function must not be called for non-physical datafiles
TRI_ASSERT(datafile->isPhysical(datafile));
char* ptr = datafile->_data;
char* end = datafile->_data + datafile->_currentSize;
TRI_voc_size_t currentSize = 0;
DatafileScan scan;
scan.currentSize = datafile->_currentSize;
scan.maximalSize = datafile->_maximalSize;
if (datafile->_currentSize == 0) {
end = datafile->_data + datafile->_maximalSize;
}
while (ptr < end) {
TRI_df_marker_t* marker = reinterpret_cast<TRI_df_marker_t*>(ptr);
DatafileScanEntry entry;
entry.position = static_cast<TRI_voc_size_t>(ptr - datafile->_data);
entry.size = marker->getSize();
entry.realSize = static_cast<TRI_voc_size_t>(DatafileHelper::AlignedMarkerSize<size_t>(marker));
entry.tick = marker->getTick();
entry.type = marker->getType();
entry.status = 1;
entry.typeName = TRI_NameMarkerDatafile(marker);
if (marker->getSize() == 0 && marker->getCrc() == 0 && marker->getType() == 0 &&
marker->getTick() == 0) {
entry.status = 2;
scan.endPosition = currentSize;
scan.entries.emplace_back(entry);
return scan;
}
++scan.numberMarkers;
if (marker->getSize() == 0) {
entry.status = 3;
scan.status = 2;
scan.endPosition = currentSize;
scan.entries.emplace_back(entry);
return scan;
}
if (marker->getSize() < sizeof(TRI_df_marker_t)) {
entry.status = 4;
entry.diagnosis = DiagnoseMarker(marker, end);
scan.endPosition = currentSize;
scan.status = 3;
scan.entries.emplace_back(entry);
return scan;
}
if (!TRI_IsValidMarkerDatafile(marker)) {
entry.status = 4;
entry.diagnosis = DiagnoseMarker(marker, end);
scan.endPosition = currentSize;
scan.status = 3;
scan.entries.emplace_back(entry);
return scan;
}
bool ok = CheckCrcMarker(marker, end);
if (!ok) {
entry.status = 5;
entry.diagnosis = DiagnoseMarker(marker, end);
scan.status = 4;
}
TRI_df_marker_type_t const type = marker->getType();
if (type == TRI_DF_MARKER_VPACK_DOCUMENT ||
type == TRI_DF_MARKER_VPACK_REMOVE) {
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(type));
TRI_ASSERT(slice.isObject());
entry.key = slice.get(StaticStrings::KeyString).copyString();
}
scan.entries.emplace_back(entry);
size_t size = DatafileHelper::AlignedMarkerSize<size_t>(marker);
currentSize += static_cast<TRI_voc_size_t>(size);
if (marker->getType() == TRI_DF_MARKER_FOOTER) {
scan.endPosition = currentSize;
scan.isSealed = true;
return scan;
}
ptr += size;
}
return scan;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns information about the datafile
////////////////////////////////////////////////////////////////////////////////
DatafileScan TRI_ScanDatafile(char const* path) {
DatafileScan scan;
// this function must not be called for non-physical datafiles
TRI_ASSERT(path != nullptr);
TRI_datafile_t* datafile = OpenDatafile(path, true);
if (datafile != nullptr) {
scan = ScanDatafile(datafile);
TRI_CloseDatafile(datafile);
TRI_FreeDatafile(datafile);
} else {
scan.currentSize = 0;
scan.maximalSize = 0;
scan.endPosition = 0;
scan.numberMarkers = 0;
scan.status = 5;
scan.isSealed = false;
}
return scan;
}
Reference in New Issue View Git Blame Copy Permalink