1
0
Fork 0

Merge branch 'mjmh' of ssh://github.com/triAGENS/ArangoDB into mjmh

This commit is contained in:
Max Neunhoeffer 2014-05-21 11:31:08 +02:00
commit 9d948618f3
6 changed files with 471 additions and 195 deletions

View File

@ -150,6 +150,7 @@ add_executable(
Wal/CollectorThread.cpp
Wal/LogfileManager.cpp
Wal/Logfile.cpp
Wal/Marker.cpp
Wal/Slot.cpp
Wal/Slots.cpp
Wal/SynchroniserThread.cpp

View File

@ -116,6 +116,7 @@ bin_arangod_SOURCES = \
arangod/Wal/CollectorThread.cpp \
arangod/Wal/LogfileManager.cpp \
arangod/Wal/Logfile.cpp \
arangod/Wal/Marker.cpp \
arangod/Wal/Slot.cpp \
arangod/Wal/Slots.cpp \
arangod/Wal/TestThread.cpp \

View File

@ -1574,7 +1574,7 @@ static int InsertIndexes (TRI_transaction_collection_t* trxCollection,
}
////////////////////////////////////////////////////////////////////////////////
/// @brief insert a shaped-json document into the WAL
/// @brief insert a shaped-json document (or edge) into the WAL
/// note: key might be NULL. in this case, a key is auto-generated
////////////////////////////////////////////////////////////////////////////////
@ -1650,7 +1650,7 @@ static int InsertDocumentShapedJson (TRI_transaction_collection_t* trxCollection
slotInfo = triagens::wal::LogfileManager::instance()->writeMarker(marker, forceSync);
}
else {
else if (markerType == TRI_DOC_MARKER_KEY_EDGE) {
// edge
assert(edge != nullptr);
@ -1665,6 +1665,11 @@ static int InsertDocumentShapedJson (TRI_transaction_collection_t* trxCollection
slotInfo = triagens::wal::LogfileManager::instance()->writeMarker(marker, forceSync);
}
else {
// invalid marker type
assert(false);
return TRI_ERROR_INTERNAL;
}
if (slotInfo.errorCode != TRI_ERROR_NO_ERROR) {

View File

@ -426,7 +426,7 @@ void LogfileManager::finalise (SlotInfo& slotInfo,
SlotInfo LogfileManager::writeMarker (Marker& marker,
bool waitForSync) {
return allocateAndWrite(marker.buffer, marker.size, waitForSync);
return allocateAndWrite(marker.mem(), marker.size(), waitForSync);
}
////////////////////////////////////////////////////////////////////////////////

334
arangod/Wal/Marker.cpp Normal file
View File

@ -0,0 +1,334 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief WAL markers
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2004-2013 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "Marker.h"
using namespace triagens::wal;
// -----------------------------------------------------------------------------
// --SECTION-- Marker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
Marker::Marker (TRI_df_marker_type_e type,
size_t size)
: _buffer(new char[size]),
_size(size) {
std::cout << "CREATING MARKER OF TYPE: " << type << "\n";
TRI_df_marker_t* m = reinterpret_cast<TRI_df_marker_t*>(base());
m->_type = type;
m->_size = static_cast<TRI_voc_size_t>(size);
m->_crc = 0;
m->_tick = 0;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
Marker::~Marker () {
if (_buffer != nullptr) {
delete _buffer;
}
}
// -----------------------------------------------------------------------------
// --SECTION-- protected methods
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief store a null-terminated string and length inside the marker
////////////////////////////////////////////////////////////////////////////////
void Marker::storeSizedString (size_t offset,
char const* value,
size_t length) {
// init key buffer
char* p = static_cast<char*>(base()) + offset;
memset(p, '\0', (1 + ((length + 1) / 8)) * 8);
// store length of key
*p = (uint8_t) length;
// store actual key
memcpy(p + 1, value, length);
}
// -----------------------------------------------------------------------------
// --SECTION-- BeginTransactionMarker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
BeginTransactionMarker::BeginTransactionMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId)
: Marker(TRI_WAL_MARKER_BEGIN_TRANSACTION,
sizeof(transaction_begin_marker_t)) {
transaction_begin_marker_t* m = reinterpret_cast<transaction_begin_marker_t*>(base());
m->_databaseId = databaseId;
m->_transactionId = transactionId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
BeginTransactionMarker::~BeginTransactionMarker () {
}
// -----------------------------------------------------------------------------
// --SECTION-- CommitTransactionMarker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
CommitTransactionMarker::CommitTransactionMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId)
: Marker(TRI_WAL_MARKER_COMMIT_TRANSACTION,
sizeof(transaction_commit_marker_t)) {
transaction_commit_marker_t* m = reinterpret_cast<transaction_commit_marker_t*>(base());
m->_databaseId = databaseId;
m->_transactionId = transactionId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
CommitTransactionMarker::~CommitTransactionMarker () {
}
// -----------------------------------------------------------------------------
// --SECTION-- AbortTransactionMarker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
AbortTransactionMarker::AbortTransactionMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId)
: Marker(TRI_WAL_MARKER_ABORT_TRANSACTION,
sizeof(transaction_abort_marker_t)) {
transaction_abort_marker_t* m = reinterpret_cast<transaction_abort_marker_t*>(base());
m->_databaseId = databaseId;
m->_transactionId = transactionId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
AbortTransactionMarker::~AbortTransactionMarker () {
}
// -----------------------------------------------------------------------------
// --SECTION-- DocumentMarker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId,
TRI_voc_rid_t revisionId,
TRI_voc_tid_t transactionId,
std::string const& key,
triagens::basics::JsonLegend& legend,
TRI_shaped_json_t const* shapedJson)
: Marker(TRI_WAL_MARKER_DOCUMENT,
sizeof(document_marker_t) + alignedSize(key.size() + 2) + legend.getSize() + shapedJson->_data.length) {
document_marker_t* m = reinterpret_cast<document_marker_t*>(base());
m->_databaseId = databaseId;
m->_collectionId = collectionId;
m->_rid = revisionId;
m->_tid = transactionId;
m->_shape = shapedJson->_sid;
m->_offsetKey = sizeof(document_marker_t); // start position of key
m->_offsetLegend = m->_offsetKey + alignedSize(key.size() + 2);
m->_offsetJson = m->_offsetLegend + alignedSize(legend.getSize());
storeSizedString(m->_offsetKey, key.c_str(), key.size());
// store legend
{
char* p = static_cast<char*>(base()) + m->_offsetLegend;
legend.dump(p);
}
// store shapedJson
{
char* p = static_cast<char*>(base()) + m->_offsetJson;
memcpy(p, shapedJson->_data.data, static_cast<size_t>(shapedJson->_data.length));
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
DocumentMarker::~DocumentMarker () {
}
// -----------------------------------------------------------------------------
// --SECTION-- EdgeMarker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId,
TRI_voc_rid_t revisionId,
TRI_voc_tid_t transactionId,
std::string const& key,
TRI_document_edge_t const* edge,
triagens::basics::JsonLegend& legend,
TRI_shaped_json_t const* shapedJson)
: Marker(TRI_WAL_MARKER_EDGE,
sizeof(edge_marker_t) + alignedSize(key.size() + 2) + alignedSize(strlen(edge->_fromKey) + 2) + alignedSize(strlen(edge->_toKey) + 2) + legend.getSize() + shapedJson->_data.length) {
document_marker_t* m = reinterpret_cast<document_marker_t*>(base());
edge_marker_t* e = reinterpret_cast<edge_marker_t*>(base());
m->_databaseId = databaseId;
m->_collectionId = collectionId;
m->_rid = revisionId;
m->_tid = transactionId;
m->_shape = shapedJson->_sid;
m->_offsetKey = sizeof(edge_marker_t); // start position of key
e->_toCid = edge->_toCid;
e->_fromCid = edge->_fromCid;
e->_offsetToKey = m->_offsetKey + alignedSize(key.size() + 2);
e->_offsetFromKey = e->_offsetToKey + alignedSize(strlen(edge->_toKey) + 2);
m->_offsetLegend = e->_offsetFromKey + alignedSize(strlen(edge->_fromKey) + 2);
m->_offsetJson = m->_offsetLegend + alignedSize(legend.getSize());
// store keys
storeSizedString(m->_offsetKey, key.c_str(), key.size());
storeSizedString(e->_offsetFromKey, edge->_fromKey, strlen(edge->_fromKey));
storeSizedString(e->_offsetToKey, edge->_toKey, strlen(edge->_toKey));
// store legend
{
char* p = static_cast<char*>(base()) + m->_offsetLegend;
legend.dump(p);
}
// store shapedJson
{
char* p = static_cast<char*>(base()) + m->_offsetJson;
memcpy(p, shapedJson->_data.data, static_cast<size_t>(shapedJson->_data.length));
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
EdgeMarker::~EdgeMarker () {
}
// -----------------------------------------------------------------------------
// --SECTION-- RemoveMarker
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors and destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create marker
////////////////////////////////////////////////////////////////////////////////
RemoveMarker::RemoveMarker (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId,
TRI_voc_rid_t revisionId,
TRI_voc_tid_t transactionId,
std::string const& key)
: Marker(TRI_WAL_MARKER_REMOVE,
sizeof(remove_marker_t) + alignedSize(key.size() + 2)) {
remove_marker_t* m = reinterpret_cast<remove_marker_t*>(base());
m->_databaseId = databaseId;
m->_collectionId = collectionId;
m->_rid = revisionId;
m->_tid = transactionId;
storeSizedString(sizeof(remove_marker_t), key.c_str(), key.size());
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy marker
////////////////////////////////////////////////////////////////////////////////
RemoveMarker::~RemoveMarker () {
}
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @addtogroup\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:

View File

@ -39,6 +39,10 @@ namespace triagens {
static_assert(sizeof(TRI_df_marker_t) == 24, "invalid base marker size");
// -----------------------------------------------------------------------------
// --SECTION-- low level structs, used for on-disk persistence
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief wal transaction begin marker
////////////////////////////////////////////////////////////////////////////////
@ -124,233 +128,164 @@ namespace triagens {
TRI_voc_tid_t _tid;
};
////////////////////////////////////////////////////////////////////////////////
/// @brief wal marker class
////////////////////////////////////////////////////////////////////////////////
// -----------------------------------------------------------------------------
// --SECTION-- Marker
// -----------------------------------------------------------------------------
struct Marker {
Marker (TRI_df_marker_type_e type,
size_t size)
: buffer(new char[size]),
size(size) {
class Marker {
std::cout << "CREATING MARKER OF TYPE: " << type << "\n";
protected:
// initialise the marker header
auto h = header();
h->_type = type;
h->_size = static_cast<TRI_voc_size_t>(size);
h->_crc = 0;
h->_tick = 0;
}
Marker (Marker const&) = delete;
Marker& operator= (Marker const&) = delete;
virtual ~Marker () {
if (buffer != nullptr) {
delete buffer;
}
}
static inline size_t alignedSize (size_t size) {
return TRI_DF_ALIGN_BLOCK(size);
}
Marker (TRI_df_marker_type_e,
size_t);
inline TRI_df_marker_t* header () const {
return (TRI_df_marker_t*) buffer;
}
inline char* base () const {
return (char*) buffer;
}
inline char* payload () const {
return base() + sizeof(TRI_df_marker_t);
}
void storeSizedString (size_t offset,
char const* value,
size_t length) {
// init key buffer
char* p = static_cast<char*>(base()) + offset;
memset(p, '\0', (1 + ((length + 1) / 8)) * 8);
// store length of key
*p = (uint8_t) length;
// store actual key
memcpy(p + 1, value, length);
}
char* buffer;
uint32_t const size;
};
struct BeginTransactionMarker : public Marker {
BeginTransactionMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId)
: Marker(TRI_WAL_MARKER_BEGIN_TRANSACTION,
sizeof(transaction_begin_marker_t)) {
virtual ~Marker ();
transaction_begin_marker_t* m = reinterpret_cast<transaction_begin_marker_t*>(base());
// -----------------------------------------------------------------------------
// --SECTION-- public methods
// -----------------------------------------------------------------------------
m->_databaseId = databaseId;
m->_transactionId = transactionId;
}
public:
static inline size_t alignedSize (size_t size) {
return TRI_DF_ALIGN_BLOCK(size);
}
inline void* mem () const {
return static_cast<void*>(_buffer);
}
~BeginTransactionMarker () {
}
inline char* base () const {
return _buffer;
}
inline char* payload () const {
return base() + sizeof(TRI_df_marker_t);
}
inline uint32_t size () const {
return _size;
}
// -----------------------------------------------------------------------------
// --SECTION-- protected methods
// -----------------------------------------------------------------------------
protected:
void storeSizedString (size_t,
char const*,
size_t);
// -----------------------------------------------------------------------------
// --SECTION-- private variables
// -----------------------------------------------------------------------------
char* _buffer;
uint32_t const _size;
};
// -----------------------------------------------------------------------------
// --SECTION-- BeginTransactionMarker
// -----------------------------------------------------------------------------
class BeginTransactionMarker : public Marker {
public:
BeginTransactionMarker (TRI_voc_tick_t,
TRI_voc_tid_t);
~BeginTransactionMarker ();
};
// -----------------------------------------------------------------------------
// --SECTION-- CommitTransactionMarker
// -----------------------------------------------------------------------------
struct CommitTransactionMarker : public Marker {
CommitTransactionMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId)
: Marker(TRI_WAL_MARKER_COMMIT_TRANSACTION,
sizeof(transaction_commit_marker_t)) {
transaction_commit_marker_t* m = reinterpret_cast<transaction_commit_marker_t*>(base());
m->_databaseId = databaseId;
m->_transactionId = transactionId;
}
class CommitTransactionMarker : public Marker {
~CommitTransactionMarker () {
}
public:
CommitTransactionMarker (TRI_voc_tick_t,
TRI_voc_tid_t);
~CommitTransactionMarker ();
};
struct AbortTransactionMarker : public Marker {
AbortTransactionMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId)
: Marker(TRI_WAL_MARKER_ABORT_TRANSACTION,
sizeof(transaction_abort_marker_t)) {
transaction_abort_marker_t* m = reinterpret_cast<transaction_abort_marker_t*>(base());
m->_databaseId = databaseId;
m->_transactionId = transactionId;
}
// -----------------------------------------------------------------------------
// --SECTION-- AbortTransactionMarker
// -----------------------------------------------------------------------------
~AbortTransactionMarker () {
}
class AbortTransactionMarker : public Marker {
public:
AbortTransactionMarker (TRI_voc_tick_t,
TRI_voc_tid_t);
~AbortTransactionMarker ();
};
struct DocumentMarker : public Marker {
DocumentMarker (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId,
TRI_voc_rid_t revisionId,
TRI_voc_tid_t transactionId,
std::string const& key,
triagens::basics::JsonLegend& legend,
TRI_shaped_json_t const* shapedJson)
: Marker(TRI_WAL_MARKER_DOCUMENT,
sizeof(document_marker_t) + alignedSize(key.size() + 2) + legend.getSize() + shapedJson->_data.length) {
// -----------------------------------------------------------------------------
// --SECTION-- DocumentMarker
// -----------------------------------------------------------------------------
document_marker_t* m = reinterpret_cast<document_marker_t*>(base());
m->_databaseId = databaseId;
m->_collectionId = collectionId;
m->_rid = revisionId;
m->_tid = transactionId;
m->_shape = shapedJson->_sid;
m->_offsetKey = sizeof(document_marker_t); // start position of key
m->_offsetLegend = m->_offsetKey + alignedSize(key.size() + 2);
m->_offsetJson = m->_offsetLegend + alignedSize(legend.getSize());
storeSizedString(m->_offsetKey, key.c_str(), key.size());
class DocumentMarker : public Marker {
// store legend
{
char* p = static_cast<char*>(base()) + m->_offsetLegend;
legend.dump(p);
}
public:
// store shapedJson
{
char* p = static_cast<char*>(base()) + m->_offsetJson;
memcpy(p, shapedJson->_data.data, static_cast<size_t>(shapedJson->_data.length));
}
}
~DocumentMarker () {
}
DocumentMarker (TRI_voc_tick_t,
TRI_voc_cid_t,
TRI_voc_rid_t,
TRI_voc_tid_t,
std::string const&,
triagens::basics::JsonLegend&,
TRI_shaped_json_t const*);
~DocumentMarker ();
};
// -----------------------------------------------------------------------------
// --SECTION-- EdgeMarker
// -----------------------------------------------------------------------------
struct EdgeMarker : public Marker {
EdgeMarker (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId,
TRI_voc_rid_t revisionId,
TRI_voc_tid_t transactionId,
std::string const& key,
TRI_document_edge_t const* edge,
triagens::basics::JsonLegend& legend,
TRI_shaped_json_t const* shapedJson)
: Marker(TRI_WAL_MARKER_EDGE,
sizeof(edge_marker_t) + alignedSize(key.size() + 2) + alignedSize(strlen(edge->_fromKey) + 2) + alignedSize(strlen(edge->_toKey) + 2) + legend.getSize() + shapedJson->_data.length) {
class EdgeMarker : public Marker {
document_marker_t* m = reinterpret_cast<document_marker_t*>(base());
edge_marker_t* e = reinterpret_cast<edge_marker_t*>(base());
public:
m->_databaseId = databaseId;
m->_collectionId = collectionId;
m->_rid = revisionId;
m->_tid = transactionId;
m->_shape = shapedJson->_sid;
m->_offsetKey = sizeof(edge_marker_t); // start position of key
e->_toCid = edge->_toCid;
e->_fromCid = edge->_fromCid;
e->_offsetToKey = m->_offsetKey + alignedSize(key.size() + 2);
e->_offsetFromKey = e->_offsetToKey + alignedSize(strlen(edge->_toKey) + 2);
m->_offsetLegend = e->_offsetFromKey + alignedSize(strlen(edge->_fromKey) + 2);
m->_offsetJson = m->_offsetLegend + alignedSize(legend.getSize());
// store keys
storeSizedString(m->_offsetKey, key.c_str(), key.size());
storeSizedString(e->_offsetFromKey, edge->_fromKey, strlen(edge->_fromKey));
storeSizedString(e->_offsetToKey, edge->_toKey, strlen(edge->_toKey));
// store legend
{
char* p = static_cast<char*>(base()) + m->_offsetLegend;
legend.dump(p);
}
// store shapedJson
{
char* p = static_cast<char*>(base()) + m->_offsetJson;
memcpy(p, shapedJson->_data.data, static_cast<size_t>(shapedJson->_data.length));
}
}
~EdgeMarker () {
}
EdgeMarker (TRI_voc_tick_t,
TRI_voc_cid_t,
TRI_voc_rid_t,
TRI_voc_tid_t,
std::string const&,
TRI_document_edge_t const*,
triagens::basics::JsonLegend&,
TRI_shaped_json_t const*);
~EdgeMarker ();
};
/*
struct RemoveMarker : public Marker {
RemoveMarker (TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId,
TRI_voc_tid_t transactionId,
std::string const& key)
: Marker(TRI_WAL_MARKER_REMOVE,
sizeof(TRI_voc_tick_t) + sizeof(TRI_voc_cid_t) + sizeof(TRI_voc_tid_t) + key.size() + 2) {
// -----------------------------------------------------------------------------
// --SECTION-- RemoveMarker
// -----------------------------------------------------------------------------
char* p = data();
store<TRI_voc_tick_t>(p, databaseId);
store<TRI_voc_cid_t>(p, collectionId);
store<TRI_voc_tid_t>(p, transactionId);
class RemoveMarker : public Marker {
// store key
store<uint8_t>(p, (uint8_t) key.size());
store(p, key.c_str(), key.size());
store<unsigned char>(p, '\0');
}
public:
~RemoveMarker () {
}
RemoveMarker (TRI_voc_tick_t,
TRI_voc_cid_t,
TRI_voc_rid_t,
TRI_voc_tid_t,
std::string const&);
~RemoveMarker ();
};
*/
}
}