Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

added startup option `--wal.suppress-shape-information`

This commit is contained in:
Jan Steemann 2014-07-21 18:02:16 +02:00
parent 712358d069
commit fe71a8f26d
8 changed files with 264 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGELOG
View File

@ -1,6 +1,17 @@
v2.2.1 (XXXX-XX-XX)
-------------------
* added startup option `--wal.suppress-shape-information`
Setting this option to `true` will reduce memory and disk space usage and require
less CPU time when modifying documents or edges. It should therefore be turned on
for standalone ArangoDB servers. However, for servers that are used as replication
masters, setting this option to `true` will effectively disable the usage of the
write-ahead log for replication, so it should be set to `false` for any replication
master servers.
The default value for this option is `false`.
* added optional `ttl` attribute to specify result cursor expiration for HTTP API method
`POST /_api/cursor`

4
Documentation/Books/Users/AdministratingArango/README.mdpp
View File

@ -82,6 +82,10 @@ logfiles:
@startDocuBlock WalLogfileAllowOversizeEntries
<!-- arangod/Wal/LogfileManager.h -->
@startDocuBlock WalLogfileSuppressShapeInformation
When data gets copied from the write-ahead logfiles into the journals or datafiles
of collections, files will be created on the collection level. How big these files
are is determined by the following global configuration value:

4
Documentation/Books/Users/ConfigureArango/Wal.mdpp
View File

@ -21,6 +21,10 @@ a replication backlog.
<!-- arangod/Wal/LogfileManager.h -->
@startDocuBlock WalLogfileAllowOversizeEntries
!SUBSECTION Suppress shape information
<!-- arangod/Wal/LogfileManager.h -->
@startDocuBlock WalLogfileSuppressShapeInformation
!SUBSECTION Number of reserve logfiles
<!-- arangod/Wal/LogfileManager.h -->
@startDocuBlock WalLogfileReserveLogfiles

203
arangod/VocBase/document-collection.cpp
View File

@ -5297,6 +5297,130 @@ int TRI_RemoveShapedJsonDocumentCollection (TRI_transaction_collection_t* trxCol
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief create a document or edge marker, without using a legend
////////////////////////////////////////////////////////////////////////////////
static int CreateMarkerNoLegend (triagens::wal::Marker*& marker,
TRI_document_collection_t* document,
TRI_voc_rid_t rid,
TRI_transaction_collection_t* trxCollection,
std::string const& keyString,
TRI_shaped_json_t const* shaped,
TRI_document_edge_t const* edge) {
TRI_ASSERT(marker == nullptr);
TRI_IF_FAILURE("InsertDocumentNoMarker") {
// test what happens when no marker can be created
return TRI_ERROR_DEBUG;
}
TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") {
// test what happens if no marker can be created
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
}
if (edge == nullptr) {
// document
marker = new triagens::wal::DocumentMarker(document->_vocbase->_id,
document->_info._cid,
rid,
TRI_MarkerIdTransaction(trxCollection->_transaction),
keyString,
8,
shaped);
}
else {
// edge
marker = new triagens::wal::EdgeMarker(document->_vocbase->_id,
document->_info._cid,
rid,
TRI_MarkerIdTransaction(trxCollection->_transaction),
keyString,
edge,
8,
shaped);
}
TRI_ASSERT(marker != nullptr);
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief create a document or edge marker, with a legend
////////////////////////////////////////////////////////////////////////////////
static int CreateMarkerWithLegend (triagens::wal::Marker*& marker,
TRI_document_collection_t* document,
TRI_voc_rid_t rid,
TRI_transaction_collection_t* trxCollection,
std::string const& keyString,
TRI_shaped_json_t const* shaped,
TRI_document_edge_t const* edge) {
// construct a legend for the shaped json
triagens::basics::JsonLegend legend(document->getShaper()); // PROTECTED by trx in trxCollection
TRI_IF_FAILURE("InsertDocumentNoLegend") {
// test what happens when no legend can be created
return TRI_ERROR_DEBUG;
}
TRI_IF_FAILURE("InsertDocumentNoLegendExcept") {
// test what happens if no legend can be created
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
}
int res = legend.addShape(shaped->_sid, &shaped->_data);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
TRI_IF_FAILURE("InsertDocumentNoMarker") {
// test what happens when no marker can be created
return TRI_ERROR_DEBUG;
}
TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") {
// test what happens if no marker can be created
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
}
if (edge == nullptr) {
// document
auto m = new triagens::wal::DocumentMarker(document->_vocbase->_id,
document->_info._cid,
rid,
TRI_MarkerIdTransaction(trxCollection->_transaction),
keyString,
legend.getSize(),
shaped);
m->storeLegend(legend);
marker = m; // reinterpret_cast<triagens::wal::Marker*>(m);
}
else {
// edge
auto m = new triagens::wal::EdgeMarker(document->_vocbase->_id,
document->_info._cid,
rid,
TRI_MarkerIdTransaction(trxCollection->_transaction),
keyString,
edge,
legend.getSize(),
shaped);
m->storeLegend(legend);
marker = m; //reinterpret_cast<triagens::wal::Marker*>(m);
}
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief insert a shaped-json document (or edge)
/// note: key might be NULL. in this case, a key is auto-generated
@ -5345,70 +5469,25 @@ int TRI_InsertShapedJsonDocumentCollection (TRI_transaction_collection_t* trxCol
keyString = key;
}
uint64_t hash = TRI_HashKeyPrimaryIndex(keyString.c_str());
// construct a legend for the shaped json
triagens::basics::JsonLegend legend(document->getShaper()); // PROTECTED by trx in trxCollection
TRI_IF_FAILURE("InsertDocumentNoLegend") {
// test what happens when no legend can be created
return TRI_ERROR_DEBUG;
}
if (marker == nullptr) {
TRI_IF_FAILURE("InsertDocumentNoLegendExcept") {
// test what happens if no legend can be created
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
}
int res = legend.addShape(shaped->_sid, &shaped->_data);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
TRI_IF_FAILURE("InsertDocumentNoMarker") {
// test what happens when no marker can be created
return TRI_ERROR_DEBUG;
}
TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") {
// test what happens if no marker can be created
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
}
if (edge == nullptr) {
// document
TRI_ASSERT(edge == nullptr);
marker = new triagens::wal::DocumentMarker(document->_vocbase->_id,
document->_info._cid,
rid,
TRI_MarkerIdTransaction(trxCollection->_transaction),
keyString,
legend,
shaped);
}
else {
// edge
marker = new triagens::wal::EdgeMarker(document->_vocbase->_id,
document->_info._cid,
rid,
TRI_MarkerIdTransaction(trxCollection->_transaction),
keyString,
edge,
legend,
shaped);
}
}
TRI_ASSERT(marker != nullptr);
int res = TRI_ERROR_NO_ERROR;
if (marker == nullptr) {
if (triagens::wal::LogfileManager::instance()->suppressShapeInformation()) {
res = CreateMarkerNoLegend(marker, document, rid, trxCollection, keyString, shaped, edge);
}
else {
res = CreateMarkerWithLegend(marker, document, rid, trxCollection, keyString, shaped, edge);
}
}
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
TRI_ASSERT(marker != nullptr);
uint64_t hash = TRI_HashKeyPrimaryIndex(keyString.c_str());
// now insert into indexes
{
TRI_IF_FAILURE("InsertDocumentNoLock") {

2
arangod/Wal/LogfileManager.cpp
View File

@ -138,6 +138,7 @@ LogfileManager::LogfileManager (TRI_server_t* server,
_allowOversizeEntries(true),
_ignoreLogfileErrors(false),
_ignoreRecoveryErrors(false),
_suppressShapeInformation(false),
_allowWrites(false), // start in read-only mode
_hasFoundLastTick(false),
_inRecovery(true),
@ -234,6 +235,7 @@ void LogfileManager::setupOptions (std::map<std::string, triagens::basics::Progr
("wal.open-logfiles", &_maxOpenLogfiles, "maximum number of parallel open logfiles")
("wal.reserve-logfiles", &_reserveLogfiles, "maximum number of reserve logfiles to maintain")
("wal.slots", &_numberOfSlots, "number of logfile slots to use")
("wal.suppress-shape-information", &_suppressShapeInformation, "do not write shape information for markers (saves a lot of disk space, but effectively disables using the write-ahead log for replication)")
("wal.sync-interval", &_syncInterval, "interval for automatic, non-requested disk syncs (in milliseconds)")
("wal.throttle-when-pending", &_throttleWhenPending, "throttle writes when at least this many operations are waiting for collection (set to 0 to deactivate write-throttling)")
("wal.throttle-wait", &_maxThrottleWait, "maximum wait time per operation when write-throttled (in milliseconds)")

27
arangod/Wal/LogfileManager.h
View File

@ -229,6 +229,15 @@ namespace triagens {
_historicLogfiles = value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not shape information should be suppress when writing
/// markers into the write-ahead log
////////////////////////////////////////////////////////////////////////////////
inline bool suppressShapeInformation () const {
return _suppressShapeInformation;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not there was a SHUTDOWN file with a tick value
/// at server start
@ -960,6 +969,24 @@ namespace triagens {
bool _ignoreRecoveryErrors;
////////////////////////////////////////////////////////////////////////////////
/// @brief suppress shape information
/// @startDocuBlock WalLogfileSuppressShapeInformation
/// `--wal.suppress-shape-information`
///
/// Setting this variable to *true* will lead to no shape information being
/// written into the write-ahead logfiles for documents or edges. While this is
/// a good optimization for a single server to save memory (and disk space), it
/// it will effectively disable using the write-ahead log as a source for
/// replicating changes to other servers.
///
/// **Do not set this variable to *true* on a server that you plan to use as a
/// replication master!**
/// @endDocuBlock
////////////////////////////////////////////////////////////////////////////////
bool _suppressShapeInformation;
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not writes to the WAL are allowed
////////////////////////////////////////////////////////////////////////////////

109
arangod/Wal/Marker.cpp
View File

@ -1092,10 +1092,10 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
TRI_voc_rid_t revisionId,
TRI_voc_tid_t transactionId,
std::string const& key,
triagens::basics::JsonLegend& legend,
size_t legendSize,
TRI_shaped_json_t const* shapedJson)
: Marker(TRI_WAL_MARKER_DOCUMENT,
sizeof(document_marker_t) + alignedSize(key.size() + 1) + legend.getSize() + shapedJson->_data.length) {
sizeof(document_marker_t) + alignedSize(key.size() + 1) + legendSize + shapedJson->_data.length) {
document_marker_t* m = reinterpret_cast<document_marker_t*>(begin());
m->_databaseId = databaseId;
m->_collectionId = collectionId;
@ -1105,14 +1105,16 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
m->_offsetKey = sizeof(document_marker_t); // start position of key
m->_offsetLegend = static_cast<uint16_t>(m->_offsetKey + alignedSize(key.size() + 1));
m->_offsetJson = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legend.getSize()));
m->_offsetJson = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legendSize));
storeSizedString(m->_offsetKey, key);
// store legend
{
TRI_ASSERT(legendSize >= 8);
char* p = static_cast<char*>(begin()) + m->_offsetLegend;
legend.dump(p);
memset(p, 0, 8); // initialise initial bytes of legend with 0s
}
// store shapedJson
@ -1133,6 +1135,14 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
DocumentMarker::~DocumentMarker () {
}
////////////////////////////////////////////////////////////////////////////////
/// @brief store legend in marker
////////////////////////////////////////////////////////////////////////////////
void DocumentMarker::storeLegend (triagens::basics::JsonLegend& legend) {
legend.dump((void*) this->legend());
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump marker
////////////////////////////////////////////////////////////////////////////////
@ -1153,8 +1163,6 @@ void DocumentMarker::dump () const {
<< "\n";
#ifdef DEBUG_WAL_DETAIL
std::cout << "LEGEND: '" << stringifyPart(legend(), legendLength()) << "'\n";
std::cout << "LEGEND HEX: '" << hexifyPart(legend(), legendLength()) << "'\n";
std::cout << "JSON: '" << stringifyPart(json(), jsonLength()) << "'\n";
std::cout << "JSON HEX: '" << hexifyPart(json(), jsonLength()) << "'\n";
@ -1179,13 +1187,15 @@ DocumentMarker* DocumentMarker::clone (TRI_df_marker_t const* other,
if (other->_type == TRI_DOC_MARKER_KEY_DOCUMENT) {
TRI_doc_document_key_marker_t const* original = reinterpret_cast<TRI_doc_document_key_marker_t const*>(other);
return new DocumentMarker(databaseId,
collectionId,
revisionId,
transactionId,
std::string(base + original->_offsetKey),
legend,
shapedJson);
auto marker = new DocumentMarker(databaseId,
collectionId,
revisionId,
transactionId,
std::string(base + original->_offsetKey),
legend.getSize(),
shapedJson);
marker->storeLegend(legend);
return marker;
}
else {
TRI_ASSERT(other->_type == TRI_WAL_MARKER_DOCUMENT);
@ -1195,13 +1205,15 @@ DocumentMarker* DocumentMarker::clone (TRI_df_marker_t const* other,
TRI_ASSERT(original->_databaseId == databaseId);
TRI_ASSERT(original->_collectionId == collectionId);
return new DocumentMarker(original->_databaseId,
original->_collectionId,
revisionId,
transactionId,
std::string(base + original->_offsetKey),
legend,
shapedJson);
auto marker = new DocumentMarker(original->_databaseId,
original->_collectionId,
revisionId,
transactionId,
std::string(base + original->_offsetKey),
legend.getSize(),
shapedJson);
marker->storeLegend(legend);
return marker;
}
}
@ -1223,10 +1235,10 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
TRI_voc_tid_t transactionId,
std::string const& key,
TRI_document_edge_t const* edge,
triagens::basics::JsonLegend& legend,
size_t legendSize,
TRI_shaped_json_t const* shapedJson)
: Marker(TRI_WAL_MARKER_EDGE,
sizeof(edge_marker_t) + alignedSize(key.size() + 1) + alignedSize(strlen(edge->_fromKey) + 1) + alignedSize(strlen(edge->_toKey) + 1) + legend.getSize() + shapedJson->_data.length) {
sizeof(edge_marker_t) + alignedSize(key.size() + 1) + alignedSize(strlen(edge->_fromKey) + 1) + alignedSize(strlen(edge->_toKey) + 1) + legendSize + shapedJson->_data.length) {
edge_marker_t* m = reinterpret_cast<edge_marker_t*>(begin());
@ -1241,7 +1253,7 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
m->_offsetToKey = static_cast<uint16_t>(m->_offsetKey + alignedSize(key.size() + 1));
m->_offsetFromKey = static_cast<uint16_t>(m->_offsetToKey + alignedSize(strlen(edge->_toKey) + 1));
m->_offsetLegend = static_cast<uint16_t>(m->_offsetFromKey + alignedSize(strlen(edge->_fromKey) + 1));
m->_offsetJson = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legend.getSize()));
m->_offsetJson = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legendSize));
// store keys
storeSizedString(m->_offsetKey, key.c_str());
@ -1250,8 +1262,9 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
// store legend
{
TRI_ASSERT(legendSize >= 8);
char* p = static_cast<char*>(begin()) + m->_offsetLegend;
legend.dump(p);
memset(p, 0, 8); // initialise initial bytes of legend with 0s
}
// store shapedJson
@ -1272,6 +1285,14 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
EdgeMarker::~EdgeMarker () {
}
////////////////////////////////////////////////////////////////////////////////
/// @brief store legend in marker
////////////////////////////////////////////////////////////////////////////////
void EdgeMarker::storeLegend (triagens::basics::JsonLegend& legend) {
legend.dump((void*) this->legend());
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump marker
////////////////////////////////////////////////////////////////////////////////
@ -1298,8 +1319,6 @@ void EdgeMarker::dump () const {
<< "\n";
#ifdef DEBUG_WAL_DETAIL
std::cout << "LEGEND: '" << stringifyPart(legend(), legendLength()) << "'\n";
std::cout << "LEGEND HEX: '" << hexifyPart(legend(), legendLength()) << "'\n";
std::cout << "JSON: '" << stringifyPart(json(), jsonLength()) << "'\n";
std::cout << "JSON HEX: '" << hexifyPart(json(), jsonLength()) << "'\n";
@ -1330,14 +1349,17 @@ EdgeMarker* EdgeMarker::clone (TRI_df_marker_t const* other,
edge._toKey = (TRI_voc_key_t) base + original->_offsetToKey;
edge._fromKey = (TRI_voc_key_t) base + original->_offsetFromKey;
return new EdgeMarker(databaseId,
collectionId,
revisionId,
transactionId,
std::string(base + original->base._offsetKey),
&edge,
legend,
shapedJson);
auto marker = new EdgeMarker(databaseId,
collectionId,
revisionId,
transactionId,
std::string(base + original->base._offsetKey),
&edge,
legend.getSize(),
shapedJson);
marker->storeLegend(legend);
return marker;
}
else {
TRI_ASSERT(other->_type == TRI_WAL_MARKER_EDGE);
@ -1353,14 +1375,17 @@ EdgeMarker* EdgeMarker::clone (TRI_df_marker_t const* other,
edge._toKey = (TRI_voc_key_t) base + original->_offsetToKey;
edge._fromKey = (TRI_voc_key_t) base + original->_offsetFromKey;
return new EdgeMarker(original->_databaseId,
original->_collectionId,
revisionId,
transactionId,
std::string(base + original->_offsetKey),
&edge,
legend,
shapedJson);
auto marker = new EdgeMarker(original->_databaseId,
original->_collectionId,
revisionId,
transactionId,
std::string(base + original->_offsetKey),
&edge,
legend.getSize(),
shapedJson);
marker->storeLegend(legend);
return marker;
}
}

12
arangod/Wal/Marker.h
View File

@ -770,9 +770,9 @@ namespace triagens {
TRI_voc_rid_t,
TRI_voc_tid_t,
std::string const&,
triagens::basics::JsonLegend&,
size_t,
TRI_shaped_json_t const*);
~DocumentMarker ();
public:
@ -814,6 +814,8 @@ namespace triagens {
return static_cast<size_t>(size() - m->_offsetJson);
}
void storeLegend (triagens::basics::JsonLegend&);
void dump () const;
static DocumentMarker* clone (TRI_df_marker_t const*,
@ -839,9 +841,9 @@ namespace triagens {
TRI_voc_tid_t,
std::string const&,
TRI_document_edge_t const*,
triagens::basics::JsonLegend&,
size_t,
TRI_shaped_json_t const*);
~EdgeMarker ();
inline TRI_voc_rid_t revisionId () const {
@ -892,6 +894,8 @@ namespace triagens {
edge_marker_t const* m = reinterpret_cast<edge_marker_t const*>(begin());
return static_cast<size_t>(size() - m->_offsetJson);
}
void storeLegend (triagens::basics::JsonLegend&);
void dump () const;