added startup option `--wal.suppress-shape-information`

CHANGELOG

@@ -1,6 +1,17 @@
 v2.2.1 (XXXX-XX-XX)
 -------------------
 
+* added startup option `--wal.suppress-shape-information`
+
+  Setting this option to `true` will reduce memory and disk space usage and require
+  less CPU time when modifying documents or edges. It should therefore be turned on
+  for standalone ArangoDB servers. However, for servers that are used as replication
+  masters, setting this option to `true` will effectively disable the usage of the
+  write-ahead log for replication, so it should be set to `false` for any replication
+  master servers.
+
+  The default value for this option is `false`. 
+
 * added optional `ttl` attribute to specify result cursor expiration for HTTP API method 
   `POST /_api/cursor` 
 

Documentation/Books/Users/AdministratingArango/README.mdpp

@@ -82,6 +82,10 @@ logfiles:
 @startDocuBlock WalLogfileAllowOversizeEntries
 
 
+<!-- arangod/Wal/LogfileManager.h -->
+@startDocuBlock WalLogfileSuppressShapeInformation
+
+
 When data gets copied from the write-ahead logfiles into the journals or datafiles
 of collections, files will be created on the collection level. How big these files
 are is determined by the following global configuration value:

Documentation/Books/Users/ConfigureArango/Wal.mdpp

@@ -21,6 +21,10 @@ a replication backlog.
 <!-- arangod/Wal/LogfileManager.h -->
 @startDocuBlock WalLogfileAllowOversizeEntries
 
+!SUBSECTION Suppress shape information
+<!-- arangod/Wal/LogfileManager.h -->
+@startDocuBlock WalLogfileSuppressShapeInformation
+
 !SUBSECTION Number of reserve logfiles
 <!-- arangod/Wal/LogfileManager.h -->
 @startDocuBlock WalLogfileReserveLogfiles

arangod/VocBase/document-collection.cpp

@@ -5297,6 +5297,130 @@ int TRI_RemoveShapedJsonDocumentCollection (TRI_transaction_collection_t* trxCol
   return res;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create a document or edge marker, without using a legend
+////////////////////////////////////////////////////////////////////////////////
+      
+static int CreateMarkerNoLegend (triagens::wal::Marker*& marker,
+                                 TRI_document_collection_t* document,
+                                 TRI_voc_rid_t rid,
+                                 TRI_transaction_collection_t* trxCollection,
+                                 std::string const& keyString,
+                                 TRI_shaped_json_t const* shaped,
+                                 TRI_document_edge_t const* edge) {
+
+  TRI_ASSERT(marker == nullptr);
+  
+  TRI_IF_FAILURE("InsertDocumentNoMarker") {
+    // test what happens when no marker can be created
+    return TRI_ERROR_DEBUG;
+  }
+
+  TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") {
+    // test what happens if no marker can be created
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+  }
+
+
+  if (edge == nullptr) {
+    // document
+    marker = new triagens::wal::DocumentMarker(document->_vocbase->_id,
+                                               document->_info._cid,
+                                               rid,
+                                               TRI_MarkerIdTransaction(trxCollection->_transaction),
+                                               keyString,
+                                               8,
+                                               shaped);
+  }
+  else {
+    // edge
+    marker = new triagens::wal::EdgeMarker(document->_vocbase->_id,
+                                           document->_info._cid,
+                                           rid,
+                                           TRI_MarkerIdTransaction(trxCollection->_transaction),
+                                           keyString,
+                                           edge,
+                                           8,
+                                           shaped);
+  }
+  
+  TRI_ASSERT(marker != nullptr);
+
+  return TRI_ERROR_NO_ERROR;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief create a document or edge marker, with a legend
+////////////////////////////////////////////////////////////////////////////////
+      
+static int CreateMarkerWithLegend (triagens::wal::Marker*& marker,
+                                   TRI_document_collection_t* document,
+                                   TRI_voc_rid_t rid,
+                                   TRI_transaction_collection_t* trxCollection,
+                                   std::string const& keyString,
+                                   TRI_shaped_json_t const* shaped,
+                                   TRI_document_edge_t const* edge) {
+  // construct a legend for the shaped json
+  triagens::basics::JsonLegend legend(document->getShaper());  // PROTECTED by trx in trxCollection
+
+  TRI_IF_FAILURE("InsertDocumentNoLegend") {
+    // test what happens when no legend can be created
+    return TRI_ERROR_DEBUG;
+  }
+    
+  TRI_IF_FAILURE("InsertDocumentNoLegendExcept") {
+    // test what happens if no legend can be created
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+  }
+
+  int res = legend.addShape(shaped->_sid, &shaped->_data);
+    
+  if (res != TRI_ERROR_NO_ERROR) {
+    return res;
+  }
+    
+  TRI_IF_FAILURE("InsertDocumentNoMarker") {
+    // test what happens when no marker can be created
+    return TRI_ERROR_DEBUG;
+  }
+
+  TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") {
+    // test what happens if no marker can be created
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
+  }
+    
+  if (edge == nullptr) {
+    // document
+    auto m = new triagens::wal::DocumentMarker(document->_vocbase->_id,
+                                               document->_info._cid,
+                                               rid,
+                                               TRI_MarkerIdTransaction(trxCollection->_transaction),
+                                               keyString,
+                                               legend.getSize(),
+                                               shaped);
+    
+    m->storeLegend(legend);
+    marker = m; // reinterpret_cast<triagens::wal::Marker*>(m);
+  }
+  else {
+    // edge
+    auto m = new triagens::wal::EdgeMarker(document->_vocbase->_id,
+                                           document->_info._cid,
+                                           rid,
+                                           TRI_MarkerIdTransaction(trxCollection->_transaction),
+                                           keyString,
+                                           edge,
+                                           legend.getSize(),
+                                           shaped);
+      
+    m->storeLegend(legend);
+
+    marker = m; //reinterpret_cast<triagens::wal::Marker*>(m);
+  }
+
+  return TRI_ERROR_NO_ERROR;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief insert a shaped-json document (or edge)
 /// note: key might be NULL. in this case, a key is auto-generated
@@ -5345,70 +5469,25 @@ int TRI_InsertShapedJsonDocumentCollection (TRI_transaction_collection_t* trxCol
     keyString = key;
   }
 
-  uint64_t hash = TRI_HashKeyPrimaryIndex(keyString.c_str());
-
-  // construct a legend for the shaped json
-  triagens::basics::JsonLegend legend(document->getShaper());  // PROTECTED by trx in trxCollection
-
-  TRI_IF_FAILURE("InsertDocumentNoLegend") {
-    // test what happens when no legend can be created
-    return TRI_ERROR_DEBUG;
-  }
-
-
-  if (marker == nullptr) {
-    TRI_IF_FAILURE("InsertDocumentNoLegendExcept") {
-      // test what happens if no legend can be created
-      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
-    }
-  
-    int res = legend.addShape(shaped->_sid, &shaped->_data);
-
-    if (res != TRI_ERROR_NO_ERROR) {
-      return res;
-    }
-
-    TRI_IF_FAILURE("InsertDocumentNoMarker") {
-      // test what happens when no marker can be created
-      return TRI_ERROR_DEBUG;
-    }
-
-    TRI_IF_FAILURE("InsertDocumentNoMarkerExcept") {
-      // test what happens if no marker can be created
-      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
-    }
-
-
-    if (edge == nullptr) {
-      // document
-      TRI_ASSERT(edge == nullptr);
-
-      marker = new triagens::wal::DocumentMarker(document->_vocbase->_id,
-                                                 document->_info._cid,
-                                                 rid,
-                                                 TRI_MarkerIdTransaction(trxCollection->_transaction),
-                                                 keyString,
-                                                 legend,
-                                                 shaped);
-    }
-    else {
-      // edge
-      marker = new triagens::wal::EdgeMarker(document->_vocbase->_id,
-                                             document->_info._cid,
-                                             rid,
-                                             TRI_MarkerIdTransaction(trxCollection->_transaction),
-                                             keyString,
-                                             edge,
-                                             legend,
-                                             shaped);
-    }
-  }
-
-
-  TRI_ASSERT(marker != nullptr);
-  
   int res = TRI_ERROR_NO_ERROR;
 
+  if (marker == nullptr) {
+    if (triagens::wal::LogfileManager::instance()->suppressShapeInformation()) {
+      res = CreateMarkerNoLegend(marker, document, rid, trxCollection, keyString, shaped, edge);
+    }
+    else {
+      res = CreateMarkerWithLegend(marker, document, rid, trxCollection, keyString, shaped, edge);
+    }
+  }
+
+  if (res != TRI_ERROR_NO_ERROR) {
+    return res;
+  }
+  
+  TRI_ASSERT(marker != nullptr);
+
+  uint64_t hash = TRI_HashKeyPrimaryIndex(keyString.c_str());
+  
   // now insert into indexes
   {
     TRI_IF_FAILURE("InsertDocumentNoLock") {

arangod/Wal/LogfileManager.cpp

@@ -138,6 +138,7 @@ LogfileManager::LogfileManager (TRI_server_t* server,
     _allowOversizeEntries(true),
     _ignoreLogfileErrors(false),
     _ignoreRecoveryErrors(false),
+    _suppressShapeInformation(false),
     _allowWrites(false), // start in read-only mode
     _hasFoundLastTick(false),
     _inRecovery(true),
@@ -234,6 +235,7 @@ void LogfileManager::setupOptions (std::map<std::string, triagens::basics::Progr
     ("wal.open-logfiles", &_maxOpenLogfiles, "maximum number of parallel open logfiles")
     ("wal.reserve-logfiles", &_reserveLogfiles, "maximum number of reserve logfiles to maintain")
     ("wal.slots", &_numberOfSlots, "number of logfile slots to use")
+    ("wal.suppress-shape-information", &_suppressShapeInformation, "do not write shape information for markers (saves a lot of disk space, but effectively disables using the write-ahead log for replication)")
     ("wal.sync-interval", &_syncInterval, "interval for automatic, non-requested disk syncs (in milliseconds)")
     ("wal.throttle-when-pending", &_throttleWhenPending, "throttle writes when at least this many operations are waiting for collection (set to 0 to deactivate write-throttling)")
     ("wal.throttle-wait", &_maxThrottleWait, "maximum wait time per operation when write-throttled (in milliseconds)")

arangod/Wal/LogfileManager.h

@@ -229,6 +229,15 @@ namespace triagens {
           _historicLogfiles = value;
         }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief whether or not shape information should be suppress when writing
+/// markers into the write-ahead log
+////////////////////////////////////////////////////////////////////////////////
+        
+        inline bool suppressShapeInformation () const {
+          return _suppressShapeInformation;
+        }
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief whether or not there was a SHUTDOWN file with a tick value
 /// at server start
@@ -960,6 +969,24 @@ namespace triagens {
 
         bool _ignoreRecoveryErrors;
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief suppress shape information
+/// @startDocuBlock WalLogfileSuppressShapeInformation
+/// `--wal.suppress-shape-information`
+///
+/// Setting this variable to *true* will lead to no shape information being 
+/// written into the write-ahead logfiles for documents or edges. While this is 
+/// a good optimization for a single server to save memory (and disk space), it
+/// it will effectively disable using the write-ahead log as a source for 
+/// replicating changes to other servers. 
+///
+/// **Do not set this variable to *true* on a server that you plan to use as a 
+/// replication master!**
+/// @endDocuBlock
+////////////////////////////////////////////////////////////////////////////////
+
+        bool _suppressShapeInformation;
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief whether or not writes to the WAL are allowed
 ////////////////////////////////////////////////////////////////////////////////

arangod/Wal/Marker.cpp

@@ -1092,10 +1092,10 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
                                 TRI_voc_rid_t revisionId,
                                 TRI_voc_tid_t transactionId,
                                 std::string const& key,
-                                triagens::basics::JsonLegend& legend,
+                                size_t legendSize,
                                 TRI_shaped_json_t const* shapedJson)
   : Marker(TRI_WAL_MARKER_DOCUMENT,
-    sizeof(document_marker_t) + alignedSize(key.size() + 1) + legend.getSize() + shapedJson->_data.length) {
+    sizeof(document_marker_t) + alignedSize(key.size() + 1) + legendSize + shapedJson->_data.length) {
   document_marker_t* m = reinterpret_cast<document_marker_t*>(begin());
   m->_databaseId    = databaseId;
   m->_collectionId  = collectionId;
@@ -1105,14 +1105,16 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
 
   m->_offsetKey     = sizeof(document_marker_t); // start position of key
   m->_offsetLegend  = static_cast<uint16_t>(m->_offsetKey + alignedSize(key.size() + 1));
-  m->_offsetJson    = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legend.getSize()));
+  m->_offsetJson    = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legendSize));
 
   storeSizedString(m->_offsetKey, key);
 
   // store legend
   {
+    TRI_ASSERT(legendSize >= 8);
+
     char* p = static_cast<char*>(begin()) + m->_offsetLegend;
-    legend.dump(p);
+    memset(p, 0, 8); // initialise initial bytes of legend with 0s
   }
 
   // store shapedJson
@@ -1133,6 +1135,14 @@ DocumentMarker::DocumentMarker (TRI_voc_tick_t databaseId,
 DocumentMarker::~DocumentMarker () {
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief store legend in marker
+////////////////////////////////////////////////////////////////////////////////
+
+void DocumentMarker::storeLegend (triagens::basics::JsonLegend& legend) {
+  legend.dump((void*) this->legend());
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief dump marker
 ////////////////////////////////////////////////////////////////////////////////
@@ -1153,8 +1163,6 @@ void DocumentMarker::dump () const {
             << "\n";
 
 #ifdef DEBUG_WAL_DETAIL
-  std::cout << "LEGEND:     '" << stringifyPart(legend(), legendLength()) << "'\n";
-  std::cout << "LEGEND HEX: '" << hexifyPart(legend(), legendLength()) << "'\n";
   std::cout << "JSON:       '" << stringifyPart(json(), jsonLength()) << "'\n";
   std::cout << "JSON HEX:   '" << hexifyPart(json(), jsonLength()) << "'\n";
 
@@ -1179,13 +1187,15 @@ DocumentMarker* DocumentMarker::clone (TRI_df_marker_t const* other,
   if (other->_type == TRI_DOC_MARKER_KEY_DOCUMENT) {
     TRI_doc_document_key_marker_t const* original = reinterpret_cast<TRI_doc_document_key_marker_t const*>(other);
 
-    return new DocumentMarker(databaseId,
+    auto marker = new DocumentMarker(databaseId,
-                              collectionId,
+                                     collectionId,
-                              revisionId,
+                                     revisionId,
-                              transactionId,
+                                     transactionId,
-                              std::string(base + original->_offsetKey),
+                                     std::string(base + original->_offsetKey),
-                              legend,
+                                     legend.getSize(),
-                              shapedJson);
+                                     shapedJson);
+    marker->storeLegend(legend);
+    return marker;
   }
   else {
     TRI_ASSERT(other->_type == TRI_WAL_MARKER_DOCUMENT);
@@ -1195,13 +1205,15 @@ DocumentMarker* DocumentMarker::clone (TRI_df_marker_t const* other,
     TRI_ASSERT(original->_databaseId == databaseId);
     TRI_ASSERT(original->_collectionId == collectionId);
 
-    return new DocumentMarker(original->_databaseId,
+    auto marker = new DocumentMarker(original->_databaseId,
-                              original->_collectionId,
+                                     original->_collectionId,
-                              revisionId,
+                                     revisionId,
-                              transactionId,
+                                     transactionId,
-                              std::string(base + original->_offsetKey),
+                                     std::string(base + original->_offsetKey),
-                              legend,
+                                     legend.getSize(),
-                              shapedJson);
+                                     shapedJson);
+    marker->storeLegend(legend);
+    return marker;
   }
 }
 
@@ -1223,10 +1235,10 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
                         TRI_voc_tid_t transactionId,
                         std::string const& key,
                         TRI_document_edge_t const* edge,
-                        triagens::basics::JsonLegend& legend,
+                        size_t legendSize,
                         TRI_shaped_json_t const* shapedJson)
   : Marker(TRI_WAL_MARKER_EDGE,
-    sizeof(edge_marker_t) + alignedSize(key.size() + 1) + alignedSize(strlen(edge->_fromKey) + 1) + alignedSize(strlen(edge->_toKey) + 1) + legend.getSize() + shapedJson->_data.length) {
+    sizeof(edge_marker_t) + alignedSize(key.size() + 1) + alignedSize(strlen(edge->_fromKey) + 1) + alignedSize(strlen(edge->_toKey) + 1) + legendSize + shapedJson->_data.length) {
 
   edge_marker_t* m = reinterpret_cast<edge_marker_t*>(begin());
 
@@ -1241,7 +1253,7 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
   m->_offsetToKey   = static_cast<uint16_t>(m->_offsetKey + alignedSize(key.size() + 1));
   m->_offsetFromKey = static_cast<uint16_t>(m->_offsetToKey + alignedSize(strlen(edge->_toKey) + 1));
   m->_offsetLegend  = static_cast<uint16_t>(m->_offsetFromKey + alignedSize(strlen(edge->_fromKey) + 1));
-  m->_offsetJson    = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legend.getSize()));
+  m->_offsetJson    = static_cast<uint32_t>(m->_offsetLegend + alignedSize(legendSize));
 
   // store keys
   storeSizedString(m->_offsetKey, key.c_str());
@@ -1250,8 +1262,9 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
 
   // store legend
   {
+    TRI_ASSERT(legendSize >= 8);
     char* p = static_cast<char*>(begin()) + m->_offsetLegend;
-    legend.dump(p);
+    memset(p, 0, 8); // initialise initial bytes of legend with 0s
   }
 
   // store shapedJson
@@ -1272,6 +1285,14 @@ EdgeMarker::EdgeMarker (TRI_voc_tick_t databaseId,
 EdgeMarker::~EdgeMarker () {
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief store legend in marker
+////////////////////////////////////////////////////////////////////////////////
+
+void EdgeMarker::storeLegend (triagens::basics::JsonLegend& legend) {
+  legend.dump((void*) this->legend());
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief dump marker
 ////////////////////////////////////////////////////////////////////////////////
@@ -1298,8 +1319,6 @@ void EdgeMarker::dump () const {
             << "\n";
 
 #ifdef DEBUG_WAL_DETAIL
-  std::cout << "LEGEND:     '" << stringifyPart(legend(), legendLength()) << "'\n";
-  std::cout << "LEGEND HEX: '" << hexifyPart(legend(), legendLength()) << "'\n";
   std::cout << "JSON:       '" << stringifyPart(json(), jsonLength()) << "'\n";
   std::cout << "JSON HEX:   '" << hexifyPart(json(), jsonLength()) << "'\n";
 
@@ -1330,14 +1349,17 @@ EdgeMarker* EdgeMarker::clone (TRI_df_marker_t const* other,
     edge._toKey   = (TRI_voc_key_t) base + original->_offsetToKey;
     edge._fromKey = (TRI_voc_key_t) base + original->_offsetFromKey;
 
-    return new EdgeMarker(databaseId,
+    auto marker = new EdgeMarker(databaseId,
-                          collectionId,
+                                 collectionId,
-                          revisionId,
+                                 revisionId,
-                          transactionId,
+                                 transactionId,
-                          std::string(base + original->base._offsetKey),
+                                 std::string(base + original->base._offsetKey),
-                          &edge,
+                                 &edge,
-                          legend,
+                                 legend.getSize(),
-                          shapedJson);
+                                 shapedJson);
+
+    marker->storeLegend(legend);
+    return marker;
   }
   else {
     TRI_ASSERT(other->_type == TRI_WAL_MARKER_EDGE);
@@ -1353,14 +1375,17 @@ EdgeMarker* EdgeMarker::clone (TRI_df_marker_t const* other,
     edge._toKey   = (TRI_voc_key_t) base + original->_offsetToKey;
     edge._fromKey = (TRI_voc_key_t) base + original->_offsetFromKey;
 
-    return new EdgeMarker(original->_databaseId,
+    auto marker = new EdgeMarker(original->_databaseId,
-                          original->_collectionId,
+                                 original->_collectionId,
-                          revisionId,
+                                 revisionId,
-                          transactionId,
+                                 transactionId,
-                          std::string(base + original->_offsetKey),
+                                 std::string(base + original->_offsetKey),
-                          &edge,
+                                 &edge,
-                          legend,
+                                 legend.getSize(),
-                          shapedJson);
+                                 shapedJson);
+
+    marker->storeLegend(legend);
+    return marker;
   }
 }
 

arangod/Wal/Marker.h

@@ -770,9 +770,9 @@ namespace triagens {
                         TRI_voc_rid_t,
                         TRI_voc_tid_t,
                         std::string const&,
-                        triagens::basics::JsonLegend&,
+                        size_t,
                         TRI_shaped_json_t const*);
-
+        
         ~DocumentMarker ();
 
       public:
@@ -814,6 +814,8 @@ namespace triagens {
           return static_cast<size_t>(size() - m->_offsetJson);
         }
 
+        void storeLegend (triagens::basics::JsonLegend&);
+
         void dump () const;
 
         static DocumentMarker* clone (TRI_df_marker_t const*,
@@ -839,9 +841,9 @@ namespace triagens {
                     TRI_voc_tid_t,
                     std::string const&,
                     TRI_document_edge_t const*,
-                    triagens::basics::JsonLegend&,
+                    size_t,
                     TRI_shaped_json_t const*);
-
+        
         ~EdgeMarker ();
 
         inline TRI_voc_rid_t revisionId () const {
@@ -892,6 +894,8 @@ namespace triagens {
           edge_marker_t const* m = reinterpret_cast<edge_marker_t const*>(begin());
           return static_cast<size_t>(size() - m->_offsetJson);
         }
+        
+        void storeLegend (triagens::basics::JsonLegend&);
 
         void dump () const;