Better logging

arangod/Pregel/GraphStore.cpp

@@ -370,6 +370,8 @@ void GraphStore<V, E>::_loadEdges(WorkerConfig const& state,
 
 template <typename V, typename E>
 void GraphStore<V, E>::storeResults(WorkerConfig const& state) {
+  double start = TRI_microtime();
+  
   std::vector<std::string> readColls, writeColls;
   for (auto shard : state.localVertexShardIDs()) {
     writeColls.push_back(shard);
@@ -417,6 +419,8 @@ void GraphStore<V, E>::storeResults(WorkerConfig const& state) {
   if (res != TRI_ERROR_NO_ERROR) {
     THROW_ARANGO_EXCEPTION(res);
   }
+  
+  LOG(INFO) << "Storing data took " << (TRI_microtime() - start) << "s";
 }
 
 template class arangodb::pregel::GraphStore<int64_t, int64_t>;

arangod/Pregel/MemoryMapped.cpp

@@ -6,43 +6,24 @@
 
 #include "Pregel/MemoryMapped.h"
 
+
+#include "ApplicationFeatures/PageSizeFeature.h"
+#include "Basics/FileUtils.h"
+#include "Basics/StaticStrings.h"
+#include "Basics/StringUtils.h"
+#include "Basics/files.h"
+#include "Basics/memory-map.h"
+#include "Basics/tri-strings.h"
+#include "Logger/Logger.h"
+
+#include <sstream>
+#include <iomanip>
+
 #include <cstdio>
 #include <stdexcept>
 
-// OS-specific
-#ifdef _MSC_VER
-// Windows
-#include <windows.h>
-#else
-// Linux
-// enable large file support on 32 bit systems
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE
-#endif
-#ifdef _FILE_OFFSET_BITS
-#undef _FILE_OFFSET_BITS
-#endif
-#define _FILE_OFFSET_BITS 64
-// and include needed headers
-#include <errno.h>
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#endif
-
-/// do nothing, must use open()
-MemoryMapped::MemoryMapped()
-    : _filename(),
-      _filesize(0),
-      _hint(Normal),
-      _mappedBytes(0),
-      _file(0),
-#ifdef _MSC_VER
-      _mappedFile(NULL),
-#endif
-      _mappedView(NULL) {
-}
+using namespace arangodb;
+using namespace arangodb::basics;
 
 /// open file, mappedBytes = 0 maps the whole file
 MemoryMapped::MemoryMapped(const std::string& filename, size_t mappedBytes,

arangod/Pregel/MemoryMapped.h

@@ -32,31 +32,24 @@ namespace pregel {
 /// Portable read-only memory mapping (Windows and Linux)
 /** Filesize limited by size_t, usually 2^32 or 2^64 */
 class MemoryMapped {
- public:
-  /// tweak performance
-  enum CacheHint {
-    Normal,          ///< good overall performance
-    SequentialScan,  ///< read file only once with few seeks
-    RandomAccess     ///< jump around
-  };
-
-  /// how much should be mappend
-  enum MapRange {
-    WholeFile =
-        0  ///< everything ... be careful when file is larger than memory
-  };
-
-  /// do nothing, must use open()
-  MemoryMapped();
+protected:
   /// open file, mappedBytes = 0 maps the whole file
-  MemoryMapped(const std::string& filename, size_t mappedBytes = WholeFile,
-               CacheHint hint = Normal);
+  MemoryMapped(std::string const& filename, int fd, void* mmHandle, TRI_voc_size_t maximalSize,
+               TRI_voc_size_t currentsize, TRI_voc_fid_t fid, char* data);
+  
+ public:
+  
   /// close file (see close() )
   ~MemoryMapped();
 
   /// open file, mappedBytes = 0 maps the whole file
-  bool open(const std::string& filename, size_t mappedBytes = WholeFile,
-            CacheHint hint = Normal);
+  static TRI_datafile_t* openHelper(std::string const& filename, bool ignoreErrors);
+  
+  /// @brief return whether the datafile is a physical file (true) or an
+  /// anonymous mapped region (false)
+  inline bool isPhysical() const { return !_filename.empty(); }
+
+
   /// close file
   void close();
 
@@ -80,6 +73,11 @@ class MemoryMapped {
   /// of the page size
   bool remap(uint64_t offset, size_t mappedBytes);
 
+  TRI_voc_size_t initSize() const { return _initSize; }
+  TRI_voc_size_t maximalSize() const { return _maximalSize; }
+  TRI_voc_size_t currentSize() const { return _currentSize; }
+  TRI_voc_size_t footerSize() const { return _footerSize; }
+  
  private:
   /// don't copy object
   MemoryMapped(const MemoryMapped&);
@@ -102,7 +100,6 @@ class MemoryMapped {
   TRI_voc_size_t _maximalSize;  // maximal size of the datafile (adjusted
   // (=reduced) at runtime)
   TRI_voc_size_t _currentSize;  // current size of the datafile
-  TRI_voc_size_t _footerSize;   // size of the final footer
 
   char* _data;  // start of the data array
   char* _next;  // end of the current data

arangod/Pregel/PregelFeature.cpp

@@ -57,10 +57,17 @@ PregelFeature::~PregelFeature() {
 
 PregelFeature* PregelFeature::instance() { return Instance; }
 
+static size_t _approxThreadNumber(){
+  const size_t procNum = TRI_numberProcessors();
+  if (procNum <= 1) return 1;
+  else if (procNum <= 16) return procNum / 2;
+  else return procNum;
+}
+
 void PregelFeature::start() {
   Instance = this;
 
-  const size_t threadNum = TRI_numberProcessors();
+  const size_t threadNum = _approxThreadNumber();
   _threadPool.reset(new basics::ThreadPool(threadNum, "Pregel"));
 
   ClusterFeature* cluster =

arangod/Pregel/Worker.cpp

@@ -274,7 +274,7 @@ void Worker<V, E, M>::_startProcessing() {
   size_t total = _graphStore->localVertexCount();
   size_t delta = total / pool->numThreads();
   size_t start = 0, end = delta;
-  if (total > 1000) {
+  if (delta >= 100 && total >= 100) {
     _runningThreads = total / delta;  // rounds-up unsigned integers
   } else {
     _runningThreads = 1;