Pregel Refactoring

arangod/CMakeLists.txt

@@ -367,6 +367,8 @@ SET(ARANGOD_SOURCES
   Pregel/Algos/ShortestPath.cpp
   Pregel/Algos/PageRank.cpp
   Pregel/Algos/RecoveringPageRank.cpp
+  Pregel/Algos/LineRank.cpp
+  Pregel/Algos/ConnectedComponents.cpp
   Pregel/Conductor.cpp
   Pregel/GraphStore.cpp
   Pregel/IncomingCache.cpp

arangod/Pregel/Aggregator.h

@@ -43,7 +43,7 @@ class Aggregator {
   virtual ~Aggregator() {}
 
   /// @brief Value from superstep S-1 supplied by the conductor
-  virtual void aggregate(const void* valuePtr) = 0;
+  virtual void aggregate(void const* valuePtr) = 0;
   virtual void aggregate(VPackSlice slice) = 0;
 
   virtual void const* getValue() const = 0;
@@ -54,27 +54,27 @@ class Aggregator {
   bool isPermanent() { return _permanent; }
 };
 
-class FloatMaxAggregator : public Aggregator {
-  float _value, _initial;
-
- public:
-  FloatMaxAggregator(float init) : _value(init), _initial(init) {}
-
+template <typename T>
+class MaxAggregator : public Aggregator {
+  static_assert(std::is_arithmetic<T>::value, "Type must be numeric");
+  T _value, _initial;
+  
+public:
+  MaxAggregator(T init, bool perm = false)
+  : Aggregator(perm), _value(init), _initial(init) {}
+  
   void aggregate(void const* valuePtr) override {
-    float other = *((float*)valuePtr);
+    T other = *((T*)valuePtr);
     if (other > _value) _value = other;
   };
   void aggregate(VPackSlice slice) override {
-    float f = slice.getNumber<float>();
+    T f = slice.getNumber<T>();
     aggregate(&f);
   }
-
+  
   void const* getValue() const override { return &_value; };
-  /*void setValue(VPackSlice slice) override {
-    _value = (float)slice.getDouble();
-  }*/
-  VPackValue vpackValue() override { return VPackValue((double)_value); };
-
+  VPackValue vpackValue() override { return VPackValue(_value); };
+  
   void reset() override { _value = _initial; }
 };
 
@@ -97,49 +97,41 @@ class MinAggregator : public Aggregator {
   }
 
   void const* getValue() const override { return &_value; };
-  /*void setValue(VPackSlice slice) override {
-   _value = (float)slice.getDouble();
-   }*/
   VPackValue vpackValue() override { return VPackValue(_value); };
 
   void reset() override { _value = _initial; }
 };
+  
+template <typename T>
+class SumAggregator : public Aggregator {
+  static_assert(std::is_arithmetic<T>::value, "Type must be numeric");
+  T _value, _initial;
+  
+public:
+  SumAggregator(T init, bool perm = false)
+      : Aggregator(perm), _value(init), _initial(init) {}
+  
+  void aggregate(void const* valuePtr) override { _value += *((T*)valuePtr); };
+  void aggregate(VPackSlice slice) override { _value += slice.getNumber<T>(); }
+  
+  void const* getValue() const override { return &_value; };
+  VPackValue vpackValue() override { return VPackValue(_value); };
+  
+  void reset() override { _value = _initial; }
+};
 
 template <typename T>
 class ValueAggregator : public Aggregator {
-  static_assert(std::is_arithmetic<T>::value, "Type must be numeric");
-
+  static_assert(std::is_fundamental<T>::value, "Type must be fundamental");
   T _value;
 
  public:
-  ValueAggregator(T val) : Aggregator(true), _value(val) {}
+  ValueAggregator(T val, bool perm = false) : Aggregator(perm), _value(val) {}
 
   void aggregate(void const* valuePtr) override { _value = *((T*)valuePtr); };
   void aggregate(VPackSlice slice) override { _value = slice.getNumber<T>(); }
 
   void const* getValue() const override { return &_value; };
-  /*void setValue(VPackSlice slice) override {
-   _value = (float)slice.getDouble();
-   }*/
-  VPackValue vpackValue() override { return VPackValue(_value); };
-};
-
-template <typename T>
-class SumAggregator : public Aggregator {
-  static_assert(std::is_arithmetic<T>::value, "Type must be numeric");
-
-  T _value;
-
- public:
-  SumAggregator(T val) : Aggregator(true), _value(val) {}
-
-  void aggregate(void const* valuePtr) override { _value += *((T*)valuePtr); };
-  void aggregate(VPackSlice slice) override { _value += slice.getNumber<T>(); }
-
-  void const* getValue() const override { return &_value; };
-  /*void setValue(VPackSlice slice) override {
-   _value = (float)slice.getDouble();
-   }*/
   VPackValue vpackValue() override { return VPackValue(_value); };
 };
 }

arangod/Pregel/AlgoRegistry.cpp

@@ -25,6 +25,8 @@
 #include "Pregel/Algos/RecoveringPageRank.h"
 #include "Pregel/Algos/SSSP.h"
 #include "Pregel/Algos/ShortestPath.h"
+#include "Pregel/Algos/LineRank.h"
+#include "Pregel/Algos/ConnectedComponents.h"
 #include "Pregel/Utils.h"
 
 using namespace arangodb;
@@ -40,6 +42,10 @@ IAlgorithm* AlgoRegistry::createAlgorithm(std::string const& algorithm,
     return new algos::RecoveringPageRank(userParams);
   } else if (algorithm == "shortestpath") {
     return new algos::ShortestPathAlgorithm(userParams);
+  } else if (algorithm == "linerank") {
+    return new algos::LineRank(userParams);
+  } else if (algorithm == "connectedcomponents") {
+    return new algos::ConnectedComponents(userParams);
   } else {
     THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
                                    "Unsupported Algorithm");
@@ -75,6 +81,10 @@ IWorker* AlgoRegistry::createWorker(TRI_vocbase_t* vocbase,
   } else if (algorithm == "shortestpath") {
     return createWorker(vocbase, new algos::ShortestPathAlgorithm(userParams),
                         body);
+  } else if (algorithm == "linerank") {
+    return createWorker(vocbase, new algos::LineRank(userParams), body);
+  }  else if (algorithm == "connectedcomponents") {
+    return createWorker(vocbase, new algos::ConnectedComponents(userParams), body);
   } else {
     THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER,
                                    "Unsupported Algorithm");

arangod/Pregel/Algorithm.h

@@ -30,7 +30,6 @@
 
 #include "Basics/Common.h"
 #include "GraphFormat.h"
-#include "MasterContext.h"
 #include "MessageCombiner.h"
 #include "MessageFormat.h"
 #include "WorkerContext.h"
@@ -46,6 +45,7 @@ class VertexCompensation;
 
 class Aggregator;
 class WorkerConfig;
+class MasterContext;
 
 struct IAlgorithm {
   virtual ~IAlgorithm() {}

arangod/Pregel/Algos/ConnectedComponents.cpp

@@ -0,0 +1,68 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Simon Grätzer
+////////////////////////////////////////////////////////////////////////////////
+
+#include "ConnectedComponents.h"
+#include "Pregel/Algorithm.h"
+#include "Pregel/GraphStore.h"
+#include "Pregel/IncomingCache.h"
+#include "Pregel/VertexComputation.h"
+
+using namespace arangodb::pregel;
+using namespace arangodb::pregel::algos;
+
+struct MyComputation : public VertexComputation<int64_t, int64_t, int64_t> {
+  MyComputation() {}
+  void compute(MessageIterator<int64_t> const& messages) override {
+    
+    int64_t currentComponent = vertexData();
+    for (const int64_t* msg : messages) {
+      if (*msg < currentComponent) {
+        currentComponent = *msg;
+      };
+    }
+    
+    if (currentComponent != vertexData()) {
+      sendMessageToAllEdges(currentComponent);
+    }
+    voteHalt();
+  }
+};
+
+VertexComputation<int64_t, int64_t, int64_t>* ConnectedComponents::createComputation(
+    WorkerConfig const* config) const {
+  return new MyComputation();
+}
+
+struct MyCompensation : public VertexCompensation<int64_t, int64_t, int64_t> {
+  MyCompensation() {}
+  void compensate(bool inLostPartition) override {
+    if (inLostPartition) {
+      int64_t* data = mutableVertexData();
+      *data = INT64_MAX;
+    }
+  }
+};
+
+VertexCompensation<int64_t, int64_t, int64_t>*
+ConnectedComponents::createCompensation(WorkerConfig const* config) const {
+  return new MyCompensation();
+}

arangod/Pregel/Algos/ConnectedComponents.h

@@ -0,0 +1,60 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Simon Grätzer
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGODB_PREGEL_ALGOS_WCC_H
+#define ARANGODB_PREGEL_ALGOS_WCC_H 1
+
+#include "Pregel/Algorithm.h"
+
+namespace arangodb {
+namespace pregel {
+namespace algos {
+
+/// The idea behind the algorithm is very simple: propagate the smallest
+/// vertex id along the edges to all vertices of a connected component. The
+/// number of supersteps necessary is equal to the length of the maximum
+/// diameter of all components + 1
+struct ConnectedComponents : public SimpleAlgorithm<int64_t, int64_t, int64_t> {
+ public:
+  ConnectedComponents(VPackSlice userParams) : SimpleAlgorithm("ConnectedComponents", userParams) {}
+
+  bool supportsAsyncMode() const override { return false; }
+  bool supportsCompensation() const override { return true; }
+
+  GraphFormat<int64_t, int64_t>* inputFormat() override {
+    return new NumberGraphFormat<int64_t, int64_t>(_sourceField, _resultField, INT64_MAX, 0);
+  }
+  MessageFormat<int64_t>* messageFormat() const override {
+    return new IntegerMessageFormat();
+  }
+  MessageCombiner<int64_t>* messageCombiner() const override {
+    return new MinCombiner<int64_t>();
+  }
+  VertexComputation<int64_t, int64_t, int64_t>* createComputation(
+      WorkerConfig const*) const override;
+  VertexCompensation<int64_t, int64_t, int64_t>* createCompensation(
+      WorkerConfig const*) const override;
+};
+}
+}
+}
+#endif

arangod/Pregel/Algos/LineRank.cpp

@@ -0,0 +1,106 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Simon Grätzer
+////////////////////////////////////////////////////////////////////////////////
+
+#include "LineRank.h"
+#include "Pregel/Aggregator.h"
+#include "Pregel/GraphFormat.h"
+#include "Pregel/Iterators.h"
+#include "Pregel/MasterContext.h"
+#include "Pregel/Utils.h"
+#include "Pregel/VertexComputation.h"
+
+#include "Cluster/ClusterInfo.h"
+#include "Utils/OperationCursor.h"
+#include "Utils/SingleCollectionTransaction.h"
+#include "Utils/StandaloneTransactionContext.h"
+#include "Utils/Transaction.h"
+#include "VocBase/vocbase.h"
+
+using namespace arangodb;
+using namespace arangodb::pregel;
+using namespace arangodb::pregel::algos;
+
+static std::string const kMoreIterations = "more";
+static const double RESTART_PROB = 0.15;
+static const double EPS = 0.000000001;
+
+LineRank::LineRank(arangodb::velocypack::Slice params)
+    : SimpleAlgorithm("LineRank", params) {
+  //VPackSlice t = params.get("convergenceThreshold");
+  //_threshold = t.isNumber() ? t.getNumber<float>() : 0.000002f;
+}
+
+// github.com/JananiC/NetworkCentralities/blob/master/src/main/java/linerank/LineRank.java
+struct MyComputation : public VertexComputation<float, float, float> {
+  MyComputation() {}
+  void compute(MessageIterator<float> const& messages) override {
+    
+    float startAtNodeProb = 1.0f / context()->edgeCount();
+    float* vertexValue = mutableVertexData();
+    RangeIterator<Edge<float>> edges = getEdges();
+    
+    if (*vertexValue < 0.0f) {
+      *vertexValue = startAtNodeProb;
+      aggregate(kMoreIterations, true);
+    } else {
+      
+      float newScore = 0.0f;
+      for (const float* msg : messages) {
+        newScore += *msg;
+      }
+      
+      bool const* moreIterations = getAggregatedValue<bool>(kMoreIterations);
+      if (*moreIterations == false) {
+        *vertexValue = *vertexValue * edges.size() + newScore;
+        voteHalt();
+      } else {
+        
+        if (edges.size() == 0) {
+          newScore = 0;
+        } else {
+          newScore /= edges.size();
+          newScore = startAtNodeProb * RESTART_PROB + newScore * (1.0 - RESTART_PROB);
+        }
+        
+        float diff = fabsf(newScore - *vertexValue);
+        *vertexValue = newScore;
+        
+        if (diff > EPS) {
+          aggregate(kMoreIterations, true);
+        }
+      }
+    }
+    sendMessageToAllEdges(*vertexValue);
+  }
+};
+
+VertexComputation<float, float, float>* LineRank::createComputation(
+    WorkerConfig const* config) const {
+  return new MyComputation();
+}
+
+Aggregator* LineRank::aggregator(std::string const& name) const {
+  if (name == kMoreIterations) {
+    return new ValueAggregator<bool>(false, false);// non perm
+  }
+  return nullptr;
+}

arangod/Pregel/Algos/LineRank.h

@@ -0,0 +1,61 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Simon Grätzer
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGODB_PREGEL_ALGOS_LINERANK_H
+#define ARANGODB_PREGEL_ALGOS_LINERANK_H 1
+
+#include <velocypack/Slice.h>
+#include "Pregel/Algorithm.h"
+
+namespace arangodb {
+namespace pregel {
+namespace algos {
+
+/// LineRank from "Centralities in Large Networks: Algorithms and Observations" 2011:
+/// Given a directed graph G, the LINERANK score of a node v ∈ G is computed by
+/// aggregating the stationary probabilities of its incident edges on the line graph L(G).
+/// Implementation based on
+/// github.com/JananiC/NetworkCentralities/blob/master/src/main/java/linerank/LineRank.java
+struct LineRank : public SimpleAlgorithm<float, float, float> {
+
+ public:
+  LineRank(arangodb::velocypack::Slice params);
+
+  GraphFormat<float, float>* inputFormat() override {
+    return new VertexGraphFormat<float, float>(_resultField, -1.0);
+  }
+  MessageFormat<float>* messageFormat() const override {
+    return new FloatMessageFormat();
+  }
+  
+  MessageCombiner<float>* messageCombiner() const override {
+    return new SumCombiner<float>();
+  }
+  
+  VertexComputation<float, float, float>* createComputation(
+      WorkerConfig const*) const override;
+  Aggregator* aggregator(std::string const& name) const override;
+};
+}
+}
+}
+#endif

arangod/Pregel/Algos/PageRank.cpp

@@ -22,7 +22,6 @@
 
 #include "PageRank.h"
 #include "Pregel/Aggregator.h"
-#include "Pregel/Combiners/FloatSumCombiner.h"
 #include "Pregel/GraphFormat.h"
 #include "Pregel/Iterators.h"
 #include "Pregel/MasterContext.h"
@@ -48,31 +47,6 @@ PageRank::PageRank(arangodb::velocypack::Slice params)
   _threshold = t.isNumber() ? t.getNumber<float>() : 0.000002f;
 }
 
-struct PageRankGraphFormat : public FloatGraphFormat {
-  PageRankGraphFormat(std::string const& s, std::string const& r)
-      : FloatGraphFormat(s, r, 0, 0) {}
-  size_t copyVertexData(VertexEntry const& vertex,
-                        std::string const& documentId,
-                        arangodb::velocypack::Slice document, void* targetPtr,
-                        size_t maxSize) override {
-    *((float*)targetPtr) = _vDefault;
-    return sizeof(float);
-  }
-
-  size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
-                      size_t maxSize) override {
-    return 0;
-  }
-};
-
-GraphFormat<float, float>* PageRank::inputFormat() {
-  return new PageRankGraphFormat(_sourceField, _resultField);
-}
-
-MessageCombiner<float>* PageRank::messageCombiner() const {
-  return new FloatSumCombiner();
-}
-
 struct PRComputation : public VertexComputation<float, float, float> {
   float _limit;
   PRComputation(float t) : _limit(t) {}
@@ -110,8 +84,8 @@ VertexComputation<float, float, float>* PageRank::createComputation(
 }
 
 Aggregator* PageRank::aggregator(std::string const& name) const {
-  if (name == "convergence") {
-    return new FloatMaxAggregator(-1);
+  if (name == kConvergence) {
+    return new MaxAggregator<float>(-1.0f);
   }
   return nullptr;
 }

arangod/Pregel/Algos/PageRank.h

@@ -37,12 +37,18 @@ struct PageRank : public SimpleAlgorithm<float, float, float> {
  public:
   PageRank(arangodb::velocypack::Slice params);
 
-  GraphFormat<float, float>* inputFormat() override;
+  GraphFormat<float, float>* inputFormat() override {
+    return new VertexGraphFormat<float, float>(_resultField, 0);
+  }
+  
   MessageFormat<float>* messageFormat() const override {
     return new FloatMessageFormat();
   }
   
-  MessageCombiner<float>* messageCombiner() const override;
+  MessageCombiner<float>* messageCombiner() const override {
+    return new SumCombiner<float>();
+  }
+  
   VertexComputation<float, float, float>* createComputation(
       WorkerConfig const*) const override;
   Aggregator* aggregator(std::string const& name) const override;

arangod/Pregel/Algos/RecoveringPageRank.cpp

@@ -22,7 +22,6 @@
 
 #include "RecoveringPageRank.h"
 #include "Pregel/Aggregator.h"
-#include "Pregel/Combiners/FloatSumCombiner.h"
 #include "Pregel/GraphFormat.h"
 #include "Pregel/Iterators.h"
 #include "Pregel/MasterContext.h"
@@ -54,31 +53,6 @@ RecoveringPageRank::RecoveringPageRank(arangodb::velocypack::Slice params)
   _threshold = t.isNumber() ? t.getNumber<float>() : 0.000002f;
 }
 
-struct PageRankGraphFormat : public FloatGraphFormat {
-  PageRankGraphFormat(std::string const& s, std::string const& r)
-      : FloatGraphFormat(s, r, 0, 0) {}
-  size_t copyVertexData(VertexEntry const& vertex,
-                        std::string const& documentId,
-                        arangodb::velocypack::Slice document, void* targetPtr,
-                        size_t maxSize) override {
-    *((float*)targetPtr) = _vDefault;
-    return sizeof(float);
-  }
-
-  size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
-                      size_t maxSize) override {
-    return 0;
-  }
-};
-
-GraphFormat<float, float>* RecoveringPageRank::inputFormat() {
-  return new PageRankGraphFormat(_sourceField, _resultField);
-}
-
-MessageCombiner<float>* RecoveringPageRank::messageCombiner() const {
-  return new FloatSumCombiner();
-}
-
 struct RPRComputation : public VertexComputation<float, float, float> {
   float _limit;
   RPRComputation(float t) : _limit(t) {}
@@ -122,7 +96,7 @@ VertexComputation<float, float, float>* RecoveringPageRank::createComputation(
 
 Aggregator* RecoveringPageRank::aggregator(std::string const& name) const {
   if (name == kConvergence) {
-    return new FloatMaxAggregator(-1);
+    return new MaxAggregator<float>(-1);
   } else if (name == kNonFailedCount) {
     return new SumAggregator<uint32_t>(0);
   } else if (name == kRank) {

arangod/Pregel/Algos/RecoveringPageRank.h

@@ -40,12 +40,18 @@ struct RecoveringPageRank : public SimpleAlgorithm<float, float, float> {
   bool supportsCompensation() const override { return true; }
   MasterContext* masterContext(VPackSlice userParams) const override;
 
-  GraphFormat<float, float>* inputFormat() override;
+  GraphFormat<float, float>* inputFormat() override {
+    return new VertexGraphFormat<float, float>(_resultField, 0);
+  }
+  
   MessageFormat<float>* messageFormat() const override {
     return new FloatMessageFormat();
   }
   
-  MessageCombiner<float>* messageCombiner() const override;
+  MessageCombiner<float>* messageCombiner() const override {
+    return new SumCombiner<float>();
+  }
+
   VertexComputation<float, float, float>* createComputation(
       WorkerConfig const*) const override;
   VertexCompensation<float, float, float>* createCompensation(

arangod/Pregel/Algos/SCC.cpp

@@ -1,107 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-/// DISCLAIMER
-///
-/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
-///
-/// Licensed under the Apache License, Version 2.0 (the "License");
-/// you may not use this file except in compliance with the License.
-/// You may obtain a copy of the License at
-///
-///     http://www.apache.org/licenses/LICENSE-2.0
-///
-/// Unless required by applicable law or agreed to in writing, software
-/// distributed under the License is distributed on an "AS IS" BASIS,
-/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-/// See the License for the specific language governing permissions and
-/// limitations under the License.
-///
-/// Copyright holder is ArangoDB GmbH, Cologne, Germany
-///
-/// @author Simon Grätzer
-////////////////////////////////////////////////////////////////////////////////
-
-#include "SCC.h"
-#include "Pregel/GraphFormat.h"
-#include "Pregel/Utils.h"
-#include "Pregel/VertexComputation.h"
-
-#include "Cluster/ClusterInfo.h"
-#include "Utils/OperationCursor.h"
-#include "Utils/SingleCollectionTransaction.h"
-#include "Utils/StandaloneTransactionContext.h"
-#include "Utils/Transaction.h"
-#include "Vocbase/vocbase.h"
-
-using namespace arangodb;
-using namespace arangodb::pregel;
-using namespace arangodb::pregel::algos;
-
-struct SCCGraphFormat : public GraphFormat<int64_t, int64_t> {
-  uint64_t _currentId = 0;
-  void willUseCollection(TRI_vocbase_t* vocbase, std::string const& shard,
-                         bool isEdgeCollection) override {
-    int64_t count = Utils::countDocuments(vocbase, shard);
-    _currentId = ClusterInfo::instance()->uniqid(count);
-  }
-  SCCGraphFormat() {}
-
-  size_t copyVertexData(VPackSlice document, void* targetPtr,
-                        size_t maxSize) override {
-    *((int64_t*)targetPtr) = _currentId;
-    _currentId++;
-    return sizeof(int64_t);
-  }
-
-  size_t copyEdgeData(VPackSlice document, void* targetPtr,
-                      size_t maxSize) override {
-    return 0;
-  }
-
-  int64_t readVertexData(void* ptr) override { return *((int64_t*)ptr); }
-  int64_t readEdgeData(void* ptr) override { return *((int64_t*)ptr); }
-};
-
-std::shared_ptr<GraphFormat<int64_t, int64_t>> SCCAlgorithm::inputFormat()
-    const {
-  return std::make_shared<IntegerGraphFormat>("value", -1, 1);
-}
-
-std::shared_ptr<MessageFormat<int64_t>> SCCAlgorithm::messageFormat() const {
-  return std::shared_ptr<IntegerMessageFormat>(new IntegerMessageFormat());
-}
-
-std::shared_ptr<MessageCombiner<int64_t>> SCCAlgorithm::messageCombiner()
-    const {
-  return std::shared_ptr<MinIntegerCombiner>(new IntegerMinCombiner());
-}
-
-struct SCCComputation : public VertexComputation<int64_t, int64_t, int64_t> {
-  SCCComputation() {}
-  void compute(std::string const& vertexID,
-               MessageIterator<int64_t> const& messages) override {
-    /*int64_t tmp = vertexData();
-    for (const int64_t* msg : messages) {
-      if (tmp < 0 || *msg < tmp) {
-        tmp = *msg;
-      };
-    }
-    int64_t* state = mutableVertexData();
-    if (tmp >= 0 && (getGlobalSuperstep() == 0 || tmp != *state)) {
-      LOG(INFO) << "Recomputing value for vertex " << vertexID;
-      *state = tmp;  // update state
-
-      EdgeIterator<int64_t> edges = getEdges();
-      for (EdgeEntry<int64_t>* edge : edges) {
-        int64_t val = *(edge->getData()) + tmp;
-        uint64_t toID = edge->toPregelID();
-        sendMessage(toID, val);
-      }
-    }*/
-    voteHalt();
-  }
-};
-
-std::shared_ptr<VertexComputation<int64_t, int64_t, int64_t>>
-SCCAlgorithm::createComputation() const {
-  return std::shared_ptr<SCCComputation>(new SCCComputation());
-}

arangod/Pregel/Algos/SCC.h

@@ -1,46 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-/// DISCLAIMER
-///
-/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
-///
-/// Licensed under the Apache License, Version 2.0 (the "License");
-/// you may not use this file except in compliance with the License.
-/// You may obtain a copy of the License at
-///
-///     http://www.apache.org/licenses/LICENSE-2.0
-///
-/// Unless required by applicable law or agreed to in writing, software
-/// distributed under the License is distributed on an "AS IS" BASIS,
-/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-/// See the License for the specific language governing permissions and
-/// limitations under the License.
-///
-/// Copyright holder is ArangoDB GmbH, Cologne, Germany
-///
-/// @author Simon Grätzer
-////////////////////////////////////////////////////////////////////////////////
-
-#ifndef ARANGODB_PREGEL_ALGOS_SSSP_H
-#define ARANGODB_PREGEL_ALGOS_SSSP_H 1
-
-#include "Pregel/Algorithm.h"
-
-namespace arangodb {
-namespace pregel {
-namespace algos {
-
-/// Strongly connected components.
-struct SCCAlgorithm : public Algorithm<int64_t, int64_t, int64_t> {
- public:
-  SCCAlgorithm() : Algorithm("SCC") {}
-
-  std::shared_ptr<GraphFormat<int64_t, int64_t>> inputFormat() override;
-  std::shared_ptr<MessageFormat<int64_t>> messageFormat() const override;
-  std::shared_ptr<MessageCombiner<int64_t>> messageCombiner() const override;
-  std::shared_ptr<VertexComputation<int64_t, int64_t, int64_t>>
-  createComputation(uint64_t gss) const override;
-};
-}
-}
-}
-#endif

arangod/Pregel/Algos/SSSP.cpp

@@ -53,7 +53,7 @@ struct SSSPComputation : public VertexComputation<int64_t, int64_t, int64_t> {
 };
 
 GraphFormat<int64_t, int64_t>* SSSPAlgorithm::inputFormat() {
-  return new IntegerGraphFormat(_sourceField, _resultField, INT64_MAX, 1);
+  return new NumberGraphFormat<int64_t, int64_t> (_sourceField, _resultField, INT64_MAX, 1);
 }
 
 MessageFormat<int64_t>* SSSPAlgorithm::messageFormat() const {
@@ -61,7 +61,7 @@ MessageFormat<int64_t>* SSSPAlgorithm::messageFormat() const {
 }
 
 MessageCombiner<int64_t>* SSSPAlgorithm::messageCombiner() const {
-  return new IntegerMinCombiner();
+  return new MinCombiner<int64_t>();
 }
 
 VertexComputation<int64_t, int64_t, int64_t>* SSSPAlgorithm::createComputation(

arangod/Pregel/Algos/ShortestPath.cpp

@@ -38,17 +38,19 @@ struct ShortestPathComp : public VertexComputation<int64_t, int64_t, int64_t> {
 
   ShortestPathComp(PregelID const& target) : _target(target) {}
   void compute(MessageIterator<int64_t> const& messages) override {
-    int64_t const* max = getAggregatedValue<int64_t>(spUpperPathBound);
     int64_t current = vertexData();
     for (const int64_t* msg : messages) {
       if (*msg < current) {
         current = *msg;
       };
     }
-
+    
+    // use global state to limit the computation of paths
     bool isSource = current == 0 && localSuperstep() == 0;
+    int64_t const* max = getAggregatedValue<int64_t>(spUpperPathBound);
+
     int64_t* state = mutableVertexData();
-    if ((current < *state && current < *max) || isSource) {
+    if (isSource || (current < *state && current < *max)) {
       *state = current;  // update state
 
       if (this->pregelId() == _target) {

arangod/Pregel/Algos/ShortestPath.h

@@ -49,7 +49,7 @@ struct ShortestPathAlgorithm : public Algorithm<int64_t, int64_t, int64_t> {
   }
 
   MessageCombiner<int64_t>* messageCombiner() const override {
-    return new IntegerMinCombiner();
+    return new MinCombiner<int64_t>();
   }
 
   VertexComputation<int64_t, int64_t, int64_t>* createComputation(

arangod/Pregel/Combiners/FloatSumCombiner.h

@@ -1,39 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-/// DISCLAIMER
-///
-/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
-///
-/// Licensed under the Apache License, Version 2.0 (the "License");
-/// you may not use this file except in compliance with the License.
-/// You may obtain a copy of the License at
-///
-///     http://www.apache.org/licenses/LICENSE-2.0
-///
-/// Unless required by applicable law or agreed to in writing, software
-/// distributed under the License is distributed on an "AS IS" BASIS,
-/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-/// See the License for the specific language governing permissions and
-/// limitations under the License.
-///
-/// Copyright holder is ArangoDB GmbH, Cologne, Germany
-///
-/// @author Simon Grätzer
-////////////////////////////////////////////////////////////////////////////////
-
-#include "Pregel/MessageCombiner.h"
-
-#ifndef ARANGODB_PREGEL_FLOAT_SUM_COMBINER_H
-#define ARANGODB_PREGEL_FLOAT_SUM_COMBINER_H 1
-namespace arangodb {
-namespace pregel {
-
-struct FloatSumCombiner : public MessageCombiner<float> {
-  FloatSumCombiner() {}
-  void combine(float & firstValue,
-               float const& secondValue) const override {
-    firstValue += secondValue;
-  };
-};
-}
-}
-#endif

arangod/Pregel/Conductor.cpp

@@ -24,6 +24,7 @@
 #include "Pregel/Aggregator.h"
 #include "Pregel/AlgoRegistry.h"
 #include "Pregel/Algorithm.h"
+#include "Pregel/MasterContext.h"
 #include "Pregel/PregelFeature.h"
 #include "Pregel/Recovery.h"
 #include "Pregel/Utils.h"

arangod/Pregel/GraphFormat.h

@@ -39,6 +39,7 @@ template <typename V, typename E>
 struct GraphFormat {
   virtual size_t estimatedVertexSize() const { return sizeof(V); };
   virtual size_t estimatedEdgeSize() const { return sizeof(E); };
+  virtual void willLoadVertices(size_t count) {}
 
   virtual size_t copyVertexData(VertexEntry const& vertex,
                                 std::string const& documentId,
@@ -53,92 +54,6 @@ struct GraphFormat {
                                  const void* targetPtr, size_t size) = 0;
 };
 
-class IntegerGraphFormat : public GraphFormat<int64_t, int64_t> {
-  const std::string _sourceField, _resultField;
-  const int64_t _vDefault, _eDefault;
-
- public:
-  IntegerGraphFormat(std::string const& source, std::string const& result,
-                     int64_t vertexNull, int64_t edgeNull)
-      : _sourceField(source),
-        _resultField(result),
-        _vDefault(vertexNull),
-        _eDefault(edgeNull) {}
-
-  size_t copyVertexData(VertexEntry const& vertex,
-                        std::string const& documentId,
-                        arangodb::velocypack::Slice document, void* targetPtr,
-                        size_t maxSize) override {
-    arangodb::velocypack::Slice val = document.get(_sourceField);
-    *((int64_t*)targetPtr) = val.isInteger() ? val.getInt() : _vDefault;
-    return sizeof(int64_t);
-  }
-
-  size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
-                      size_t maxSize) override {
-    arangodb::velocypack::Slice val = document.get(_sourceField);
-    *((int64_t*)targetPtr) = val.isInteger() ? val.getInt() : _eDefault;
-    return sizeof(int64_t);
-  }
-
-  bool buildVertexDocument(arangodb::velocypack::Builder& b,
-                           const void* targetPtr, size_t size) override {
-    b.add(_resultField, VPackValue(*((int64_t*)targetPtr)));
-    return true;
-  }
-
-  bool buildEdgeDocument(arangodb::velocypack::Builder& b,
-                         const void* targetPtr, size_t size) override {
-    b.add(_resultField, VPackValue(*((int64_t*)targetPtr)));
-    return true;
-  }
-};
-
-class FloatGraphFormat : public GraphFormat<float, float> {
- protected:
-  const std::string _sourceField, _resultField;
-  const float _vDefault, _eDefault;
-
- public:
-  FloatGraphFormat(std::string const& source, std::string const& result,
-                   float vertexNull, float edgeNull)
-      : _sourceField(source),
-        _resultField(result),
-        _vDefault(vertexNull),
-        _eDefault(edgeNull) {}
-
-  float readVertexData(const void* ptr) { return *((float*)ptr); }
-  float readEdgeData(const void* ptr) { return *((float*)ptr); }
-
-  size_t copyVertexData(VertexEntry const& vertex,
-                        std::string const& documentId,
-                        arangodb::velocypack::Slice document, void* targetPtr,
-                        size_t maxSize) override {
-    arangodb::velocypack::Slice val = document.get(_sourceField);
-    *((float*)targetPtr) = val.isDouble() ? (float)val.getDouble() : _vDefault;
-    return sizeof(float);
-  }
-
-  size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
-                      size_t maxSize) override {
-    arangodb::velocypack::Slice val = document.get(_sourceField);
-    *((float*)targetPtr) = val.isDouble() ? (float)val.getDouble() : _eDefault;
-    return sizeof(float);
-  }
-
-  bool buildVertexDocument(arangodb::velocypack::Builder& b,
-                           const void* targetPtr, size_t size) override {
-    b.add(_resultField, VPackValue(readVertexData(targetPtr)));
-    return true;
-  }
-
-  bool buildEdgeDocument(arangodb::velocypack::Builder& b,
-                         const void* targetPtr, size_t size) override {
-    b.add(_resultField, VPackValue(readEdgeData(targetPtr)));
-    return true;
-  }
-};
-/*
 template <typename V, typename E>
 class NumberGraphFormat : public GraphFormat<V, E> {
   static_assert(std::is_arithmetic<V>::value, "Vertex type must be numeric");
@@ -157,20 +72,19 @@ public:
   _vDefault(vertexNull),
   _eDefault(edgeNull) {}
 
-  V readVertexData(void* ptr) override { return *((V*)ptr); }
-  E readEdgeData(void* ptr) override { return *((E*)ptr); }
-
-  size_t copyVertexData(arangodb::velocypack::Slice document, void* targetPtr,
-                        size_t maxSize) override {
+  size_t copyVertexData(VertexEntry const& vertex,
+                        std::string const& documentId,
+                        arangodb::velocypack::Slice document,
+                        void* targetPtr, size_t maxSize) override {
     arangodb::velocypack::Slice val = document.get(_sourceField);
-    if (std::is_integral<V>()) {
-      if (std::is_signed<V>()) {
+    if (std::is_integral<V>::value) {
+      if (std::is_signed<V>::value) {
         *((V*)targetPtr) = val.isInteger() ? val.getInt() : _vDefault;
       } else {
         *((V*)targetPtr) = val.isInteger() ? val.getUInt() : _vDefault;
       }
     } else {
-      *((V*)targetPtr) = val.isDouble() ? val.getDouble() : _vDefault;
+      *((V*)targetPtr) = val.isNumber() ? val.getNumber<V>() : _vDefault;
     }
     return sizeof(V);
   }
@@ -178,30 +92,109 @@ public:
   size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
                       size_t maxSize) override {
     arangodb::velocypack::Slice val = document.get(_sourceField);
-    if (std::is_integral<E>()) {
-      if (std::is_signed<E>()) {
+    if (std::is_integral<E>::value) {
+      if (std::is_signed<E>::value) {// getNumber does range checks
         *((E*)targetPtr) = val.isInteger() ? val.getInt() : _eDefault;
       } else {
         *((E*)targetPtr) = val.isInteger() ? val.getUInt() : _eDefault;
       }
     } else {
-      *((E*)targetPtr) = val.isDouble() ? val.getDouble() : _eDefault;
+      *((E*)targetPtr) = val.isNumber() ? val.getNumber<E>() : _eDefault;
     }
     return sizeof(E);
   }
 
-  void buildVertexDocument(arangodb::velocypack::Builder& b, const void*
-targetPtr,
+  bool buildVertexDocument(arangodb::velocypack::Builder& b, const void* ptr,
                            size_t size) override {
-    b.add(_resultField, VPackValue(readVertexData(targetPtr)));
+    b.add(_resultField, VPackValue(*((V*)ptr)));
+    return true;
   }
 
-  void buildEdgeDocument(arangodb::velocypack::Builder& b, const void*
-targetPtr,
+  bool buildEdgeDocument(arangodb::velocypack::Builder& b, const void* ptr,
                          size_t size) override {
-    b.add(_resultField, VPackValue(readEdgeData(targetPtr)));
+    b.add(_resultField, VPackValue(*((E*)ptr)));
+    return true;
   }
-};//*/
+};
+
+template <typename V, typename E>
+class InitGraphFormat : public GraphFormat<V, E> {
+  protected:
+    const std::string _resultField;
+    const V _vDefault;
+    const E _eDefault;
+    
+  public:
+  
+  InitGraphFormat(std::string const& result, V vertexNull, E edgeNull)
+  : _resultField(result), _vDefault(vertexNull), _eDefault(edgeNull) {}
+  
+  size_t copyVertexData(VertexEntry const& vertex,
+                        std::string const& documentId,
+                        arangodb::velocypack::Slice document,
+                        void* targetPtr, size_t maxSize) override {
+    *((V*)targetPtr) = _vDefault;
+    return sizeof(V);
+  }
+  
+  size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
+                      size_t maxSize) override {
+    *((E*)targetPtr) = _eDefault;
+    return sizeof(E);
+  }
+  
+  bool buildVertexDocument(arangodb::velocypack::Builder& b,
+                           const void* ptr,
+                           size_t size) override {
+    b.add(_resultField, VPackValue(*((V*)ptr)));
+    return true;
+  }
+  
+  bool buildEdgeDocument(arangodb::velocypack::Builder& b, const void* ptr,
+                         size_t size) override {
+    b.add(_resultField, VPackValue(*((E*)ptr)));
+    return true;
+  }
+};
+  
+  
+template <typename V, typename E>
+class VertexGraphFormat : public GraphFormat<V, E> {
+protected:
+  const std::string _resultField;
+  const V _vDefault;
+  
+public:
+  
+  VertexGraphFormat(std::string const& result, V vertexNull)
+  : _resultField(result), _vDefault(vertexNull) {}
+  
+  size_t copyVertexData(VertexEntry const& vertex,
+                        std::string const& documentId,
+                        arangodb::velocypack::Slice document,
+                        void* targetPtr, size_t maxSize) override {
+    *((V*)targetPtr) = _vDefault;
+    return sizeof(V);
+  }
+  
+  size_t copyEdgeData(arangodb::velocypack::Slice document, void* targetPtr,
+                      size_t maxSize) override {
+    return 0;
+  }
+  
+  bool buildVertexDocument(arangodb::velocypack::Builder& b,
+                           const void* ptr,
+                           size_t size) override {
+    b.add(_resultField, VPackValue(*((V*)ptr)));
+    return true;
+  }
+  
+  bool buildEdgeDocument(arangodb::velocypack::Builder& b, const void* ptr,
+                        size_t size) override {
+    return false;
+  }
+};
+  
 }
 }
 #endif

arangod/Pregel/GraphStore.cpp

@@ -126,8 +126,9 @@ void GraphStore<V, E>::loadDocument(WorkerConfig const& config,
   std::string documentId = _readTrx->extractIdString(opResult.slice());
   VertexEntry entry(sourceShard, _key);
   V vertexData;
-  size_t size = _graphFormat->copyVertexData(
-      entry, documentId, opResult.slice(), &vertexData, sizeof(V));
+  size_t size = _graphFormat->copyVertexData(entry, documentId,
+                                             opResult.slice(),
+                                             &vertexData, sizeof(V));
   if (size > 0) {
     entry._vertexDataOffset = _vertexData.size();
     _vertexData.push_back(vertexData);

arangod/Pregel/IncomingCache.cpp

@@ -239,9 +239,12 @@ void CombiningInCache<M>::forEach(
 }
 
 // template types to create
+template class arangodb::pregel::InCache<int32_t>;
 template class arangodb::pregel::InCache<int64_t>;
 template class arangodb::pregel::InCache<float>;
+template class arangodb::pregel::ArrayInCache<int32_t>;
 template class arangodb::pregel::ArrayInCache<int64_t>;
 template class arangodb::pregel::ArrayInCache<float>;
+template class arangodb::pregel::CombiningInCache<int32_t>;
 template class arangodb::pregel::CombiningInCache<int64_t>;
 template class arangodb::pregel::CombiningInCache<float>;

arangod/Pregel/MasterContext.h

@@ -27,7 +27,6 @@
 #include <velocypack/velocypack-aliases.h>
 #include "Basics/Common.h"
 #include "Pregel/AggregatorHandler.h"
-#include "Pregel/Utils.h"
 
 namespace arangodb {
 namespace pregel {

arangod/Pregel/MessageCombiner.h

@@ -34,14 +34,25 @@ struct MessageCombiner {
   virtual void combine(M& firstValue, M const& secondValue) const = 0;
 };
 
-struct IntegerMinCombiner : public MessageCombiner<int64_t> {
-  IntegerMinCombiner() {}
-  void combine(int64_t& firstValue, int64_t const& secondValue) const override {
+template <typename M>
+struct MinCombiner : public MessageCombiner<M> {
+  static_assert(std::is_arithmetic<M>::value, "Message type must be numeric");
+  MinCombiner() {}
+  void combine(M& firstValue, M const& secondValue) const override {
     if (firstValue > secondValue) {
       firstValue = secondValue;
     }
   };
 };
+
+template <typename M>
+struct SumCombiner : public MessageCombiner<M> {
+  static_assert(std::is_arithmetic<M>::value, "Message type must be numeric");
+  SumCombiner() {}
+  void combine(M& firstValue, M const& secondValue) const {
+    firstValue += secondValue;
+  }
+};
 }
 }
 #endif

arangod/Pregel/MessageFormat.h

@@ -58,23 +58,19 @@ struct FloatMessageFormat : public MessageFormat<float> {
     arrayBuilder.add(VPackValue(val));
   }
 };
-
-/*
- template <typename M>
- struct NumberMessageFormat : public MessageFormat<M> {
- static_assert(std::is_arithmetic<M>::value, "Type must be numeric");
- NumberMessageFormat() {}
- bool unwrapValue(VPackSlice s, int64_t& value) const override {
- if (s.isNumber()) {
- value = s.getNumber<M>();
- return true;
- }
- return false;
- }
- void addValue(VPackBuilder& arrayBuilder, int64_t const& val) const override {
- arrayBuilder.add(VPackValue(val));
- }
- };*/
+  
+template <typename M>
+struct NumberMessageFormat : public MessageFormat<M> {
+  static_assert(std::is_arithmetic<M>::value, "Message type must be numeric");
+  NumberMessageFormat() {}
+  void unwrapValue(VPackSlice s, M& value) const override {
+    value = s.getNumber<M>();
+  }
+  void addValue(VPackBuilder& arrayBuilder, M const& val) const override {
+    arrayBuilder.add(VPackValue(val));
+  }
+};
+  
 }
 }
 #endif

arangod/Pregel/VertexComputation.h

@@ -59,9 +59,14 @@ class VertexContext {
   }
 
   template <typename T>
-  inline void aggregate(std::string const& name, const T* valuePtr) {
+  inline void aggregate(std::string const& name, T const* valuePtr) {
     _workerAggregators->aggregate(name, valuePtr);
   }
+  
+  template <typename T>
+  inline void aggregate(std::string const& name, T const& valuePtr) {
+    _workerAggregators->aggregate(name, &valuePtr);
+  }
 
   inline WorkerContext const* context() { return _context; }
 
@@ -101,6 +106,14 @@ class VertexComputation : public VertexContext<V, E, M> {
   void sendMessage(Edge<E> const* edge, M const& data) {
     _cache->appendMessage(edge->targetShard(), edge->toKey(), data);
   }
+  
+  // TODO optimize outgoing cache somehow
+  void sendMessageToAllEdges(M const& data) {
+    RangeIterator<Edge<E>> edges = this->getEdges();
+    for (Edge<E> const* edge : edges) {
+      _cache->appendMessage(edge->targetShard(), edge->toKey(), data);
+    }
+  }
 
   void enterNextPhase() {
     if (!_nextPhase) {