From 2b4c89cf491b93066f8070467fd06599692f06cd Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Mon, 11 Apr 2016 16:59:43 +0200
Subject: [PATCH 01/10] Catching up with highest log entry on leader

---
 arangod/Agency/Agent.cpp       |  5 +++--
 arangod/Agency/Constituent.cpp | 22 ++++++++++------------
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp
index cc3740aea8..a825bd74b2 100644
--- a/arangod/Agency/Agent.cpp
+++ b/arangod/Agency/Agent.cpp
@@ -305,8 +305,9 @@ bool Agent::load () {
   }
 
   LOG_TOPIC(INFO, Logger::AGENCY) << "Reassembling spearhead and read stores.";
-//  _read_db.apply(_state.slices());
   _spearhead.apply(_state.slices(_last_commit_index+1));
+  reportIn(id(),_state.lastLog().index);
+  //_cv.signal();
 
   LOG_TOPIC(INFO, Logger::AGENCY) << "Starting spearhead worker.";
   _spearhead.start(this);
@@ -337,7 +338,7 @@ write_ret_t Agent::write (query_t const& query)  {
       _cv.signal();                                  // Wake up run
     }
 
-    reportIn(0,maxind);
+    reportIn(id(),maxind);
     
     return write_ret_t(true,id(),applied,indices); // Indices to wait for to rest
     
diff --git a/arangod/Agency/Constituent.cpp b/arangod/Agency/Constituent.cpp
index b37f40283f..6528276d9e 100644
--- a/arangod/Agency/Constituent.cpp
+++ b/arangod/Agency/Constituent.cpp
@@ -268,8 +268,8 @@ void Constituent::callElection() {
   std::vector<ClusterCommResult> results(_agent->config().end_points.size());
   std::stringstream path;
   
-  path << "/_api/agency_priv/requestVote?term=" << _term << "&candidateId=" << _id
-       << "&prevLogIndex=" << _agent->lastLog().index << "&prevLogTerm="
+  path << "/_api/agency_priv/requestVote?term=" << _term << "&candidateId="
+       << _id << "&prevLogIndex=" << _agent->lastLog().index << "&prevLogTerm="
        << _agent->lastLog().term;
 
   // Ask everyone for their vote
@@ -286,8 +286,7 @@ void Constituent::callElection() {
 
   // Wait randomized timeout
   std::this_thread::sleep_for(
-    sleepFor(.5*_agent->config().min_ping,
-             .8*_agent->config().min_ping));
+    sleepFor(.5*_agent->config().min_ping, .8*_agent->config().min_ping));
 
   // Collect votes
   for (id_t i = 0; i < _agent->config().end_points.size(); ++i) { 
@@ -341,8 +340,6 @@ void Constituent::beginShutdown() {
 }
 
 
-#include <iostream>
-
 bool Constituent::start (TRI_vocbase_t* vocbase,
                          ApplicationV8* applicationV8,
                          aql::QueryRegistry* queryRegistry) {
@@ -356,6 +353,7 @@ bool Constituent::start (TRI_vocbase_t* vocbase,
 
 
 void Constituent::run() {
+
   TRI_ASSERT(_vocbase != nullptr);
   auto bindVars = std::make_shared<VPackBuilder>();
   bindVars->openObject();
@@ -367,8 +365,8 @@ void Constituent::run() {
   // Query
   std::string const aql ("FOR l IN election SORT l._key DESC LIMIT 1 RETURN l");
   arangodb::aql::Query query(_applicationV8, false, _vocbase,
-			     aql.c_str(), aql.size(), bindVars, nullptr,
-			     arangodb::aql::PART_MAIN);
+                             aql.c_str(), aql.size(), bindVars, nullptr,
+                             arangodb::aql::PART_MAIN);
   
   auto queryResult = query.execute(_queryRegistry);
   if (queryResult.code != TRI_ERROR_NO_ERROR) {
@@ -376,19 +374,19 @@ void Constituent::run() {
   }
   
   VPackSlice result = queryResult.result->slice();
-
+  
   if (result.isArray()) {
     for (auto const& i : VPackArrayIterator(result)) {
       try {
-      _term = i.get("term").getUInt();
-      _voted_for = i.get("voted_for").getUInt();
+        _term = i.get("term").getUInt();
+        _voted_for = i.get("voted_for").getUInt();
       } catch (std::exception const& e) {
         LOG_TOPIC(ERR, Logger::AGENCY)
           << "Persisted election entries corrupt! Defaulting term,vote (0,0)";
       }
     }
   }
-
+  
   // Always start off as follower
   while (!this->isStopping() && size() > 1) { 
     if (_role == FOLLOWER) {

From bf952614934bcd2f174cc92c0870b3567acd8dbb Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 10:25:52 +0200
Subject: [PATCH 02/10] introducing cluster sanity checking to agency

---
 arangod/Agency/Agent.cpp             | 16 ++++++---
 arangod/Agency/Agent.h               | 12 +++++--
 arangod/Agency/ApplicationAgency.cpp | 26 ++++++++------
 arangod/Agency/Constituent.cpp       | 54 +++++++++++++++++++---------
 arangod/Agency/Constituent.h         |  5 +--
 arangod/Agency/SanityCheck.cpp       | 45 ++++++++++++++++++-----
 arangod/Agency/SanityCheck.h         | 18 +++++-----
 arangod/Agency/State.cpp             | 33 +++++++++--------
 arangod/Agency/Store.cpp             | 20 +++++------
 9 files changed, 150 insertions(+), 79 deletions(-)

diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp
index 6a75a12467..3b5c606b54 100644
--- a/arangod/Agency/Agent.cpp
+++ b/arangod/Agency/Agent.cpp
@@ -38,7 +38,8 @@ namespace arangodb {
 namespace consensus {
 
 //  Agent configuration
-Agent::Agent (TRI_server_t* server, config_t const& config, ApplicationV8* applicationV8, aql::QueryRegistry* queryRegistry) 
+Agent::Agent (TRI_server_t* server, config_t const& config,
+              ApplicationV8* applicationV8, aql::QueryRegistry* queryRegistry) 
     : Thread ("Agent"), 
       _server(server), 
       _vocbase(nullptr), 
@@ -218,7 +219,8 @@ bool Agent::recvAppendEntriesRPC (term_t term, id_t leaderId, index_t prevIndex,
   if (queries->slice().length()) {
     LOG_TOPIC(INFO, Logger::AGENCY) << "Appending "<< queries->slice().length()
                                     << " entries to state machine.";
-    /* bool success = */_state.log (queries, term, leaderId, prevIndex, prevTerm);
+    /* bool success = */
+    _state.log (queries, term, leaderId, prevIndex, prevTerm);
   } else { 
     // heart-beat
   }
@@ -298,13 +300,13 @@ bool Agent::load () {
 
   LOG_TOPIC(INFO, Logger::AGENCY) << "Loading persistent state.";
   if (!_state.loadCollections(_vocbase, _applicationV8, _queryRegistry)) {
-    LOG_TOPIC(WARN, Logger::AGENCY) << "Failed to load persistent state on statup.";
+    LOG_TOPIC(WARN, Logger::AGENCY)
+      << "Failed to load persistent state on statup.";
   }
 
   LOG_TOPIC(INFO, Logger::AGENCY) << "Reassembling spearhead and read stores.";
   _spearhead.apply(_state.slices(_last_commit_index+1));
   reportIn(id(),_state.lastLog().index);
-  //_cv.signal();
 
   LOG_TOPIC(INFO, Logger::AGENCY) << "Starting spearhead worker.";
   _spearhead.start(this);
@@ -312,6 +314,11 @@ bool Agent::load () {
 
   LOG_TOPIC(INFO, Logger::AGENCY) << "Starting constituent personality.";
   _constituent.start(_vocbase, _applicationV8, _queryRegistry);
+
+  if (_config.sanity_check) {
+    LOG_TOPIC(INFO, Logger::AGENCY) << "Starting cluster sanity facilities";
+    _sanity_check.start(this);
+  }
   
   return true;
 }
@@ -387,6 +394,7 @@ void Agent::beginShutdown() {
   _constituent.beginShutdown();
   _spearhead.beginShutdown();
   _read_db.beginShutdown();
+  _sanity_check.beginShutdown();
   
   // Wake up all waiting REST handler (waitFor)
   CONDITION_LOCKER(guard, _cv);
diff --git a/arangod/Agency/Agent.h b/arangod/Agency/Agent.h
index 5f9ee1a2f5..0e14afdd3f 100644
--- a/arangod/Agency/Agent.h
+++ b/arangod/Agency/Agent.h
@@ -27,6 +27,7 @@
 #include "AgencyCommon.h"
 #include "AgentCallback.h"
 #include "Constituent.h"
+#include "SanityCheck.h"
 #include "State.h"
 #include "Store.h"
 
@@ -42,7 +43,8 @@ class QueryRegistry;
 namespace consensus {
 
 class Agent : public arangodb::Thread {
- public:
+
+public:
   /// @brief Construct with program options
   Agent(TRI_server_t*, config_t const&, ApplicationV8*, aql::QueryRegistry*);
 
@@ -77,6 +79,8 @@ class Agent : public arangodb::Thread {
 
   /// @brief Leader ID
   id_t leaderID() const;
+
+  /// @brief Are we leading?
   bool leading() const;
 
   /// @brief Pick up leadership tasks
@@ -148,7 +152,9 @@ class Agent : public arangodb::Thread {
   aql::QueryRegistry* _queryRegistry;
 
   Constituent _constituent; /**< @brief Leader election delegate */
+  SanityCheck _sanity_check; /**< @brief sanitychecking */
   State _state;             /**< @brief Log replica              */
+  
   config_t _config;         /**< @brief Command line arguments   */
 
   std::atomic<index_t> _last_commit_index; /**< @brief Last commit index */
@@ -165,7 +171,7 @@ class Agent : public arangodb::Thread {
       _confirmed;          /**< @brief Confirmed log index of each slave */
   arangodb::Mutex _ioLock; /**< @brief Read/Write lock */
 };
-}
-}
+
+}}
 
 #endif
diff --git a/arangod/Agency/ApplicationAgency.cpp b/arangod/Agency/ApplicationAgency.cpp
index e9f1c0e3a2..9c7ab5882d 100644
--- a/arangod/Agency/ApplicationAgency.cpp
+++ b/arangod/Agency/ApplicationAgency.cpp
@@ -38,17 +38,21 @@ using namespace arangodb::basics;
 using namespace arangodb::rest;
 using namespace arangodb;
 
-ApplicationAgency::ApplicationAgency(TRI_server_t* server, 
-                                     ApplicationEndpointServer* aes, 
-                                     ApplicationV8* applicationV8, 
-                                     aql::QueryRegistry* queryRegistry)
-  : ApplicationFeature("agency"), _server(server), _size(1),
-    _min_election_timeout(0.15), _max_election_timeout(1.0),
-    _election_call_rate_mul(0.85), _notify(false), _sanity_check(false),
-          _agent_id((std::numeric_limits<uint32_t>::max)()), 
-          _endpointServer(aes), 
-          _applicationV8(applicationV8), 
-          _queryRegistry(queryRegistry) {
+ApplicationAgency::ApplicationAgency(
+  TRI_server_t* server, ApplicationEndpointServer* aes,
+  ApplicationV8* applicationV8, aql::QueryRegistry* queryRegistry)
+  : ApplicationFeature("agency"),
+    _server(server),
+    _size(1),
+    _min_election_timeout(0.15),
+    _max_election_timeout(1.0),
+    _election_call_rate_mul(0.85),
+    _notify(false),
+    _sanity_check(false),
+    _agent_id((std::numeric_limits<uint32_t>::max)()), 
+    _endpointServer(aes), 
+    _applicationV8(applicationV8), 
+    _queryRegistry(queryRegistry) {
 }
 
 
diff --git a/arangod/Agency/Constituent.cpp b/arangod/Agency/Constituent.cpp
index 6528276d9e..0ea68ca8d8 100644
--- a/arangod/Agency/Constituent.cpp
+++ b/arangod/Agency/Constituent.cpp
@@ -1,3 +1,4 @@
+
 ////////////////////////////////////////////////////////////////////////////////
 /// DISCLAIMER
 ///
@@ -67,14 +68,28 @@ void Constituent::configure(Agent* agent) {
 
 // Default ctor
 Constituent::Constituent() :
-  Thread("Constituent"), _term(0), _leader_id(0), _id(0), _gen(std::random_device()()),
-  _role(FOLLOWER), _agent(0), _voted_for(0) {}
+  Thread("Constituent"),
+  _vocbase(nullptr),
+  _applicationV8(nullptr),
+  _queryRegistry(nullptr),
+  _term(0),
+  _leader_id(0),
+  _id(0),
+  _gen(std::random_device()()),
+  _role(FOLLOWER),
+  _agent(0),
+  _voted_for(0) {}
 
 // Shutdown if not already
 Constituent::~Constituent() {
   shutdown();
 }
 
+// Configuration
+config_t const& Constituent::config () const {
+  return _agent->config();
+}
+
 // Random sleep times in election process
 duration_t Constituent::sleepFor (double min_t, double max_t) {
   dist_t dis(min_t, max_t);
@@ -105,8 +120,10 @@ void Constituent::term(term_t t) {
     body.close();
     
     TRI_ASSERT(_vocbase != nullptr);
-    auto transactionContext = std::make_shared<StandaloneTransactionContext>(_vocbase);
-    SingleCollectionTransaction trx(transactionContext, "election", TRI_TRANSACTION_WRITE);
+    auto transactionContext =
+      std::make_shared<StandaloneTransactionContext>(_vocbase);
+    SingleCollectionTransaction trx(transactionContext, "election",
+                                    TRI_TRANSACTION_WRITE);
     
     int res = trx.begin();
     
@@ -133,7 +150,8 @@ role_t Constituent::role () const {
 /// @brief Become follower in term 
 void Constituent::follow (term_t t) {
   if (_role != FOLLOWER) {
-    LOG_TOPIC(INFO, Logger::AGENCY) << "Role change: Converted to follower in term " << t;
+    LOG_TOPIC(INFO, Logger::AGENCY)
+      << "Role change: Converted to follower in term " << t;
   }
   this->term(t);
   _role = FOLLOWER;
@@ -142,7 +160,8 @@ void Constituent::follow (term_t t) {
 /// @brief Become leader
 void Constituent::lead () {
   if (_role != LEADER) {
-    LOG_TOPIC(INFO, Logger::AGENCY) << "Role change: Converted to leader in term " << _term ;
+    LOG_TOPIC(INFO, Logger::AGENCY)
+      << "Role change: Converted to leader in term " << _term ;
     _agent->lead(); // We need to rebuild spear_head and read_db;
   }
   _role = LEADER;
@@ -152,7 +171,8 @@ void Constituent::lead () {
 /// @brief Become follower
 void Constituent::candidate () {
   if (_role != CANDIDATE)
-    LOG_TOPIC(INFO, Logger::AGENCY) << "Role change: Converted to candidate in term " << _term ;
+    LOG_TOPIC(INFO, Logger::AGENCY)
+      << "Role change: Converted to candidate in term " << _term ;
   _role = CANDIDATE;
 }
 
@@ -178,17 +198,17 @@ id_t Constituent::leaderID ()  const {
 
 /// @brief Agency size
 size_t Constituent::size() const {
-  return _agent->config().size();
+  return config().size();
 }
 
 /// @brief Get endpoint to an id 
 std::string const& Constituent::end_point(id_t id) const {
-  return _agent->config().end_points[id];
+  return config().end_points[id];
 }
 
 /// @brief Get all endpoints
 std::vector<std::string> const& Constituent::end_points() const {
-  return _agent->config().end_points;
+  return config().end_points;
 }
 
 /// @brief Notify peers of updated endpoints
@@ -265,7 +285,7 @@ void Constituent::callElection() {
   }
   
   std::string body;
-  std::vector<ClusterCommResult> results(_agent->config().end_points.size());
+  std::vector<ClusterCommResult> results(config().end_points.size());
   std::stringstream path;
   
   path << "/_api/agency_priv/requestVote?term=" << _term << "&candidateId="
@@ -273,23 +293,23 @@ void Constituent::callElection() {
        << _agent->lastLog().term;
 
   // Ask everyone for their vote
-  for (id_t i = 0; i < _agent->config().end_points.size(); ++i) { 
+  for (id_t i = 0; i < config().end_points.size(); ++i) { 
     if (i != _id && end_point(i) != "") {
       std::unique_ptr<std::map<std::string, std::string>> headerFields =
         std::make_unique<std::map<std::string, std::string> >();
       results[i] = arangodb::ClusterComm::instance()->asyncRequest(
-        "1", 1, _agent->config().end_points[i], GeneralRequest::RequestType::GET,
+        "1", 1, config().end_points[i], GeneralRequest::RequestType::GET,
         path.str(), std::make_shared<std::string>(body), headerFields, nullptr,
-        _agent->config().min_ping, true);
+        config().min_ping, true);
     }
   }
 
   // Wait randomized timeout
   std::this_thread::sleep_for(
-    sleepFor(.5*_agent->config().min_ping, .8*_agent->config().min_ping));
+    sleepFor(.5*config().min_ping, .8*config().min_ping));
 
   // Collect votes
-  for (id_t i = 0; i < _agent->config().end_points.size(); ++i) { 
+  for (id_t i = 0; i < config().end_points.size(); ++i) { 
     if (i != _id && end_point(i) != "") {
       ClusterCommResult res = arangodb::ClusterComm::instance()->
         enquire(results[i].operationID);
@@ -392,7 +412,7 @@ void Constituent::run() {
     if (_role == FOLLOWER) {
       _cast = false;                           // New round set not cast vote
       std::this_thread::sleep_for(             // Sleep for random time
-        sleepFor(_agent->config().min_ping, _agent->config().max_ping)); 
+        sleepFor(config().min_ping, config().max_ping)); 
       if (!_cast) {
         candidate();                           // Next round, we are running
       }
diff --git a/arangod/Agency/Constituent.h b/arangod/Agency/Constituent.h
index a10df4efe1..714df335f6 100644
--- a/arangod/Agency/Constituent.h
+++ b/arangod/Agency/Constituent.h
@@ -33,7 +33,6 @@
 #include "AgencyCommon.h"
 #include "Basics/Thread.h"
 
-struct TRI_server_t;
 struct TRI_vocbase_t;
 
 namespace arangodb {
@@ -93,6 +92,9 @@ public:
   /// @brief Who is leading
   id_t leaderID () const;
 
+  /// @brief Configuration
+  config_t const& config () const;
+
   /// @brief Become follower
   void follow(term_t);
 
@@ -135,7 +137,6 @@ private:
   /// @brief Sleep for how long
   duration_t sleepFor(double, double);
 
-  TRI_server_t* _server;
   TRI_vocbase_t* _vocbase; 
   ApplicationV8* _applicationV8;
   aql::QueryRegistry* _queryRegistry;
diff --git a/arangod/Agency/SanityCheck.cpp b/arangod/Agency/SanityCheck.cpp
index d24d196316..5c69288647 100644
--- a/arangod/Agency/SanityCheck.cpp
+++ b/arangod/Agency/SanityCheck.cpp
@@ -1,28 +1,55 @@
 #include "SanityCheck.h"
-
 #include "Agent.h"
 
+#include "Basics/ConditionLocker.h"
+
 using namespace arangodb::consensus;
 
 SanityCheck::SanityCheck() : arangodb::Thread("SanityCheck"), _agent(nullptr) {}
 
 SanityCheck::~SanityCheck() {};
 
-void SanityCheck::configure(Agent* agent) {
-  _agent = agent;
-}
-
 void SanityCheck::wakeUp () {
+  _cv.signal();
 }
 
-void SanityCheck::passOut () {
+bool SanityCheck::doChecks (bool timedout) {
+  return true;
 }
 
 void SanityCheck::run() {
+
+  CONDITION_LOCKER(guard, _cv);
+  TRI_ASSERT(_agent!=nullptr);
+  bool timedout = false;
+  
+  while (!this->isStopping()) {
+    
+    if (_agent->leading()) {
+      timedout = _cv.wait(1000000);
+    } else {
+      _cv.wait();
+    }
+    
+    doChecks(timedout);
+    
+  }
+  
+}
+
+// Start thread
+bool SanityCheck::start () {
+  Thread::start();
+  return true;
+}
+
+// Start thread with agent
+bool SanityCheck::start (Agent* agent) {
+  _agent = agent;
+  return start();
 }
 
 void SanityCheck::beginShutdown() {
+  // Personal hygiene
+  Thread::beginShutdown();
 }
-
-
-
diff --git a/arangod/Agency/SanityCheck.h b/arangod/Agency/SanityCheck.h
index 8a21d339e6..36fb547d0e 100644
--- a/arangod/Agency/SanityCheck.h
+++ b/arangod/Agency/SanityCheck.h
@@ -42,9 +42,12 @@ public:
   /// @brief Default dtor
   ~SanityCheck ();
   
-  /// @brief Configure with agent
-  void configure (Agent* agent);
-  
+  /// @brief Start thread
+  bool start ();
+
+  /// @brief Start thread with access to agent
+  bool start (Agent*);
+
   /// @brief Run woker
   void run() override final;
   
@@ -54,16 +57,15 @@ public:
   /// @brief Wake up to task
   void wakeUp ();
 
-  /// @brief Stop task and wait
-  void passOut ();
-  
 private:
+
+  /// @brief Perform sanity checking
+  bool doChecks(bool);
   
-  Agent* _agent;
+  Agent* _agent; /**< @brief My agent */
 
   arangodb::basics::ConditionVariable _cv; /**< @brief Control if thread should run */
   
-
   
 };
 
diff --git a/arangod/Agency/State.cpp b/arangod/Agency/State.cpp
index 5cc9df9e36..9e75c996c9 100644
--- a/arangod/Agency/State.cpp
+++ b/arangod/Agency/State.cpp
@@ -75,8 +75,10 @@ bool State::persist(index_t index, term_t term, id_t lid,
   body.close();
 
   TRI_ASSERT(_vocbase != nullptr);
-  auto transactionContext = std::make_shared<StandaloneTransactionContext>(_vocbase);
-  SingleCollectionTransaction trx(transactionContext, "log", TRI_TRANSACTION_WRITE);
+  auto transactionContext =
+    std::make_shared<StandaloneTransactionContext>(_vocbase);
+  SingleCollectionTransaction trx (
+    transactionContext, "log", TRI_TRANSACTION_WRITE);
   
   int res = trx.begin();
 
@@ -131,7 +133,7 @@ bool State::log(query_t const& queries, term_t term, id_t lid,
       buf->append((char const*)i.get("query").begin(),
                   i.get("query").byteSize());
       _log.push_back(log_t(i.get("index").getUInt(), term, lid, buf));
-      persist(i.get("index").getUInt(), term, lid, i.get("query")); // log to disk
+      persist(i.get("index").getUInt(), term, lid, i.get("query")); // to disk
     } catch (std::exception const& e) {
       LOG(ERR) << e.what();
     }
@@ -194,7 +196,8 @@ bool State::createCollections() {
 
 bool State::checkCollection(std::string const& name) {
   if (!_collections_checked) {
-    return (TRI_LookupCollectionByNameVocBase(_vocbase, name.c_str()) != nullptr);
+    return (
+      TRI_LookupCollectionByNameVocBase(_vocbase, name.c_str()) != nullptr);
   }
   return true;
 }
@@ -204,7 +207,8 @@ bool State::createCollection(std::string const& name) {
   body.add(VPackValue(VPackValueType::Object));
   body.close();
 
-  VocbaseCollectionInfo parameters(_vocbase, name.c_str(), TRI_COL_TYPE_DOCUMENT, body.slice());
+  VocbaseCollectionInfo parameters(_vocbase, name.c_str(),
+                                   TRI_COL_TYPE_DOCUMENT, body.slice());
   TRI_vocbase_col_t const* collection =
       TRI_CreateCollectionVocBase(_vocbase, parameters, parameters.id(), true);
   
@@ -216,8 +220,7 @@ bool State::createCollection(std::string const& name) {
 
 }
 
-bool State::loadCollections(TRI_vocbase_t* vocbase, 
-                            ApplicationV8* applicationV8, 
+bool State::loadCollections(TRI_vocbase_t* vocbase, ApplicationV8* applicationV8, 
                             aql::QueryRegistry* queryRegistry) {
   _vocbase = vocbase;
   _applicationV8 = applicationV8;
@@ -227,20 +230,21 @@ bool State::loadCollections(TRI_vocbase_t* vocbase,
 
 bool State::loadCollection(std::string const& name) {
   TRI_ASSERT(_vocbase != nullptr);
-
+  
   if (checkCollection(name)) {
     auto bindVars = std::make_shared<VPackBuilder>();
     bindVars->openObject();
     bindVars->close();
     // ^^^ TODO: check if bindvars are actually needed
-
+    
     TRI_ASSERT(_applicationV8 != nullptr);
     TRI_ASSERT(_queryRegistry != nullptr);
-    std::string const aql(std::string("FOR l IN ") + name + " SORT l._key RETURN l");
+    std::string const aql(std::string("FOR l IN ") + name
+                          + " SORT l._key RETURN l");
     arangodb::aql::Query query(_applicationV8, false, _vocbase,
                                aql.c_str(), aql.size(), bindVars, nullptr,
                                arangodb::aql::PART_MAIN);
-  
+    
     auto queryResult = query.execute(_queryRegistry);
 
     if (queryResult.code != TRI_ERROR_NO_ERROR) {
@@ -255,9 +259,10 @@ bool State::loadCollection(std::string const& name) {
               std::make_shared<arangodb::velocypack::Buffer<uint8_t>>();
         VPackSlice req = i.get("request");
         tmp->append(req.startAs<char const>(), req.byteSize());
-        _log.push_back(log_t(std::stoi(i.get(TRI_VOC_ATTRIBUTE_KEY).copyString()),
-                                 i.get("term").getUInt(),
-                                 i.get("leader").getUInt(), tmp));
+        _log.push_back(
+          log_t(std::stoi(i.get(TRI_VOC_ATTRIBUTE_KEY).copyString()),
+                i.get("term").getUInt(),
+                i.get("leader").getUInt(), tmp));
       }
     }
 
diff --git a/arangod/Agency/Store.cpp b/arangod/Agency/Store.cpp
index 5de4c87fed..610727fb52 100644
--- a/arangod/Agency/Store.cpp
+++ b/arangod/Agency/Store.cpp
@@ -647,7 +647,8 @@ std::vector<bool> Store::apply (query_t const& query) {
 }
 
 //template<class T, class U> std::multimap<std::string, std::string>
-std::ostream& operator<< (std::ostream& os, std::multimap<std::string,std::string> const& m) {
+std::ostream& operator<< (
+  std::ostream& os, std::multimap<std::string,std::string> const& m) {
   for (auto const& i : m) {
     os << i.first << ": " << i.second << std::endl;
   }
@@ -700,10 +701,11 @@ std::vector<bool> Store::apply (
   }
 
   std::vector<std::string> urls;
-  for(auto it = in.begin(), end = in.end(); it != end; it = in.upper_bound(it->first)) {
+  for (auto it = in.begin(), end = in.end(); it != end;
+       it = in.upper_bound(it->first)) {
     urls.push_back(it->first);
   }
-
+  
   for (auto const& url : urls) {
     Builder tmp; // host
     tmp.openObject();
@@ -712,18 +714,14 @@ std::vector<bool> Store::apply (
     auto ret = in.equal_range(url);
     
     for (auto it = ret.first; it!=ret.second; ++it) {
-      //tmp.add(url,VPackValue(VPackValueType::Object));
       tmp.add(it->second->key,VPackValue(VPackValueType::Object));
       tmp.add("op",VPackValue(it->second->oper));
-      //tmp.close();
       tmp.close();
-      }
+    }
     
     tmp.close();
-      std::cout << tmp.toJson() << std::endl;
-
   }
-
+  
   return applied;
 }
 
@@ -791,7 +789,7 @@ std::vector<bool> Store::read (query_t const& queries, query_t& result) const {
 
 // read single query into ret
 bool Store::read (VPackSlice const& query, Builder& ret) const {
-
+  
   bool success = true;
   
   // Collect all paths
@@ -837,7 +835,7 @@ bool Store::read (VPackSlice const& query, Builder& ret) const {
       }
     }
   }
-
+  
   // Into result builder
   copy.toBuilder(ret);
   

From b7c2a03558270b90c5e355a8ed18e57c3e137129 Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 10:59:18 +0200
Subject: [PATCH 03/10] introducing cluster sanity checking to agency

---
 arangod/Agency/Agent.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/arangod/Agency/Agent.cpp b/arangod/Agency/Agent.cpp
index 3b5c606b54..f5e3161ce3 100644
--- a/arangod/Agency/Agent.cpp
+++ b/arangod/Agency/Agent.cpp
@@ -394,7 +394,9 @@ void Agent::beginShutdown() {
   _constituent.beginShutdown();
   _spearhead.beginShutdown();
   _read_db.beginShutdown();
-  _sanity_check.beginShutdown();
+  if (_config.sanity_check) {
+    _sanity_check.beginShutdown();
+  }
   
   // Wake up all waiting REST handler (waitFor)
   CONDITION_LOCKER(guard, _cv);

From 9a608f3037354da5166a49206ecc116e809a517a Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 11:04:37 +0200
Subject: [PATCH 04/10] introducing cluster sanity checking to agency

---
 arangod/Agency/SanityCheck.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/arangod/Agency/SanityCheck.cpp b/arangod/Agency/SanityCheck.cpp
index 5c69288647..7075f7b96c 100644
--- a/arangod/Agency/SanityCheck.cpp
+++ b/arangod/Agency/SanityCheck.cpp
@@ -7,7 +7,9 @@ using namespace arangodb::consensus;
 
 SanityCheck::SanityCheck() : arangodb::Thread("SanityCheck"), _agent(nullptr) {}
 
-SanityCheck::~SanityCheck() {};
+SanityCheck::~SanityCheck() {
+  shutdown();
+};
 
 void SanityCheck::wakeUp () {
   _cv.signal();

From 66636cdb7d952999f709606c4117314752088f11 Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 11:35:27 +0200
Subject: [PATCH 05/10] introducing cluster sanity checking to agency

---
 arangod/Agency/SanityCheck.cpp   |  1 +
 arangod/Agency/Store.cpp         | 42 ++++++++++++++++++++++++++
 arangod/Agency/StoreCallback.cpp | 12 ++++++++
 arangod/Agency/StoreCallback.h   | 51 ++++++++++++++++++++++++++++++++
 arangod/CMakeLists.txt           |  3 +-
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arangod/Agency/StoreCallback.cpp
 create mode 100644 arangod/Agency/StoreCallback.h

diff --git a/arangod/Agency/SanityCheck.cpp b/arangod/Agency/SanityCheck.cpp
index 7075f7b96c..ca9e2db7bb 100644
--- a/arangod/Agency/SanityCheck.cpp
+++ b/arangod/Agency/SanityCheck.cpp
@@ -16,6 +16,7 @@ void SanityCheck::wakeUp () {
 }
 
 bool SanityCheck::doChecks (bool timedout) {
+  LOG_TOPIC(INFO, Logger::AGENCY) << "Sanity checks";
   return true;
 }
 
diff --git a/arangod/Agency/Store.cpp b/arangod/Agency/Store.cpp
index 610727fb52..72d75815b3 100644
--- a/arangod/Agency/Store.cpp
+++ b/arangod/Agency/Store.cpp
@@ -37,6 +37,34 @@
 
 using namespace arangodb::consensus;
 
+inline static enpointPathFromUrl (
+  std::string const& url, std::string& enpoint, std::string& path) {
+
+  std::stringstream ep;
+  path = "/";
+  size_t pos = 7;
+  if (i.find("http://")==0) {
+    ep << "tcp://";
+  } else if (i.find("https://")==0) {
+    ep << "ssl://";
+    ++pos;
+  } else {
+    LOG_TOPIC(WARN,Logger::AGENCY) << "Malformed notification URL " << i;
+    return;
+  }
+  
+  size_t slash_p = i.find("/",pos);
+  if ((slash_p==std::string::npos)) {
+    ep << i.substr(pos);
+  } else {
+    ep << i.substr(pos,slash_p-pos);
+    path = i.substr(slash_p);
+  }
+
+  endpoint = ep.str();
+    
+}
+
 struct NotEmpty {
   bool operator()(const std::string& s) { return !s.empty(); }
 };
@@ -707,6 +735,7 @@ std::vector<bool> Store::apply (
   }
   
   for (auto const& url : urls) {
+
     Builder tmp; // host
     tmp.openObject();
     tmp.add("term",VPackValue(0));
@@ -720,6 +749,19 @@ std::vector<bool> Store::apply (
     }
     
     tmp.close();
+
+    std::string endpoint, path;
+    endpointPathFromUrl (url,endpoint,path);
+
+    std::unique_ptr<std::map<std::string, std::string>> headerFields =
+      std::make_unique<std::map<std::string, std::string> >();
+    
+    ClusterCommResult res =
+      arangodb::ClusterComm::instance()->asyncRequest(
+        "1", 1, endpoint, GeneralRequest::RequestType::POST, path,
+        std::make_shared<std::string>(body.toString()), headerFields,
+        nullptr, 0.0, true);
+
   }
   
   return applied;
diff --git a/arangod/Agency/StoreCallback.cpp b/arangod/Agency/StoreCallback.cpp
new file mode 100644
index 0000000000..853c6b3aff
--- /dev/null
+++ b/arangod/Agency/StoreCallback.cpp
@@ -0,0 +1,12 @@
+#include "StoreCallback.h"
+#include "Store.h"
+
+using namespace arangodb::consensus;
+using namespace arangodb::velocypack;
+
+StoreCallback::StoreCallback() {}
+
+bool StoreCallback::operator()(arangodb::ClusterCommResult* res) {
+  return true;
+}
+  
diff --git a/arangod/Agency/StoreCallback.h b/arangod/Agency/StoreCallback.h
new file mode 100644
index 0000000000..6c909c9e9d
--- /dev/null
+++ b/arangod/Agency/StoreCallback.h
@@ -0,0 +1,51 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Kaveh Vahedipour
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef __ARANGODB_CONSENSUS_STORE_CALLBACK__
+#define __ARANGODB_CONSENSUS_STORE_CALLBACK__
+
+#include "Cluster/ClusterComm.h"
+#include "AgencyCommon.h"
+
+namespace arangodb {
+namespace consensus {
+
+class StoreCallback : public arangodb::ClusterCommCallback {
+
+public:
+  
+  StoreCallback();
+
+  explicit StoreCallback();
+
+  virtual bool operator()(arangodb::ClusterCommResult*) override final;
+
+  void shutdown();
+
+private:
+  
+};
+
+}} // namespace
+
+#endif
diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt
index f54478ce6d..0fd113b3c0 100644
--- a/arangod/CMakeLists.txt
+++ b/arangod/CMakeLists.txt
@@ -69,12 +69,13 @@ add_executable(${BIN_ARANGOD}
   Actions/RestActionHandler.cpp
   Actions/actions.cpp
   Agency/Agent.cpp
+  Agency/AgentCallback.cpp
   Agency/ApplicationAgency.cpp
   Agency/Constituent.cpp
   Agency/SanityCheck.cpp
   Agency/State.cpp
   Agency/Store.cpp
-  Agency/AgentCallback.cpp
+  Agency/StoreCallback.cpp
   ApplicationServer/ApplicationFeature.cpp
   ApplicationServer/ApplicationServer.cpp
   Aql/Aggregator.cpp

From 9853b9662cbc442b3e2b0b244acaac4f2abcf019 Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 12:10:46 +0200
Subject: [PATCH 06/10] introducing cluster sanity checking to agency

---
 arangod/Agency/AgentCallback.h            |  2 +-
 arangod/Agency/Store.cpp                  | 64 +++++++++++++----------
 arangod/Agency/StoreCallback.cpp          |  5 +-
 arangod/Agency/StoreCallback.h            |  3 --
 arangod/RestHandler/RestAgencyHandler.cpp |  8 ++-
 5 files changed, 47 insertions(+), 35 deletions(-)

diff --git a/arangod/Agency/AgentCallback.h b/arangod/Agency/AgentCallback.h
index c974563c0d..64754766c2 100644
--- a/arangod/Agency/AgentCallback.h
+++ b/arangod/Agency/AgentCallback.h
@@ -38,7 +38,7 @@ public:
   
   AgentCallback();
 
-  explicit AgentCallback(Agent* agent, id_t slave_id, index_t last);
+  AgentCallback(Agent* agent, id_t slave_id, index_t last);
 
   virtual bool operator()(arangodb::ClusterCommResult*) override final;
 
diff --git a/arangod/Agency/Store.cpp b/arangod/Agency/Store.cpp
index 72d75815b3..ee9b1a9303 100644
--- a/arangod/Agency/Store.cpp
+++ b/arangod/Agency/Store.cpp
@@ -22,6 +22,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "Store.h"
+#include "StoreCallback.h"
 #include "Agency/Agent.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/VelocyPackHelper.h"
@@ -37,31 +38,36 @@
 
 using namespace arangodb::consensus;
 
-inline static enpointPathFromUrl (
-  std::string const& url, std::string& enpoint, std::string& path) {
+inline static bool endpointPathFromUrl (
+  std::string const& url, std::string& endpoint, std::string& path) {
 
   std::stringstream ep;
   path = "/";
   size_t pos = 7;
-  if (i.find("http://")==0) {
+  if (url.find("http://")==0) {
     ep << "tcp://";
-  } else if (i.find("https://")==0) {
+  } else if (url.find("https://")==0) {
     ep << "ssl://";
     ++pos;
   } else {
-    LOG_TOPIC(WARN,Logger::AGENCY) << "Malformed notification URL " << i;
-    return;
+    return false;
   }
   
-  size_t slash_p = i.find("/",pos);
+  size_t slash_p = url.find("/",pos);
   if ((slash_p==std::string::npos)) {
-    ep << i.substr(pos);
+    ep << url.substr(pos);
   } else {
-    ep << i.substr(pos,slash_p-pos);
-    path = i.substr(slash_p);
+    ep << url.substr(pos,slash_p-pos);
+    path = url.substr(slash_p);
+  }
+
+  if (ep.str().find(':')==std::string::npos) {
+    ep << ":8529";
   }
 
   endpoint = ep.str();
+
+  return true;
     
 }
 
@@ -736,31 +742,35 @@ std::vector<bool> Store::apply (
   
   for (auto const& url : urls) {
 
-    Builder tmp; // host
-    tmp.openObject();
-    tmp.add("term",VPackValue(0));
-    tmp.add("index",VPackValue(0));
+    Builder body; // host
+    body.openObject();
+    body.add("term",VPackValue(0));
+    body.add("index",VPackValue(0));
     auto ret = in.equal_range(url);
     
     for (auto it = ret.first; it!=ret.second; ++it) {
-      tmp.add(it->second->key,VPackValue(VPackValueType::Object));
-      tmp.add("op",VPackValue(it->second->oper));
-      tmp.close();
+      body.add(it->second->key,VPackValue(VPackValueType::Object));
+      body.add("op",VPackValue(it->second->oper));
+      body.close();
     }
     
-    tmp.close();
+    body.close();
 
     std::string endpoint, path;
-    endpointPathFromUrl (url,endpoint,path);
+    if (endpointPathFromUrl (url,endpoint,path)) {
 
-    std::unique_ptr<std::map<std::string, std::string>> headerFields =
-      std::make_unique<std::map<std::string, std::string> >();
-    
-    ClusterCommResult res =
-      arangodb::ClusterComm::instance()->asyncRequest(
-        "1", 1, endpoint, GeneralRequest::RequestType::POST, path,
-        std::make_shared<std::string>(body.toString()), headerFields,
-        nullptr, 0.0, true);
+      std::unique_ptr<std::map<std::string, std::string>> headerFields =
+        std::make_unique<std::map<std::string, std::string> >();
+      
+      ClusterCommResult res =
+        arangodb::ClusterComm::instance()->asyncRequest(
+          "1", 1, endpoint, GeneralRequest::RequestType::POST, path,
+          std::make_shared<std::string>(body.toString()), headerFields,
+          std::make_shared<StoreCallback>(), 0.0, true);
+      
+    } else {
+      LOG_TOPIC(WARN, Logger::AGENCY) << "Malformed URL " << url;
+    }
 
   }
   
diff --git a/arangod/Agency/StoreCallback.cpp b/arangod/Agency/StoreCallback.cpp
index 853c6b3aff..f2828c1393 100644
--- a/arangod/Agency/StoreCallback.cpp
+++ b/arangod/Agency/StoreCallback.cpp
@@ -1,12 +1,13 @@
 #include "StoreCallback.h"
-#include "Store.h"
 
 using namespace arangodb::consensus;
 using namespace arangodb::velocypack;
 
-StoreCallback::StoreCallback() {}
 
+StoreCallback::StoreCallback() {}
+#include <iostream>
 bool StoreCallback::operator()(arangodb::ClusterCommResult* res) {
+  std::cout << "CALLED" << std::endl;
   return true;
 }
   
diff --git a/arangod/Agency/StoreCallback.h b/arangod/Agency/StoreCallback.h
index 6c909c9e9d..ff0d855d2f 100644
--- a/arangod/Agency/StoreCallback.h
+++ b/arangod/Agency/StoreCallback.h
@@ -25,7 +25,6 @@
 #define __ARANGODB_CONSENSUS_STORE_CALLBACK__
 
 #include "Cluster/ClusterComm.h"
-#include "AgencyCommon.h"
 
 namespace arangodb {
 namespace consensus {
@@ -36,8 +35,6 @@ public:
   
   StoreCallback();
 
-  explicit StoreCallback();
-
   virtual bool operator()(arangodb::ClusterCommResult*) override final;
 
   void shutdown();
diff --git a/arangod/RestHandler/RestAgencyHandler.cpp b/arangod/RestHandler/RestAgencyHandler.cpp
index f9872542e9..dc274d6134 100644
--- a/arangod/RestHandler/RestAgencyHandler.cpp
+++ b/arangod/RestHandler/RestAgencyHandler.cpp
@@ -76,8 +76,12 @@ void RestAgencyHandler::redirectRequest(id_t leaderId) {
   std::shared_ptr<Endpoint> ep (
     Endpoint::clientFactory (_agent->config().end_points.at(leaderId)));
   std::stringstream url;
-  url << ep->transport() << "://" << ep->hostAndPort()
-      << _request->requestPath();
+
+  url << ep->transport() << "://";
+  if (ep->encryption() == arangodb::Endpoint::EncryptionType::SSL) {
+    url << "s";
+  }
+  url << ep->hostAndPort() << _request->requestPath();
   
   createResponse(GeneralResponse::ResponseCode::TEMPORARY_REDIRECT);
   static std::string const location = "location";

From d80df7407171c4373d8a4e2a4fafa9baee2acc9f Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 12:52:15 +0200
Subject: [PATCH 07/10] introducing cluster sanity checking to agency

---
 arangod/Agency/StoreCallback.cpp | 4 +---
 arangod/Agency/StoreCallback.h   | 8 ++++----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/arangod/Agency/StoreCallback.cpp b/arangod/Agency/StoreCallback.cpp
index f2828c1393..b62c4644cf 100644
--- a/arangod/Agency/StoreCallback.cpp
+++ b/arangod/Agency/StoreCallback.cpp
@@ -3,11 +3,9 @@
 using namespace arangodb::consensus;
 using namespace arangodb::velocypack;
 
-
 StoreCallback::StoreCallback() {}
-#include <iostream>
+
 bool StoreCallback::operator()(arangodb::ClusterCommResult* res) {
-  std::cout << "CALLED" << std::endl;
   return true;
 }
   
diff --git a/arangod/Agency/StoreCallback.h b/arangod/Agency/StoreCallback.h
index ff0d855d2f..b281a9ee0a 100644
--- a/arangod/Agency/StoreCallback.h
+++ b/arangod/Agency/StoreCallback.h
@@ -30,15 +30,15 @@ namespace arangodb {
 namespace consensus {
 
 class StoreCallback : public arangodb::ClusterCommCallback {
-
+  
 public:
   
   StoreCallback();
-
+  
   virtual bool operator()(arangodb::ClusterCommResult*) override final;
-
+  
   void shutdown();
-
+  
 private:
   
 };

From 7d608b160f365f10fc3f9f0cc2285367cf977748 Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 12:56:12 +0200
Subject: [PATCH 08/10] introducing cluster sanity checking to agency

---
 arangod/Agency/Store.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arangod/Agency/Store.cpp b/arangod/Agency/Store.cpp
index ee9b1a9303..8251a52d03 100644
--- a/arangod/Agency/Store.cpp
+++ b/arangod/Agency/Store.cpp
@@ -54,7 +54,7 @@ inline static bool endpointPathFromUrl (
   }
   
   size_t slash_p = url.find("/",pos);
-  if ((slash_p==std::string::npos)) {
+  if (slash_p==std::string::npos) {
     ep << url.substr(pos);
   } else {
     ep << url.substr(pos,slash_p-pos);

From 0efa889f437a19b1d99a679d0b175ca442a3ff18 Mon Sep 17 00:00:00 2001
From: Kaveh Vahedipour <kaveh@arangodb.com>
Date: Tue, 12 Apr 2016 14:10:18 +0200
Subject: [PATCH 09/10] new simple URI form from endpoint

---
 arangod/RestHandler/RestAgencyHandler.cpp |  6 +++++-
 lib/Endpoint/Endpoint.cpp                 | 25 +++++++++++++++++++++--
 lib/Endpoint/Endpoint.h                   |  1 +
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/arangod/RestHandler/RestAgencyHandler.cpp b/arangod/RestHandler/RestAgencyHandler.cpp
index dc274d6134..992929882a 100644
--- a/arangod/RestHandler/RestAgencyHandler.cpp
+++ b/arangod/RestHandler/RestAgencyHandler.cpp
@@ -73,6 +73,7 @@ inline HttpHandler::status_t RestAgencyHandler::reportUnknownMethod() {
 
 void RestAgencyHandler::redirectRequest(id_t leaderId) {
 
+  /*
   std::shared_ptr<Endpoint> ep (
     Endpoint::clientFactory (_agent->config().end_points.at(leaderId)));
   std::stringstream url;
@@ -82,10 +83,13 @@ void RestAgencyHandler::redirectRequest(id_t leaderId) {
     url << "s";
   }
   url << ep->hostAndPort() << _request->requestPath();
+  */
+
+  std::string url = Endpoint::uriForm(_agent->config().end_points.at(leaderId));
   
   createResponse(GeneralResponse::ResponseCode::TEMPORARY_REDIRECT);
   static std::string const location = "location";
-  _response->setHeaderNC(location, url.str());
+  _response->setHeaderNC(location, url);
 }
 
 HttpHandler::status_t RestAgencyHandler::handleStores () {
diff --git a/lib/Endpoint/Endpoint.cpp b/lib/Endpoint/Endpoint.cpp
index 092be6acfc..e60c0dec26 100644
--- a/lib/Endpoint/Endpoint.cpp
+++ b/lib/Endpoint/Endpoint.cpp
@@ -51,6 +51,27 @@ Endpoint::Endpoint(DomainType domainType, EndpointType type,
   TRI_invalidatesocket(&_socket);
 }
 
+
+std::string Endpoint::uriForm (std::string const& endpoint) {
+
+  std::stringstream url;
+  size_t const prefix_len = 6;
+  
+  if (StringUtils::isPrefix(endpoint, "tcp://")) {
+    url << "http://";
+  } else if (StringUtils::isPrefix(endpoint, "ssl://")) {
+    url << "https://";
+  } else {
+    throw arangodb::basics::Exception (
+      0, std::string("malformed URL ") + endpoint
+      + ". Support only for ssl:// and tcp:// endpoints." , __FILE__, __LINE__);
+  }
+
+  url << endpoint.substr(prefix_len,endpoint.size()+1-(prefix_len+1));
+  
+  return url.str();
+  
+}
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief return the endpoint specification in a unified form
 ////////////////////////////////////////////////////////////////////////////////
@@ -68,9 +89,9 @@ std::string Endpoint::unifiedForm(std::string const& specification) {
   std::string copy = StringUtils::tolower(specification);
   StringUtils::trimInPlace(copy);
 
-  if (specification[specification.size() - 1] == '/') {
+  if (specification.back() == '/') {
     // address ends with a slash => remove
-    copy = copy.substr(0, copy.size() - 1);
+    copy.pop_back();
   }
 
   // read protocol from string
diff --git a/lib/Endpoint/Endpoint.h b/lib/Endpoint/Endpoint.h
index 3e63d5acf1..d0c73c4f08 100644
--- a/lib/Endpoint/Endpoint.h
+++ b/lib/Endpoint/Endpoint.h
@@ -52,6 +52,7 @@ class Endpoint {
   virtual ~Endpoint() {}
 
  public:
+  static std::string uriForm(std::string const&);
   static std::string unifiedForm(std::string const&);
   static Endpoint* serverFactory(std::string const&, int, bool reuseAddress);
   static Endpoint* clientFactory(std::string const&);

From 7b90f3040c09b923e5d99667430b660c325679c5 Mon Sep 17 00:00:00 2001
From: Andreas Streichardt <andreas@arangodb.com>
Date: Tue, 12 Apr 2016 14:10:38 +0200
Subject: [PATCH 10/10] parallelize cluster instance startup during test

---
 js/client/modules/@arangodb/testing.js | 67 ++++++++++++++++++--------
 1 file changed, 47 insertions(+), 20 deletions(-)

diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js
index 390e17e826..ec28f89d11 100644
--- a/js/client/modules/@arangodb/testing.js
+++ b/js/client/modules/@arangodb/testing.js
@@ -1232,6 +1232,26 @@ function startInstanceCluster(instanceInfo, protocol, options,
   httpOptions.method = 'POST';
   httpOptions.returnBodyOnError = true;
   
+  let count = 0;
+  instanceInfo.arangods.forEach(arangod => {
+    while (true) {
+      const reply = download(arangod.url + "/_api/version", "", makeAuthorizationHeaders(options));
+
+      if (!reply.error && reply.code === 200) {
+        break;
+      }
+
+      ++count;
+
+      if (count % 60 === 0) {
+        if (!checkArangoAlive(arangod, options)) {
+          throw new Error("startup failed! bailing out!");
+        }
+      }
+      wait(0.5, false);
+    }
+  });
+  
   response = download(coordinatorUrl + '/_admin/cluster/bootstrapDbServers', '{"isRelaunch":false}', httpOptions);
 
   while (response.code !== 200) {
@@ -1294,24 +1314,6 @@ function startArango(protocol, options, addArgs, name, rootDir) {
   const startTime = time();
   instanceInfo.pid = executeValgrind(ARANGOD_BIN, toArgv(args), options, name).pid;
 
-  let count = 0;
-  while (true) {
-    wait(0.5, false);
-
-    const reply = download(instanceInfo.url + "/_api/version", "", makeAuthorizationHeaders(options));
-
-    if (!reply.error && reply.code === 200) {
-      break;
-    }
-
-    ++count;
-
-    if (count % 60 === 0) {
-      if (!checkArangoAlive(instanceInfo, options)) {
-        throw new Error("startup failed! bailing out!");
-      }
-    }
-  }
   if (platform.substr(0, 3) === 'win') {
     const procdumpArgs = [
       '-accepteula',
@@ -1371,6 +1373,7 @@ function startInstanceSingleServer(instanceInfo, protocol, options,
   addArgs, testname, rootDir) {
 
   instanceInfo.arangods.push(startArango(protocol, options, addArgs, testname, rootDir));
+  
   instanceInfo.endpoint = instanceInfo.arangods[instanceInfo.arangods.length - 1].endpoint;
   instanceInfo.url = instanceInfo.arangods[instanceInfo.arangods.length - 1].url;
 
@@ -1380,7 +1383,7 @@ function startInstanceSingleServer(instanceInfo, protocol, options,
 function startInstance(protocol, options, addArgs, testname, tmpDir) {
   let rootDir = fs.join(tmpDir || fs.getTempFile(), testname);
   let instanceInfo = {rootDir, arangods: []};
-
+  
   try {
     if (options.cluster) {
       startInstanceCluster(instanceInfo, protocol, options,
@@ -1390,12 +1393,36 @@ function startInstance(protocol, options, addArgs, testname, tmpDir) {
           addArgs, testname, rootDir);
     } else {
       startInstanceSingleServer(instanceInfo, protocol, options,
-            addArgs, testname, rootDir);
+          addArgs, testname, rootDir);
+    }
+    
+    if (!options.cluster) {
+      let count = 0;
+      instanceInfo.arangods.forEach(arangod => {
+        while (true) {
+          const reply = download(arangod.url + "/_api/version", "", makeAuthorizationHeaders(options));
+
+          if (!reply.error && reply.code === 200) {
+            break;
+          }
+
+          ++count;
+
+          if (count % 60 === 0) {
+            if (!checkArangoAlive(arangod, options)) {
+              throw new Error("startup failed! bailing out!");
+            }
+          }
+          wait(0.5, false);
+        }
+      });
     }
   } catch (e) {
     print(e, e.stack);
     return false;
   }
+  
+  
 
   return instanceInfo;
 }