agencycommanager ran in locks when ::redirect called ::failed

arangod/Agency/AgencyComm.cpp

@@ -484,6 +484,14 @@ void AgencyCommManager::failed(
   std::unique_ptr<httpclient::GeneralClientConnection> connection,
   std::string const& endpoint) {
   MUTEX_LOCKER(locker, _lock);
+  failedNonLocking(std::move(connection), endpoint);
+
+}
+
+
+void AgencyCommManager::failedNonLocking(
+  std::unique_ptr<httpclient::GeneralClientConnection> connection,
+  std::string const& endpoint) {
   
   if (_endpoints.front() == endpoint) {
     LOG_TOPIC(TRACE, Logger::AGENCYCOMM)
@@ -501,6 +509,8 @@ void AgencyCommManager::failed(
   
 }
 
+
+
 std::string AgencyCommManager::redirect(
     std::unique_ptr<httpclient::GeneralClientConnection> connection,
     std::string const& endpoint, std::string const& location,
@@ -520,7 +530,7 @@ std::string AgencyCommManager::redirect(
 
   // invalid location header
   if (delim == std::string::npos) {
-    failed(std::move(connection), endpoint);
+    failedNonLocking(std::move(connection), endpoint);
     return "";
   }
 
@@ -534,7 +544,7 @@ std::string AgencyCommManager::redirect(
   if (endpoint == specification) {
     LOG_TOPIC(WARN, Logger::AGENCYCOMM)
       << "got an agency redirect back to the old agency '" << endpoint << "'";
-    failed(std::move(connection), endpoint);
+    failedNonLocking(std::move(connection), endpoint);
     return "";
   }
 
@@ -560,6 +570,7 @@ std::string AgencyCommManager::redirect(
                                       << "'";
 
   _endpoints.push_front(specification);
+  LOG(WARN) << __FILE__ << __LINE__;
   return specification;
 }
 
@@ -1219,9 +1230,9 @@ AgencyCommResult AgencyComm::sendWithFailover(
     // in this case we have to pick it up and use the new location returned
     if (result._statusCode ==
         (int)arangodb::rest::ResponseCode::TEMPORARY_REDIRECT) {
-      std::string red = AgencyCommManager::MANAGER->redirect(
+      endpoint = AgencyCommManager::MANAGER->redirect(
         std::move(connection), endpoint, result._location, url);
-      connection = AgencyCommManager::MANAGER->acquire(red);
+      connection = AgencyCommManager::MANAGER->acquire(endpoint);
       continue;
     }
 

arangod/Agency/AgencyComm.h

@@ -441,6 +441,11 @@ class AgencyCommManager {
   std::vector<std::string> endpoints() const;
 
  private:
+
+  // caller must hold _lock
+  void failedNonLocking(std::unique_ptr<httpclient::GeneralClientConnection>,
+              std::string const& endpoint);
+
   // caller must hold _lock
   std::unique_ptr<httpclient::GeneralClientConnection> createNewConnection();
 

scripts/startLocalCluster.sh

@@ -177,6 +177,7 @@ fi
 echo Starting agency ... 
 for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do
     port=$(( $BASE + $aid ))
+    AGENCY_ENDPOINTS+="--cluster.agency-endpoint $TRANSPORT://localhost:$port "
     ${BUILD}/bin/arangod \
         -c none \
         --agency.activate true \
@@ -221,7 +222,7 @@ start() {
     ${BUILD}/bin/arangod \
        -c none \
        --database.directory cluster/data$PORT \
-       --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \
+       $AGENCY_ENDPOINTS \
        --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \
        --server.endpoint $TRANSPORT://0.0.0.0:$PORT \
        --cluster.my-role $ROLE \
@@ -233,6 +234,7 @@ start() {
        --javascript.module-directory ./enterprise/js \
        --javascript.app-path cluster/apps$PORT \
        --log.force-direct true \
+       --log.thread true \
         $LOG_LEVEL_CLUSTER \
         $AUTHENTICATION \
         $SSLKEYFILE \
@@ -252,7 +254,7 @@ startTerminal() {
     $XTERM $XTERMOPTIONS -e ${BUILD}/bin/arangod \
         -c none \
         --database.directory cluster/data$PORT \
-        --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \
+        $AGENCY_ENDPOINTS \
         --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \
         --server.endpoint $TRANSPORT://0.0.0.0:$PORT \
         --cluster.my-role $ROLE \
@@ -281,7 +283,7 @@ startDebugger() {
     ${BUILD}/bin/arangod \
       -c none \
       --database.directory cluster/data$PORT \
-      --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \
+      $AGENCY_ENDPOINTS \
       --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \
       --server.endpoint $TRANSPORT://0.0.0.0:$PORT \
       --cluster.my-role $ROLE \
@@ -310,7 +312,7 @@ startRR() {
     $XTERM $XTERMOPTIONS -e rr ${BUILD}/bin/arangod \
         -c none \
         --database.directory cluster/data$PORT \
-        --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \
+        $AGENCY_ENDPOINTS \
         --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \
         --server.endpoint $TRANSPORT://0.0.0.0:$PORT \
         --cluster.my-role $ROLE \
@@ -395,7 +397,7 @@ if [ "$SECONDARIES" == "1" ] ; then
         ${BUILD}/bin/arangod \
             -c none \
             --database.directory cluster/data$PORT \
-            --cluster.agency-endpoint $TRANSPORT://127.0.0.1:$BASE \
+            $AGENCY_ENDPOINTS \
             --cluster.my-address $TRANSPORT://127.0.0.1:$PORT \
             --server.endpoint $TRANSPORT://0.0.0.0:$PORT \
             --cluster.my-id $CLUSTER_ID \