Merge branch 'devel' of https://github.com/arangodb/arangodb into generic-col-types

3rdParty/V8/CMakeLists.txt

@@ -61,10 +61,11 @@ list(APPEND V8_GYP_ARGS
 )
 
 if (CROSS_COMPILING)
-  list(APPEND V8_GYP_ARGS -DGYP_CROSSCOMPILE=1)
+  list(APPEND V8_GYP_ARGS
+  -Dhost_arch=${V8_PROC_ARCH}
+  -DGYP_CROSSCOMPILE=1)
 endif()
 
-
 ################################################################################
 ## ICU EXPORTS
 ################################################################################

Installation/Jenkins/build.sh

@@ -300,6 +300,16 @@ while [ $# -gt 0 ];  do
             CLEAN_IT=1
             shift
             ;;
+        --cxArmV8)
+            ARMV8=1
+            CXGCC=1
+            shift
+            ;;
+        --cxArmV7)
+            ARMV7=1
+            CXGCC=1
+            shift
+            ;;
         *)
             echo "Unknown option: $1"
             exit 1
@@ -330,9 +340,31 @@ elif [ "$CLANG36" == 1 ]; then
     CC=/usr/bin/clang-3.6
     CXX=/usr/bin/clang++-3.6
     CXXFLAGS="${CXXFLAGS} -std=c++11"
+elif [ "${CXGCC}" = 1 ]; then
+    if [ "${ARMV8}" = 1 ]; then
+        export TOOL_PREFIX=aarch64-linux-gnu
+        BUILD_DIR="${BUILD_DIR}-ARMV8"
+    elif [ "${ARMV7}" = 1 ]; then
+        export TOOL_PREFIX=aarch64-linux-gnu
+        BUILD_DIR="${BUILD_DIR}-ARMV7"
+    else
+        echo "Unknown CX-Compiler!"
+        exit 1;
+    fi
+
+    export CXX=$TOOL_PREFIX-g++
+    export AR=$TOOL_PREFIX-ar
+    export RANLIB=$TOOL_PREFIX-ranlib
+    export CC=$TOOL_PREFIX-gcc
+    export LD=$TOOL_PREFIX-g++
+    export LINK=$TOOL_PREFIX-g++
+    export STRIP=$TOOL_PREFIX-strip
+    CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true"
 fi
 
 
+
+
 if [ "$SANITIZE" == 1 ]; then
     if [ "$GCC5" == 1 ]; then
         CFLAGS="${CFLAGS} -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=vptr"
@@ -415,7 +447,7 @@ SOURCE_DIR=`compute_relative ${DST}/ ${SRC}/`
 
 if [ ! -f Makefile -o ! -f CMakeCache.txt ];  then
     CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" LDFLAGS="${LDFLAGS}" LIBS="${LIBS}" \
-          cmake ${SOURCE_DIR} ${CONFIGURE_OPTIONS} -G "${GENERATOR}"
+          cmake ${SOURCE_DIR} ${CONFIGURE_OPTIONS} -G "${GENERATOR}" || exit 1
 fi
 
 ${MAKE_CMD_PREFIX} ${MAKE} ${MAKE_PARAMS}

arangod/Agency/AgencyCommon.h

@@ -44,6 +44,8 @@ typedef uint64_t term_t;
 /// @brief Agent roles
 enum role_t {FOLLOWER, CANDIDATE, LEADER};
 
+static const std::string NO_LEADER = "";
+
 
 /// @brief Duration type
 typedef std::chrono::duration<long, std::ratio<1, 1000>> duration_t;

arangod/Agency/Agent.cpp

@@ -199,6 +199,8 @@ void Agent::reportIn(std::string const& id, index_t index) {
 
   MUTEX_LOCKER(mutexLocker, _ioLock);
 
+  _lastAcked[id] = std::chrono::system_clock::now();
+
   if (index > _confirmed[id]) {  // progress this follower?
     _confirmed[id] = index;
   }
@@ -342,7 +344,7 @@ priv_rpc_ret_t Agent::sendAppendEntriesRPC(std::string const& follower_id) {
   builder.close();
 
   // Verbose output
-  if (unconfirmed.size() > 1) {
+ if (unconfirmed.size() > 1) {
     LOG_TOPIC(DEBUG, Logger::AGENCY) << "Appending " << unconfirmed.size() - 1
                                      << " entries up to index " << highest
                                      << " to follower " << follower_id;
@@ -477,11 +479,27 @@ write_ret_t Agent::write(query_t const& query) {
 /// Read from store
 read_ret_t Agent::read(query_t const& query) {
 
+  MUTEX_LOCKER(mutexLocker, _ioLock);
+
   // Only leader else redirect
   if (!_constituent.leading()) {
     return read_ret_t(false, _constituent.leaderID());
   }
 
+  // Still leading?
+  size_t good = 0; 
+  for (auto const& i : _lastAcked) {
+    std::chrono::duration<double> m =
+      std::chrono::system_clock::now() - i.second;
+    if(0.9*_config.minPing() > m.count()) {
+      ++good;
+    }
+  }
+  
+  if (good < size() / 2) {
+    _constituent.candidate();
+  }
+  
   // Retrieve data from readDB
   auto result = std::make_shared<arangodb::velocypack::Builder>();
   std::vector<bool> success = _readDB.read(query, result);
@@ -558,7 +576,27 @@ bool Agent::lead() {
     CONDITION_LOCKER(guard, _appendCV);
     guard.broadcast();
   }
+
+  for (auto const& i : _config.active()) {
+    _lastAcked[i] = std::chrono::system_clock::now();
+  }
   
+  // Agency configuration
+  auto agency = std::make_shared<Builder>();
+  agency->openArray();
+  agency->openArray();
+  agency->openObject();
+  agency->add(".agency", VPackValue(VPackValueType::Object));
+  agency->add("term", VPackValue(term()));
+  agency->add("id", VPackValue(id()));
+  agency->add("active", _config.activeToBuilder()->slice());
+  agency->add("pool", _config.poolToBuilder()->slice());
+  agency->close();  
+  agency->close();  
+  agency->close();
+  agency->close();
+  write(agency);
+
   // Wake up supervision
   _supervision.wakeUp();
 

arangod/Agency/Agent.h

@@ -220,6 +220,7 @@ class Agent : public arangodb::Thread {
   std::map<std::string, index_t> _confirmed;
   std::map<std::string, index_t> _lastHighest;
 
+  std::map<std::string, TimePoint> _lastAcked;
   std::map<std::string, TimePoint> _lastSent;
   arangodb::Mutex _ioLock; /**< @brief Read/Write lock */
 

arangod/Agency/AgentConfiguration.cpp

@@ -243,6 +243,19 @@ query_t config_t::activeToBuilder () const {
   return ret;
 }
 
+query_t config_t::activeAgentsToBuilder () const {
+  query_t ret = std::make_shared<arangodb::velocypack::Builder>();
+  ret->openObject();
+  {
+    READ_LOCKER(readLocker, _lock);
+    for (auto const& i : _active) {
+      ret->add(i, VPackValue(_pool.at(i)));
+    }
+  }
+  ret->close();
+  return ret;
+}
+
 query_t config_t::poolToBuilder () const {
   query_t ret = std::make_shared<arangodb::velocypack::Builder>();
   ret->openObject();

arangod/Agency/AgentConfiguration.h

@@ -142,6 +142,7 @@ struct config_t {
 
   /// @brief of active agents
   query_t activeToBuilder () const;
+  query_t activeAgentsToBuilder () const;
   query_t poolToBuilder () const;
 
   

arangod/Agency/Constituent.cpp

@@ -49,7 +49,6 @@ using namespace arangodb::rest;
 using namespace arangodb::velocypack;
 using namespace arangodb;
 
-static const std::string NO_LEADER = "";
 //  (std::numeric_limits<std::string>::max)();
 
 /// Raft role names for display purposes 

arangod/Agency/Inception.cpp

@@ -51,18 +51,17 @@ bool Inception::start() { return Thread::start(); }
 /// - Get snapshot of gossip peers and agent pool
 /// - Create outgoing gossip.
 /// - Send to all peers
-void Inception::run() {
 
-  TRI_ASSERT(_agent != nullptr);
+void Inception::gossip() {
 
   auto s = std::chrono::system_clock::now();
   std::chrono::seconds timeout(120);
   size_t i = 0;
-  //bool cs = false;
+
   while (!this->isStopping()) {
 
     config_t config = _agent->config(); // get a copy of conf
-
+    
     query_t out = std::make_shared<Builder>();
     out->openObject();
     out->add("endpoint", VPackValue(config.endpoint()));
@@ -73,7 +72,7 @@ void Inception::run() {
     }
     out->close();
     out->close();
-
+    
     std::string path = "/_api/agency_priv/gossip";
 
     for (auto const& p : config.gossipPeers()) { // gossip peers
@@ -99,9 +98,9 @@ void Inception::run() {
           std::make_shared<GossipCallback>(_agent), 1.0, true);
       }
     }
-
+    
     std::this_thread::sleep_for(std::chrono::milliseconds(250));
-
+    
     if ((std::chrono::system_clock::now()-s) > timeout) {
       if (config.poolComplete()) {
         LOG_TOPIC(DEBUG, Logger::AGENCY) << "Stopping active gossipping!";
@@ -111,17 +110,64 @@ void Inception::run() {
       }
       break;
     }
-
+    
     if (config.poolComplete()) {
-      //if(!cs) {
-        _agent->startConstituent();
-        break;
-        //cs = true;
-        //}
+      _agent->startConstituent();
+      break;
     }
-
+    
   }
+  
+}
 
+void Inception::activeAgency() { // Do we have an active agency?
+/*
+  config_t config = _agent->config(); // get a copy of conf
+  size_t i = 0;
+  std::string const path = "/_api/agency/activeAgents";
+
+  for (auto const& endpoint : config.gossipPeers()) { // gossip peers
+    if (endpoint != config.endpoint()) {
+      std::string clientid = config.id() + std::to_string(i++);
+      auto hf = std::make_unique<std::unordered_map<std::string, std::string>>();
+      arangodb::ClusterComm::instance()->asyncRequest(
+        clientid, 1, endpoint, GeneralRequest::RequestType::POST, path,
+        std::make_shared<std::string>(out->toJson()), hf,
+        std::make_shared<GossipCallback>(_agent), 1.0, true);
+    }
+  }
+  
+  for (auto const& pair : config.pool()) { // pool entries
+    if (pair.second != config.endpoint()) {
+      std::string clientid = config.id() + std::to_string(i++);
+      auto hf = std::make_unique<std::unordered_map<std::string, std::string>>();
+      arangodb::ClusterComm::instance()->asyncRequest(
+        clientid, 1, pair.second, GeneralRequest::RequestType::POST, path,
+        std::make_shared<std::string>(out->toJson()), hf,
+        std::make_shared<GossipCallback>(_agent), 1.0, true);
+    }
+  }
+*/  
+  // start in pool/gossi peers start check if active agency
+
+  // if not if i have persisted agency 
+  // if member 
+  // contact other agents.
+  // if agreement raft
+
+  // complete pool?
+  
+}
+
+void Inception::run() {
+
+  //activeAgency();
+
+  config_t config = _agent->config();
+  if (!config.poolComplete()) {
+    gossip();
+  }
+  
 }
 
 

arangod/Agency/Inception.h

@@ -45,7 +45,7 @@ public:
   Inception();
   explicit Inception(Agent*);
   virtual ~Inception();
-  
+
   void run() override;
   bool start();
   
@@ -54,6 +54,9 @@ public:
   
 private:
 
+  void activeAgency();
+  void gossip();
+
   Agent* _agent;
   
 };

arangod/Agency/RestAgencyPrivHandler.cpp

@@ -130,6 +130,9 @@ RestHandler::status RestAgencyPrivHandler::execute() {
           return reportBadQuery();  // bad query
         }
       } else if (_request->suffix()[0] == "gossip") {
+        if (_request->requestType() != rest::RequestType::POST) {
+          return reportMethodNotAllowed();
+        }
         arangodb::velocypack::Options options;
         query_t query = _request->toVelocyPackBuilderPtr(&options);
         try {
@@ -140,6 +143,13 @@ RestHandler::status RestAgencyPrivHandler::execute() {
         } catch (std::exception const& e) {
           return reportBadQuery(e.what());
         }
+      } else if (_request->suffix()[0] == "activeAgents") {
+        if (_request->requestType() != rest::RequestType::GET) {
+          return reportMethodNotAllowed();
+        }
+        if (_agent->leaderID() != NO_LEADER) {
+          result.add("active", _agent->config().activeAgentsToBuilder()->slice());
+        }
       } else if (_request->suffix()[0] == "inform") {
         arangodb::velocypack::Options options;
         query_t query = _request->toVelocyPackBuilderPtr(&options);

arangod/Agency/Supervision.cpp

@@ -316,7 +316,9 @@ bool Supervision::updateSnapshot() {
   if (_agent == nullptr || this->isStopping()) {
     return false;
   }
-  _snapshot = _agent->readDB().get(_agencyPrefix);
+  try {
+    _snapshot = _agent->readDB().get(_agencyPrefix);
+  } catch (...) {}
   return true;
 }
 
@@ -339,7 +341,7 @@ void Supervision::run() {
     // make sense at all without other ArangoDB servers, we wait pretty
     // long here before giving up:
     if (!updateAgencyPrefix(1000, 1)) {
-      LOG_TOPIC(ERR, Logger::AGENCY)
+      LOG_TOPIC(DEBUG, Logger::AGENCY)
         << "Cannot get prefix from Agency. Stopping supervision for good.";
       return;
     }

arangod/Cluster/ClusterComm.cpp

@@ -1348,6 +1348,11 @@ size_t ClusterComm::performSingleRequest(
     THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_BACKEND_UNAVAILABLE);
   }
 
+  if (req.result.status == CL_COMM_ERROR && req.result.result != nullptr
+      && req.result.result->getHttpReturnCode() == 503) {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_CLUSTER_BACKEND_UNAVAILABLE);
+  }
+  
   // Add correct recognition of content type later.
   req.result.status = CL_COMM_RECEIVED;  // a fake, but a good one
   req.done = true;

arangod/RestServer/arangod.cpp

@@ -179,6 +179,10 @@ static int runServer(int argc, char** argv) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangod terminated because of an unhandled exception: "
              << ex.what();

arangod/VocBase/LogicalCollection.cpp

@@ -420,7 +420,7 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, VPackSlice info)
 
     auto indexesSlice = info.get("indexes");
     if (indexesSlice.isArray()) {
-      TRI_ASSERT(ServerState::instance()->isRunningInCluster());
+      bool const isCluster = ServerState::instance()->isRunningInCluster();
       for (auto const& v : VPackArrayIterator(indexesSlice)) {
         if (arangodb::basics::VelocyPackHelper::getBooleanValue(
                 v, "error", false)) {
@@ -430,7 +430,11 @@ LogicalCollection::LogicalCollection(TRI_vocbase_t* vocbase, VPackSlice info)
           continue;
         }
         auto idx = PrepareIndexFromSlice(v, false, this, true);
-        addIndexCoordinator(idx, false);
+        if (isCluster) {
+          addIndexCoordinator(idx, false);
+        } else {
+          addIndex(idx);
+        }
       }
     }
 

arangod/VocBase/collection.cpp

@@ -103,8 +103,6 @@ TRI_collection_t::~TRI_collection_t() {
 
 /// @brief whether or not a collection is fully collected
 bool TRI_collection_t::isFullyCollected() {
-  READ_LOCKER(readLocker, _lock);
-
   int64_t uncollected = _uncollectedLogfileEntries.load();
 
   return (uncollected == 0);

arangod/VocBase/collection.h

@@ -25,7 +25,6 @@
 #define ARANGOD_VOC_BASE_COLLECTION_H 1
 
 #include "Basics/Common.h"
-#include "Basics/ReadWriteLock.h"
 #include "Cluster/ClusterInfo.h"
 #include "VocBase/Ditch.h"
 #include "VocBase/MasterPointer.h"
@@ -194,7 +193,6 @@ struct TRI_collection_t {
   
  private:
   mutable arangodb::Ditches _ditches;
-  mutable arangodb::basics::ReadWriteLock _lock;  // lock protecting the indexes
 
   arangodb::Mutex _compactionStatusLock;
   size_t _nextCompactionStartIndex;

arangosh/Benchmark/arangobench.cpp

@@ -66,6 +66,10 @@ int main(int argc, char* argv[]) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangobench terminated because of an unhandled exception: "
              << ex.what();

arangosh/Dump/arangodump.cpp

@@ -62,6 +62,10 @@ int main(int argc, char* argv[]) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangodump terminated because of an unhandled exception: "
              << ex.what();

arangosh/Import/arangoimp.cpp

@@ -64,6 +64,10 @@ int main(int argc, char* argv[]) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangoimp terminated because of an unhandled exception: "
              << ex.what();

arangosh/Restore/arangorestore.cpp

@@ -64,6 +64,10 @@ int main(int argc, char* argv[]) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangorestore terminated because of an unhandled exception: "
              << ex.what();

arangosh/Shell/arangosh.cpp

@@ -74,6 +74,10 @@ int main(int argc, char* argv[]) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangosh terminated because of an unhandled exception: "
              << ex.what();

arangosh/VPack/arangovpack.cpp

@@ -58,6 +58,10 @@ int main(int argc, char* argv[]) {
 
   try {
     server.run(argc, argv);
+    if (server.helpShown()) {
+      // --help was displayed
+      ret = EXIT_SUCCESS;
+    }
   } catch (std::exception const& ex) {
     LOG(ERR) << "arangovpack terminated because of an unhandled exception: "
              << ex.what();

lib/ApplicationFeatures/ApplicationServer.cpp

@@ -166,6 +166,11 @@ void ApplicationServer::run(int argc, char* argv[]) {
   // file(s)
   parseOptions(argc, argv);
 
+  if (!_helpSection.empty()) {
+    // help shown. we can exit early
+    return;
+  }
+
   // seal the options
   _options->seal();
 
@@ -285,17 +290,17 @@ void ApplicationServer::collectOptions() {
 void ApplicationServer::parseOptions(int argc, char* argv[]) {
   ArgumentParser parser(_options.get());
 
-  std::string helpSection = parser.helpSection(argc, argv);
+  _helpSection = parser.helpSection(argc, argv);
 
-  if (!helpSection.empty()) {
+  if (!_helpSection.empty()) {
     // user asked for "--help"
 
     // translate "all" to "*"
-    if (helpSection == "all") {
-      helpSection = "*";
+    if (_helpSection == "all") {
+      _helpSection = "*";
     }
-    _options->printHelp(helpSection);
-    exit(EXIT_SUCCESS);
+    _options->printHelp(_helpSection);
+    return;
   }
 
   if (!parser.parse(argc, argv)) {

lib/ApplicationFeatures/ApplicationServer.h

@@ -166,6 +166,9 @@ class ApplicationServer {
 
   ~ApplicationServer();
 
+  std::string helpSection() const { return _helpSection; }
+  bool helpShown() const { return !_helpSection.empty(); }
+
   // adds a feature to the application server. the application server
   // will take ownership of the feature object and destroy it in its
   // destructor
@@ -297,6 +300,9 @@ class ApplicationServer {
 
   // reporter for progress
   std::vector<ProgressHandler> _progressReports;
+
+  // help section displayed
+  std::string _helpSection;
 };
 }
 }

scripts/startLocalCluster.sh

@@ -69,7 +69,7 @@ for aid in `seq 0 $(( $NRAGENTS - 1 ))`; do
         --agency.pool-size $NRAGENTS \
         --agency.supervision true \
         --agency.supervision-frequency $SFRE \
-        --agency.wait-for-sync true \
+        --agency.wait-for-sync false \
         --agency.election-timeout-min $MINP \
         --agency.election-timeout-max $MAXP \
         --database.directory cluster/data$port \