Merge branch 'devel' of github.com:arangodb/arangodb into devel

Documentation/Books/Manual/Graphs/README.mdpp

@@ -100,6 +100,8 @@ ArangoDB comes with a set of easily graspable graphs that are used to demonstrat
 You can use the `add samples` tab in the `create graph` window in the webinterface, or load the module `@arangodb/graph-examples/example-graph` in arangosh and use it to create instances of these graphs in your ArangoDB.
 Once you've created them, you can [inspect them in the webinterface](../Administration/WebInterface/Graphs.md) - which was used to create the pictures below.
 
+You [can easily look into the innards of this script](https://github.com/arangodb/arangodb/blob/devel/js/common/modules/%40arangodb/graph-examples/example-graph.js) for reference about howto manage graphs programatically.
+
 !SUBSUBSECTION The Knows\_Graph
 
 A set of persons knowing each other:

Documentation/Books/Manual/Indexing/Fulltext.mdpp

@@ -1,9 +1,9 @@
 !CHAPTER Fulltext indexes
 
-!SUBSECTION Introduction to Fulltext Indexes
-
 This is an introduction to ArangoDB's fulltext indexes.
 
+!SUBSECTION Introduction to Fulltext Indexes
+
 A fulltext index can be used to find words, or prefixes of words inside documents. 
 
 A fulltext index can be defined on one attribute only, and will include all words contained in 
@@ -46,6 +46,7 @@ Other data types are ignored and not indexed.
 <!-- js/server/modules/@arangodb/arango-collection.js -->
 
 Ensures that a fulltext index exists:
+
 `collection.ensureIndex({ type: "fulltext", fields: [ "field" ], minLength: minLength })`
 
 Creates a fulltext index on all documents on attribute *field*.
@@ -84,6 +85,7 @@ details is returned.
 
 
 Looks up a fulltext index:
+
 `collection.lookupFulltextIndex(attribute, minLength)`
 
 Checks whether a fulltext index on the given attribute *attribute* exists.

arangod/Agency/Node.cpp

@@ -730,6 +730,14 @@ uint64_t Node::getUInt() const {
   
 }
 
+bool Node::getBool() const {
+  if (type() == NODE) {
+    throw StoreException("Must not convert NODE type to bool");
+  }
+  return slice().getBool();
+  
+}
+
 double Node::getDouble() const {
 
   if (type() == NODE) {

arangod/Agency/Node.h

@@ -258,6 +258,8 @@ public:
   /// @brief Get insigned value (throws if type NODE or if conversion fails)
   uint64_t getUInt() const;
 
+  /// @brief Get bool value (throws if type NODE or if conversion fails)
+  bool getBool() const;
   
   /// @brief Get double value (throws if type NODE or if conversion fails)
   double getDouble() const;

arangod/Agency/Supervision.cpp

@@ -31,6 +31,7 @@
 #include "Job.h"
 #include "Store.h"
 
+#include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/ConditionLocker.h"
 #include "VocBase/server.h"
 
@@ -38,6 +39,7 @@
 using namespace arangodb;
 
 using namespace arangodb::consensus;
+using namespace arangodb::application_features;
 
 std::string Supervision::_agencyPrefix = "/arango";
 
@@ -326,45 +328,73 @@ bool Supervision::doChecks() {
 }
 
 void Supervision::run() {
-
   CONDITION_LOCKER(guard, _cv);
   TRI_ASSERT(_agent != nullptr);
   
-  while (!this->isStopping()) {
+  // Get agency prefix after cluster init
+  if (_jobId == 0) {
+    // We need the agency prefix to work, but it is only initialized by
+    // some other server in the cluster. Since the supervision does not
+    // make sense at all without other ArangoDB servers, we wait pretty
+    // long here before giving up:
+    if (!updateAgencyPrefix(1000, 1)) {
+      LOG_TOPIC(ERR, Logger::AGENCY)
+        << "Cannot get prefix from Agency. Stopping supervision for good.";
+      return;
+    }
+  }
   
-    // Get agency prefix after cluster init
+  while (!this->isStopping()) {
-    if (_jobId == 0) {
+    updateSnapshot();
-      // We need the agency prefix to work, but it is only initialized by
+    if (isShuttingDown()) {
-      // some other server in the cluster. Since the supervision does not
+      handleShutdown();
-      // make sense at all without other ArangoDB servers, we wait pretty
+    } else if (_agent->leading()) {
-      // long here before giving up:
+      if (!handleJobs()) {
-      if (!updateAgencyPrefix(1000, 1)) {
-        LOG_TOPIC(DEBUG, Logger::AGENCY)
-          << "Cannot get prefix from Agency. Stopping supervision for good.";
         break;
       }
     }
+    _cv.wait(_frequency * 1000000);
+  }
+}
 
-    // Get bunch of job IDs from agency for future jobs
+bool Supervision::isShuttingDown() {
-    if (_jobId == 0 || _jobId == _jobIdMax) {
+  try {
-      getUniqueIds();  // cannot fail but only hang
+    return _snapshot("/Shutdown").getBool();
+  } catch (...) {
+    return false;
+  }
+}
+
+void Supervision::handleShutdown() {
+  LOG_TOPIC(DEBUG, Logger::AGENCY) << "Initiating shutdown";
+  Node::Children const& serversRegistered = _snapshot(currentServersRegisteredPrefix).children();
+  bool serversCleared = true;
+  for (auto const& server : serversRegistered) {
+    if (server.first == "Version") {
+      continue;
     }
-    
+    LOG_TOPIC(DEBUG, Logger::AGENCY)
-    // Do nothing unless leader 
+      << "Waiting for " << server.first << " to shutdown";
-    if (_agent->leading()) {
+    serversCleared = false;
-      _cv.wait(_frequency * 1000000); 
-    } else {
-      _cv.wait();
-    }
-    
-    // Do supervision
-    updateSnapshot();
-    doChecks();
-    shrinkCluster();
-    workJobs();
-    
   }
 
+  if (serversCleared) {
+    ApplicationServer::server->beginShutdown();
+  }
+}
+
+bool Supervision::handleJobs() {
+  // Get bunch of job IDs from agency for future jobs
+  if (_jobId == 0 || _jobId == _jobIdMax) {
+    getUniqueIds();  // cannot fail but only hang
+  }
+
+  // Do supervision
+  doChecks();
+  shrinkCluster();
+  workJobs();
+  
+  return true;
 }
 
 void Supervision::workJobs() {

arangod/Agency/Supervision.h

@@ -145,6 +145,11 @@ class Supervision : public arangodb::Thread {
 
   void shrinkCluster();
 
+  bool isShuttingDown();
+  
+  bool handleJobs();
+  void handleShutdown();
+
   Agent* _agent; /**< @brief My agent */
   Node _snapshot;
 

arangod/Cluster/HeartbeatThread.cpp

@@ -48,6 +48,7 @@
 #include "VocBase/vocbase.h"
 
 using namespace arangodb;
+using namespace arangodb::application_features;
 
 std::atomic<bool> HeartbeatThread::HasRunOnce(false);
 
@@ -190,23 +191,30 @@ void HeartbeatThread::runDBServer() {
       LOG_TOPIC(TRACE, Logger::HEARTBEAT)
           << "Looking at Sync/Commands/" + _myId;
     
-      AgencyCommResult result = _agency.getValues("Sync/Commands/" + _myId);
+      AgencyReadTransaction trx(std::vector<std::string>(
+        {_agency.prefixPath() + "Shutdown",
+         _agency.prefixPath() + "Current/Version",
+         _agency.prefixPath() + "Sync/Commands/" + _myId
+         }));
 
-      if (result.successful()) {
+      AgencyCommResult result = _agency.sendTransactionWithFailover(trx);
-        handleStateChange(result);
+      if (!result.successful()) {
-      }
+        LOG_TOPIC(WARN, Logger::HEARTBEAT)
-
+            << "Heartbeat: Could not read from agency!";
-      if (isStopping()) {
-        break;
-      }
-
-      LOG_TOPIC(TRACE, Logger::HEARTBEAT) << "Refetching Current/Version...";
-      AgencyCommResult res = _agency.getValues("Current/Version");
-      if (!res.successful()) {
-        LOG_TOPIC(ERR, Logger::HEARTBEAT)
-            << "Could not read Current/Version from agency.";
       } else {
-        VPackSlice s = res.slice()[0].get(
+        VPackSlice shutdownSlice = result.slice()[0].get(
+            std::vector<std::string>({_agency.prefix(), "Shutdown"})
+        );
+
+        if (shutdownSlice.isBool() && shutdownSlice.getBool()) {
+          ApplicationServer::server->beginShutdown();
+          break;
+        }
+        LOG_TOPIC(TRACE, Logger::HEARTBEAT)
+            << "Looking at Sync/Commands/" + _myId;
+        handleStateChange(result);
+        
+        VPackSlice s = result.slice()[0].get(
             std::vector<std::string>({_agency.prefix(), std::string("Current"),
                                       std::string("Version")}));
         if (!s.isInteger()) {
@@ -322,7 +330,8 @@ void HeartbeatThread::runCoordinator() {
     }
 
     AgencyReadTransaction trx(std::vector<std::string>(
-        {_agency.prefixPath() + "Plan/Version",
+        {_agency.prefixPath() + "Shutdown",
+         _agency.prefixPath() + "Plan/Version",
          _agency.prefixPath() + "Current/Version",
          _agency.prefixPath() + "Current/Foxxmaster",
          _agency.prefixPath() + "Current/FoxxmasterQueueupdate",
@@ -334,6 +343,15 @@ void HeartbeatThread::runCoordinator() {
       LOG_TOPIC(WARN, Logger::HEARTBEAT)
           << "Heartbeat: Could not read from agency!";
     } else {
+      VPackSlice shutdownSlice = result.slice()[0].get(
+          std::vector<std::string>({_agency.prefix(), "Shutdown"})
+      );
+
+      if (shutdownSlice.isBool() && shutdownSlice.getBool()) {
+        ApplicationServer::server->beginShutdown();
+        break;
+      }
+
       LOG_TOPIC(TRACE, Logger::HEARTBEAT)
           << "Looking at Sync/Commands/" + _myId;
 

arangod/RestHandler/RestShutdownHandler.cpp

@@ -24,6 +24,7 @@
 #include "RestShutdownHandler.h"
 
 #include "Rest/HttpRequest.h"
+#include "Cluster/AgencyComm.h"
 #include "Cluster/ClusterFeature.h"
 
 #include <velocypack/Builder.h>
@@ -47,10 +48,25 @@ RestHandler::status RestShutdownHandler::execute() {
     generateError(GeneralResponse::ResponseCode::METHOD_NOT_ALLOWED, 405);
     return status::DONE;
   }
+  bool removeFromCluster;
+  std::string const& remove = _request->value("remove_from_cluster", removeFromCluster);
+  removeFromCluster = removeFromCluster && remove == "1";
 
-  bool found;
+  bool shutdownClusterFound;
-  std::string const& remove = _request->value("remove_from_cluster", found);
+  std::string const& shutdownCluster = _request->value("shutdown_cluster", shutdownClusterFound);
-  if (found && remove == "1") {
+  if (shutdownClusterFound && shutdownCluster == "1") {
+    AgencyComm agency;
+
+    VPackBuilder builder;
+    builder.add(VPackValue(true));
+    AgencyCommResult result = agency.setValue("Shutdown", builder.slice(), 0.0);
+    if (!result.successful()) {
+      generateError(GeneralResponse::ResponseCode::SERVER_ERROR, 500);
+      return status::DONE;
+    }
+    removeFromCluster = true;
+  }
+  if (removeFromCluster) {
     ClusterFeature* clusterFeature = ApplicationServer::getFeature<ClusterFeature>("Cluster");
     clusterFeature->setUnregisterOnShutdown(true);
   }

js/apps/system/_admin/aardvark/APP/foxxes.js

@@ -64,7 +64,7 @@ router.use(foxxRouter)
 
 const installer = createRouter();
 foxxRouter.use(installer)
-.queryParam('legacy', joi.boolean().default(true), dd`
+.queryParam('legacy', joi.boolean().default(false), dd`
   Flag to install the service in legacy mode.
 `)
 .queryParam('upgrade', joi.boolean().default(false), dd`