Merge branch 'devel' of https://github.com/arangodb/arangodb into engine-api

Documentation/Books/Manual/Administration/Arangoexport.mdpp

@@ -50,6 +50,16 @@ Export JSONL
 
 This exports the collection *test* into the output directory *export* as [jsonl](http://jsonlines.org). Every line in the export is one document from the collection *test* as json.
 
+Export CSV
+----------
+
+    unix> arangoexport --type CSV --collection test --fields _key,_id,_rev
+
+This exports the collection *test* into the output directory *export* as CSV. The first
+line contains the header with all field names. Each line is one document represented as
+CSV and separated with a comma. Objects and Arrays are represented as a JSON string.
+
+
 Export XML
 ----------
 

arangod/Agency/AgencyComm.cpp

@@ -1252,10 +1252,8 @@ void AgencyComm::updateEndpoints(arangodb::velocypack::Slice const& current) {
       << "Removing endpoint " << i << " from agent pool";
     AgencyCommManager::MANAGER->removeEndpoint(i);
   }
-  
 }
 
-
 AgencyCommResult AgencyComm::sendWithFailover(
     arangodb::rest::RequestType method, double const timeout,
     std::string const& initialUrl, std::string const& body,
@@ -1400,11 +1398,6 @@ AgencyCommResult AgencyComm::sendWithFailover(
           "Inquiry failed (" << inq._statusCode << "). Keep trying ...";
         continue;
       }
-    
-      AgencyCommManager::MANAGER->failed(std::move(connection), endpoint);
-      endpoint.clear();
-      connection = AgencyCommManager::MANAGER->acquire(endpoint);
-      continue;
     }
     
     // sometimes the agency will return a 307 (temporary redirect)
@@ -1424,11 +1417,11 @@ AgencyCommResult AgencyComm::sendWithFailover(
       break;
     }
 
-    if (tries%50 == 0) {
+    if (tries % 50 == 0) {
       LOG_TOPIC(WARN, Logger::AGENCYCOMM)
         << "Bad agency communiction! Unsuccessful consecutive tries:"
         << tries << " (" << elapsed << "s). Network checks needed!";
-    } else if (tries%15 == 0) {
+    } else if (tries % 15 == 0) {
       LOG_TOPIC(INFO, Logger::AGENCYCOMM)
         << "Flaky agency communication. Unsuccessful consecutive tries: "
         << tries << " (" << elapsed << "s). Network checks advised.";

arangod/Aql/Graphs.cpp

@@ -99,9 +99,8 @@ AstNode* EdgeConditionBuilder::getInboundCondition() {
 EdgeConditionBuilderContainer::EdgeConditionBuilderContainer() :
   EdgeConditionBuilder(nullptr) {
     auto node = std::make_unique<AstNode>(NODE_TYPE_OPERATOR_NARY_AND);
-    _modCondition = node.get();
     _astNodes.emplace_back(node.get());
-    node.release();
+    _modCondition = node.release();
 
     auto comp = std::make_unique<AstNode>(NODE_TYPE_VALUE);
     comp->setValueType(VALUE_TYPE_STRING);

arangod/Scheduler/JobGuard.h

@@ -35,6 +35,9 @@ class Scheduler;
 
 class JobGuard {
  public:
+  JobGuard(JobGuard const&) = delete;
+  JobGuard& operator=(JobGuard const&) = delete;
+
   explicit JobGuard(EventLoop const& loop) : _scheduler(loop._scheduler) {}
   explicit JobGuard(rest::Scheduler* scheduler) : _scheduler(scheduler) {}
   ~JobGuard() { release(); }
@@ -43,6 +46,8 @@ class JobGuard {
   bool isIdle() { return _scheduler->isIdle(); }
 
   void work() {
+    TRI_ASSERT(!_isWorkingFlag);
+
     if (0 == _isWorking) {
       _scheduler->workThread();
     }
@@ -52,6 +57,8 @@ class JobGuard {
   }
 
   void block() {
+    TRI_ASSERT(!_isBlockedFlag);
+
     if (0 == _isBlocked) {
       _scheduler->blockThread();
     }

arangosh/Export/ExportFeature.cpp

@@ -32,7 +32,9 @@
 #include "SimpleHttpClient/SimpleHttpClient.h"
 #include "SimpleHttpClient/SimpleHttpResult.h"
 
+#include <regex>
 #include <boost/property_tree/detail/xml_parser_utils.hpp>
+#include <boost/algorithm/string.hpp>
 
 using namespace arangodb;
 using namespace arangodb::basics;
@@ -47,6 +49,8 @@ ExportFeature::ExportFeature(application_features::ApplicationServer* server,
       _graphName(),
       _xgmmlLabelAttribute("label"),
       _typeExport("json"),
+      _csvFieldOptions(),
+      _csvFields(),
       _xgmmlLabelOnly(false),
       _outputDirectory(),
       _overwrite(false),
@@ -92,7 +96,10 @@ void ExportFeature::collectOptions(
   options->addOption("--progress", "show progress",
                      new BooleanParameter(&_progress));
 
-  std::unordered_set<std::string> exports = {"json", "jsonl", "xgmml", "xml"};
+  options->addOption("--fields", "comma separated list of fileds to export into a csv file",
+                     new StringParameter(&_csvFieldOptions));
+
+  std::unordered_set<std::string> exports = {"csv", "json", "jsonl", "xgmml", "xml"};
   options->addOption(
       "--type", "type of export", new DiscreteValuesParameter<StringParameter>(&_typeExport, exports));
 }
@@ -123,10 +130,25 @@ void ExportFeature::validateOptions(
     FATAL_ERROR_EXIT();
   }
 
-  if (_typeExport == "xgmml" && _graphName.empty() ) {
+  if (_typeExport == "xgmml" && _graphName.empty()) {
     LOG_TOPIC(FATAL, Logger::CONFIG) << "expecting a graph name to dump a graph";
     FATAL_ERROR_EXIT();
   }
+
+  if ( (_typeExport == "json" || _typeExport == "jsonl" || _typeExport == "csv") &&
+        _collections.empty()) {
+          LOG_TOPIC(FATAL, Logger::CONFIG) << "expecting at least one collection";
+          FATAL_ERROR_EXIT();
+  }
+
+  if (_typeExport == "csv") {
+    if (_csvFieldOptions.empty()) {
+      LOG_TOPIC(FATAL, Logger::CONFIG) << "expecting at least one field definition";
+      FATAL_ERROR_EXIT();
+    }
+
+    boost::split(_csvFields, _csvFieldOptions, boost::is_any_of(","));
+  }
 }
 
 void ExportFeature::prepare() {
@@ -209,7 +231,7 @@ void ExportFeature::start() {
 
   uint64_t exportedSize = 0;
 
-  if (_typeExport == "json" || _typeExport == "jsonl" || _typeExport == "xml") {
+  if (_typeExport == "json" || _typeExport == "jsonl" || _typeExport == "xml" || _typeExport == "csv") {
     if (_collections.size()) {
       collectionExport(httpClient.get());
 
@@ -282,12 +304,27 @@ void ExportFeature::collectionExport(SimpleHttpClient* httpClient) {
     if (_typeExport == "json") {
       std::string openingBracket = "[";
       writeToFile(fd, openingBracket, fileName);
+
     } else if (_typeExport == "xml") {
       std::string xmlHeader = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n"
                               "<collection name=\"";
       xmlHeader.append(encode_char_entities(collection));
       xmlHeader.append("\">\n");
       writeToFile(fd, xmlHeader, fileName);
+
+    } else if (_typeExport == "csv") {
+      std::string firstLine = "";
+      bool isFirstValue = true;
+      for(auto const& str : _csvFields) {
+        if (isFirstValue) {
+          firstLine += str;
+          isFirstValue = false;
+        } else {
+          firstLine += "," + str;
+        }
+      }
+      firstLine += "\n";
+      writeToFile(fd, firstLine, fileName);
     }
 
     writeCollectionBatch(fd, VPackArrayIterator(body.get("result")), fileName);
@@ -333,6 +370,41 @@ void ExportFeature::writeCollectionBatch(int fd, VPackArrayIterator it, std::str
       line += doc.toJson();
       writeToFile(fd, line, fileName);
     }
+  } else if (_typeExport == "csv") {
+    for (auto const& doc : it) {
+      line.clear();
+      bool isFirstValue = true;
+
+      for(auto const& key : _csvFields) {
+        std::string value = "";
+
+        if (isFirstValue) {
+          isFirstValue = false;
+        } else {
+          line.append(",");
+        }
+
+        if (doc.hasKey(key)) {
+          VPackSlice val = doc.get(key);
+
+          if (val.isArray() || val.isObject()) {
+            value = val.toJson();
+          } else {
+            if (val.isString()) {
+              value = val.copyString();
+            } else {
+              value = val.toString();
+            }
+          }
+
+          value = std::regex_replace(value, std::regex("\""), "\"\"");
+          value = std::regex_replace(value, std::regex(","), "\",");
+        }
+        line.append(value);
+      }
+      line.append("\n");
+      writeToFile(fd, line, fileName);
+    }
   } else if (_typeExport == "xml") {
     for (auto const& doc : it) {
       line.clear();
@@ -378,7 +450,7 @@ std::shared_ptr<VPackBuilder> ExportFeature::httpCall(SimpleHttpClient* httpClie
       if (_currentGraph.size()) {
         LOG_TOPIC(FATAL, Logger::CONFIG) << "Graph '" << _currentGraph << "' not found.";
       } else if (_currentCollection.size()) {
-        LOG_TOPIC(FATAL, Logger::CONFIG) << "Collection " << _currentCollection << "not found.";
+        LOG_TOPIC(FATAL, Logger::CONFIG) << "Collection " << _currentCollection << " not found.";
       }
 
       FATAL_ERROR_EXIT();

arangosh/Export/ExportFeature.h

@@ -64,6 +64,8 @@ class ExportFeature final : public application_features::ApplicationFeature,
   std::string _graphName;
   std::string _xgmmlLabelAttribute;
   std::string _typeExport;
+  std::string _csvFieldOptions;
+  std::vector<std::string> _csvFields;
   bool        _xgmmlLabelOnly;
 
   std::string _outputDirectory;