Merge branch 'devel' of https://github.com/arangodb/arangodb into cmakification

UnitTests/unittest.js

@@ -3,18 +3,19 @@
 'use strict';
 
 const yaml = require("js-yaml");
+const _ = require("lodash");
 
 const UnitTest = require("@arangodb/testing");
 
 const internalMembers = UnitTest.internalMembers;
 const fs = require("fs");
 const internal = require("internal");
+const inspect = internal.inspect;
 
 function makePathGeneric(path) {
   return path.split(fs.pathSeparator);
 }
 
-function resultsToXml(results, baseName, cluster) {
 function xmlEscape(s) {
   return s.replace(/[<>&"]/g, function(c) {
     return "&" + {
@@ -57,63 +58,78 @@ function resultsToXml(results, baseName, cluster) {
   return xml;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief converts results to XML representation
+////////////////////////////////////////////////////////////////////////////////
+
+function resultsToXml(results, baseName, cluster) {
   let clprefix = '';
 
   if (cluster) {
     clprefix = 'CL_';
   }
 
-  for (let testrun in results) {
+  const isSignificant = function(a, b) {
-    if ((internalMembers.indexOf(testrun) === -1) && (results.hasOwnProperty(testrun))) {
+    return (internalMembers.indexOf(b) === -1) && a.hasOwnProperty(b);
-      for (let test in results[testrun]) {
+  };
-        if ((internalMembers.indexOf(test) === -1) &&
+
-          results[testrun].hasOwnProperty(test) &&
+  for (let resultName in results) {
-          !results[testrun][test].hasOwnProperty('skipped')) {
+    if (isSignificant(results, resultName)) {
+      let run = results[resultName];
+
+      for (let runName in run) {
+        if (isSignificant(run, runName)) {
+          const current = run[runName];
+
+          if (current.skipped) {
+            continue;
+          }
 
           let xml = buildXml();
           let failuresFound = "";
 
-          if (results[testrun][test].hasOwnProperty('failed')) {
+          if (current.hasOwnProperty('failed')) {
-            failuresFound = results[testrun][test].failed;
+            failuresFound = current.failed;
           }
 
           xml.elem("testsuite", {
             errors: 0,
             failures: failuresFound,
-            name: clprefix + test,
+            name: clprefix + runName,
-            tests: results[testrun][test].total,
+            tests: current.total,
-            time: results[testrun][test].duration
+            time: current.duration
           });
 
-          for (let oneTest in results[testrun][test]) {
+          for (let oneTestName in current) {
-            if (internalMembers.indexOf(oneTest) === -1) {
+            if (isSignificant(current, oneTestName)) {
-              const result = results[testrun][test][oneTest].status;
+              const oneTest = current[oneTestName];
+              const result = oneTest.status || false;
               const success = (typeof(result) === 'boolean') ? result : false;
 
               xml.elem("testcase", {
-                name: clprefix + oneTest,
+                name: clprefix + oneTestName,
-                time: results[testrun][test][oneTest].duration
+                time: oneTest.duration
               }, success);
 
               if (!success) {
                 xml.elem("failure");
-                xml.text('<![CDATA[' + results[testrun][test][oneTest].message + ']]>\n');
+                xml.text('<![CDATA[' + oneTest.message + ']]>\n');
                 xml.elem("/failure");
                 xml.elem("/testcase");
               }
             }
           }
 
-          if ((!results[testrun][test].status) &&
+          if (!current.status) {
-            results[testrun][test].hasOwnProperty('message')) {
-
             xml.elem("testcase", {
-              name: 'all tests in ' + clprefix + test,
+              name: 'all tests in ' + clprefix + runName,
-              time: results[testrun][test].duration
+              time: current.duration
             }, false);
 
             xml.elem("failure");
-            xml.text('<![CDATA[' + JSON.stringify(results[testrun][test].message) + ']]>\n');
+            xml.text('<![CDATA[' +
+              JSON.stringify(current.message || "unknown failure reason") +
+              ']]>\n');
             xml.elem("/failure");
             xml.elem("/testcase");
           }
@@ -121,7 +137,7 @@ function resultsToXml(results, baseName, cluster) {
           xml.elem("/testsuite");
 
           const fn = makePathGeneric(baseName + clprefix +
-            testrun + '_' + test + ".xml").join('_');
+            resultName + '_' + runName + ".xml").join('_');
 
           fs.write("out/" + fn, xml.join(""));
         }
@@ -130,6 +146,10 @@ function resultsToXml(results, baseName, cluster) {
   }
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief runs the test using testing.js
+////////////////////////////////////////////////////////////////////////////////
+
 function main(argv) {
   const test = argv[0];
   let options = {};
@@ -153,6 +173,7 @@ function main(argv) {
 
   start_pretty_print();
 
+  // run the test and store the result
   let r = {};
 
   try {
@@ -174,15 +195,17 @@ function main(argv) {
     print(JSON.stringify(r));
   }
 
+  _.defaults(r, {
+    all_ok: false,
+    crashed: true
+  });
+
   // whether or not there was an error 
-  fs.write("out/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json", JSON.stringify(r.all_ok));
+  fs.write("out/UNITTEST_RESULT_EXECUTIVE_SUMMARY.json", String(r.all_ok));
 
   if (options.writeXmlReport) {
-    fs.write("out/UNITTEST_RESULT.json", JSON.stringify(r));
+    fs.write("out/UNITTEST_RESULT.json", inspect(r));
-
+    fs.write("out/UNITTEST_RESULT_CRASHED.txt", String(r.crashed));
-    // should be renamed to UNITTEST_RESULT_CRASHED, because that's what
-    // it actually contains
-    fs.write("out/UNITTEST_RESULT_SUMMARY.txt", JSON.stringify(!r.crashed));
 
     try {
       resultsToXml(r,

arangod/Aql/AstNode.h

@@ -219,25 +219,25 @@ struct AstNode {
   /// @brief create a node, with defining a value type
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit AstNode(AstNodeType, AstNodeValueType);
+  AstNode(AstNodeType, AstNodeValueType);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief create a boolean node, with defining a value type
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit AstNode(bool, AstNodeValueType);
+  AstNode(bool, AstNodeValueType);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief create a boolean node, with defining a value type
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit AstNode(int64_t, AstNodeValueType);
+  AstNode(int64_t, AstNodeValueType);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief create a string node, with defining a value type
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit AstNode(char const*, size_t, AstNodeValueType);
+  AstNode(char const*, size_t, AstNodeValueType);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief create the node from JSON

arangod/Indexes/PathBasedIndex.h

@@ -69,7 +69,7 @@ class PathBasedIndex : public Index {
       std::vector<std::vector<arangodb::basics::AttributeName>> const&,
       bool unique, bool sparse, bool allowPartialIndex);
 
-  explicit PathBasedIndex(VPackSlice const&, bool);
+  PathBasedIndex(VPackSlice const&, bool);
 
   ~PathBasedIndex();
 

arangod/RestHandler/RestVocbaseBaseHandler.cpp

@@ -446,6 +446,22 @@ void RestVocbaseBaseHandler::generateTransactionError(
       return;
     }
     
+    case TRI_ERROR_FORBIDDEN: {
+      generateError(HttpResponse::FORBIDDEN, res);
+      return;
+    }
+        
+    case TRI_ERROR_OUT_OF_MEMORY: 
+    case TRI_ERROR_LOCK_TIMEOUT: 
+    case TRI_ERROR_AID_NOT_FOUND:
+    case TRI_ERROR_DEBUG:
+    case TRI_ERROR_LEGEND_NOT_IN_WAL_FILE:
+    case TRI_ERROR_LOCKED:
+    case TRI_ERROR_DEADLOCK: {
+      generateError(HttpResponse::SERVER_ERROR, res);
+      return;
+    }
+
     default:
       generateError(HttpResponse::SERVER_ERROR, TRI_ERROR_INTERNAL,
                     "failed with error: " + std::string(TRI_errno_string(res)));

arangod/RestServer/ArangoServer.cpp

@@ -1168,6 +1168,13 @@ int ArangoServer::startupServer() {
     }
   }
 
+
+  // active deadlock detection in case we're not running in cluster mode
+  if (!arangodb::ServerState::instance()->isRunningInCluster()) {
+    TRI_EnableDeadlockDetectionDatabasesServer(_server);
+  }
+
+
   // .............................................................................
   // start the work monitor
   // .............................................................................

arangod/Scheduler/SocketTask.h

@@ -63,7 +63,7 @@ class SocketTask : virtual public Task, public ConnectionStatisticsAgent {
   //////////////////////////////////////////////////////////////////////////////
 
  public:
-  explicit SocketTask(TRI_socket_t, double);
+  SocketTask(TRI_socket_t, double);
 
   //////////////////////////////////////////////////////////////////////////////
   /// @brief deletes a socket task

arangod/V8Server/V8Traverser.h

@@ -203,7 +203,7 @@ class DepthFirstTraverser : public Traverser {
 
   class EdgeGetter {
    public:
-    explicit EdgeGetter(DepthFirstTraverser* traverser,
+    EdgeGetter(DepthFirstTraverser* traverser,
                         TraverserOptions const& opts,
                         CollectionNameResolver* resolver, Transaction* trx)
         : _traverser(traverser), _resolver(resolver), _opts(opts), _trx(trx) {}

arangod/V8Server/v8-vocbase.cpp

@@ -3362,6 +3362,8 @@ static void JS_CreateDatabase(v8::FunctionCallbackInfo<v8::Value> const& args) {
 
   TRI_ASSERT(database != nullptr);
 
+  database->_deadlockDetector.enabled(!arangodb::ServerState::instance()->isRunningInCluster());
+
   // copy users into context
   if (args.Length() >= 3 && args[2]->IsArray()) {
     v8::Handle<v8::Object> users = v8::Object::New(isolate);

arangod/VocBase/server.cpp

@@ -1693,6 +1693,23 @@ int TRI_CreateDatabaseServer(TRI_server_t* server, TRI_voc_tick_t databaseId,
   return res;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief activates or deactivates deadlock detection in all existing 
+/// databases
+////////////////////////////////////////////////////////////////////////////////
+
+void TRI_EnableDeadlockDetectionDatabasesServer(TRI_server_t* server) {
+  auto unuser(server->_databasesProtector.use());
+  auto theLists = server->_databasesLists.load();
+
+  for (auto& p : theLists->_databases) {
+    TRI_vocbase_t* vocbase = p.second;
+    TRI_ASSERT(vocbase != nullptr);
+
+    vocbase->_deadlockDetector.enabled(true);
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief get the ids of all local coordinator databases
 /// the caller is responsible for freeing the result

arangod/VocBase/server.h

@@ -153,6 +153,13 @@ int TRI_CreateDatabaseServer(TRI_server_t*, TRI_voc_tick_t, char const*,
                              TRI_vocbase_defaults_t const*, TRI_vocbase_t**,
                              bool);
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief activates or deactivates deadlock detection in all existing 
+/// databases
+////////////////////////////////////////////////////////////////////////////////
+
+void TRI_EnableDeadlockDetectionDatabasesServer(TRI_server_t*);
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief get the ids of all local coordinator databases
 /// the caller is responsible for freeing the result

arangod/VocBase/vocbase.cpp

@@ -2349,6 +2349,7 @@ TRI_vocbase_t::TRI_vocbase_t(TRI_server_t* server, TRI_vocbase_type_e type,
       _type(type),
       _refCount(0),
       _server(server),
+      _deadlockDetector(false),
       _userStructures(nullptr),
       _queries(nullptr),
       _cursorRepository(nullptr),

arangod/Wal/CollectorThread.h

@@ -71,7 +71,7 @@ struct CollectorCache {
   CollectorCache(CollectorCache const&) = delete;
   CollectorCache& operator=(CollectorCache const&) = delete;
 
-  explicit CollectorCache(TRI_voc_cid_t collectionId, TRI_voc_tick_t databaseId,
+  CollectorCache(TRI_voc_cid_t collectionId, TRI_voc_tick_t databaseId,
                           Logfile* logfile, int64_t totalOperationsCount,
                           size_t operationsSize)
       : collectionId(collectionId),

arangod/Wal/Marker.h

@@ -387,7 +387,7 @@ class Marker {
 
 class EnvelopeMarker : public Marker {
  public:
-  explicit EnvelopeMarker(TRI_df_marker_t const*, TRI_voc_fid_t);
+  EnvelopeMarker(TRI_df_marker_t const*, TRI_voc_fid_t);
 
   ~EnvelopeMarker();
 };

js/server/tests/shell/shell-hash-index-failures.js

@@ -130,6 +130,7 @@ function HashIndexMultiFailuresSuite () {
 ////////////////////////////////////////////////////////////////////////////////
 
     setUp : function () {
+      internal.debugClearFailAt();
       internal.db._drop(cn);
       collection = internal.db._create(cn);
       collection.ensureHashIndex("a");
@@ -140,8 +141,8 @@ function HashIndexMultiFailuresSuite () {
 ////////////////////////////////////////////////////////////////////////////////
 
     tearDown : function () {
-      internal.db._drop(cn);
       internal.debugClearFailAt();
+      internal.db._drop(cn);
     },
 
 ////////////////////////////////////////////////////////////////////////////////

lib/Basics/DeadlockDetector.h

@@ -35,7 +35,7 @@ namespace basics {
 template <typename T>
 class DeadlockDetector {
  public:
-  DeadlockDetector() = default;
+  explicit DeadlockDetector(bool enabled) : _enabled(enabled) {};
   ~DeadlockDetector() = default;
 
   DeadlockDetector(DeadlockDetector const&) = delete;
@@ -105,12 +105,34 @@ class DeadlockDetector {
 
   void unsetWriter(T const* value) { unsetActive(value, true); }
 
+  ////////////////////////////////////////////////////////////////////////////////
+  /// @brief enable / disable
+  ////////////////////////////////////////////////////////////////////////////////
+
+  void enabled(bool value) {
+    MUTEX_LOCKER(mutexLocker, _lock);
+    _enabled = value;
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  /// @brief return the enabled status
+  ////////////////////////////////////////////////////////////////////////////////
+
+  bool enabled() {
+    MUTEX_LOCKER(mutexLocker, _lock);
+    return _enabled;
+  }
+
  private:
   ////////////////////////////////////////////////////////////////////////////////
   /// @brief add a thread to the list of blocked threads
   ////////////////////////////////////////////////////////////////////////////////
 
   int detectDeadlock(T const* value, TRI_tid_t tid, bool isWrite) const {
+    if (!_enabled) {
+      return TRI_ERROR_NO_ERROR;
+    }
+
     struct StackValue {
       StackValue(T const* value, TRI_tid_t tid, bool isWrite)
           : value(value), tid(tid), isWrite(isWrite) {}
@@ -189,6 +211,10 @@ class DeadlockDetector {
 
     MUTEX_LOCKER(mutexLocker, _lock);
 
+    if (!_enabled) {
+      return TRI_ERROR_NO_ERROR;
+    }
+
     auto it = _blocked.find(tid);
 
     if (it != _blocked.end()) {
@@ -223,6 +249,10 @@ class DeadlockDetector {
 
     MUTEX_LOCKER(mutexLocker, _lock);
 
+    if (!_enabled) {
+      return;
+    }
+
     _blocked.erase(tid);
   }
 
@@ -234,6 +264,11 @@ class DeadlockDetector {
     auto tid = TRI_CurrentThreadId();
 
     MUTEX_LOCKER(mutexLocker, _lock);
+
+    if (!_enabled) {
+      return;
+    }
+
     auto it = _active.find(value);
 
     if (it == _active.end()) {
@@ -266,6 +301,11 @@ class DeadlockDetector {
     auto tid = TRI_CurrentThreadId();
 
     MUTEX_LOCKER(mutexLocker, _lock);
+
+    if (!_enabled) {
+      return;
+    }
+
     auto it = _active.find(value);
 
     if (it == _active.end()) {
@@ -303,6 +343,12 @@ class DeadlockDetector {
 
   std::unordered_map<T const*, std::pair<std::unordered_set<TRI_tid_t>, bool>>
       _active;
+
+  ////////////////////////////////////////////////////////////////////////////////
+  /// @brief whether or not the detector is enabled
+  ////////////////////////////////////////////////////////////////////////////////
+
+  bool _enabled;
 };
 
 }  // namespace arangodb::basics

lib/Basics/JsonHelper.h

@@ -345,7 +345,7 @@ class Json {
   //////////////////////////////////////////////////////////////////////////////
 
  public:
-  explicit Json()
+  Json()
       : _json(nullptr),
         _zone(TRI_MemoryZoneId(TRI_UNKNOWN_MEM_ZONE)),
         _autofree(AUTOFREE) {}
@@ -365,7 +365,7 @@ class Json {
   /// @brief generic constructor for a memzone and a type_e
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, type_e t, autofree_e autofree = AUTOFREE)
+  Json(TRI_memory_zone_t* z, type_e t, autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     make(t, 0);
   }
@@ -374,7 +374,7 @@ class Json {
   /// @brief generic constructor for a type_e with a size hint
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(type_e t, size_t sizeHint, autofree_e autofree = AUTOFREE)
+  Json(type_e t, size_t sizeHint, autofree_e autofree = AUTOFREE)
       : _json(nullptr),
         _zone(TRI_MemoryZoneId(TRI_UNKNOWN_MEM_ZONE)),
         _autofree(autofree) {
@@ -385,7 +385,7 @@ class Json {
   /// @brief generic constructor for a memzone, a type_e and a size hint
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, type_e t, size_t sizeHint,
+  Json(TRI_memory_zone_t* z, type_e t, size_t sizeHint,
                 autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     make(t, sizeHint);
@@ -410,7 +410,7 @@ class Json {
   /// @brief constructor for a memzone and a bool
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, bool x, autofree_e autofree = AUTOFREE)
+  Json(TRI_memory_zone_t* z, bool x, autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     _json = TRI_CreateBooleanJson(z, x);
 
@@ -438,7 +438,7 @@ class Json {
   /// @brief constructor for a memzone and an int32_t
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, int32_t x, autofree_e autofree = AUTOFREE)
+  Json(TRI_memory_zone_t* z, int32_t x, autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     _json = TRI_CreateNumberJson(z, static_cast<double>(x));
 
@@ -466,7 +466,7 @@ class Json {
   /// @brief constructor for a memzone and a double
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, double x, autofree_e autofree = AUTOFREE)
+  Json(TRI_memory_zone_t* z, double x, autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     _json = TRI_CreateNumberJson(z, x);
 
@@ -494,7 +494,7 @@ class Json {
   /// @brief constructor for a char const*
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(char const* x, size_t length, autofree_e autofree = AUTOFREE)
+  Json(char const* x, size_t length, autofree_e autofree = AUTOFREE)
       : _json(nullptr),
         _zone(TRI_MemoryZoneId(TRI_UNKNOWN_MEM_ZONE)),
         _autofree(autofree) {
@@ -509,7 +509,7 @@ class Json {
   /// @brief constructor for a memzone and a char const*
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, char const* x,
+  Json(TRI_memory_zone_t* z, char const* x,
                 autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     _json = TRI_CreateStringCopyJson(z, x, strlen(x));
@@ -523,7 +523,7 @@ class Json {
   /// @brief constructor for a memzone and a char const*
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, char const* x, size_t length,
+  Json(TRI_memory_zone_t* z, char const* x, size_t length,
                 autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     _json = TRI_CreateStringCopyJson(z, x, length);
@@ -552,7 +552,7 @@ class Json {
   /// @brief constructor for a memzone and a string
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, std::string const& x,
+  Json(TRI_memory_zone_t* z, std::string const& x,
                 autofree_e autofree = AUTOFREE)
       : _json(nullptr), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {
     _json = TRI_CreateStringCopyJson(z, x.c_str(), x.size());
@@ -566,7 +566,7 @@ class Json {
   /// @brief constructor for a memzone and a TRI_json_t*
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, TRI_json_t* j,
+  Json(TRI_memory_zone_t* z, TRI_json_t* j,
                 autofree_e autofree = AUTOFREE)
       : _json(j), _zone(TRI_MemoryZoneId(z)), _autofree(autofree) {}
 
@@ -574,7 +574,7 @@ class Json {
   /// @brief constructor for a memzone and a const TRI_json_t*
   //////////////////////////////////////////////////////////////////////////////
 
-  explicit Json(TRI_memory_zone_t* z, TRI_json_t const* j,
+  Json(TRI_memory_zone_t* z, TRI_json_t const* j,
                 autofree_e autofree = NOFREE)
       : _json(const_cast<TRI_json_t*>(j)),
         _zone(TRI_MemoryZoneId(z)),