Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

CHANGELOG

@@ -49,6 +49,22 @@ edge attribute `label`.
 * process.stdout.isTTY now returns `true` in arangosh and when running arangod with the `--console` flag
 
 
+v3.1.8 (XXXX-XX-XX)
+-------------------
+
+* fixed issue #2238
+
+* fixed issue #2234
+
+
+v3.1.7 (2016-12-29)
+-------------------
+
+* fixed one too many elections in RAFT
+
+* new agency comm backported from devel
+
+
 v3.1.6 (2016-12-20)
 -------------------
 

arangod/Aql/AstNode.cpp

@@ -1925,6 +1925,8 @@ AstNode* AstNode::clone(Ast* ast) const { return ast->clone(this); }
 /// the string representation does not need to be JavaScript-compatible
 /// except for node types NODE_TYPE_VALUE, NODE_TYPE_ARRAY and NODE_TYPE_OBJECT
 /// (only for objects that do not contain dynamic attributes)
+/// note that this may throw and that the caller is responsible for
+/// catching the error
 void AstNode::stringify(arangodb::basics::StringBuffer* buffer, bool verbose,
                         bool failIfLong) const {
   // any arrays/objects with more values than this will not be stringified if
@@ -2199,6 +2201,8 @@ void AstNode::stringify(arangodb::basics::StringBuffer* buffer, bool verbose,
   THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, message);
 }
 
+/// note that this may throw and that the caller is responsible for
+/// catching the error
 std::string AstNode::toString() const {
   arangodb::basics::StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE);
   stringify(&buffer, false, false);

arangod/Aql/AstNode.h

@@ -639,9 +639,16 @@ struct AstNode {
   /// @brief clone a node, recursively
   AstNode* clone(Ast*) const;
 
-  /// @brief append a JavaScript representation of the node into a string buffer
+  /// @brief append a string representation of the node into a string buffer
+  /// the string representation does not need to be JavaScript-compatible
+  /// except for node types NODE_TYPE_VALUE, NODE_TYPE_ARRAY and NODE_TYPE_OBJECT
+  /// (only for objects that do not contain dynamic attributes)
+  /// note that this may throw and that the caller is responsible for
+  /// catching the error
   void stringify(arangodb::basics::StringBuffer*, bool, bool) const;
 
+  /// note that this may throw and that the caller is responsible for
+  /// catching the error
   std::string toString() const;
 
   /// @brief stringify the value of a node into a string buffer

arangod/Aql/Condition.cpp

@@ -991,54 +991,58 @@ bool Condition::canRemove(ExecutionPlan const* plan, ConditionPart const& me,
     return node->toString();
   };
 
-  for (size_t i = 0; i < n; ++i) {
+  try {
-    auto operand = andNode->getMemberUnchecked(i);
+    for (size_t i = 0; i < n; ++i) {
+      auto operand = andNode->getMemberUnchecked(i);
 
-    if (operand->isComparisonOperator()) {
+      if (operand->isComparisonOperator()) {
-      auto lhs = operand->getMember(0);
+        auto lhs = operand->getMember(0);
-      auto rhs = operand->getMember(1);
+        auto rhs = operand->getMember(1);
 
-      if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) {
+        if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) {
-        clearAttributeAccess(result);
+          clearAttributeAccess(result);
 
-        if (lhs->isAttributeAccessForVariable(result)) {
+          if (lhs->isAttributeAccessForVariable(result)) {
-          if (rhs->isConstant()) {
+            if (rhs->isConstant()) {
-            ConditionPart indexCondition(result.first, result.second, operand,
+              ConditionPart indexCondition(result.first, result.second, operand,
-                                         ATTRIBUTE_LEFT, nullptr);
+                                          ATTRIBUTE_LEFT, nullptr);
 
-            if (me.isCoveredBy(indexCondition, false)) {
+              if (me.isCoveredBy(indexCondition, false)) {
+                return true;
+              }
+            }
+            // non-constant condition
+            else if (me.operatorType == operand->type &&
+                    normalize(me.valueNode) == normalize(rhs)) {
               return true;
             }
           }
-          // non-constant condition
-          else if (me.operatorType == operand->type &&
-                   normalize(me.valueNode) == normalize(rhs)) {
-            return true;
-          }
         }
-      }
 
-      if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
+        if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
-          rhs->type == NODE_TYPE_EXPANSION) {
+            rhs->type == NODE_TYPE_EXPANSION) {
-        clearAttributeAccess(result);
+          clearAttributeAccess(result);
 
-        if (rhs->isAttributeAccessForVariable(result)) {
+          if (rhs->isAttributeAccessForVariable(result)) {
-          if (lhs->isConstant()) {
+            if (lhs->isConstant()) {
-            ConditionPart indexCondition(result.first, result.second, operand,
+              ConditionPart indexCondition(result.first, result.second, operand,
-                                         ATTRIBUTE_RIGHT, nullptr);
+                                          ATTRIBUTE_RIGHT, nullptr);
 
-            if (me.isCoveredBy(indexCondition, true)) {
+              if (me.isCoveredBy(indexCondition, true)) {
+                return true;
+              }
+            }
+            // non-constant condition
+            else if (me.operatorType == operand->type &&
+                    normalize(me.valueNode) == normalize(lhs)) {
               return true;
             }
           }
-          // non-constant condition
-          else if (me.operatorType == operand->type &&
-                   normalize(me.valueNode) == normalize(lhs)) {
-            return true;
-          }
         }
       }
     }
+  } catch (...) {
+    // simply ignore any errors and return false
   }
 
   return false;
@@ -1125,6 +1129,7 @@ AstNode* Condition::transformNode(AstNode* node) {
 
     // create a new n-ary node
     node = _ast->createNode(Ast::NaryOperatorType(old->type));
+    node->reserve(2);
     node->addMember(old->getMember(0));
     node->addMember(old->getMember(1));
   }
@@ -1142,11 +1147,9 @@ AstNode* Condition::transformNode(AstNode* node) {
       auto sub = transformNode(node->getMemberUnchecked(i));
       node->changeMember(i, sub);
 
-      if (sub->type == NODE_TYPE_OPERATOR_NARY_OR ||
+      if (sub->type == NODE_TYPE_OPERATOR_NARY_OR) {
-          sub->type == NODE_TYPE_OPERATOR_BINARY_OR) {
         processChildren = true;
-      } else if (sub->type == NODE_TYPE_OPERATOR_NARY_AND ||
+      } else if (sub->type == NODE_TYPE_OPERATOR_NARY_AND) {
-                 sub->type == NODE_TYPE_OPERATOR_BINARY_AND) {
         mustCollapse = true;
       }
     }
@@ -1165,14 +1168,15 @@ AstNode* Condition::transformNode(AstNode* node) {
       auto newOperator = _ast->createNode(NODE_TYPE_OPERATOR_NARY_OR);
 
       std::vector<PermutationState> permutationStates;
+      permutationStates.reserve(n);
+
       for (size_t i = 0; i < n; ++i) {
         auto sub = node->getMemberUnchecked(i);
 
         if (sub->type == NODE_TYPE_OPERATOR_NARY_OR) {  
-          permutationStates.emplace_back(
+          permutationStates.emplace_back(sub, sub->numMembers());
-              PermutationState(sub, sub->numMembers()));
         } else {
-          permutationStates.emplace_back(PermutationState(sub, 1));
+          permutationStates.emplace_back(sub, 1);
         }
       }
 
@@ -1182,9 +1186,10 @@ AstNode* Condition::transformNode(AstNode* node) {
 
       while (!done) {
         auto andOperator = _ast->createNode(NODE_TYPE_OPERATOR_NARY_AND);
+        andOperator->reserve(numPermutations);
 
         for (size_t i = 0; i < numPermutations; ++i) {
-          auto state = permutationStates[i];
+          auto const& state = permutationStates[i];
           andOperator->addMember(state.getValue()->clone(_ast));
         }
 

arangod/Aql/OptimizerRules.cpp

@@ -3190,7 +3190,7 @@ struct OrSimplifier {
     }
     catch (...) {
     }
-    return "";
+    return std::string();
   }
 
   bool qualifies(AstNode const* node, std::string& attributeName) const {
@@ -3400,6 +3400,18 @@ struct RemoveRedundantOr {
   AstNode const* commonNode = nullptr;
   std::string commonName;
   
+  bool hasRedundantCondition(AstNode const* node) {
+    try {
+      if (finder.find(node, NODE_TYPE_OPERATOR_BINARY_LT, commonNode,
+                      commonName)) {
+        return hasRedundantConditionWalker(node);
+      } 
+    } catch (...) {
+      // ignore errors and simply return false
+    }
+    return false;
+  }
+
   AstNode* createReplacementNode(Ast* ast) {
     TRI_ASSERT(commonNode != nullptr);
     TRI_ASSERT(bestValue != nullptr);
@@ -3408,6 +3420,7 @@ struct RemoveRedundantOr {
                                          bestValue);
   }
 
+ private:
   bool isInclusiveBound(AstNodeType type) {
     return (type == NODE_TYPE_OPERATOR_BINARY_GE ||
             type == NODE_TYPE_OPERATOR_BINARY_LE);
@@ -3429,8 +3442,7 @@ struct RemoveRedundantOr {
   }
 
   // returns false if the existing value is better and true if the input value
-  // is
+  // is better
-  // better
   bool compareBounds(AstNodeType type, AstNode const* value, int lowhigh) {
     int cmp = CompareAstNodes(bestValue, value, true);
 
@@ -3440,14 +3452,6 @@ struct RemoveRedundantOr {
     return (cmp * lowhigh == 1);
   }
 
-  bool hasRedundantCondition(AstNode const* node) {
-    if (finder.find(node, NODE_TYPE_OPERATOR_BINARY_LT, commonNode,
-                    commonName)) {
-      return hasRedundantConditionWalker(node);
-    }
-    return false;
-  }
-
   bool hasRedundantConditionWalker(AstNode const* node) {
     AstNodeType type = node->type;
 

arangod/Aql/QueryResources.cpp

@@ -62,19 +62,25 @@ void QueryResources::addNode(AstNode* node) {
 
   if (_nodes.empty()) {
     // reserve some initial space for nodes 
-    capacity = 16;
+    capacity = 64;
   } else {
-    capacity = (std::max)(_nodes.size() + 8, _nodes.capacity());
+    capacity = _nodes.size() + 1;
+    if (capacity > _nodes.capacity()) {
+      capacity *= 2;
+    }
   }
 
   TRI_ASSERT(capacity > _nodes.size());
-  TRI_ASSERT(capacity >= _nodes.capacity());
 
-  // reserve space
+  // reserve space for pointers
-  _resourceMonitor->increaseMemoryUsage((capacity - _nodes.capacity()) * sizeof(AstNode*));
+  if (capacity > _nodes.capacity()) {
-  _nodes.reserve(capacity);
+    _resourceMonitor->increaseMemoryUsage((capacity - _nodes.capacity()) * sizeof(AstNode*));
+    _nodes.reserve(capacity);
+  }
 
+  // may throw
   _resourceMonitor->increaseMemoryUsage(sizeof(AstNode));
+  
   // will not fail
   _nodes.emplace_back(node); 
 }

arangod/StorageEngine/MMFilesCompactorThread.cpp

@@ -307,7 +307,7 @@ MMFilesCompactorThread::CompactionInitialContext MMFilesCompactorThread::getComp
     bool ok;
     {
       bool const useDeadlockDetector = false;
-      int res = collection->beginReadTimed(useDeadlockDetector, 86400ULL * 1000ULL * 1000ULL, TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
+      int res = collection->beginReadTimed(useDeadlockDetector, 86400.0);
 
       if (res != TRI_ERROR_NO_ERROR) {
         ok = false;

arangod/Utils/CollectionReadLocker.h

@@ -40,8 +40,7 @@ class CollectionReadLocker {
   CollectionReadLocker(LogicalCollection* collection, bool useDeadlockDetector, bool doLock)
       : _collection(collection), _useDeadlockDetector(useDeadlockDetector), _doLock(false) {
     if (doLock) {
-      int res = _collection->beginReadTimed(_useDeadlockDetector,
+      int res = _collection->beginReadTimed(_useDeadlockDetector);
-          0, TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
 
       if (res != TRI_ERROR_NO_ERROR) {
         THROW_ARANGO_EXCEPTION(res);

arangod/Utils/CollectionWriteLocker.h

@@ -40,7 +40,7 @@ class CollectionWriteLocker {
   CollectionWriteLocker(arangodb::LogicalCollection* collection, bool useDeadlockDetector, bool doLock)
       : _collection(collection), _useDeadlockDetector(useDeadlockDetector), _doLock(false) {
     if (doLock) {
-      int res = _collection->beginWriteTimed(_useDeadlockDetector, 0, TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
+      int res = _collection->beginWriteTimed(_useDeadlockDetector);
 
       if (res != TRI_ERROR_NO_ERROR) {
         THROW_ARANGO_EXCEPTION(res);

arangod/VocBase/LogicalCollection.cpp

@@ -2922,7 +2922,7 @@ int LogicalCollection::endWrite(bool useDeadlockDetector) {
 }
 
 /// @brief read locks a collection, with a timeout (in µseconds)
-int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout, uint64_t sleepPeriod) {
+int LogicalCollection::beginReadTimed(bool useDeadlockDetector, double timeout) {
   if (arangodb::Transaction::_makeNolockHeaders != nullptr) {
     auto it = arangodb::Transaction::_makeNolockHeaders->find(name());
     if (it != arangodb::Transaction::_makeNolockHeaders->end()) {
@@ -2933,16 +2933,12 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
       return TRI_ERROR_NO_ERROR;
     }
   }
-  uint64_t waited = 0;
-  if (timeout == 0) {
-    // we don't allow looping forever. limit waiting to 15 minutes max.
-    timeout = 15 * 60 * 1000 * 1000;
-  }
 
   // LOCKING-DEBUG
   // std::cout << "BeginReadTimed: " << _name << std::endl;
   int iterations = 0;
   bool wasBlocked = false;
+  double end = 0.0;
 
   while (true) {
     TRY_READ_LOCKER(locker, _idxLock);
@@ -2970,6 +2966,7 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
             return TRI_ERROR_DEADLOCK;
           }
           LOG(TRACE) << "waiting for read-lock on collection '" << name() << "'";
+          // fall-through intentional
         } else if (++iterations >= 5) {
           // periodically check for deadlocks
           TRI_ASSERT(wasBlocked);
@@ -2992,15 +2989,20 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
       }
     }
 
-#ifdef _WIN32
+    if (end == 0.0) {
-    usleep((unsigned long)sleepPeriod);
+      // set end time for lock waiting
-#else
+      if (timeout <= 0.0) {
-    usleep((useconds_t)sleepPeriod);
+        timeout = 15.0 * 60.0;
-#endif
+      }
+      end = TRI_microtime() + timeout;
+      TRI_ASSERT(end > 0.0);
+    }
 
-    waited += sleepPeriod;
+    std::this_thread::yield();
     
-    if (waited > timeout) {
+    TRI_ASSERT(end > 0.0);
+
+    if (TRI_microtime() > end) {
       if (useDeadlockDetector) {
         _vocbase->_deadlockDetector.unsetReaderBlocked(this);
       }
@@ -3011,7 +3013,7 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
 }
 
 /// @brief write locks a collection, with a timeout
-int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, uint64_t timeout, uint64_t sleepPeriod) {
+int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, double timeout) {
   if (arangodb::Transaction::_makeNolockHeaders != nullptr) {
     auto it = arangodb::Transaction::_makeNolockHeaders->find(name());
     if (it != arangodb::Transaction::_makeNolockHeaders->end()) {
@@ -3022,16 +3024,12 @@ int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, uint64_t timeou
       return TRI_ERROR_NO_ERROR;
     }
   }
-  uint64_t waited = 0;
-  if (timeout == 0) {
-    // we don't allow looping forever. limit waiting to 15 minutes max.
-    timeout = 15 * 60 * 1000 * 1000;
-  }
 
   // LOCKING-DEBUG
   // std::cout << "BeginWriteTimed: " << document->_info._name << std::endl;
   int iterations = 0;
   bool wasBlocked = false;
+  double end = 0.0;
 
   while (true) {
     TRY_WRITE_LOCKER(locker, _idxLock);
@@ -3080,15 +3078,22 @@ int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, uint64_t timeou
       }
     }
 
-#ifdef _WIN32
+    std::this_thread::yield();
-    usleep((unsigned long)sleepPeriod);
-#else
-    usleep((useconds_t)sleepPeriod);
-#endif
     
-    waited += sleepPeriod;
+    if (end == 0.0) {
+      // set end time for lock waiting
+      if (timeout <= 0.0) {
+        timeout = 15.0 * 60.0;
+      }
+      end = TRI_microtime() + timeout;
+      TRI_ASSERT(end > 0.0);
+    }
 
-    if (waited > timeout) {
+    std::this_thread::yield();
+    
+    TRI_ASSERT(end > 0.0);
+    
+    if (TRI_microtime() > end) {
       if (useDeadlockDetector) {
         _vocbase->_deadlockDetector.unsetWriterBlocked(this);
       }

arangod/VocBase/LogicalCollection.h

@@ -380,8 +380,8 @@ class LogicalCollection {
   int fillIndex(arangodb::Transaction*, arangodb::Index*,
                 bool skipPersistent = true);
 
-  int beginReadTimed(bool useDeadlockDetector, uint64_t, uint64_t);
+  int beginReadTimed(bool useDeadlockDetector, double timeout = 0.0);
-  int beginWriteTimed(bool useDeadlockDetector, uint64_t, uint64_t);
+  int beginWriteTimed(bool useDeadlockDetector, double timeout = 0.0);
   int endRead(bool useDeadlockDetector);
   int endWrite(bool useDeadlockDetector);
   

arangod/VocBase/transaction.cpp

@@ -286,10 +286,10 @@ static int LockCollection(TRI_transaction_collection_t* trxCollection,
 
   LogicalCollection* collection = trxCollection->_collection;
   TRI_ASSERT(collection != nullptr);
-  uint64_t timeout = trx->_timeout;
+  double timeout = trx->_timeout;
   if (HasHint(trxCollection->_transaction, TRI_TRANSACTION_HINT_TRY_LOCK)) {
-    // give up if we cannot acquire the lock instantly
+    // give up early if we cannot acquire the lock instantly
-    timeout = 1 * 100;
+    timeout = 0.00000001;
   }
   
   bool const useDeadlockDetector = !IsSingleOperationTransaction(trx);
@@ -297,13 +297,11 @@ static int LockCollection(TRI_transaction_collection_t* trxCollection,
   int res;
   if (type == TRI_TRANSACTION_READ) {
     LOG_TRX(trx, nestingLevel) << "read-locking collection " << trxCollection->_cid;
-    res = collection->beginReadTimed(useDeadlockDetector, timeout,
+    res = collection->beginReadTimed(useDeadlockDetector, timeout);
-                                     TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
   } else {
     LOG_TRX(trx, nestingLevel) << "write-locking collection "
                                << trxCollection->_cid;
-    res = collection->beginWriteTimed(useDeadlockDetector, timeout,
+    res = collection->beginWriteTimed(useDeadlockDetector, timeout);
-                                      TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
   }
 
   if (res == TRI_ERROR_NO_ERROR) {
@@ -1263,9 +1261,7 @@ TRI_transaction_t::TRI_transaction_t(TRI_vocbase_t* vocbase, double timeout, boo
       _timeout(TRI_TRANSACTION_DEFAULT_LOCK_TIMEOUT) {
   
   if (timeout > 0.0) {
-    _timeout = (uint64_t)(timeout * 1000000.0);
+    _timeout = timeout;
-  } else if (timeout == 0.0) {
-    _timeout = static_cast<uint64_t>(0);
   } 
 }
 

arangod/VocBase/transaction.h

@@ -49,13 +49,7 @@ struct TRI_vocbase_t;
 /// @brief time (in µs) that is spent waiting for a lock
 ////////////////////////////////////////////////////////////////////////////////
 
-#define TRI_TRANSACTION_DEFAULT_LOCK_TIMEOUT 30000000ULL
+#define TRI_TRANSACTION_DEFAULT_LOCK_TIMEOUT 30.0
-
-////////////////////////////////////////////////////////////////////////////////
-/// @brief sleep time (in µs) while waiting for lock acquisition
-////////////////////////////////////////////////////////////////////////////////
-
-#define TRI_TRANSACTION_DEFAULT_SLEEP_DURATION 10000ULL
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief transaction type
@@ -128,7 +122,7 @@ struct TRI_transaction_t {
   bool _hasOperations;
   bool _waitForSync;   // whether or not the collection had a synchronous op
   bool _beginWritten;  // whether or not the begin marker was already written
-  uint64_t _timeout;   // timeout for lock acquisition
+  double _timeout;     // timeout for lock acquisition
 };
 
 ////////////////////////////////////////////////////////////////////////////////