Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

This commit is contained in:
Kaveh Vahedipour 2017-01-02 15:58:52 +01:00
parent a2ee40d4f3 3c9d71fdcf
commit 4671f7ce45
13 changed files with 145 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG
View File

@ -49,6 +49,22 @@ edge attribute `label`.
* process.stdout.isTTY now returns `true` in arangosh and when running arangod with the `--console` flag
v3.1.8 (XXXX-XX-XX)
-------------------
* fixed issue #2238
* fixed issue #2234
v3.1.7 (2016-12-29)
-------------------
* fixed one too many elections in RAFT
* new agency comm backported from devel
v3.1.6 (2016-12-20)
-------------------

4
arangod/Aql/AstNode.cpp
View File

@ -1925,6 +1925,8 @@ AstNode* AstNode::clone(Ast* ast) const { return ast->clone(this); }
/// the string representation does not need to be JavaScript-compatible
/// except for node types NODE_TYPE_VALUE, NODE_TYPE_ARRAY and NODE_TYPE_OBJECT
/// (only for objects that do not contain dynamic attributes)
/// note that this may throw and that the caller is responsible for
/// catching the error
void AstNode::stringify(arangodb::basics::StringBuffer* buffer, bool verbose,
bool failIfLong) const {
// any arrays/objects with more values than this will not be stringified if
@ -2199,6 +2201,8 @@ void AstNode::stringify(arangodb::basics::StringBuffer* buffer, bool verbose,
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, message);
}
/// note that this may throw and that the caller is responsible for
/// catching the error
std::string AstNode::toString() const {
arangodb::basics::StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE);
stringify(&buffer, false, false);

9
arangod/Aql/AstNode.h
View File

@ -639,9 +639,16 @@ struct AstNode {
/// @brief clone a node, recursively
AstNode* clone(Ast*) const;
/// @brief append a JavaScript representation of the node into a string buffer
/// @brief append a string representation of the node into a string buffer
/// the string representation does not need to be JavaScript-compatible
/// except for node types NODE_TYPE_VALUE, NODE_TYPE_ARRAY and NODE_TYPE_OBJECT
/// (only for objects that do not contain dynamic attributes)
/// note that this may throw and that the caller is responsible for
/// catching the error
void stringify(arangodb::basics::StringBuffer*, bool, bool) const;
/// note that this may throw and that the caller is responsible for
/// catching the error
std::string toString() const;
/// @brief stringify the value of a node into a string buffer

83
arangod/Aql/Condition.cpp
View File

@ -991,54 +991,58 @@ bool Condition::canRemove(ExecutionPlan const* plan, ConditionPart const& me,
return node->toString();
};
for (size_t i = 0; i < n; ++i) {
auto operand = andNode->getMemberUnchecked(i);
try {
for (size_t i = 0; i < n; ++i) {
auto operand = andNode->getMemberUnchecked(i);
if (operand->isComparisonOperator()) {
auto lhs = operand->getMember(0);
auto rhs = operand->getMember(1);
if (operand->isComparisonOperator()) {
auto lhs = operand->getMember(0);
auto rhs = operand->getMember(1);
if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) {
clearAttributeAccess(result);
if (lhs->type == NODE_TYPE_ATTRIBUTE_ACCESS) {
clearAttributeAccess(result);
if (lhs->isAttributeAccessForVariable(result)) {
if (rhs->isConstant()) {
ConditionPart indexCondition(result.first, result.second, operand,
ATTRIBUTE_LEFT, nullptr);
if (lhs->isAttributeAccessForVariable(result)) {
if (rhs->isConstant()) {
ConditionPart indexCondition(result.first, result.second, operand,
ATTRIBUTE_LEFT, nullptr);
if (me.isCoveredBy(indexCondition, false)) {
if (me.isCoveredBy(indexCondition, false)) {
return true;
}
}
// non-constant condition
else if (me.operatorType == operand->type &&
normalize(me.valueNode) == normalize(rhs)) {
return true;
}
}
// non-constant condition
else if (me.operatorType == operand->type &&
normalize(me.valueNode) == normalize(rhs)) {
return true;
}
}
}
if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
rhs->type == NODE_TYPE_EXPANSION) {
clearAttributeAccess(result);
if (rhs->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
rhs->type == NODE_TYPE_EXPANSION) {
clearAttributeAccess(result);
if (rhs->isAttributeAccessForVariable(result)) {
if (lhs->isConstant()) {
ConditionPart indexCondition(result.first, result.second, operand,
ATTRIBUTE_RIGHT, nullptr);
if (rhs->isAttributeAccessForVariable(result)) {
if (lhs->isConstant()) {
ConditionPart indexCondition(result.first, result.second, operand,
ATTRIBUTE_RIGHT, nullptr);
if (me.isCoveredBy(indexCondition, true)) {
if (me.isCoveredBy(indexCondition, true)) {
return true;
}
}
// non-constant condition
else if (me.operatorType == operand->type &&
normalize(me.valueNode) == normalize(lhs)) {
return true;
}
}
// non-constant condition
else if (me.operatorType == operand->type &&
normalize(me.valueNode) == normalize(lhs)) {
return true;
}
}
}
}
} catch (...) {
// simply ignore any errors and return false
}
return false;
@ -1125,6 +1129,7 @@ AstNode* Condition::transformNode(AstNode* node) {
// create a new n-ary node
node = _ast->createNode(Ast::NaryOperatorType(old->type));
node->reserve(2);
node->addMember(old->getMember(0));
node->addMember(old->getMember(1));
}
@ -1142,11 +1147,9 @@ AstNode* Condition::transformNode(AstNode* node) {
auto sub = transformNode(node->getMemberUnchecked(i));
node->changeMember(i, sub);
if (sub->type == NODE_TYPE_OPERATOR_NARY_OR ||
sub->type == NODE_TYPE_OPERATOR_BINARY_OR) {
if (sub->type == NODE_TYPE_OPERATOR_NARY_OR) {
processChildren = true;
} else if (sub->type == NODE_TYPE_OPERATOR_NARY_AND ||
sub->type == NODE_TYPE_OPERATOR_BINARY_AND) {
} else if (sub->type == NODE_TYPE_OPERATOR_NARY_AND) {
mustCollapse = true;
}
}
@ -1165,14 +1168,15 @@ AstNode* Condition::transformNode(AstNode* node) {
auto newOperator = _ast->createNode(NODE_TYPE_OPERATOR_NARY_OR);
std::vector<PermutationState> permutationStates;
permutationStates.reserve(n);
for (size_t i = 0; i < n; ++i) {
auto sub = node->getMemberUnchecked(i);
if (sub->type == NODE_TYPE_OPERATOR_NARY_OR) {
permutationStates.emplace_back(
PermutationState(sub, sub->numMembers()));
permutationStates.emplace_back(sub, sub->numMembers());
} else {
permutationStates.emplace_back(PermutationState(sub, 1));
permutationStates.emplace_back(sub, 1);
}
}
@ -1182,9 +1186,10 @@ AstNode* Condition::transformNode(AstNode* node) {
while (!done) {
auto andOperator = _ast->createNode(NODE_TYPE_OPERATOR_NARY_AND);
andOperator->reserve(numPermutations);
for (size_t i = 0; i < numPermutations; ++i) {
auto state = permutationStates[i];
auto const& state = permutationStates[i];
andOperator->addMember(state.getValue()->clone(_ast));
}

26
arangod/Aql/OptimizerRules.cpp
View File

@ -3190,7 +3190,7 @@ struct OrSimplifier {
}
catch (...) {
}
return "";
return std::string();
}
bool qualifies(AstNode const* node, std::string& attributeName) const {
@ -3399,6 +3399,18 @@ struct RemoveRedundantOr {
CommonNodeFinder finder;
AstNode const* commonNode = nullptr;
std::string commonName;
bool hasRedundantCondition(AstNode const* node) {
try {
if (finder.find(node, NODE_TYPE_OPERATOR_BINARY_LT, commonNode,
commonName)) {
return hasRedundantConditionWalker(node);
}
} catch (...) {
// ignore errors and simply return false
}
return false;
}
AstNode* createReplacementNode(Ast* ast) {
TRI_ASSERT(commonNode != nullptr);
@ -3408,6 +3420,7 @@ struct RemoveRedundantOr {
bestValue);
}
private:
bool isInclusiveBound(AstNodeType type) {
return (type == NODE_TYPE_OPERATOR_BINARY_GE ||
type == NODE_TYPE_OPERATOR_BINARY_LE);
@ -3429,8 +3442,7 @@ struct RemoveRedundantOr {
}
// returns false if the existing value is better and true if the input value
// is
// better
// is better
bool compareBounds(AstNodeType type, AstNode const* value, int lowhigh) {
int cmp = CompareAstNodes(bestValue, value, true);
@ -3440,14 +3452,6 @@ struct RemoveRedundantOr {
return (cmp * lowhigh == 1);
}
bool hasRedundantCondition(AstNode const* node) {
if (finder.find(node, NODE_TYPE_OPERATOR_BINARY_LT, commonNode,
commonName)) {
return hasRedundantConditionWalker(node);
}
return false;
}
bool hasRedundantConditionWalker(AstNode const* node) {
AstNodeType type = node->type;

18
arangod/Aql/QueryResources.cpp
View File

@ -62,19 +62,25 @@ void QueryResources::addNode(AstNode* node) {
if (_nodes.empty()) {
// reserve some initial space for nodes
capacity = 16;
capacity = 64;
} else {
capacity = (std::max)(_nodes.size() + 8, _nodes.capacity());
capacity = _nodes.size() + 1;
if (capacity > _nodes.capacity()) {
capacity *= 2;
}
}
TRI_ASSERT(capacity > _nodes.size());
TRI_ASSERT(capacity >= _nodes.capacity());
// reserve space
_resourceMonitor->increaseMemoryUsage((capacity - _nodes.capacity()) * sizeof(AstNode*));
_nodes.reserve(capacity);
// reserve space for pointers
if (capacity > _nodes.capacity()) {
_resourceMonitor->increaseMemoryUsage((capacity - _nodes.capacity()) * sizeof(AstNode*));
_nodes.reserve(capacity);
}
// may throw
_resourceMonitor->increaseMemoryUsage(sizeof(AstNode));
// will not fail
_nodes.emplace_back(node);
}

2
arangod/StorageEngine/MMFilesCompactorThread.cpp
View File

@ -307,7 +307,7 @@ MMFilesCompactorThread::CompactionInitialContext MMFilesCompactorThread::getComp
bool ok;
{
bool const useDeadlockDetector = false;
int res = collection->beginReadTimed(useDeadlockDetector, 86400ULL * 1000ULL * 1000ULL, TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
int res = collection->beginReadTimed(useDeadlockDetector, 86400.0);
if (res != TRI_ERROR_NO_ERROR) {
ok = false;

3
arangod/Utils/CollectionReadLocker.h
View File

@ -40,8 +40,7 @@ class CollectionReadLocker {
CollectionReadLocker(LogicalCollection* collection, bool useDeadlockDetector, bool doLock)
: _collection(collection), _useDeadlockDetector(useDeadlockDetector), _doLock(false) {
if (doLock) {
int res = _collection->beginReadTimed(_useDeadlockDetector,
0, TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
int res = _collection->beginReadTimed(_useDeadlockDetector);
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);

2
arangod/Utils/CollectionWriteLocker.h
View File

@ -40,7 +40,7 @@ class CollectionWriteLocker {
CollectionWriteLocker(arangodb::LogicalCollection* collection, bool useDeadlockDetector, bool doLock)
: _collection(collection), _useDeadlockDetector(useDeadlockDetector), _doLock(false) {
if (doLock) {
int res = _collection->beginWriteTimed(_useDeadlockDetector, 0, TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
int res = _collection->beginWriteTimed(_useDeadlockDetector);
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);

59
arangod/VocBase/LogicalCollection.cpp
View File

@ -2922,7 +2922,7 @@ int LogicalCollection::endWrite(bool useDeadlockDetector) {
}
/// @brief read locks a collection, with a timeout (in µseconds)
int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout, uint64_t sleepPeriod) {
int LogicalCollection::beginReadTimed(bool useDeadlockDetector, double timeout) {
if (arangodb::Transaction::_makeNolockHeaders != nullptr) {
auto it = arangodb::Transaction::_makeNolockHeaders->find(name());
if (it != arangodb::Transaction::_makeNolockHeaders->end()) {
@ -2933,16 +2933,12 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
return TRI_ERROR_NO_ERROR;
}
}
uint64_t waited = 0;
if (timeout == 0) {
// we don't allow looping forever. limit waiting to 15 minutes max.
timeout = 15 * 60 * 1000 * 1000;
}
// LOCKING-DEBUG
// std::cout << "BeginReadTimed: " << _name << std::endl;
int iterations = 0;
bool wasBlocked = false;
double end = 0.0;
while (true) {
TRY_READ_LOCKER(locker, _idxLock);
@ -2970,6 +2966,7 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
return TRI_ERROR_DEADLOCK;
}
LOG(TRACE) << "waiting for read-lock on collection '" << name() << "'";
// fall-through intentional
} else if (++iterations >= 5) {
// periodically check for deadlocks
TRI_ASSERT(wasBlocked);
@ -2992,15 +2989,20 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
}
}
#ifdef _WIN32
usleep((unsigned long)sleepPeriod);
#else
usleep((useconds_t)sleepPeriod);
#endif
if (end == 0.0) {
// set end time for lock waiting
if (timeout <= 0.0) {
timeout = 15.0 * 60.0;
}
end = TRI_microtime() + timeout;
TRI_ASSERT(end > 0.0);
}
waited += sleepPeriod;
std::this_thread::yield();
TRI_ASSERT(end > 0.0);
if (waited > timeout) {
if (TRI_microtime() > end) {
if (useDeadlockDetector) {
_vocbase->_deadlockDetector.unsetReaderBlocked(this);
}
@ -3011,7 +3013,7 @@ int LogicalCollection::beginReadTimed(bool useDeadlockDetector, uint64_t timeout
}
/// @brief write locks a collection, with a timeout
int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, uint64_t timeout, uint64_t sleepPeriod) {
int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, double timeout) {
if (arangodb::Transaction::_makeNolockHeaders != nullptr) {
auto it = arangodb::Transaction::_makeNolockHeaders->find(name());
if (it != arangodb::Transaction::_makeNolockHeaders->end()) {
@ -3022,16 +3024,12 @@ int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, uint64_t timeou
return TRI_ERROR_NO_ERROR;
}
}
uint64_t waited = 0;
if (timeout == 0) {
// we don't allow looping forever. limit waiting to 15 minutes max.
timeout = 15 * 60 * 1000 * 1000;
}
// LOCKING-DEBUG
// std::cout << "BeginWriteTimed: " << document->_info._name << std::endl;
int iterations = 0;
bool wasBlocked = false;
double end = 0.0;
while (true) {
TRY_WRITE_LOCKER(locker, _idxLock);
@ -3080,15 +3078,22 @@ int LogicalCollection::beginWriteTimed(bool useDeadlockDetector, uint64_t timeou
}
}
#ifdef _WIN32
usleep((unsigned long)sleepPeriod);
#else
usleep((useconds_t)sleepPeriod);
#endif
std::this_thread::yield();
if (end == 0.0) {
// set end time for lock waiting
if (timeout <= 0.0) {
timeout = 15.0 * 60.0;
}
end = TRI_microtime() + timeout;
TRI_ASSERT(end > 0.0);
}
waited += sleepPeriod;
if (waited > timeout) {
std::this_thread::yield();
TRI_ASSERT(end > 0.0);
if (TRI_microtime() > end) {
if (useDeadlockDetector) {
_vocbase->_deadlockDetector.unsetWriterBlocked(this);
}

4
arangod/VocBase/LogicalCollection.h
View File

@ -380,8 +380,8 @@ class LogicalCollection {
int fillIndex(arangodb::Transaction*, arangodb::Index*,
bool skipPersistent = true);
int beginReadTimed(bool useDeadlockDetector, uint64_t, uint64_t);
int beginWriteTimed(bool useDeadlockDetector, uint64_t, uint64_t);
int beginReadTimed(bool useDeadlockDetector, double timeout = 0.0);
int beginWriteTimed(bool useDeadlockDetector, double timeout = 0.0);
int endRead(bool useDeadlockDetector);
int endWrite(bool useDeadlockDetector);

18
arangod/VocBase/transaction.cpp
View File

@ -286,10 +286,10 @@ static int LockCollection(TRI_transaction_collection_t* trxCollection,
LogicalCollection* collection = trxCollection->_collection;
TRI_ASSERT(collection != nullptr);
uint64_t timeout = trx->_timeout;
double timeout = trx->_timeout;
if (HasHint(trxCollection->_transaction, TRI_TRANSACTION_HINT_TRY_LOCK)) {
// give up if we cannot acquire the lock instantly
timeout = 1 * 100;
// give up early if we cannot acquire the lock instantly
timeout = 0.00000001;
}
bool const useDeadlockDetector = !IsSingleOperationTransaction(trx);
@ -297,13 +297,11 @@ static int LockCollection(TRI_transaction_collection_t* trxCollection,
int res;
if (type == TRI_TRANSACTION_READ) {
LOG_TRX(trx, nestingLevel) << "read-locking collection " << trxCollection->_cid;
res = collection->beginReadTimed(useDeadlockDetector, timeout,
TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
res = collection->beginReadTimed(useDeadlockDetector, timeout);
} else {
LOG_TRX(trx, nestingLevel) << "write-locking collection "
<< trxCollection->_cid;
res = collection->beginWriteTimed(useDeadlockDetector, timeout,
TRI_TRANSACTION_DEFAULT_SLEEP_DURATION);
res = collection->beginWriteTimed(useDeadlockDetector, timeout);
}
if (res == TRI_ERROR_NO_ERROR) {
@ -1263,10 +1261,8 @@ TRI_transaction_t::TRI_transaction_t(TRI_vocbase_t* vocbase, double timeout, boo
_timeout(TRI_TRANSACTION_DEFAULT_LOCK_TIMEOUT) {
if (timeout > 0.0) {
_timeout = (uint64_t)(timeout * 1000000.0);
} else if (timeout == 0.0) {
_timeout = static_cast<uint64_t>(0);
}
_timeout = timeout;
}
}
////////////////////////////////////////////////////////////////////////////////

10
arangod/VocBase/transaction.h
View File

@ -49,13 +49,7 @@ struct TRI_vocbase_t;
/// @brief time (in µs) that is spent waiting for a lock
////////////////////////////////////////////////////////////////////////////////
#define TRI_TRANSACTION_DEFAULT_LOCK_TIMEOUT 30000000ULL
////////////////////////////////////////////////////////////////////////////////
/// @brief sleep time (in µs) while waiting for lock acquisition
////////////////////////////////////////////////////////////////////////////////
#define TRI_TRANSACTION_DEFAULT_SLEEP_DURATION 10000ULL
#define TRI_TRANSACTION_DEFAULT_LOCK_TIMEOUT 30.0
////////////////////////////////////////////////////////////////////////////////
/// @brief transaction type
@ -128,7 +122,7 @@ struct TRI_transaction_t {
bool _hasOperations;
bool _waitForSync; // whether or not the collection had a synchronous op
bool _beginWritten; // whether or not the begin marker was already written
uint64_t _timeout; // timeout for lock acquisition
double _timeout; // timeout for lock acquisition
};
////////////////////////////////////////////////////////////////////////////////