Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

This commit is contained in:
Simon Grätzer 2017-01-31 13:38:17 +01:00
parent 6abfd6c3bd 659699a11e
commit bcc1a83ae5
31 changed files with 374 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
CHANGELOG
View File

@ -5,6 +5,18 @@ devel
* changed index filling to make it more parallel, dispatch tasks to boost::asio
* more detailed stacktraces in Foxx apps
v3.1.10 (2017-XX-XX)
--------------------
* updated versions of bundled node modules:
- joi: from 8.4.2 to 9.2.0
- joi-to-json-schema: from 2.2.0 to 2.3.0
- sinon: from 1.17.4 to 1.17.6
- lodash: from 4.13.1 to 4.16.6
* added shortcut for AQL ternary operator
instead of `condition ? true-part : false-part` it is now possible to also use a
shortcut variant `condition ? : false-part`, e.g.
@ -74,6 +86,13 @@ edge attribute `label`.
* added option -D to define a configuration file environment key=value
* changed encoding behavior for URLs encoded in the C++ code of ArangoDB:
previously the special characters `-`, `_`, `~` and `.` were returned as-is
after URL-encoding, now `.` will be encoded to be `%2e`.
This also changes the behavior of how incoming URIs are processed: previously
occurrences of `..` in incoming request URIs were collapsed (e.g. `a/../b/` was
collapsed to a plain `b/`). Now `..` in incoming request URIs are not collapsed.
* Foxx request URL suffix is no longer unescaped
* @arangodb/request option json now defaults to `true` if the response body is not empty and encoding is not explicitly set to `null` (binary).

10
Documentation/DocuBlocks/schedulerBackend.md
View File

@ -1,10 +0,0 @@
@brief scheduler backend
`--scheduler.backend arg`
The I/O method used by the event handler. The default (if this option is
not specified) is to try all recommended backends. This is platform
specific. See libev for further details and the meaning of select, poll
and epoll.

3
Installation/systemd/arangodb3.service.in
View File

@ -30,7 +30,8 @@ ExecStartPre=@CHOWN_EXECUTABLE@ -R arangodb:arangodb /var/lib/arangodb3-apps
ExecStartPre=@CHMOD_EXECUTABLE@ 700 /var/lib/arangodb3-apps
ExecStartPre=/usr/sbin/arangod --uid arangodb --gid arangodb --pid-file /var/run/arangodb3/arangod.pid --server.rest-server false --database.auto-upgrade true
ExecStart=/usr/sbin/arangod --uid arangodb --gid arangodb --pid-file /var/run/arangodb3/arangod.pid --temp.path /var/tmp/arangodb3 --supervisor --log.foreground-tty false
TimeoutStopSec=120
TimeoutStopSec=3600
TimeoutSec=3600
Restart=on-failure
[Install]

5
LICENSES-OTHER-COMPONENTS.md
View File

@ -51,11 +51,6 @@
* Project Home: http://site.icu-project.org/
* License: [ICU License](http://source.icu-project.org/repos/icu/icu/trunk/license.html)
### libev 4.11
* Project Home: http://software.schmorp.de/pkg/libev.html
* License: Dual-License [BSD-style 2-Clause License](http://cvs.schmorp.de/libev/LICENSE?revision=1.11&view=markup)
### linenoise-ng
* GitHub: https://github.com/arangodb/linenoise-ng

44
arangod/Agency/AgencyComm.cpp
View File

@ -612,7 +612,7 @@ std::string AgencyCommManager::redirect(
<< specification << ", url = " << rest;
if (endpoint == specification) {
LOG_TOPIC(WARN, Logger::AGENCYCOMM)
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< "got an agency redirect back to the old agency '" << endpoint << "'";
failedNonLocking(std::move(connection), endpoint);
return "";
@ -632,7 +632,7 @@ std::string AgencyCommManager::redirect(
std::remove(_endpoints.begin(), _endpoints.end(), specification),
_endpoints.end());
LOG_TOPIC(WARN, Logger::AGENCYCOMM)
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< "Got an agency redirect from '" << endpoint
<< "' to '" << specification << "'";
@ -1359,17 +1359,11 @@ AgencyCommResult AgencyComm::sendWithFailover(
LOG_TOPIC(ERR, Logger::AGENCYCOMM) << result._message;
return result;
}
if (1 < tries) {
LOG_TOPIC(WARN, Logger::AGENCYCOMM)
<< "Retrying agency communication at '" << endpoint
<< "', tries: " << tries << " ("
<< 1.e-2 * (
std::round(
1.e+2 * std::chrono::duration<double>(
std::chrono::steady_clock::now() - started).count())) << "s)";
}
double elapsed = 1.e-2 * (
std::round(1.e+2 * std::chrono::duration<double>(
std::chrono::steady_clock::now() - started).count()));
// try to send; if we fail completely, do not retry
try {
result = send(connection.get(), method, conTimeout, url, body, clientId);
@ -1379,7 +1373,7 @@ AgencyCommResult AgencyComm::sendWithFailover(
connection = AgencyCommManager::MANAGER->acquire(endpoint);
continue;
}
// got a result, we are done
if (result.successful()) {
AgencyCommManager::MANAGER->release(std::move(connection), endpoint);
@ -1389,7 +1383,7 @@ AgencyCommResult AgencyComm::sendWithFailover(
// break on a watch timeout (drop connection)
if (!clientId.empty() && result._sent &&
(result._statusCode == 0 || result._statusCode == 503)) {
VPackBuilder b;
{
VPackArrayBuilder ab(&b);
@ -1470,6 +1464,26 @@ AgencyCommResult AgencyComm::sendWithFailover(
break;
}
if (tries%50 == 0) {
LOG_TOPIC(WARN, Logger::AGENCYCOMM)
<< "Bad agency communiction! Unsuccessful consecutive tries:"
<< tries << " (" << elapsed << "s). Network checks needed!";
} else if (tries%15 == 0) {
LOG_TOPIC(INFO, Logger::AGENCYCOMM)
<< "Flaky agency communication. Unsuccessful consecutive tries: "
<< tries << " (" << elapsed << "s). Network checks advised.";
}
if (1 < tries) {
LOG_TOPIC(DEBUG, Logger::AGENCYCOMM)
<< "Retrying agency communication at '" << endpoint
<< "', tries: " << tries << " ("
<< 1.e-2 * (
std::round(
1.e+2 * std::chrono::duration<double>(
std::chrono::steady_clock::now() - started).count())) << "s)";
}
// here we have failed and want to try next endpoint
AgencyCommManager::MANAGER->failed(std::move(connection), endpoint);
endpoint.clear();

36
arangod/Aql/AqlItemBlock.cpp
View File

@ -22,6 +22,8 @@
////////////////////////////////////////////////////////////////////////////////
#include "AqlItemBlock.h"
#include "Aql/BlockCollector.h"
#include "Aql/ExecutionBlock.h"
#include "Aql/ExecutionNode.h"
#include "Basics/VelocyPackHelper.h"
@ -430,6 +432,37 @@ AqlItemBlock* AqlItemBlock::steal(std::vector<size_t> const& chosen, size_t from
return res.release();
}
/// @brief concatenate multiple blocks
AqlItemBlock* AqlItemBlock::concatenate(ResourceMonitor* resourceMonitor,
BlockCollector* collector) {
size_t totalSize = collector->totalSize();
RegisterId nrRegs = collector->nrRegs();
TRI_ASSERT(totalSize > 0);
TRI_ASSERT(nrRegs > 0);
auto res = std::make_unique<AqlItemBlock>(resourceMonitor, totalSize, nrRegs);
size_t pos = 0;
for (auto& it : collector->_blocks) {
size_t const n = it->size();
for (size_t row = 0; row < n; ++row) {
for (RegisterId col = 0; col < nrRegs; ++col) {
// copy over value
AqlValue const& a = it->getValueReference(row, col);
if (!a.isEmpty()) {
res->setValue(pos + row, col, a);
}
}
}
it->eraseAll();
pos += n;
}
return res.release();
}
/// @brief concatenate multiple blocks, note that the new block now owns all
/// AqlValue pointers in the old blocks, therefore, the latter are all
/// set to nullptr, just to be sure.
@ -455,7 +488,6 @@ AqlItemBlock* AqlItemBlock::concatenate(ResourceMonitor* resourceMonitor,
size_t pos = 0;
for (auto& it : blocks) {
TRI_ASSERT(it != res.get());
size_t const n = it->size();
for (size_t row = 0; row < n; ++row) {
for (RegisterId col = 0; col < nrRegs; ++col) {
@ -467,7 +499,7 @@ AqlItemBlock* AqlItemBlock::concatenate(ResourceMonitor* resourceMonitor,
}
}
it->eraseAll();
pos += it->size();
pos += n;
}
return res.release();

4
arangod/Aql/AqlItemBlock.h
View File

@ -32,6 +32,7 @@
namespace arangodb {
namespace aql {
class BlockCollector;
// an <AqlItemBlock> is a <nrItems>x<nrRegs> vector of <AqlValue>s (not
// pointers). The size of an <AqlItemBlock> is the number of items.
@ -254,6 +255,9 @@ class AqlItemBlock {
/// after this operation, because it is unclear, when the values
/// to which our AqlValues point will vanish.
AqlItemBlock* steal(std::vector<size_t> const& chosen, size_t from, size_t to);
/// @brief concatenate multiple blocks from a collector
static AqlItemBlock* concatenate(ResourceMonitor*, BlockCollector* collector);
/// @brief concatenate multiple blocks, note that the new block now owns all
/// AqlValue pointers in the old blocks, therefore, the latter are all

6
arangod/Aql/Ast.cpp
View File

@ -2749,6 +2749,12 @@ AstNode* Ast::optimizeFunctionCall(AstNode* node) {
return createNodeFunctionCall("COLLECTION_COUNT", countArgs);
}
}
} else if (func->externalName == "IS_NULL") {
auto args = node->getMember(0);
if (args->numMembers() == 1) {
// replace IS_NULL(x) function call with `x == null`
return createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, args->getMemberUnchecked(0), createNodeValueNull());
}
}
if (!func->isDeterministic) {

11
arangod/Aql/BasicBlocks.cpp
View File

@ -22,11 +22,17 @@
////////////////////////////////////////////////////////////////////////////////
#include "BasicBlocks.h"
#include "Aql/AqlItemBlock.h"
#include "Aql/ExecutionEngine.h"
#include "Basics/Exceptions.h"
#include "VocBase/vocbase.h"
using namespace arangodb::aql;
void SingletonBlock::deleteInputVariables() {
delete _inputRegisterValues;
_inputRegisterValues = nullptr;
}
void SingletonBlock::buildWhitelist() {
if (!_whitelistBuilt) {
@ -154,6 +160,11 @@ FilterBlock::FilterBlock(ExecutionEngine* engine, FilterNode const* en)
}
FilterBlock::~FilterBlock() {}
/// @brief internal function to actually decide if the document should be used
bool FilterBlock::takeItem(AqlItemBlock* items, size_t index) const {
return items->getValueReference(index, _inReg).toBoolean();
}
/// @brief internal function to get another block
bool FilterBlock::getBlock(size_t atLeast, size_t atMost) {

9
arangod/Aql/BasicBlocks.h
View File

@ -59,10 +59,7 @@ class SingletonBlock : public ExecutionBlock {
int64_t remaining() override final { return _done ? 0 : 1; }
private:
void deleteInputVariables() {
delete _inputRegisterValues;
_inputRegisterValues = nullptr;
}
void deleteInputVariables();
void buildWhitelist();
@ -85,9 +82,7 @@ class FilterBlock : public ExecutionBlock {
private:
/// @brief internal function to actually decide if the document should be used
inline bool takeItem(AqlItemBlock* items, size_t index) const {
return items->getValueReference(index, _inReg).toBoolean();
}
bool takeItem(AqlItemBlock* items, size_t index) const;
/// @brief internal function to get another block
bool getBlock(size_t atLeast, size_t atMost);

84
arangod/Aql/BlockCollector.cpp Normal file
View File

@ -0,0 +1,84 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief Infrastructure for ExecutionPlans
///
/// DISCLAIMER
///
/// Copyright 2010-2014 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, triagens GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "BlockCollector.h"
#include "Aql/AqlItemBlock.h"
#include "Aql/ResourceUsage.h"
using namespace arangodb::aql;
BlockCollector::BlockCollector() : _totalSize(0) {}
BlockCollector::~BlockCollector() { clear(); }
size_t BlockCollector::totalSize() const { return _totalSize; }
size_t BlockCollector::nrRegs() const {
TRI_ASSERT(_totalSize > 0);
TRI_ASSERT(!_blocks.empty());
return _blocks[0]->getNrRegs();
}
void BlockCollector::clear() {
for (auto& it : _blocks) {
it->eraseAll();
delete it;
}
_blocks.clear();
_totalSize = 0;
}
void BlockCollector::add(std::unique_ptr<AqlItemBlock> block) {
TRI_ASSERT(block->size() > 0);
_blocks.push_back(block.get());
_totalSize += block->size();
block.release();
}
AqlItemBlock* BlockCollector::steal(ResourceMonitor* resourceMonitor) {
if (_blocks.empty()) {
return nullptr;
}
TRI_ASSERT(_totalSize > 0);
AqlItemBlock* result = nullptr;
if (_blocks.size() == 1) {
// only got a single result. return it as it is
result = _blocks[0];
} else {
result = AqlItemBlock::concatenate(resourceMonitor, this);
for (auto& it : _blocks) {
delete it;
}
}
// ownership is now passed to result
_totalSize = 0;
_blocks.clear();
return result;
}

61
arangod/Aql/BlockCollector.h Normal file
View File

@ -0,0 +1,61 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_AQL_BLOCK_COLLECTOR_H
#define ARANGOD_AQL_BLOCK_COLLECTOR_H 1
#include "Basics/Common.h"
namespace arangodb {
namespace aql {
class AqlItemBlock;
class ResourceMonitor;
class BlockCollector {
friend class AqlItemBlock;
public:
BlockCollector(BlockCollector const&) = delete;
BlockCollector& operator=(BlockCollector const&) = delete;
BlockCollector();
~BlockCollector();
size_t totalSize() const;
size_t nrRegs() const;
void clear();
void add(std::unique_ptr<AqlItemBlock> block);
AqlItemBlock* steal(ResourceMonitor*);
private:
std::vector<AqlItemBlock*> _blocks;
size_t _totalSize;
};
} // namespace arangodb::aql
} // namespace arangodb
#endif

1
arangod/Aql/ClusterBlocks.cpp
View File

@ -29,6 +29,7 @@
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>
#include "Aql/AqlItemBlock.h"
#include "Aql/AqlValue.h"
#include "Aql/ExecutionEngine.h"
#include "Basics/Exceptions.h"

5
arangod/Aql/ExecutionBlock.cpp
View File

@ -23,11 +23,12 @@
////////////////////////////////////////////////////////////////////////////////
#include "ExecutionBlock.h"
#include "Aql/ExecutionEngine.h"
#include "Aql/AqlItemBlock.h"
#include "Aql/Ast.h"
#include "Aql/ExecutionEngine.h"
using namespace arangodb::aql;
ExecutionBlock::ExecutionBlock(ExecutionEngine* engine, ExecutionNode const* ep)
: _engine(engine),
_trx(engine->getQuery()->trx()),

2
arangod/Aql/ExecutionBlock.h
View File

@ -24,7 +24,6 @@
#ifndef ARANGOD_AQL_EXECUTION_BLOCK_H
#define ARANGOD_AQL_EXECUTION_BLOCK_H 1
#include "AqlItemBlock.h"
#include "Aql/ExecutionNode.h"
#include "Aql/Variable.h"
@ -61,6 +60,7 @@ namespace arangodb {
class Transaction;
namespace aql {
class AqlItemBlock;
/// @brief sort element for block, consisting of register, sort direction,
/// and a possible attribute path to dig into the document

2
arangod/Aql/ExecutionEngine.cpp
View File

@ -1042,7 +1042,7 @@ struct CoordinatorInstanciator : public WalkerWorker<ExecutionNode> {
id = TRI_NewTickServer();
try {
queryRegistry->insert(id, engine->getQuery(), 3600.0);
queryRegistry->insert(id, engine->getQuery(), 600.0);
} catch (...) {
delete engine->getQuery();
// This deletes the new query as well as the engine

19
arangod/Aql/IndexBlock.cpp
View File

@ -470,6 +470,7 @@ bool IndexBlock::readIndex(size_t atMost) {
int IndexBlock::initializeCursor(AqlItemBlock* items, size_t pos) {
DEBUG_BEGIN_BLOCK();
_collector.clear();
int res = ExecutionBlock::initializeCursor(items, pos);
if (res != TRI_ERROR_NO_ERROR) {
@ -492,9 +493,9 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
traceGetSomeBegin();
if (_done) {
traceGetSomeEnd(nullptr);
return nullptr;
return _collector.steal(_engine->getQuery()->resourceMonitor());
}
std::unique_ptr<AqlItemBlock> res;
do {
@ -508,7 +509,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
if (!ExecutionBlock::getBlock(toFetch, toFetch) || (!initIndexes())) {
_done = true;
traceGetSomeEnd(nullptr);
return nullptr;
return _collector.steal(_engine->getQuery()->resourceMonitor());
}
_pos = 0; // this is in the first block
@ -530,7 +531,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
if (!ExecutionBlock::getBlock(DefaultBatchSize(), DefaultBatchSize())) {
_done = true;
traceGetSomeEnd(nullptr);
return nullptr;
return _collector.steal(_engine->getQuery()->resourceMonitor());
}
_pos = 0; // this is in the first block
}
@ -538,7 +539,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
if (!initIndexes()) {
_done = true;
traceGetSomeEnd(nullptr);
return nullptr;
return _collector.steal(_engine->getQuery()->resourceMonitor());
}
readIndex(atMost);
}
@ -581,6 +582,13 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
res->copyValuesFromFirstRow(j, static_cast<RegisterId>(curRegs));
}
}
_collector.add(std::move(res));
TRI_ASSERT(res.get() == nullptr);
if (_collector.totalSize() >= atMost) {
res.reset(_collector.steal(_engine->getQuery()->resourceMonitor()));
}
}
} while (res.get() == nullptr);
@ -588,6 +596,7 @@ AqlItemBlock* IndexBlock::getSome(size_t atLeast, size_t atMost) {
// Clear out registers no longer needed later:
clearRegisters(res.get());
traceGetSomeEnd(res.get());
return res.release();
// cppcheck-suppress style

3
arangod/Aql/IndexBlock.h
View File

@ -25,6 +25,7 @@
#ifndef ARANGOD_AQL_INDEX_BLOCK_H
#define ARANGOD_AQL_INDEX_BLOCK_H 1
#include "Aql/BlockCollector.h"
#include "Aql/ExecutionBlock.h"
#include "Aql/ExecutionNode.h"
#include "Aql/IndexNode.h"
@ -155,6 +156,8 @@ class IndexBlock : public ExecutionBlock {
bool _hasV8Expression;
std::unique_ptr<ManagedDocumentResult> _mmdr;
BlockCollector _collector;
};
} // namespace arangodb::aql

2
arangod/Aql/QueryRegistry.h
View File

@ -45,7 +45,7 @@ class QueryRegistry {
/// a query for this <vocbase> and <id> combination and an exception will
/// be thrown in that case. The time to live <ttl> is in seconds and the
/// query will be deleted if it is not opened for that amount of time.
void insert(QueryId id, Query* query, double ttl = 3600.0);
void insert(QueryId id, Query* query, double ttl = 600.0);
/// @brief open, find a query in the registry, if none is found, a nullptr
/// is returned, otherwise, ownership of the query is transferred to the

4
arangod/Aql/RestAqlHandler.cpp
View File

@ -105,7 +105,7 @@ void RestAqlHandler::createQueryFromVelocyPack() {
}
// Now the query is ready to go, store it in the registry and return:
double ttl = 3600.0;
double ttl = 600.0;
bool found;
std::string const& ttlstring = _request->header("ttl", found);
@ -317,7 +317,7 @@ void RestAqlHandler::createQueryFromString() {
}
// Now the query is ready to go, store it in the registry and return:
double ttl = 3600.0;
double ttl = 600.0;
bool found;
std::string const& ttlstring = _request->header("ttl", found);

1
arangod/CMakeLists.txt
View File

@ -115,6 +115,7 @@ SET(ARANGOD_SOURCES
Aql/BaseExpressionContext.cpp
Aql/BasicBlocks.cpp
Aql/BindParameters.cpp
Aql/BlockCollector.cpp
Aql/CalculationBlock.cpp
Aql/ClusterBlocks.cpp
Aql/ClusterNodes.cpp

97
arangod/Cluster/HeartbeatThread.cpp
View File

@ -87,6 +87,60 @@ HeartbeatThread::HeartbeatThread(AgencyCallbackRegistry* agencyCallbackRegistry,
HeartbeatThread::~HeartbeatThread() { shutdown(); }
////////////////////////////////////////////////////////////////////////////////
/// @brief running of heartbeat background jobs (in JavaScript), we run
/// these by instantiating an object in class HeartbeatBackgroundJob,
/// which is a std::function<void()> and holds a shared_ptr to the
/// HeartbeatThread singleton itself. This instance is then posted to
/// the io_service for execution in the thread pool. Should the heartbeat
/// thread itself terminate during shutdown, then the HeartbeatThread
/// singleton itself is still kept alive by the shared_ptr in the instance
/// of HeartbeatBackgroundJob. The operator() method simply calls the
/// runBackgroundJob() method of the heartbeat thread. Should this have
/// to schedule another background job, then it can simply create a new
/// HeartbeatBackgroundJob instance, again using shared_from_this() to
/// create a new shared_ptr keeping the HeartbeatThread object alive.
////////////////////////////////////////////////////////////////////////////////
class HeartbeatBackgroundJob {
std::shared_ptr<HeartbeatThread> _heartbeatThread;
public:
explicit HeartbeatBackgroundJob(std::shared_ptr<HeartbeatThread> hbt)
: _heartbeatThread(hbt) {}
void operator()() {
_heartbeatThread->runBackgroundJob();
}
};
////////////////////////////////////////////////////////////////////////////////
/// @brief method runBackgroundJob()
////////////////////////////////////////////////////////////////////////////////
void HeartbeatThread::runBackgroundJob() {
uint64_t jobNr = ++_backgroundJobsLaunched;
LOG_TOPIC(INFO, Logger::HEARTBEAT) << "sync callback started " << jobNr;
{
DBServerAgencySync job(this);
job.work();
}
LOG_TOPIC(INFO, Logger::HEARTBEAT) << "sync callback ended " << jobNr;
{
MUTEX_LOCKER(mutexLocker, *_statusLock);
TRI_ASSERT(_backgroundJobScheduledOrRunning);
if (_launchAnotherBackgroundJob) {
jobNr = ++_backgroundJobsPosted;
LOG_TOPIC(INFO, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr;
_launchAnotherBackgroundJob = false;
_ioService->post(HeartbeatBackgroundJob(shared_from_this()));
} else {
_backgroundJobScheduledOrRunning = false;
_launchAnotherBackgroundJob = false;
}
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief heartbeat main loop
/// the heartbeat thread constantly reports the current server status to the
@ -104,31 +158,6 @@ void HeartbeatThread::run() {
if (ServerState::instance()->isCoordinator()) {
runCoordinator();
} else {
// Set the member variable that holds a closure to run background
// jobs in JS:
auto self = shared_from_this();
_backgroundJob = [self, this]() {
uint64_t jobNr = ++_backgroundJobsLaunched;
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback started " << jobNr;
{
DBServerAgencySync job(this);
job.work();
}
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "sync callback ended " << jobNr;
{
MUTEX_LOCKER(mutexLocker, *_statusLock);
if (_launchAnotherBackgroundJob) {
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail "
<< ++_backgroundJobsPosted;
_launchAnotherBackgroundJob = false;
_ioService->post(_backgroundJob);
} else {
_backgroundJobScheduledOrRunning = false;
_launchAnotherBackgroundJob = false;
}
}
};
runDBServer();
}
}
@ -340,16 +369,6 @@ void HeartbeatThread::runDBServer() {
}
_agencyCallbackRegistry->unregisterCallback(planAgencyCallback);
int count = 0;
while (++count < 3000) {
{
MUTEX_LOCKER(mutexLocker, *_statusLock);
if (!_backgroundJobScheduledOrRunning) {
break;
}
}
usleep(100000);
}
LOG_TOPIC(TRACE, Logger::HEARTBEAT)
<< "stopped heartbeat thread (DBServer version)";
}
@ -593,7 +612,7 @@ bool HeartbeatThread::init() {
////////////////////////////////////////////////////////////////////////////////
void HeartbeatThread::dispatchedJobResult(DBServerAgencySyncResult result) {
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "Dispatched job returned!";
LOG_TOPIC(INFO, Logger::HEARTBEAT) << "Dispatched job returned!";
bool doSleep = false;
{
MUTEX_LOCKER(mutexLocker, *_statusLock);
@ -777,10 +796,10 @@ void HeartbeatThread::syncDBServerStatusQuo() {
}
// schedule a job for the change:
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync "
<< ++_backgroundJobsPosted;
uint64_t jobNr = ++_backgroundJobsPosted;
LOG_TOPIC(INFO, Logger::HEARTBEAT) << "dispatching sync " << jobNr;
_backgroundJobScheduledOrRunning = true;
_ioService->post(_backgroundJob);
_ioService->post(HeartbeatBackgroundJob(shared_from_this()));
}
////////////////////////////////////////////////////////////////////////////////

8
arangod/Cluster/HeartbeatThread.h
View File

@ -74,6 +74,8 @@ class HeartbeatThread : public Thread,
void setReady() { _ready.store(true); }
void runBackgroundJob();
void dispatchedJobResult(DBServerAgencySyncResult);
//////////////////////////////////////////////////////////////////////////////
@ -253,12 +255,6 @@ class HeartbeatThread : public Thread,
//////////////////////////////////////////////////////////////////////////////
bool _launchAnotherBackgroundJob;
//////////////////////////////////////////////////////////////////////////////
/// @brief _backgroundJob, the closure that does the work
//////////////////////////////////////////////////////////////////////////////
std::function<void()> _backgroundJob;
};
}

2
arangod/Indexes/PathBasedIndex.cpp
View File

@ -124,7 +124,7 @@ int PathBasedIndex::fillElement(std::vector<T*>& elements,
auto slices = buildIndexValue(doc);
if (slices.size() == n) {
// if shapes.size() != n, then the value is not inserted into the index
// if slices.size() != n, then the value is not inserted into the index
// because of index sparsity!
T* element = static_cast<T*>(_allocator->allocate());
TRI_ASSERT(element != nullptr);

14
arangod/Wal/LogfileManager.cpp
View File

@ -876,7 +876,7 @@ int LogfileManager::flush(bool waitForSync, bool waitForCollector,
res = this->waitForCollector(lastOpenLogfileId, maxWaitTime);
if (res == TRI_ERROR_LOCK_TIMEOUT) {
LOG(ERR) << "got lock timeout when waiting for WAL flush. lastOpenLogfileId: " << lastOpenLogfileId;
LOG(DEBUG) << "got lock timeout when waiting for WAL flush. lastOpenLogfileId: " << lastOpenLogfileId;
}
} else if (res == TRI_ERROR_ARANGO_DATAFILE_EMPTY) {
// current logfile is empty and cannot be collected
@ -887,7 +887,7 @@ int LogfileManager::flush(bool waitForSync, bool waitForCollector,
res = this->waitForCollector(lastSealedLogfileId, maxWaitTime);
if (res == TRI_ERROR_LOCK_TIMEOUT) {
LOG(ERR) << "got lock timeout when waiting for WAL flush. lastSealedLogfileId: " << lastSealedLogfileId;
LOG(DEBUG) << "got lock timeout when waiting for WAL flush. lastSealedLogfileId: " << lastSealedLogfileId;
}
}
}
@ -1731,8 +1731,7 @@ int LogfileManager::waitForCollector(Logfile::IdType logfileId,
// try again
}
// TODO: remove debug info here
LOG(ERR) << "going into lock timeout. having waited for logfile: " << logfileId << ", maxWaitTime: " << maxWaitTime;
LOG(DEBUG) << "going into lock timeout. having waited for logfile: " << logfileId << ", maxWaitTime: " << maxWaitTime;
logStatus();
// waited for too long
@ -1740,12 +1739,11 @@ int LogfileManager::waitForCollector(Logfile::IdType logfileId,
}
void LogfileManager::logStatus() {
// TODO: remove debug info here
LOG(ERR) << "logfile manager status report: lastCollectedId: " << _lastCollectedId.load() << ", lastSealedId: " << _lastSealedId.load();
LOG(DEBUG) << "logfile manager status report: lastCollectedId: " << _lastCollectedId.load() << ", lastSealedId: " << _lastSealedId.load();
READ_LOCKER(locker, _logfilesLock);
for (auto logfile : _logfiles) {
LOG(ERR) << "- logfile " << logfile.second->id() << ", filename '" << logfile.second->filename()
<< "', status " << logfile.second->statusText();
LOG(DEBUG) << "- logfile " << logfile.second->id() << ", filename '" << logfile.second->filename()
<< "', status " << logfile.second->statusText();
}
}

1
etc/testing/arangod-common.conf
View File

@ -2,6 +2,7 @@
force-direct = true
level = info
level = replication=warn
level = threads=debug
[database]
force-sync-properties = false

14
js/server/modules/@arangodb/cluster.js
View File

@ -745,7 +745,7 @@ function executePlanForCollections(plannedCollections) {
let collection;
if (!localCollections.hasOwnProperty(shardName)) {
// must create this shard
console.info("creating local shard '%s/%s' for central '%s/%s'",
console.debug("creating local shard '%s/%s' for central '%s/%s'",
database,
shardName,
database,
@ -813,7 +813,7 @@ function executePlanForCollections(plannedCollections) {
}, {});
if (Object.keys(properties).length > 0) {
console.info("updating properties for local shard '%s/%s'",
console.debug("updating properties for local shard '%s/%s'",
database,
shardName);
@ -831,17 +831,17 @@ function executePlanForCollections(plannedCollections) {
// Now check whether the status is OK:
if (collectionStatus !== collectionInfo.status) {
console.info("detected status change for local shard '%s/%s'",
console.debug("detected status change for local shard '%s/%s'",
database,
shardName);
if (collectionInfo.status === ArangoCollection.STATUS_UNLOADED) {
console.info("unloading local shard '%s/%s'",
console.debug("unloading local shard '%s/%s'",
database,
shardName);
collection.unload();
} else if (collectionInfo.status === ArangoCollection.STATUS_LOADED) {
console.info("loading local shard '%s/%s'",
console.debug("loading local shard '%s/%s'",
database,
shardName);
collection.load();
@ -1264,7 +1264,7 @@ function executePlanForDatabases(plannedDatabases) {
if (!plannedDatabases.hasOwnProperty(name) && name.substr(0, 1) !== '_') {
// must drop database
console.info("dropping local database '%s'", name);
console.debug("dropping local database '%s'", name);
// Do we have to stop a replication applier first?
if (ArangoServerState.role() === 'SECONDARY') {
@ -1273,7 +1273,7 @@ function executePlanForDatabases(plannedDatabases) {
var rep = require('@arangodb/replication');
var state = rep.applier.state();
if (state.state.running === true) {
console.info('stopping replication applier first');
console.debug('stopping replication applier first');
rep.applier.stop();
}
}

40
js/server/tests/aql/aql-optimizer-geoindex.js
View File

@ -53,23 +53,8 @@ function optimizerRuleTestSuite() {
sorted : true
};
var ruleName = "use-geoindex";
var secondRuleName = "use-geoindexes";
var removeCalculationNodes = "remove-unnecessary-calculations-2";
var ruleName = "geoindex";
var colName = "UnitTestsAqlOptimizer" + ruleName.replace(/-/g, "_");
var colNameOther = colName + "_XX";
// various choices to control the optimizer:
var paramNone = { optimizer: { rules: [ "-all" ] } };
var paramIndexFromSort = { optimizer: { rules: [ "-all", "+" + ruleName ] } };
var paramIndexRange = { optimizer: { rules: [ "-all", "+" + secondRuleName ] } };
var paramIndexFromSort_IndexRange = { optimizer: { rules: [ "-all", "+" + ruleName, "+" + secondRuleName ] } };
var paramIndexFromSort_IndexRange_RemoveCalculations = {
optimizer: { rules: [ "-all", "+" + ruleName, "+" + secondRuleName, "+" + removeCalculationNodes ] }
};
var paramIndexFromSort_RemoveCalculations = {
optimizer: { rules: [ "-all", "+" + ruleName, "+" + removeCalculationNodes ] }
};
var geocol;
var sortArray = function (l, r) {
@ -113,19 +98,6 @@ function optimizerRuleTestSuite() {
};
var geodistance = function(latitude1, longitude1, latitude2, longitude2) {
//if (TYPEWEIGHT(latitude1) !== TYPEWEIGHT_NUMBER ||
// TYPEWEIGHT(longitude1) !== TYPEWEIGHT_NUMBER ||
// TYPEWEIGHT(latitude2) !== TYPEWEIGHT_NUMBER ||
// TYPEWEIGHT(longitude2) !== TYPEWEIGHT_NUMBER) {
// WARN('DISTANCE', INTERNAL.errors.ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH);
// return null;
//}
//var p1 = AQL_TO_NUMBER(latitude1) * (Math.PI / 180.0);
//var p2 = AQL_TO_NUMBER(latitude2) * (Math.PI / 180.0);
//var d1 = AQL_TO_NUMBER(latitude2 - latitude1) * (Math.PI / 180.0);
//var d2 = AQL_TO_NUMBER(longitude2 - longitude1) * (Math.PI / 180.0);
var p1 = (latitude1) * (Math.PI / 180.0);
var p2 = (latitude2) * (Math.PI / 180.0);
var d1 = (latitude2 - latitude1) * (Math.PI / 180.0);
@ -165,7 +137,6 @@ function optimizerRuleTestSuite() {
tearDown : function () {
internal.db._drop(colName);
internal.db._drop(colNameOther);
geocol = null;
},
@ -215,14 +186,6 @@ function optimizerRuleTestSuite() {
queries.forEach(function(query) {
var result = AQL_EXPLAIN(query.string);
// //optimized on cluster
// if (query[1]) {
// assertNotEqual(-1, removeAlwaysOnClusterRules(result.plan.rules).indexOf(ruleName), query[0]);
// }
// else {
// assertEqual(-1, removeAlwaysOnClusterRules(result.plan.rules).indexOf(ruleName), query[0]);
// }
//sort nodes
if (query.sort) {
hasSortNode(result,query);
@ -268,7 +231,6 @@ function optimizerRuleTestSuite() {
var pairs = result.json.map(function(res){
return [res.lat,res.lon];
});
//internal.print(pairs)
assertEqual(expected[qindex].sort(),pairs.sort());
//expect(expected[qindex].sort()).to.be.equal(result.json.sort())
});

2
js/server/tests/resilience/moving-shards-cluster.js
View File

@ -163,7 +163,7 @@ function MovingShardsSuite () {
}
if (!ok) {
console.info(
console.error(
"Failed: Server " + id + " was not cleaned out. List of cleaned servers: ["
+ obj.cleanedServers + "]");
}

17
lib/Basics/ArangoGlobalContext.cpp
View File

@ -289,6 +289,23 @@ void ArangoGlobalContext::runStartupChecks() {
"necessary to set the value in '"
<< filename << "' to 2";
}
std::string const proc_cpuinfo_filename("/proc/cpuinfo");
try {
std::string const cpuInfo =
arangodb::basics::FileUtils::slurp(proc_cpuinfo_filename);
auto start = cpuInfo.find("ARMv6");
if (start != std::string::npos) {
LOG(FATAL)
<< "possibly incompatible ARMv6 CPU detected.";
FATAL_ERROR_EXIT();
}
} catch (...) {
// ignore that we cannot detect the alignment
LOG(TRACE)
<< "unable to detect CPU type '"
<< filename << "'";
}
}
#endif
}

2
lib/Logger/Logger.cpp
View File

@ -84,7 +84,7 @@ void Logger::setLogLevel(std::string const& levelName) {
if (l == "fatal") {
level = LogLevel::FATAL;
} else if (l == "error") {
} else if (l == "error" || l == "err") {
level = LogLevel::ERR;
} else if (l == "warning" || l == "warn") {
level = LogLevel::WARN;