Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add diagnostics to see background jobs that do not run. 

If a job is only executed more than 5 seconds later than it was
scheduled, an error is logged.
This commit is contained in:
Max Neunhoeffer 2017-02-01 09:35:16 +01:00
parent 73cf8e9e04
commit 48be25bdde
1 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
arangod/Cluster/HeartbeatThread.cpp
View File

@ -105,11 +105,24 @@ HeartbeatThread::~HeartbeatThread() { shutdown(); }
class HeartbeatBackgroundJob { class HeartbeatBackgroundJob {
std::shared_ptr<HeartbeatThread> _heartbeatThread; std::shared_ptr<HeartbeatThread> _heartbeatThread;
double _startTime;
std::string _schedulerInfo;
public: public:
explicit HeartbeatBackgroundJob(std::shared_ptr<HeartbeatThread> hbt) explicit HeartbeatBackgroundJob(std::shared_ptr<HeartbeatThread> hbt,
: _heartbeatThread(hbt) {} double startTime)
: _heartbeatThread(hbt), _startTime(startTime) {
_schedulerInfo = SchedulerFeature::SCHEDULER->infoStatus();
}
void operator()() { void operator()() {
double now = TRI_microtime();
if (now > _startTime + 5.0) {
LOG_TOPIC(ERR, Logger::HEARTBEAT) << "ALARM: Scheduling background job "
"took " << now - _startTime
<< " seconds, scheduler info at schedule time: " << _schedulerInfo
<< ", scheduler info now: "
<< SchedulerFeature::SCHEDULER->infoStatus();
}
_heartbeatThread->runBackgroundJob(); _heartbeatThread->runBackgroundJob();
} }
}; };
@ -138,7 +151,8 @@ void HeartbeatThread::runBackgroundJob() {
jobNr = ++_backgroundJobsPosted; jobNr = ++_backgroundJobsPosted;
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr; LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync tail " << jobNr;
_launchAnotherBackgroundJob = false; _launchAnotherBackgroundJob = false;
_ioService->post(HeartbeatBackgroundJob(shared_from_this())); _ioService->post(HeartbeatBackgroundJob(shared_from_this(),
TRI_microtime()));
} else { } else {
_backgroundJobScheduledOrRunning = false; _backgroundJobScheduledOrRunning = false;
_launchAnotherBackgroundJob = false; _launchAnotherBackgroundJob = false;
@ -790,7 +804,7 @@ void HeartbeatThread::syncDBServerStatusQuo() {
uint64_t jobNr = ++_backgroundJobsPosted; uint64_t jobNr = ++_backgroundJobsPosted;
LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " << jobNr; LOG_TOPIC(DEBUG, Logger::HEARTBEAT) << "dispatching sync " << jobNr;
_backgroundJobScheduledOrRunning = true; _backgroundJobScheduledOrRunning = true;
_ioService->post(HeartbeatBackgroundJob(shared_from_this())); _ioService->post(HeartbeatBackgroundJob(shared_from_this(), TRI_microtime()));
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////