reduce log spam for the deadlock watchdog

This commit is contained in:
Brad Hefta-Gaub 2016-04-07 10:25:08 -07:00
parent 50a081db00
commit 2c3aeddbe4
3 changed files with 31 additions and 13 deletions

View file

@ -250,37 +250,49 @@ public:
auto lastHeartbeatAge = (now > lastHeartbeat) ? now - lastHeartbeat : 0;
auto sinceLastReport = (now > _lastReport) ? now - _lastReport : 0;
auto elapsedMovingAverage = _movingAverage.getAverage();
auto menu = Menu::getInstance();
bool suppressStatusMessages = menu ? menu->isOptionChecked(MenuOption::SupressDeadlockWatchdogStatus) : false;
if (elapsedMovingAverage > _maxElapsedAverage) {
qDebug() << "DEADLOCK WATCHDOG NEW maxElapsedAverage:"
qDebug() << "DEADLOCK WATCHDOG WARNING:"
<< "lastHeartbeatAge:" << lastHeartbeatAge
<< "elapsedMovingAverage:" << elapsedMovingAverage
<< "maxElapsed:" << _maxElapsed
<< "PREVIOUS maxElapsedAverage:" << _maxElapsedAverage
<< "NEW maxElapsedAverage:" << elapsedMovingAverage
<< "NEW maxElapsedAverage:" << elapsedMovingAverage << "** NEW MAX ELAPSED AVERAGE **"
<< "samples:" << _movingAverage.getSamples();
_maxElapsedAverage = elapsedMovingAverage;
}
if (lastHeartbeatAge > _maxElapsed) {
qDebug() << "DEADLOCK WATCHDOG NEW maxElapsed:"
qDebug() << "DEADLOCK WATCHDOG WARNING:"
<< "lastHeartbeatAge:" << lastHeartbeatAge
<< "elapsedMovingAverage:" << elapsedMovingAverage
<< "PREVIOUS maxElapsed:" << _maxElapsed
<< "NEW maxElapsed:" << lastHeartbeatAge
<< "NEW maxElapsed:" << lastHeartbeatAge << "** NEW MAX ELAPSED **"
<< "maxElapsedAverage:" << _maxElapsedAverage
<< "samples:" << _movingAverage.getSamples();
_maxElapsed = lastHeartbeatAge;
}
if ((sinceLastReport > HEARTBEAT_REPORT_INTERVAL_USECS) || (elapsedMovingAverage > WARNING_ELAPSED_HEARTBEAT)) {
qDebug() << "DEADLOCK WATCHDOG STATUS -- lastHeartbeatAge:" << lastHeartbeatAge
<< "elapsedMovingAverage:" << elapsedMovingAverage
<< "maxElapsed:" << _maxElapsed
<< "maxElapsedAverage:" << _maxElapsedAverage
<< "samples:" << _movingAverage.getSamples();
if (elapsedMovingAverage > WARNING_ELAPSED_HEARTBEAT) {
qDebug() << "DEADLOCK WATCHDOG WARNING:"
<< "lastHeartbeatAge:" << lastHeartbeatAge
<< "elapsedMovingAverage:" << elapsedMovingAverage << "** OVER EXPECTED VALUE**"
<< "maxElapsed:" << _maxElapsed
<< "maxElapsedAverage:" << _maxElapsedAverage
<< "samples:" << _movingAverage.getSamples();
_lastReport = now;
}
if (!suppressStatusMessages && sinceLastReport > HEARTBEAT_REPORT_INTERVAL_USECS) {
qDebug() << "DEADLOCK WATCHDOG STATUS:"
<< "lastHeartbeatAge:" << lastHeartbeatAge
<< "elapsedMovingAverage:" << elapsedMovingAverage
<< "maxElapsed:" << _maxElapsed
<< "maxElapsedAverage:" << _maxElapsedAverage
<< "samples:" << _movingAverage.getSamples();
_lastReport = now;
}
#ifdef NDEBUG
if (lastHeartbeatAge > MAX_HEARTBEAT_AGE_USECS) {
qDebug() << "DEADLOCK DETECTED -- "
<< "lastHeartbeatAge:" << lastHeartbeatAge
@ -290,9 +302,13 @@ public:
<< "maxElapsed:" << _maxElapsed
<< "maxElapsedAverage:" << _maxElapsedAverage
<< "samples:" << _movingAverage.getSamples();
deadlockDetectionCrash();
// Don't actually crash in debug builds, in case this apparent deadlock is simply from
// the developer actively debugging code
#ifdef NDEBUG
deadlockDetectionCrash();
#endif
}
#endif
}
}

View file

@ -530,6 +530,7 @@ Menu::Menu() {
addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::PipelineWarnings);
addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::LogExtraTimings);
addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::SuppressShortTimings);
addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::SupressDeadlockWatchdogStatus, 0, false);
// Developer > Audio >>>
MenuWrapper* audioDebugMenu = developerMenu->addMenu("Audio");

View file

@ -162,6 +162,7 @@ namespace MenuOption {
const QString Stats = "Stats";
const QString StopAllScripts = "Stop All Scripts";
const QString SuppressShortTimings = "Suppress Timings Less than 10ms";
const QString SupressDeadlockWatchdogStatus = "Supress Deadlock Watchdog Status";
const QString ThirdPerson = "Third Person";
const QString ThreePointCalibration = "3 Point Calibration";
const QString ThrottleFPSIfNotFocus = "Throttle FPS If Not Focus"; // FIXME - this value duplicated in Basic2DWindowOpenGLDisplayPlugin.cpp