diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 280f2c5d1f..2512dc8b26 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -250,37 +250,49 @@ public: auto lastHeartbeatAge = (now > lastHeartbeat) ? now - lastHeartbeat : 0; auto sinceLastReport = (now > _lastReport) ? now - _lastReport : 0; auto elapsedMovingAverage = _movingAverage.getAverage(); + auto menu = Menu::getInstance(); + bool suppressStatusMessages = menu ? menu->isOptionChecked(MenuOption::SupressDeadlockWatchdogStatus) : false; if (elapsedMovingAverage > _maxElapsedAverage) { - qDebug() << "DEADLOCK WATCHDOG NEW maxElapsedAverage:" + qDebug() << "DEADLOCK WATCHDOG WARNING:" << "lastHeartbeatAge:" << lastHeartbeatAge << "elapsedMovingAverage:" << elapsedMovingAverage << "maxElapsed:" << _maxElapsed << "PREVIOUS maxElapsedAverage:" << _maxElapsedAverage - << "NEW maxElapsedAverage:" << elapsedMovingAverage + << "NEW maxElapsedAverage:" << elapsedMovingAverage << "** NEW MAX ELAPSED AVERAGE **" << "samples:" << _movingAverage.getSamples(); _maxElapsedAverage = elapsedMovingAverage; } if (lastHeartbeatAge > _maxElapsed) { - qDebug() << "DEADLOCK WATCHDOG NEW maxElapsed:" + qDebug() << "DEADLOCK WATCHDOG WARNING:" << "lastHeartbeatAge:" << lastHeartbeatAge << "elapsedMovingAverage:" << elapsedMovingAverage << "PREVIOUS maxElapsed:" << _maxElapsed - << "NEW maxElapsed:" << lastHeartbeatAge + << "NEW maxElapsed:" << lastHeartbeatAge << "** NEW MAX ELAPSED **" << "maxElapsedAverage:" << _maxElapsedAverage << "samples:" << _movingAverage.getSamples(); _maxElapsed = lastHeartbeatAge; } - if ((sinceLastReport > HEARTBEAT_REPORT_INTERVAL_USECS) || (elapsedMovingAverage > WARNING_ELAPSED_HEARTBEAT)) { - qDebug() << "DEADLOCK WATCHDOG STATUS -- lastHeartbeatAge:" << lastHeartbeatAge - << "elapsedMovingAverage:" << elapsedMovingAverage - << "maxElapsed:" << _maxElapsed - << "maxElapsedAverage:" << _maxElapsedAverage - << "samples:" << _movingAverage.getSamples(); + if (elapsedMovingAverage > WARNING_ELAPSED_HEARTBEAT) { + qDebug() << "DEADLOCK WATCHDOG WARNING:" + << "lastHeartbeatAge:" << lastHeartbeatAge + << "elapsedMovingAverage:" << elapsedMovingAverage << "** OVER EXPECTED VALUE**" + << "maxElapsed:" << _maxElapsed + << "maxElapsedAverage:" << _maxElapsedAverage + << "samples:" << _movingAverage.getSamples(); + _lastReport = now; + } + + if (!suppressStatusMessages && sinceLastReport > HEARTBEAT_REPORT_INTERVAL_USECS) { + qDebug() << "DEADLOCK WATCHDOG STATUS:" + << "lastHeartbeatAge:" << lastHeartbeatAge + << "elapsedMovingAverage:" << elapsedMovingAverage + << "maxElapsed:" << _maxElapsed + << "maxElapsedAverage:" << _maxElapsedAverage + << "samples:" << _movingAverage.getSamples(); _lastReport = now; } -#ifdef NDEBUG if (lastHeartbeatAge > MAX_HEARTBEAT_AGE_USECS) { qDebug() << "DEADLOCK DETECTED -- " << "lastHeartbeatAge:" << lastHeartbeatAge @@ -290,9 +302,13 @@ public: << "maxElapsed:" << _maxElapsed << "maxElapsedAverage:" << _maxElapsedAverage << "samples:" << _movingAverage.getSamples(); - deadlockDetectionCrash(); + + // Don't actually crash in debug builds, in case this apparent deadlock is simply from + // the developer actively debugging code + #ifdef NDEBUG + deadlockDetectionCrash(); + #endif } -#endif } } diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index d69796b453..4ea69af2b8 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -530,6 +530,7 @@ Menu::Menu() { addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::PipelineWarnings); addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::LogExtraTimings); addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::SuppressShortTimings); + addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::SupressDeadlockWatchdogStatus, 0, false); // Developer > Audio >>> MenuWrapper* audioDebugMenu = developerMenu->addMenu("Audio"); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 3488f03b0e..48bda01076 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -162,6 +162,7 @@ namespace MenuOption { const QString Stats = "Stats"; const QString StopAllScripts = "Stop All Scripts"; const QString SuppressShortTimings = "Suppress Timings Less than 10ms"; + const QString SupressDeadlockWatchdogStatus = "Supress Deadlock Watchdog Status"; const QString ThirdPerson = "Third Person"; const QString ThreePointCalibration = "3 Point Calibration"; const QString ThrottleFPSIfNotFocus = "Throttle FPS If Not Focus"; // FIXME - this value duplicated in Basic2DWindowOpenGLDisplayPlugin.cpp