From 47a98d9d0f49dd2621102c34475bae4112d50419 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 21 Feb 2018 10:12:04 -0800 Subject: [PATCH 1/3] relax the deadlock time to 2 minutes --- interface/src/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 3aa0f3d889..2b87456459 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -392,7 +392,7 @@ const QHash Application::_acceptedExtensi class DeadlockWatchdogThread : public QThread { public: static const unsigned long HEARTBEAT_UPDATE_INTERVAL_SECS = 1; - static const unsigned long MAX_HEARTBEAT_AGE_USECS = 30 * USECS_PER_SECOND; + static const unsigned long MAX_HEARTBEAT_AGE_USECS = 120 * USECS_PER_SECOND; // 2 mins with no checkin probably a deadlock static const int WARNING_ELAPSED_HEARTBEAT = 500 * USECS_PER_MSEC; // warn if elapsed heartbeat average is large static const int HEARTBEAT_SAMPLES = 100000; // ~5 seconds worth of samples From e9cad947d7f429be3a50e98c0acf5e0feeafd496 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 22 Feb 2018 14:01:56 -0800 Subject: [PATCH 2/3] add the deadlock watchdog stats to the stats metaverse checkin data --- interface/src/Application.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 2878515240..8c3e6f130d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1719,6 +1719,10 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo properties["has_async_reprojection"] = displayPlugin->hasAsyncReprojection(); properties["hardware_stats"] = displayPlugin->getHardwareStats(); + // deadlock watchdog related stats + properties["deadlock_watchdog_maxElapsed"] = (int)DeadlockWatchdogThread::_maxElapsed; + properties["deadlock_watchdog_maxElapsedAverage"] = (int)DeadlockWatchdogThread::_maxElapsedAverage; + auto bandwidthRecorder = DependencyManager::get(); properties["packet_rate_in"] = bandwidthRecorder->getCachedTotalAverageInputPacketsPerSecond(); properties["packet_rate_out"] = bandwidthRecorder->getCachedTotalAverageOutputPacketsPerSecond(); From 9f68accb302e6ea59120a9bb5717df00e2788fc7 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 22 Feb 2018 17:34:00 -0800 Subject: [PATCH 3/3] add menu for testing unresponsive interface --- interface/src/Application.cpp | 12 ++++++++++++ interface/src/Application.h | 1 + interface/src/Menu.cpp | 1 + interface/src/Menu.h | 1 + 4 files changed, 15 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8c3e6f130d..e8264a398a 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -7542,6 +7542,18 @@ void Application::deadlockApplication() { } } +// cause main thread to be unresponsive for 35 seconds +void Application::unresponsiveApplication() { + // to avoid compiler warnings about a loop that will never exit + uint64_t start = usecTimestampNow(); + uint64_t UNRESPONSIVE_FOR_SECONDS = 35; + uint64_t UNRESPONSIVE_FOR_USECS = UNRESPONSIVE_FOR_SECONDS * USECS_PER_SECOND; + qCDebug(interfaceapp) << "Intentionally cause Interface to be unresponsive for " << UNRESPONSIVE_FOR_SECONDS << " seconds"; + while (usecTimestampNow() - start < UNRESPONSIVE_FOR_USECS) { + QThread::sleep(1); + } +} + void Application::setActiveDisplayPlugin(const QString& pluginName) { auto menu = Menu::getInstance(); foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getDisplayPlugins()) { diff --git a/interface/src/Application.h b/interface/src/Application.h index d4041aa3be..5ce9a1619a 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -367,6 +367,7 @@ public slots: void updateHeartbeat() const; static void deadlockApplication(); + static void unresponsiveApplication(); // cause main thread to be unresponsive for 35 seconds void rotationModeChanged() const; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 464de87fdb..5abc59356e 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -713,6 +713,7 @@ Menu::Menu() { MenuWrapper* crashMenu = developerMenu->addMenu("Crash"); addActionToQMenuAndActionHash(crashMenu, MenuOption::DeadlockInterface, 0, qApp, SLOT(deadlockApplication())); + addActionToQMenuAndActionHash(crashMenu, MenuOption::UnresponsiveInterface, 0, qApp, SLOT(unresponsiveApplication())); action = addActionToQMenuAndActionHash(crashMenu, MenuOption::CrashPureVirtualFunction); connect(action, &QAction::triggered, qApp, []() { crash::pureVirtualCall(); }); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 8cb1804fd4..9c4efd6ac7 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -77,6 +77,7 @@ namespace MenuOption { const QString CrashNewFault = "New Fault"; const QString CrashNewFaultThreaded = "New Fault (threaded)"; const QString DeadlockInterface = "Deadlock Interface"; + const QString UnresponsiveInterface = "Unresponsive Interface"; const QString DecreaseAvatarSize = "Decrease Avatar Size"; const QString DefaultSkybox = "Default Skybox"; const QString DeleteAvatarBookmark = "Delete Avatar Bookmark...";