Add a watchdog thread to trigger a crash on detecting a deadlock

interface/src/Application.cpp

@@ -235,6 +235,42 @@ const QHash<QString, Application::AcceptURLMethod> Application::_acceptedExtensi
     { AVA_JSON_EXTENSION, &Application::askToWearAvatarAttachmentUrl }
 };
 
+class DeadlockWatchdogThread : public QThread {
+public:
+    static const unsigned long HEARTBEAT_CHECK_INTERVAL_SECS = 1;
+    static const unsigned long HEARTBEAT_UPDATE_INTERVAL_SECS = 1;
+    static const unsigned long MAX_HEARTBEAT_AGE_USECS = 10 * USECS_PER_SECOND;
+
+    // Set the heartbeat on launch
+    DeadlockWatchdogThread() {
+        QTimer* heartbeatTimer = new QTimer();
+        connect(heartbeatTimer, &QTimer::timeout, [this] {
+            _heartbeat = usecTimestampNow();
+        });
+        heartbeatTimer->start(HEARTBEAT_UPDATE_INTERVAL_SECS * MSECS_PER_SECOND);
+    }
+
+    void deadlockDetectionCrash() {
+        uint32_t* crashTrigger = nullptr;
+        *crashTrigger = 0xDEAD10CC;
+    }
+
+    void run() override {
+        while (!qApp->isAboutToQuit()) {
+            QThread::sleep(HEARTBEAT_UPDATE_INTERVAL_SECS);
+            auto now = usecTimestampNow();
+            auto lastHeartbeatAge = now - _heartbeat;
+            if (lastHeartbeatAge > MAX_HEARTBEAT_AGE_USECS) {
+                deadlockDetectionCrash();
+            }
+        }
+    }
+
+    static std::atomic<uint64_t> _heartbeat;
+};
+
+std::atomic<uint64_t> DeadlockWatchdogThread::_heartbeat;
+
 #ifdef Q_OS_WIN
 class MyNativeEventFilter : public QAbstractNativeEventFilter {
 public:
@@ -457,6 +493,9 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
 
     auto nodeList = DependencyManager::get<NodeList>();
 
+    // Set up a watchdog thread to intentionally crash the application on deadlocks
+    (new DeadlockWatchdogThread())->start();
+
     qCDebug(interfaceapp) << "[VERSION] Build sequence:" << qPrintable(applicationVersion());
 
     _bookmarks = new Bookmarks();  // Before setting up the menu
@@ -4960,6 +4999,15 @@ void Application::crashApplication() {
     Q_UNUSED(value);
 }
 
+void Application::deadlockApplication() {
+    qCDebug(interfaceapp) << "Intentionally deadlocked Interface";
+    // Using a loop that will *technically* eventually exit (in ~600 billion years)
+    // to avoid compiler warnings about a loop that will never exit
+    for (uint64_t i = 1; i != 0; ++i) {
+        QThread::sleep(1);
+    }
+}
+
 void Application::setActiveDisplayPlugin(const QString& pluginName) {
     auto menu = Menu::getInstance();
     foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getDisplayPlugins()) {

interface/src/Application.h

@@ -276,6 +276,7 @@ public slots:
     void reloadResourceCaches();
 
     void crashApplication();
+    void deadlockApplication();
 
     void rotationModeChanged();
 

interface/src/Menu.cpp

@@ -590,6 +590,8 @@ Menu::Menu() {
     addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::DisplayCrashOptions, 0, true);
     // Developer > Crash Application
     addActionToQMenuAndActionHash(developerMenu, MenuOption::CrashInterface, 0, qApp, SLOT(crashApplication()));
+    // Developer > Deadlock Application
+    addActionToQMenuAndActionHash(developerMenu, MenuOption::DeadlockInterface, 0, qApp, SLOT(deadlockApplication()));
 
     // Developer > Log...
     addActionToQMenuAndActionHash(developerMenu, MenuOption::Log, Qt::CTRL | Qt::SHIFT | Qt::Key_L,

interface/src/Menu.h

@@ -66,6 +66,7 @@ namespace MenuOption {
     const QString CopyPath = "Copy Path to Clipboard";
     const QString CoupleEyelids = "Couple Eyelids";
     const QString CrashInterface = "Crash Interface";
+    const QString DeadlockInterface = "Deadlock Interface";
     const QString DecreaseAvatarSize = "Decrease Avatar Size";
     const QString DeleteBookmark = "Delete Bookmark...";
     const QString DisableActivityLogger = "Disable Activity Logger";