Sandbox restarts crashed processes

assignment-client/src/AssignmentClientApp.cpp

@@ -9,8 +9,12 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
-#include <QCommandLineParser>
-#include <QThread>
+#include "AssignmentClientApp.h"
+
+#include <QtCore/QCommandLineParser>
+#include <QtCore/QDir>
+#include <QtCore/QStandardPaths>
+#include <QtCore/QThread>
 
 #include <LogHandler.h>
 #include <SharedUtil.h>
@@ -20,30 +24,6 @@
 #include "Assignment.h"
 #include "AssignmentClient.h"
 #include "AssignmentClientMonitor.h"
-#include "AssignmentClientApp.h"
-#include <QtCore/QDir>
-#include <QtCore/QStandardPaths>
-
-#ifdef WIN32
-VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) {
-    if (!timerOrWaitFired && qApp) {
-        qDebug() << "Parent process died, quitting";
-        qApp->quit();
-    }
-}
-
-void watchParentProcess(int parentPID) {
-    DWORD processID = parentPID;
-    HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID);
-
-    HANDLE newHandle;
-    RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE);
-}
-#else
-void watchParentProcess(int parentPID) {
-    qWarning() << "Parent PID option not implemented on this plateform";
-}
-#endif // WIN32
 
 AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) :
     QCoreApplication(argc, argv)
@@ -107,7 +87,7 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) :
     const QCommandLineOption logDirectoryOption(ASSIGNMENT_LOG_DIRECTORY, "directory to store logs", "log-directory");
     parser.addOption(logDirectoryOption);
 
-    const QCommandLineOption parentPIDOption(ASSIGNMENT_PARENT_PID, "PID of the parent process", "parent-pid");
+    const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid");
     parser.addOption(parentPIDOption);
 
     if (!parser.parse(QCoreApplication::arguments())) {
@@ -231,6 +211,7 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) :
         int parentPID = parser.value(parentPIDOption).toInt(&ok);
 
         if (ok) {
+            qDebug() << "Parent process PID is" << parentPID;
             watchParentProcess(parentPID);
         }
     }

assignment-client/src/AssignmentClientApp.h

@@ -27,7 +27,6 @@ const QString ASSIGNMENT_MAX_FORKS_OPTION = "max";
 const QString ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION = "monitor-port";
 const QString ASSIGNMENT_HTTP_STATUS_PORT = "http-status-port";
 const QString ASSIGNMENT_LOG_DIRECTORY = "log-directory";
-const QString ASSIGNMENT_PARENT_PID = "parent-pid";
 
 class AssignmentClientApp : public QCoreApplication {
     Q_OBJECT

assignment-client/src/AssignmentClientMonitor.cpp

@@ -131,7 +131,6 @@ void AssignmentClientMonitor::aboutToQuit() {
 void AssignmentClientMonitor::spawnChildClient() {
     QProcess* assignmentClient = new QProcess(this);
 
-
     // unparse the parts of the command-line that the child cares about
     QStringList _childArguments;
     if (_assignmentPool != "") {
@@ -159,9 +158,8 @@ void AssignmentClientMonitor::spawnChildClient() {
     // for now they simply talk to us on localhost
     _childArguments.append("--" + ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION);
     _childArguments.append(QString::number(DependencyManager::get<NodeList>()->getLocalSockAddr().getPort()));
-    
 
-    _childArguments.append("--" + ASSIGNMENT_PARENT_PID);
+    _childArguments.append("--" + PARENT_PID_OPTION);
     _childArguments.append(QString::number(QCoreApplication::applicationPid()));
 
     QString nowString, stdoutFilenameTemp, stderrFilenameTemp, stdoutPathTemp, stderrPathTemp;

domain-server/src/DomainServer.cpp

@@ -221,6 +221,8 @@ void DomainServer::parseCommandLine() {
     const QCommandLineOption masterConfigOption("master-config", "Deprecated config-file option");
     parser.addOption(masterConfigOption);
 
+    const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid");
+    parser.addOption(parentPIDOption);
 
     if (!parser.parse(QCoreApplication::arguments())) {
         qWarning() << parser.errorText() << endl;
@@ -249,6 +251,17 @@ void DomainServer::parseCommandLine() {
         _overrideDomainID = true;
         qDebug() << "domain-server ID is" << _overridingDomainID;
     }
+
+
+    if (parser.isSet(parentPIDOption)) {
+        bool ok = false;
+        int parentPID = parser.value(parentPIDOption).toInt(&ok);
+
+        if (ok) {
+            qDebug() << "Parent process PID is" << parentPID;
+            watchParentProcess(parentPID);
+        }
+    }
 }
 
 DomainServer::~DomainServer() {

libraries/shared/src/SharedUtil.cpp

@@ -1076,3 +1076,24 @@ void setMaxCores(uint8_t maxCores) {
     SetProcessAffinityMask(process, newProcessAffinity);
 #endif
 }
+
+#ifdef Q_OS_WIN
+VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) {
+    if (!timerOrWaitFired && qApp) {
+        qDebug() << "Parent process died, quitting";
+        qApp->quit();
+    }
+}
+
+void watchParentProcess(int parentPID) {
+    DWORD processID = parentPID;
+    HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID);
+
+    HANDLE newHandle;
+    RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE);
+}
+#else
+void watchParentProcess(int parentPID) {
+    qWarning() << "Parent PID option not implemented on this plateform";
+}
+#endif // Q_OS_WIN

libraries/shared/src/SharedUtil.h

@@ -235,4 +235,7 @@ const QString& getInterfaceSharedMemoryName();
 
 void setMaxCores(uint8_t maxCores);
 
+const QString PARENT_PID_OPTION = "parent-pid";
+void watchParentProcess(int parentPID);
+
 #endif // hifi_SharedUtil_h

server-console/src/main.js

@@ -892,10 +892,12 @@ function onContentLoaded() {
     deleteOldFiles(logPath, DELETE_LOG_FILES_OLDER_THAN_X_SECONDS, LOG_FILE_REGEX);
 
     if (dsPath && acPath) {
-        domainServer = new Process('domain-server', dsPath, ["--get-temp-name"], logPath);
+        domainServer = new Process('domain-server', dsPath, ['--get-temp-name',
+                                                             '--parent-pid', process.pid], logPath);
         acMonitor = new ACMonitorProcess('ac-monitor', acPath, ['-n7',
                                                                 '--log-directory', logPath,
-                                                                '--http-status-port', httpStatusPort], httpStatusPort, logPath);
+                                                                '--http-status-port', httpStatusPort,
+                                                                '--parent-pid', process.pid], httpStatusPort, logPath);
         homeServer = new ProcessGroup('home', [domainServer, acMonitor]);
         logWindow = new LogWindow(acMonitor, domainServer);
 

server-console/src/modules/hf-process.js

@@ -267,7 +267,15 @@ Process.prototype = extend(Process.prototype, {
             clearTimeout(this.stoppingTimeoutID);
             this.stoppingTimeoutID = null;
         }
+        // Grab current state berofe updating it.
+        var unexpectedShutdown = this.state != ProcessStates.STOPPING;
         this.updateState(ProcessStates.STOPPED);
+
+        if (unexpectedShutdown) {
+            log.warn("Child stopped unexpectedly, restarting.");
+            this.start();
+            return;
+        }
     }
 });