Merge pull request #10859 from Atlante45/fix/ac-cleanup

Fix AC cleanup

assignment-client/src/AssignmentClientApp.cpp

@@ -9,8 +9,12 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
-#include <QCommandLineParser>
+#include "AssignmentClientApp.h"
-#include <QThread>
+
+#include <QtCore/QCommandLineParser>
+#include <QtCore/QDir>
+#include <QtCore/QStandardPaths>
+#include <QtCore/QThread>
 
 #include <LogHandler.h>
 #include <SharedUtil.h>
@@ -20,10 +24,6 @@
 #include "Assignment.h"
 #include "AssignmentClient.h"
 #include "AssignmentClientMonitor.h"
-#include "AssignmentClientApp.h"
-#include <QtCore/QDir>
-#include <QtCore/QStandardPaths>
-
 
 AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) :
     QCoreApplication(argc, argv)
@@ -87,6 +87,9 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) :
     const QCommandLineOption logDirectoryOption(ASSIGNMENT_LOG_DIRECTORY, "directory to store logs", "log-directory");
     parser.addOption(logDirectoryOption);
 
+    const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid");
+    parser.addOption(parentPIDOption);
+
     if (!parser.parse(QCoreApplication::arguments())) {
         qCritical() << parser.errorText() << endl;
         parser.showHelp();
@@ -203,6 +206,16 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) :
         }
     }
 
+    if (parser.isSet(parentPIDOption)) {
+        bool ok = false;
+        int parentPID = parser.value(parentPIDOption).toInt(&ok);
+
+        if (ok) {
+            qDebug() << "Parent process PID is" << parentPID;
+            watchParentProcess(parentPID);
+        }
+    }
+
     QThread::currentThread()->setObjectName("main thread");
 
     DependencyManager::registerInheritance<LimitedNodeList, NodeList>();

assignment-client/src/AssignmentClientMonitor.cpp

@@ -131,7 +131,6 @@ void AssignmentClientMonitor::aboutToQuit() {
 void AssignmentClientMonitor::spawnChildClient() {
     QProcess* assignmentClient = new QProcess(this);
 
-
     // unparse the parts of the command-line that the child cares about
     QStringList _childArguments;
     if (_assignmentPool != "") {
@@ -160,6 +159,9 @@ void AssignmentClientMonitor::spawnChildClient() {
     _childArguments.append("--" + ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION);
     _childArguments.append(QString::number(DependencyManager::get<NodeList>()->getLocalSockAddr().getPort()));
 
+    _childArguments.append("--" + PARENT_PID_OPTION);
+    _childArguments.append(QString::number(QCoreApplication::applicationPid()));
+
     QString nowString, stdoutFilenameTemp, stderrFilenameTemp, stdoutPathTemp, stderrPathTemp;
 
 

domain-server/src/DomainServer.cpp

@@ -221,6 +221,8 @@ void DomainServer::parseCommandLine() {
     const QCommandLineOption masterConfigOption("master-config", "Deprecated config-file option");
     parser.addOption(masterConfigOption);
 
+    const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid");
+    parser.addOption(parentPIDOption);
 
     if (!parser.parse(QCoreApplication::arguments())) {
         qWarning() << parser.errorText() << endl;
@@ -249,6 +251,17 @@ void DomainServer::parseCommandLine() {
         _overrideDomainID = true;
         qDebug() << "domain-server ID is" << _overridingDomainID;
     }
+
+
+    if (parser.isSet(parentPIDOption)) {
+        bool ok = false;
+        int parentPID = parser.value(parentPIDOption).toInt(&ok);
+
+        if (ok) {
+            qDebug() << "Parent process PID is" << parentPID;
+            watchParentProcess(parentPID);
+        }
+    }
 }
 
 DomainServer::~DomainServer() {

libraries/shared/src/SharedUtil.cpp

@@ -1076,3 +1076,24 @@ void setMaxCores(uint8_t maxCores) {
     SetProcessAffinityMask(process, newProcessAffinity);
 #endif
 }
+
+#ifdef Q_OS_WIN
+VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) {
+    if (!timerOrWaitFired && qApp) {
+        qDebug() << "Parent process died, quitting";
+        qApp->quit();
+    }
+}
+
+void watchParentProcess(int parentPID) {
+    DWORD processID = parentPID;
+    HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID);
+
+    HANDLE newHandle;
+    RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE);
+}
+#else
+void watchParentProcess(int parentPID) {
+    qWarning() << "Parent PID option not implemented on this plateform";
+}
+#endif // Q_OS_WIN

libraries/shared/src/SharedUtil.h

@@ -235,4 +235,7 @@ const QString& getInterfaceSharedMemoryName();
 
 void setMaxCores(uint8_t maxCores);
 
+const QString PARENT_PID_OPTION = "parent-pid";
+void watchParentProcess(int parentPID);
+
 #endif // hifi_SharedUtil_h

server-console/src/main.js

@@ -334,6 +334,7 @@ function startInterface(url) {
 
     // create a new Interface instance - Interface makes sure only one is running at a time
     var pInterface = new Process('interface', interfacePath, argArray);
+    pInterface.detached = true;
     pInterface.start();
 }
 
@@ -892,10 +893,19 @@ function onContentLoaded() {
     deleteOldFiles(logPath, DELETE_LOG_FILES_OLDER_THAN_X_SECONDS, LOG_FILE_REGEX);
 
     if (dsPath && acPath) {
-        domainServer = new Process('domain-server', dsPath, ["--get-temp-name"], logPath);
+        var dsArguments = ['--get-temp-name',
-        acMonitor = new ACMonitorProcess('ac-monitor', acPath, ['-n7',
+                           '--parent-pid', process.pid];
-                                                                '--log-directory', logPath,
+        domainServer = new Process('domain-server', dsPath, dsArguments, logPath);
-                                                                '--http-status-port', httpStatusPort], httpStatusPort, logPath);
+        domainServer.restartOnCrash = true;
+
+        var acArguments = ['-n7',
+                           '--log-directory', logPath,
+                           '--http-status-port', httpStatusPort,
+                           '--parent-pid', process.pid];
+        acMonitor = new ACMonitorProcess('ac-monitor', acPath, acArguments,
+                                         httpStatusPort, logPath);
+        acMonitor.restartOnCrash = true;
+
         homeServer = new ProcessGroup('home', [domainServer, acMonitor]);
         logWindow = new LogWindow(acMonitor, domainServer);
 

server-console/src/modules/hf-process.js

@@ -113,6 +113,10 @@ function Process(name, command, commandArgs, logDirectory) {
     this.logDirectory = logDirectory;
     this.logStdout = null;
     this.logStderr = null;
+    this.detached = false;
+    this.restartOnCrash = false;
+    this.restartCount = 0;
+    this.firstRestartTimestamp = Date.now();
 
     this.state = ProcessStates.STOPPED;
 };
@@ -165,9 +169,10 @@ Process.prototype = extend(Process.prototype, {
 
         try {
             this.child = childProcess.spawn(this.command, this.commandArgs, {
-                detached: false,
+                detached: this.detached,
                 stdio: ['ignore', logStdout, logStderr]
             });
+            log.debug("Spawned " + this.command + " with pid " + this.child.pid);
         } catch (e) {
             log.debug("Got error starting child process for " + this.name, e);
             this.child = null;
@@ -266,7 +271,30 @@ Process.prototype = extend(Process.prototype, {
             clearTimeout(this.stoppingTimeoutID);
             this.stoppingTimeoutID = null;
         }
+        // Grab current state before updating it.
+        var unexpectedShutdown = this.state != ProcessStates.STOPPING;
         this.updateState(ProcessStates.STOPPED);
+
+        if (unexpectedShutdown && this.restartOnCrash) {
+            var MAX_RESTARTS = 10;
+            var MAX_RESTARTS_PERIOD = 10; // 10 min
+            var MSEC_PER_MIN = 1000 * 60;
+            var now = Date.now();
+            var timeDiff = (now - this.firstRestartTimestamp) / MSEC_PER_MIN;
+            if (timeDiff > MAX_RESTARTS_PERIOD) {
+                this.firstRestartTimestamp = now;
+                this.restartCount = 0;
+            }
+
+            if (this.restartCount < 10) {
+                this.restartCount++;
+
+                log.warn("Child stopped unexpectedly, restarting.");
+                this.start();
+            } else {
+                log.warn("Child stopped unexpectedly too many times, not restarting.");
+            }
+        }
     }
 });