diff --git a/assignment-client/src/AssignmentClientApp.cpp b/assignment-client/src/AssignmentClientApp.cpp index 7e9042e609..bd656ceb09 100644 --- a/assignment-client/src/AssignmentClientApp.cpp +++ b/assignment-client/src/AssignmentClientApp.cpp @@ -9,8 +9,12 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include -#include +#include "AssignmentClientApp.h" + +#include +#include +#include +#include #include #include @@ -20,10 +24,6 @@ #include "Assignment.h" #include "AssignmentClient.h" #include "AssignmentClientMonitor.h" -#include "AssignmentClientApp.h" -#include -#include - AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : QCoreApplication(argc, argv) @@ -87,6 +87,9 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : const QCommandLineOption logDirectoryOption(ASSIGNMENT_LOG_DIRECTORY, "directory to store logs", "log-directory"); parser.addOption(logDirectoryOption); + const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid"); + parser.addOption(parentPIDOption); + if (!parser.parse(QCoreApplication::arguments())) { qCritical() << parser.errorText() << endl; parser.showHelp(); @@ -203,6 +206,16 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : } } + if (parser.isSet(parentPIDOption)) { + bool ok = false; + int parentPID = parser.value(parentPIDOption).toInt(&ok); + + if (ok) { + qDebug() << "Parent process PID is" << parentPID; + watchParentProcess(parentPID); + } + } + QThread::currentThread()->setObjectName("main thread"); DependencyManager::registerInheritance(); diff --git a/assignment-client/src/AssignmentClientMonitor.cpp b/assignment-client/src/AssignmentClientMonitor.cpp index 1ee876ceea..070034d54b 100644 --- a/assignment-client/src/AssignmentClientMonitor.cpp +++ b/assignment-client/src/AssignmentClientMonitor.cpp @@ -131,7 +131,6 @@ void AssignmentClientMonitor::aboutToQuit() { void AssignmentClientMonitor::spawnChildClient() { QProcess* assignmentClient = new QProcess(this); - // unparse the parts of the command-line that the child cares about QStringList _childArguments; if (_assignmentPool != "") { @@ -160,6 +159,9 @@ void AssignmentClientMonitor::spawnChildClient() { _childArguments.append("--" + ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION); _childArguments.append(QString::number(DependencyManager::get()->getLocalSockAddr().getPort())); + _childArguments.append("--" + PARENT_PID_OPTION); + _childArguments.append(QString::number(QCoreApplication::applicationPid())); + QString nowString, stdoutFilenameTemp, stderrFilenameTemp, stdoutPathTemp, stderrPathTemp; diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 8e3d04897b..095613a473 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -221,6 +221,8 @@ void DomainServer::parseCommandLine() { const QCommandLineOption masterConfigOption("master-config", "Deprecated config-file option"); parser.addOption(masterConfigOption); + const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid"); + parser.addOption(parentPIDOption); if (!parser.parse(QCoreApplication::arguments())) { qWarning() << parser.errorText() << endl; @@ -249,6 +251,17 @@ void DomainServer::parseCommandLine() { _overrideDomainID = true; qDebug() << "domain-server ID is" << _overridingDomainID; } + + + if (parser.isSet(parentPIDOption)) { + bool ok = false; + int parentPID = parser.value(parentPIDOption).toInt(&ok); + + if (ok) { + qDebug() << "Parent process PID is" << parentPID; + watchParentProcess(parentPID); + } + } } DomainServer::~DomainServer() { diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index a68d27e620..58b8aead45 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -1076,3 +1076,24 @@ void setMaxCores(uint8_t maxCores) { SetProcessAffinityMask(process, newProcessAffinity); #endif } + +#ifdef Q_OS_WIN +VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) { + if (!timerOrWaitFired && qApp) { + qDebug() << "Parent process died, quitting"; + qApp->quit(); + } +} + +void watchParentProcess(int parentPID) { + DWORD processID = parentPID; + HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID); + + HANDLE newHandle; + RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE); +} +#else +void watchParentProcess(int parentPID) { + qWarning() << "Parent PID option not implemented on this plateform"; +} +#endif // Q_OS_WIN \ No newline at end of file diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index 0f30842f47..681418a263 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -235,4 +235,7 @@ const QString& getInterfaceSharedMemoryName(); void setMaxCores(uint8_t maxCores); +const QString PARENT_PID_OPTION = "parent-pid"; +void watchParentProcess(int parentPID); + #endif // hifi_SharedUtil_h diff --git a/server-console/src/main.js b/server-console/src/main.js index 95fb0d81b2..efa04a8512 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -334,6 +334,7 @@ function startInterface(url) { // create a new Interface instance - Interface makes sure only one is running at a time var pInterface = new Process('interface', interfacePath, argArray); + pInterface.detached = true; pInterface.start(); } @@ -892,10 +893,19 @@ function onContentLoaded() { deleteOldFiles(logPath, DELETE_LOG_FILES_OLDER_THAN_X_SECONDS, LOG_FILE_REGEX); if (dsPath && acPath) { - domainServer = new Process('domain-server', dsPath, ["--get-temp-name"], logPath); - acMonitor = new ACMonitorProcess('ac-monitor', acPath, ['-n7', - '--log-directory', logPath, - '--http-status-port', httpStatusPort], httpStatusPort, logPath); + var dsArguments = ['--get-temp-name', + '--parent-pid', process.pid]; + domainServer = new Process('domain-server', dsPath, dsArguments, logPath); + domainServer.restartOnCrash = true; + + var acArguments = ['-n7', + '--log-directory', logPath, + '--http-status-port', httpStatusPort, + '--parent-pid', process.pid]; + acMonitor = new ACMonitorProcess('ac-monitor', acPath, acArguments, + httpStatusPort, logPath); + acMonitor.restartOnCrash = true; + homeServer = new ProcessGroup('home', [domainServer, acMonitor]); logWindow = new LogWindow(acMonitor, domainServer); diff --git a/server-console/src/modules/hf-process.js b/server-console/src/modules/hf-process.js index 20b8966148..767befec7b 100644 --- a/server-console/src/modules/hf-process.js +++ b/server-console/src/modules/hf-process.js @@ -113,6 +113,10 @@ function Process(name, command, commandArgs, logDirectory) { this.logDirectory = logDirectory; this.logStdout = null; this.logStderr = null; + this.detached = false; + this.restartOnCrash = false; + this.restartCount = 0; + this.firstRestartTimestamp = Date.now(); this.state = ProcessStates.STOPPED; }; @@ -165,9 +169,10 @@ Process.prototype = extend(Process.prototype, { try { this.child = childProcess.spawn(this.command, this.commandArgs, { - detached: false, + detached: this.detached, stdio: ['ignore', logStdout, logStderr] }); + log.debug("Spawned " + this.command + " with pid " + this.child.pid); } catch (e) { log.debug("Got error starting child process for " + this.name, e); this.child = null; @@ -266,7 +271,30 @@ Process.prototype = extend(Process.prototype, { clearTimeout(this.stoppingTimeoutID); this.stoppingTimeoutID = null; } + // Grab current state before updating it. + var unexpectedShutdown = this.state != ProcessStates.STOPPING; this.updateState(ProcessStates.STOPPED); + + if (unexpectedShutdown && this.restartOnCrash) { + var MAX_RESTARTS = 10; + var MAX_RESTARTS_PERIOD = 10; // 10 min + var MSEC_PER_MIN = 1000 * 60; + var now = Date.now(); + var timeDiff = (now - this.firstRestartTimestamp) / MSEC_PER_MIN; + if (timeDiff > MAX_RESTARTS_PERIOD) { + this.firstRestartTimestamp = now; + this.restartCount = 0; + } + + if (this.restartCount < 10) { + this.restartCount++; + + log.warn("Child stopped unexpectedly, restarting."); + this.start(); + } else { + log.warn("Child stopped unexpectedly too many times, not restarting."); + } + } } });