From a66e1148ba84abee7364339c979d742e101828b2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 28 Jun 2017 19:20:18 -0700 Subject: [PATCH 1/6] Preliminary work on AC auto cleanup --- assignment-client/src/AssignmentClientApp.cpp | 22 +++++++++++++++++++ assignment-client/src/AssignmentClientApp.h | 1 + .../src/AssignmentClientMonitor.cpp | 4 ++++ server-console/src/modules/hf-process.js | 1 + 4 files changed, 28 insertions(+) diff --git a/assignment-client/src/AssignmentClientApp.cpp b/assignment-client/src/AssignmentClientApp.cpp index 7e9042e609..ed684dcb65 100644 --- a/assignment-client/src/AssignmentClientApp.cpp +++ b/assignment-client/src/AssignmentClientApp.cpp @@ -24,6 +24,12 @@ #include #include +VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) { + if (!timerOrWaitFired) { + qDebug() << "Parent process died, quitting"; + qApp->quit(); + } +} AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : QCoreApplication(argc, argv) @@ -87,6 +93,9 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : const QCommandLineOption logDirectoryOption(ASSIGNMENT_LOG_DIRECTORY, "directory to store logs", "log-directory"); parser.addOption(logDirectoryOption); + const QCommandLineOption parentPIDOption(ASSIGNMENT_PARENT_PID, "PID of the parent process", "parent-pid"); + parser.addOption(parentPIDOption); + if (!parser.parse(QCoreApplication::arguments())) { qCritical() << parser.errorText() << endl; parser.showHelp(); @@ -203,6 +212,19 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : } } + DWORD processID; + HANDLE procHandle; + HANDLE newHandle; + if (parser.isSet(parentPIDOption)) { + bool ok = false; + processID = parser.value(parentPIDOption).toInt(&ok); + + if (ok) { + procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID); + RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE); + } + } + QThread::currentThread()->setObjectName("main thread"); DependencyManager::registerInheritance(); diff --git a/assignment-client/src/AssignmentClientApp.h b/assignment-client/src/AssignmentClientApp.h index 37d3b9cc1d..a7c8cdb54d 100644 --- a/assignment-client/src/AssignmentClientApp.h +++ b/assignment-client/src/AssignmentClientApp.h @@ -27,6 +27,7 @@ const QString ASSIGNMENT_MAX_FORKS_OPTION = "max"; const QString ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION = "monitor-port"; const QString ASSIGNMENT_HTTP_STATUS_PORT = "http-status-port"; const QString ASSIGNMENT_LOG_DIRECTORY = "log-directory"; +const QString ASSIGNMENT_PARENT_PID = "parent-pid"; class AssignmentClientApp : public QCoreApplication { Q_OBJECT diff --git a/assignment-client/src/AssignmentClientMonitor.cpp b/assignment-client/src/AssignmentClientMonitor.cpp index 1ee876ceea..568a29c8b1 100644 --- a/assignment-client/src/AssignmentClientMonitor.cpp +++ b/assignment-client/src/AssignmentClientMonitor.cpp @@ -159,6 +159,10 @@ void AssignmentClientMonitor::spawnChildClient() { // for now they simply talk to us on localhost _childArguments.append("--" + ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION); _childArguments.append(QString::number(DependencyManager::get()->getLocalSockAddr().getPort())); + + + _childArguments.append("--" + ASSIGNMENT_PARENT_PID); + _childArguments.append(QString::number(QCoreApplication::applicationPid())); QString nowString, stdoutFilenameTemp, stderrFilenameTemp, stdoutPathTemp, stderrPathTemp; diff --git a/server-console/src/modules/hf-process.js b/server-console/src/modules/hf-process.js index 20b8966148..3c1200fecc 100644 --- a/server-console/src/modules/hf-process.js +++ b/server-console/src/modules/hf-process.js @@ -168,6 +168,7 @@ Process.prototype = extend(Process.prototype, { detached: false, stdio: ['ignore', logStdout, logStderr] }); + log.debug("Spawned " + this.command + " with pid " + this.child.pid); } catch (e) { log.debug("Got error starting child process for " + this.name, e); this.child = null; From 76e39a3ebdc8b0865c4dc420ad71e186d91467b3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 29 Jun 2017 10:54:44 -0700 Subject: [PATCH 2/6] Make code windows specific --- assignment-client/src/AssignmentClientApp.cpp | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/AssignmentClientApp.cpp b/assignment-client/src/AssignmentClientApp.cpp index ed684dcb65..bc03808c1e 100644 --- a/assignment-client/src/AssignmentClientApp.cpp +++ b/assignment-client/src/AssignmentClientApp.cpp @@ -24,13 +24,27 @@ #include #include +#ifdef WIN32 VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) { - if (!timerOrWaitFired) { + if (!timerOrWaitFired && qApp) { qDebug() << "Parent process died, quitting"; qApp->quit(); } } +void watchParentProcess(int parentPID) { + DWORD processID = parentPID; + HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID); + + HANDLE newHandle; + RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE); +} +#else +void watchParentProcess(int parentPID) { + qWarning() << "Parent PID option not implemented on this plateform"; +} +#endif // WIN32 + AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : QCoreApplication(argc, argv) { @@ -212,16 +226,12 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : } } - DWORD processID; - HANDLE procHandle; - HANDLE newHandle; if (parser.isSet(parentPIDOption)) { bool ok = false; - processID = parser.value(parentPIDOption).toInt(&ok); + int parentPID = parser.value(parentPIDOption).toInt(&ok); if (ok) { - procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID); - RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE); + watchParentProcess(parentPID); } } From 5c731636b1c12b2073b68ab38c5a14d487a52ae4 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 29 Jun 2017 11:44:05 -0700 Subject: [PATCH 3/6] Sandbox restarts crashed processes --- assignment-client/src/AssignmentClientApp.cpp | 35 +++++-------------- assignment-client/src/AssignmentClientApp.h | 1 - .../src/AssignmentClientMonitor.cpp | 4 +-- domain-server/src/DomainServer.cpp | 13 +++++++ libraries/shared/src/SharedUtil.cpp | 21 +++++++++++ libraries/shared/src/SharedUtil.h | 3 ++ server-console/src/main.js | 6 ++-- server-console/src/modules/hf-process.js | 8 +++++ 8 files changed, 58 insertions(+), 33 deletions(-) diff --git a/assignment-client/src/AssignmentClientApp.cpp b/assignment-client/src/AssignmentClientApp.cpp index bc03808c1e..bd656ceb09 100644 --- a/assignment-client/src/AssignmentClientApp.cpp +++ b/assignment-client/src/AssignmentClientApp.cpp @@ -9,8 +9,12 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include -#include +#include "AssignmentClientApp.h" + +#include +#include +#include +#include #include #include @@ -20,30 +24,6 @@ #include "Assignment.h" #include "AssignmentClient.h" #include "AssignmentClientMonitor.h" -#include "AssignmentClientApp.h" -#include -#include - -#ifdef WIN32 -VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) { - if (!timerOrWaitFired && qApp) { - qDebug() << "Parent process died, quitting"; - qApp->quit(); - } -} - -void watchParentProcess(int parentPID) { - DWORD processID = parentPID; - HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID); - - HANDLE newHandle; - RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE); -} -#else -void watchParentProcess(int parentPID) { - qWarning() << "Parent PID option not implemented on this plateform"; -} -#endif // WIN32 AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : QCoreApplication(argc, argv) @@ -107,7 +87,7 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : const QCommandLineOption logDirectoryOption(ASSIGNMENT_LOG_DIRECTORY, "directory to store logs", "log-directory"); parser.addOption(logDirectoryOption); - const QCommandLineOption parentPIDOption(ASSIGNMENT_PARENT_PID, "PID of the parent process", "parent-pid"); + const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid"); parser.addOption(parentPIDOption); if (!parser.parse(QCoreApplication::arguments())) { @@ -231,6 +211,7 @@ AssignmentClientApp::AssignmentClientApp(int argc, char* argv[]) : int parentPID = parser.value(parentPIDOption).toInt(&ok); if (ok) { + qDebug() << "Parent process PID is" << parentPID; watchParentProcess(parentPID); } } diff --git a/assignment-client/src/AssignmentClientApp.h b/assignment-client/src/AssignmentClientApp.h index a7c8cdb54d..37d3b9cc1d 100644 --- a/assignment-client/src/AssignmentClientApp.h +++ b/assignment-client/src/AssignmentClientApp.h @@ -27,7 +27,6 @@ const QString ASSIGNMENT_MAX_FORKS_OPTION = "max"; const QString ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION = "monitor-port"; const QString ASSIGNMENT_HTTP_STATUS_PORT = "http-status-port"; const QString ASSIGNMENT_LOG_DIRECTORY = "log-directory"; -const QString ASSIGNMENT_PARENT_PID = "parent-pid"; class AssignmentClientApp : public QCoreApplication { Q_OBJECT diff --git a/assignment-client/src/AssignmentClientMonitor.cpp b/assignment-client/src/AssignmentClientMonitor.cpp index 568a29c8b1..070034d54b 100644 --- a/assignment-client/src/AssignmentClientMonitor.cpp +++ b/assignment-client/src/AssignmentClientMonitor.cpp @@ -131,7 +131,6 @@ void AssignmentClientMonitor::aboutToQuit() { void AssignmentClientMonitor::spawnChildClient() { QProcess* assignmentClient = new QProcess(this); - // unparse the parts of the command-line that the child cares about QStringList _childArguments; if (_assignmentPool != "") { @@ -159,9 +158,8 @@ void AssignmentClientMonitor::spawnChildClient() { // for now they simply talk to us on localhost _childArguments.append("--" + ASSIGNMENT_CLIENT_MONITOR_PORT_OPTION); _childArguments.append(QString::number(DependencyManager::get()->getLocalSockAddr().getPort())); - - _childArguments.append("--" + ASSIGNMENT_PARENT_PID); + _childArguments.append("--" + PARENT_PID_OPTION); _childArguments.append(QString::number(QCoreApplication::applicationPid())); QString nowString, stdoutFilenameTemp, stderrFilenameTemp, stdoutPathTemp, stderrPathTemp; diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 8e3d04897b..095613a473 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -221,6 +221,8 @@ void DomainServer::parseCommandLine() { const QCommandLineOption masterConfigOption("master-config", "Deprecated config-file option"); parser.addOption(masterConfigOption); + const QCommandLineOption parentPIDOption(PARENT_PID_OPTION, "PID of the parent process", "parent-pid"); + parser.addOption(parentPIDOption); if (!parser.parse(QCoreApplication::arguments())) { qWarning() << parser.errorText() << endl; @@ -249,6 +251,17 @@ void DomainServer::parseCommandLine() { _overrideDomainID = true; qDebug() << "domain-server ID is" << _overridingDomainID; } + + + if (parser.isSet(parentPIDOption)) { + bool ok = false; + int parentPID = parser.value(parentPIDOption).toInt(&ok); + + if (ok) { + qDebug() << "Parent process PID is" << parentPID; + watchParentProcess(parentPID); + } + } } DomainServer::~DomainServer() { diff --git a/libraries/shared/src/SharedUtil.cpp b/libraries/shared/src/SharedUtil.cpp index a68d27e620..58b8aead45 100644 --- a/libraries/shared/src/SharedUtil.cpp +++ b/libraries/shared/src/SharedUtil.cpp @@ -1076,3 +1076,24 @@ void setMaxCores(uint8_t maxCores) { SetProcessAffinityMask(process, newProcessAffinity); #endif } + +#ifdef Q_OS_WIN +VOID CALLBACK parentDiedCallback(PVOID lpParameter, BOOLEAN timerOrWaitFired) { + if (!timerOrWaitFired && qApp) { + qDebug() << "Parent process died, quitting"; + qApp->quit(); + } +} + +void watchParentProcess(int parentPID) { + DWORD processID = parentPID; + HANDLE procHandle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID); + + HANDLE newHandle; + RegisterWaitForSingleObject(&newHandle, procHandle, parentDiedCallback, NULL, INFINITE, WT_EXECUTEONLYONCE); +} +#else +void watchParentProcess(int parentPID) { + qWarning() << "Parent PID option not implemented on this plateform"; +} +#endif // Q_OS_WIN \ No newline at end of file diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index 0f30842f47..681418a263 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -235,4 +235,7 @@ const QString& getInterfaceSharedMemoryName(); void setMaxCores(uint8_t maxCores); +const QString PARENT_PID_OPTION = "parent-pid"; +void watchParentProcess(int parentPID); + #endif // hifi_SharedUtil_h diff --git a/server-console/src/main.js b/server-console/src/main.js index 95fb0d81b2..9cbdc38f47 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -892,10 +892,12 @@ function onContentLoaded() { deleteOldFiles(logPath, DELETE_LOG_FILES_OLDER_THAN_X_SECONDS, LOG_FILE_REGEX); if (dsPath && acPath) { - domainServer = new Process('domain-server', dsPath, ["--get-temp-name"], logPath); + domainServer = new Process('domain-server', dsPath, ['--get-temp-name', + '--parent-pid', process.pid], logPath); acMonitor = new ACMonitorProcess('ac-monitor', acPath, ['-n7', '--log-directory', logPath, - '--http-status-port', httpStatusPort], httpStatusPort, logPath); + '--http-status-port', httpStatusPort, + '--parent-pid', process.pid], httpStatusPort, logPath); homeServer = new ProcessGroup('home', [domainServer, acMonitor]); logWindow = new LogWindow(acMonitor, domainServer); diff --git a/server-console/src/modules/hf-process.js b/server-console/src/modules/hf-process.js index 3c1200fecc..1de0630a0a 100644 --- a/server-console/src/modules/hf-process.js +++ b/server-console/src/modules/hf-process.js @@ -267,7 +267,15 @@ Process.prototype = extend(Process.prototype, { clearTimeout(this.stoppingTimeoutID); this.stoppingTimeoutID = null; } + // Grab current state berofe updating it. + var unexpectedShutdown = this.state != ProcessStates.STOPPING; this.updateState(ProcessStates.STOPPED); + + if (unexpectedShutdown) { + log.warn("Child stopped unexpectedly, restarting."); + this.start(); + return; + } } }); From ec7fe81b87f87958821b6497e08d92d32f45b318 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 29 Jun 2017 13:22:11 -0700 Subject: [PATCH 4/6] Sandbox doesn't restart Interface. --- server-console/src/main.js | 18 ++++++++++++------ server-console/src/modules/hf-process.js | 9 +++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/server-console/src/main.js b/server-console/src/main.js index 9cbdc38f47..b99d1387b5 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -892,12 +892,18 @@ function onContentLoaded() { deleteOldFiles(logPath, DELETE_LOG_FILES_OLDER_THAN_X_SECONDS, LOG_FILE_REGEX); if (dsPath && acPath) { - domainServer = new Process('domain-server', dsPath, ['--get-temp-name', - '--parent-pid', process.pid], logPath); - acMonitor = new ACMonitorProcess('ac-monitor', acPath, ['-n7', - '--log-directory', logPath, - '--http-status-port', httpStatusPort, - '--parent-pid', process.pid], httpStatusPort, logPath); + var dsArguments = ['--get-temp-name', + '--parent-pid', process.pid]; + domainServer = new Process('domain-server', dsPath, dsArguments, + logPath, true); + + var acArguments = ['-n7', + '--log-directory', logPath, + '--http-status-port', httpStatusPort, + '--parent-pid', process.pid]; + acMonitor = new ACMonitorProcess('ac-monitor', acPath, acArguments, + httpStatusPort, logPath, true); + homeServer = new ProcessGroup('home', [domainServer, acMonitor]); logWindow = new LogWindow(acMonitor, domainServer); diff --git a/server-console/src/modules/hf-process.js b/server-console/src/modules/hf-process.js index 1de0630a0a..af728eeda7 100644 --- a/server-console/src/modules/hf-process.js +++ b/server-console/src/modules/hf-process.js @@ -102,7 +102,7 @@ ProcessGroup.prototype = extend(ProcessGroup.prototype, { }); var ID = 0; -function Process(name, command, commandArgs, logDirectory) { +function Process(name, command, commandArgs, logDirectory, restartOnCrash) { events.EventEmitter.call(this); this.id = ++ID; @@ -113,6 +113,7 @@ function Process(name, command, commandArgs, logDirectory) { this.logDirectory = logDirectory; this.logStdout = null; this.logStderr = null; + this.restartOnCrash = restartOnCrash; this.state = ProcessStates.STOPPED; }; @@ -271,7 +272,7 @@ Process.prototype = extend(Process.prototype, { var unexpectedShutdown = this.state != ProcessStates.STOPPING; this.updateState(ProcessStates.STOPPED); - if (unexpectedShutdown) { + if (unexpectedShutdown && this.restartOnCrash) { log.warn("Child stopped unexpectedly, restarting."); this.start(); return; @@ -282,8 +283,8 @@ Process.prototype = extend(Process.prototype, { // ACMonitorProcess is an extension of Process that keeps track of the AC Montior's // children status and log locations. const CHECK_AC_STATUS_INTERVAL = 1000; -function ACMonitorProcess(name, path, args, httpStatusPort, logPath) { - Process.call(this, name, path, args, logPath); +function ACMonitorProcess(name, path, args, httpStatusPort, logPath, restartOnCrash) { + Process.call(this, name, path, args, logPath, restartOnCrash); this.httpStatusPort = httpStatusPort; From 22099a14a867fa5235449c543b95ec71344f649d Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 29 Jun 2017 13:50:40 -0700 Subject: [PATCH 5/6] Go Home spawns interface detached --- server-console/src/main.js | 8 +++++--- server-console/src/modules/hf-process.js | 11 ++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server-console/src/main.js b/server-console/src/main.js index b99d1387b5..efa04a8512 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -334,6 +334,7 @@ function startInterface(url) { // create a new Interface instance - Interface makes sure only one is running at a time var pInterface = new Process('interface', interfacePath, argArray); + pInterface.detached = true; pInterface.start(); } @@ -894,15 +895,16 @@ function onContentLoaded() { if (dsPath && acPath) { var dsArguments = ['--get-temp-name', '--parent-pid', process.pid]; - domainServer = new Process('domain-server', dsPath, dsArguments, - logPath, true); + domainServer = new Process('domain-server', dsPath, dsArguments, logPath); + domainServer.restartOnCrash = true; var acArguments = ['-n7', '--log-directory', logPath, '--http-status-port', httpStatusPort, '--parent-pid', process.pid]; acMonitor = new ACMonitorProcess('ac-monitor', acPath, acArguments, - httpStatusPort, logPath, true); + httpStatusPort, logPath); + acMonitor.restartOnCrash = true; homeServer = new ProcessGroup('home', [domainServer, acMonitor]); logWindow = new LogWindow(acMonitor, domainServer); diff --git a/server-console/src/modules/hf-process.js b/server-console/src/modules/hf-process.js index af728eeda7..b75835c06f 100644 --- a/server-console/src/modules/hf-process.js +++ b/server-console/src/modules/hf-process.js @@ -102,7 +102,7 @@ ProcessGroup.prototype = extend(ProcessGroup.prototype, { }); var ID = 0; -function Process(name, command, commandArgs, logDirectory, restartOnCrash) { +function Process(name, command, commandArgs, logDirectory) { events.EventEmitter.call(this); this.id = ++ID; @@ -113,7 +113,8 @@ function Process(name, command, commandArgs, logDirectory, restartOnCrash) { this.logDirectory = logDirectory; this.logStdout = null; this.logStderr = null; - this.restartOnCrash = restartOnCrash; + this.detached = false; + this.restartOnCrash = false; this.state = ProcessStates.STOPPED; }; @@ -166,7 +167,7 @@ Process.prototype = extend(Process.prototype, { try { this.child = childProcess.spawn(this.command, this.commandArgs, { - detached: false, + detached: this.detached, stdio: ['ignore', logStdout, logStderr] }); log.debug("Spawned " + this.command + " with pid " + this.child.pid); @@ -283,8 +284,8 @@ Process.prototype = extend(Process.prototype, { // ACMonitorProcess is an extension of Process that keeps track of the AC Montior's // children status and log locations. const CHECK_AC_STATUS_INTERVAL = 1000; -function ACMonitorProcess(name, path, args, httpStatusPort, logPath, restartOnCrash) { - Process.call(this, name, path, args, logPath, restartOnCrash); +function ACMonitorProcess(name, path, args, httpStatusPort, logPath) { + Process.call(this, name, path, args, logPath); this.httpStatusPort = httpStatusPort; From 85601669e10780c3d9cf05690191049139783210 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 30 Jun 2017 14:09:49 -0700 Subject: [PATCH 6/6] Prevent thrashing by limiting to 10 restarts every 10 minutes --- server-console/src/modules/hf-process.js | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/server-console/src/modules/hf-process.js b/server-console/src/modules/hf-process.js index b75835c06f..767befec7b 100644 --- a/server-console/src/modules/hf-process.js +++ b/server-console/src/modules/hf-process.js @@ -115,6 +115,8 @@ function Process(name, command, commandArgs, logDirectory) { this.logStderr = null; this.detached = false; this.restartOnCrash = false; + this.restartCount = 0; + this.firstRestartTimestamp = Date.now(); this.state = ProcessStates.STOPPED; }; @@ -269,14 +271,29 @@ Process.prototype = extend(Process.prototype, { clearTimeout(this.stoppingTimeoutID); this.stoppingTimeoutID = null; } - // Grab current state berofe updating it. + // Grab current state before updating it. var unexpectedShutdown = this.state != ProcessStates.STOPPING; this.updateState(ProcessStates.STOPPED); if (unexpectedShutdown && this.restartOnCrash) { - log.warn("Child stopped unexpectedly, restarting."); - this.start(); - return; + var MAX_RESTARTS = 10; + var MAX_RESTARTS_PERIOD = 10; // 10 min + var MSEC_PER_MIN = 1000 * 60; + var now = Date.now(); + var timeDiff = (now - this.firstRestartTimestamp) / MSEC_PER_MIN; + if (timeDiff > MAX_RESTARTS_PERIOD) { + this.firstRestartTimestamp = now; + this.restartCount = 0; + } + + if (this.restartCount < 10) { + this.restartCount++; + + log.warn("Child stopped unexpectedly, restarting."); + this.start(); + } else { + log.warn("Child stopped unexpectedly too many times, not restarting."); + } } } });