Do not close when evaluation complete.

tools/auto-tester/src/Test.cpp

@@ -78,7 +78,7 @@ bool Test::compareImageLists() {
 
         double similarityIndex;  // in [-1.0 .. 1.0], where 1.0 means images are identical
 
-        bool isInteractiveMode = (!_isRunningFromCommandLine && _checkBoxInteractiveMode->isChecked());
+        bool isInteractiveMode = (!_isRunningFromCommandLine && _checkBoxInteractiveMode->isChecked() && !_isRunningInAutomaticTestRun);
                                  
         // similarityIndex is set to -100.0 to indicate images are not the same size
         if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) {
@@ -178,14 +178,16 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa
     comparisonImage.save(failureFolderPath + "/" + "Difference Image.png");
 }
 
-void Test::startTestsEvaluation(const bool isRunningFromCommandLine, 
+void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
-                                const QString& testFolder,
+                                const bool isRunningInAutomaticTestRun, 
+                                const QString& snapshotDirectory,
                                 const QString& branchFromCommandLine,
                                 const QString& userFromCommandLine
 ) {
     _isRunningFromCommandLine = isRunningFromCommandLine;
+    _isRunningInAutomaticTestRun = isRunningInAutomaticTestRun;
 
-    if (testFolder.isNull()) {
+    if (snapshotDirectory.isNull()) {
         // Get list of JPEG images in folder, sorted by name
         QString previousSelection = _snapshotDirectory;
         QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
@@ -201,8 +203,8 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
             return;
         }
     } else {
-        _snapshotDirectory = testFolder;
+        _snapshotDirectory = snapshotDirectory;
-        _exitWhenComplete = true;
+        _exitWhenComplete = (isRunningFromCommandLine && !isRunningInAutomaticTestRun);
     }
 
     // Quit if test results folder could not be created
@@ -254,7 +256,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
 void Test::finishTestsEvaluation() {
     bool success = compareImageLists();
     
-    if (!_isRunningFromCommandLine) {
+    if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) {
         if (success) {
             QMessageBox::information(0, "Success", "All images are as expected");
         } else {
@@ -267,6 +269,10 @@ void Test::finishTestsEvaluation() {
     if (_exitWhenComplete) {
         exit(0);
     }
+
+    if (_isRunningInAutomaticTestRun) {
+        autoTester->automaticTestRunEvaluationComplete();
+    }
 }
 
 bool Test::isAValidDirectory(const QString& pathname) {

tools/auto-tester/src/Test.h

@@ -44,7 +44,8 @@ public:
     Test(QProgressBar* progressBar, QCheckBox* checkBoxInteractiveMode);
 
     void startTestsEvaluation(const bool isRunningFromCommandLine,
-                              const QString& testFolder = QString(),
+                              const bool isRunningInAutomaticTestRun, 
+                              const QString& snapshotDirectory = QString(),
                               const QString& branchFromCommandLine = QString(),
                               const QString& userFromCommandLine = QString());
 
@@ -101,6 +102,7 @@ private:
     QCheckBox* _checkBoxInteractiveMode;
 
     bool _isRunningFromCommandLine{ false };
+    bool _isRunningInAutomaticTestRun{ false };
 
     const QString TEST_FILENAME { "test.js" };
     const QString TEST_RESULTS_FOLDER { "TestResults" };

tools/auto-tester/src/TestRunner.cpp

@@ -56,7 +56,7 @@ void TestRunner::installerDownloadComplete() {
 
     evaluateResults();
 
-    restoreHighFidelityAppDataFolder();
+    // The High Fidelity AppData folder will be restored after evaluation has completed
 }
 
 void TestRunner::runInstaller() {
@@ -173,7 +173,7 @@ void TestRunner::runInterfaceWithTestScript() {
 }
 
 void TestRunner::evaluateResults() {
-    autoTester->runFromCommandLine(_snapshotFolder, _branch, _user);
+    autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user);
 }
 
 // Copies a folder recursively
@@ -206,4 +206,8 @@ void TestRunner::copyFolder(const QString& source, const QString& destination) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "unknown error");
         exit(-1);
     }
-}
+}
+
+void TestRunner::automaticTestRunEvaluationComplete() {
+    restoreHighFidelityAppDataFolder();
+}

tools/auto-tester/src/TestRunner.h

@@ -35,6 +35,7 @@ public:
     void startLocalServerProcesses();
     void runInterfaceWithTestScript();
     void evaluateResults();
+    void automaticTestRunEvaluationComplete();
 
     void copyFolder(const QString& source, const QString& destination);
 

tools/auto-tester/src/main.cpp

@@ -66,7 +66,7 @@ int main(int argc, char *argv[]) {
     autoTester->setup();
 
     if (!testFolder.isNull()) {
-        autoTester->runFromCommandLine(testFolder, branch, user);
+        autoTester->startTestsEvaluation(true ,false, testFolder, branch, user);
     } else {
         autoTester->show();
     }

tools/auto-tester/src/ui/AutoTester.cpp

@@ -41,8 +41,12 @@ void AutoTester::setup() {
     _testRunner = new TestRunner();
 }
 
-void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) {
+void AutoTester::startTestsEvaluation(const bool isRunningFromCommandLine,
-    _test->startTestsEvaluation(true, testFolder, branch, user);
+                                      const bool isRunningInAutomaticTestRun,
+                                      const QString& snapshotDirectory,
+                                      const QString& branch,
+                                      const QString& user) {
+    _test->startTestsEvaluation(isRunningFromCommandLine, isRunningInAutomaticTestRun, snapshotDirectory, branch, user);
 }
 
 void AutoTester::on_tabWidget_currentChanged(int index) {
@@ -56,7 +60,7 @@ void AutoTester::on_tabWidget_currentChanged(int index) {
 }
 
 void AutoTester::on_evaluateTestsButton_clicked() {
-    _test->startTestsEvaluation(false);
+    _test->startTestsEvaluation(false, false);
 }
 
 void AutoTester::on_createRecursiveScriptButton_clicked() {
@@ -103,6 +107,10 @@ void AutoTester::on_runNowButton_clicked() {
     _testRunner->run();
 }
 
+void AutoTester::automaticTestRunEvaluationComplete() {
+    _testRunner->automaticTestRunEvaluationComplete();
+}
+
 void AutoTester::on_updateTestRailRunResultsButton_clicked() {
     _test->updateTestRailRunResult();
 }

tools/auto-tester/src/ui/AutoTester.h

@@ -30,7 +30,13 @@ public:
 
     void setup();
 
-    void runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user);
+    void startTestsEvaluation(const bool isRunningFromCommandLine,
+                              const bool isRunningInAutomaticTestRun,
+                              const QString& snapshotDirectory,
+                              const QString& branch,
+                              const QString& user);
+
+    void automaticTestRunEvaluationComplete();
 
     void downloadFile(const QUrl& url);
     void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void *caller);