Added number of failures to log.

tools/auto-tester/src/Test.cpp

@@ -63,7 +63,7 @@ QString Test::zipAndDeleteTestResultsFolder() {
     return zippedResultsFileName;
 }
 
-bool Test::compareImageLists() {
+int Test::compareImageLists() {
     _progressBar->setMinimum(0);
     _progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1);
     _progressBar->setValue(0);
@@ -71,8 +71,8 @@ bool Test::compareImageLists() {
 
     // Loop over both lists and compare each pair of images
     // Quit loop if user has aborted due to a failed test.
-    bool success{ true };
     bool keepOn{ true };
+    int numberOfFailures{ 0 };
     for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) {
         // First check that images are the same size
         QImage resultImage(_resultImagesFullFilenames[i]);
@@ -91,6 +91,7 @@ bool Test::compareImageLists() {
         }
 
         if (similarityIndex < THRESHOLD) {
+            ++numberOfFailures;
             TestFailure testFailure = TestFailure{
                 (float)similarityIndex,
                 _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
@@ -102,7 +103,6 @@ bool Test::compareImageLists() {
 
             if (!isInteractiveMode) {
                 appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
-                success = false;
             } else {
                 _mismatchWindow.exec();
 
@@ -111,11 +111,9 @@ bool Test::compareImageLists() {
                         break;
                     case USE_RESPONSE_FAIL:
                         appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
-                        success = false;
                         break;
                     case USER_RESPONSE_ABORT:
                         keepOn = false;
-                        success = false;
                         break;
                     default:
                         assert(false);
@@ -128,7 +126,7 @@ bool Test::compareImageLists() {
     }
 
     _progressBar->setVisible(false);
-    return success;
+    return numberOfFailures;
 }
 
 void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
@@ -256,10 +254,10 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine,
     autoTester->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this);
 }
 void Test::finishTestsEvaluation() {
-    bool success = compareImageLists();
+    int numberOfFailures = compareImageLists();
     
     if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) {
-        if (success) {
+        if (numberOfFailures == 0) {
             QMessageBox::information(0, "Success", "All images are as expected");
         } else {
             QMessageBox::information(0, "Failure", "One or more images are not as expected");
@@ -273,7 +271,7 @@ void Test::finishTestsEvaluation() {
     }
 
     if (_isRunningInAutomaticTestRun) {
-        autoTester->automaticTestRunEvaluationComplete(zippedFolderName);
+        autoTester->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
     }
 }
 

tools/auto-tester/src/Test.h

@@ -75,7 +75,7 @@ public:
     void createAllRecursiveScripts();
     void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
 
-    bool compareImageLists();
+    int compareImageLists();
 
     QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory);
 

tools/auto-tester/src/TestRunner.cpp

@@ -119,7 +119,6 @@ void TestRunner::run() {
         filenames << _installerFilename;
     }
 
-
     updateStatusLabel("Downloading installer");
 
     autoTester->downloadFiles(urls, _workingFolder, filenames, (void*)this);
@@ -174,7 +173,12 @@ void TestRunner::saveExistingHighFidelityAppDataFolder() {
     dataDirectory = qgetenv("USERPROFILE") + "\\AppData\\Roaming";
 #endif
 
-    _appDataFolder = dataDirectory + "\\High Fidelity";
+    if (!_runLatest->isChecked()) {
+        // We are running a PR build
+        _appDataFolder = dataDirectory + "\\High Fidelity - " + getPRNumberFromURL(_url->toPlainText());
+    } else {
+        _appDataFolder = dataDirectory + "\\High Fidelity";
+    }
 
     if (_appDataFolder.exists()) {
         // The original folder is saved in a unique name
@@ -270,8 +274,7 @@ void TestRunner::startLocalServerProcesses() {
 }
 
 void TestRunner::runInterfaceWithTestScript() {
-    QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) +
-                      "\\interface.exe\"";
+    QString exeFile = QString("\"") + QDir::toNativeSeparators(_installationFolder) + "\\interface.exe\"";
 
     QString url = QString("hifi://localhost");
     if (_runServerless->isChecked()) {
@@ -302,7 +305,7 @@ void TestRunner::evaluateResults() {
     autoTester->startTestsEvaluation(false, true, _snapshotFolder, _branch, _user);
 }
 
-void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) {
+void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder, int numberOfFailures) {
     addBuildNumberToResults(zippedFolder);
     restoreHighFidelityAppDataFolder();
 
@@ -310,9 +313,18 @@ void TestRunner::automaticTestRunEvaluationComplete(QString zippedFolder) {
 
     QDateTime currentDateTime = QDateTime::currentDateTime();
 
-    appendLog(QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" +
-              QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
-              currentDateTime.date().toString("ddd, MMM d, yyyy"));
+    QString completionText = QString("Tests completed at ") + QString::number(currentDateTime.time().hour()) + ":" +
+                             QString("%1").arg(currentDateTime.time().minute(), 2, 10, QChar('0')) + ", on " +
+                             currentDateTime.date().toString("ddd, MMM d, yyyy");
+
+    if (numberOfFailures == 0) {
+        completionText += "; no failures";
+    } else if (numberOfFailures == 1) {
+        completionText += "; 1 failure";
+    } else {
+        completionText += QString("; ") + QString::number(numberOfFailures) + " failures";
+    }
+    appendLog(completionText);
 
     _automatedTestIsRunning = false;
 }
@@ -488,7 +500,7 @@ QString TestRunner::getInstallerNameFromURL(const QString& url) {
     // An example URL: https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe
     try {
         QStringList urlParts = url.split("/");
-        int rr = urlParts.size(); 
+        int rr = urlParts.size();
         if (urlParts.size() != 8) {
             throw "URL not in expected format, should look like `https://deployment.highfidelity.com/jobs/pr-build/label%3Dwindows/13023/HighFidelity-Beta-Interface-PR14006-be76c43.exe`";
         }

tools/auto-tester/src/TestRunner.h

@@ -54,7 +54,7 @@ public:
     void runInterfaceWithTestScript();
 
     void evaluateResults();
-    void automaticTestRunEvaluationComplete(QString zippedFolderName);
+    void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
     void addBuildNumberToResults(QString zippedFolderName);
 
     void copyFolder(const QString& source, const QString& destination);

tools/auto-tester/src/ui/AutoTester.cpp

@@ -166,8 +166,8 @@ void AutoTester::on_checkBoxRunLatest_clicked() {
     _ui.urlTextEdit->setEnabled(!_ui.checkBoxRunLatest->isChecked());
 }
 
-void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName) {
-    _testRunner->automaticTestRunEvaluationComplete(zippedFolderName);
+void AutoTester::automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures) {
+    _testRunner->automaticTestRunEvaluationComplete(zippedFolderName, numberOfFailures);
 }
 
 void AutoTester::on_updateTestRailRunResultsButton_clicked() {

tools/auto-tester/src/ui/AutoTester.h

@@ -36,7 +36,7 @@ public:
                               const QString& branch,
                               const QString& user);
 
-    void automaticTestRunEvaluationComplete(QString zippedFolderName);
+    void automaticTestRunEvaluationComplete(QString zippedFolderName, int numberOfFailures);
 
     void downloadFile(const QUrl& url);
     void downloadFiles(const QStringList& URLs, const QString& directoryName, const QStringList& filenames, void* caller);

tools/auto-tester/src/ui/AutoTester.ui

@@ -495,7 +495,7 @@
        <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
       </property>
       <property name="text">
-       <string>Serveless</string>
+       <string>Server-less</string>
       </property>
       <property name="checked">
        <bool>false</bool>