Auto-tester now saves results in a folder hierarchy.

tools/auto-tester/src/Test.cpp

@@ -22,10 +22,10 @@ Test::Test() {
 }
 
 bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory) {
-    // Delete any previous test results, if user agrees
-    QString s = testDirectory + "/" + testResultsFolder;
-    QFileInfo fileInfo(testDirectory + "/" + testResultsFolder);
-    while (fileInfo.exists()) {
+    // If a previous test results folder is found then wait for the user to delete it, or cancel
+    // (e.g. the user may want to move the folder elsewhere)
+    QString testResultsFolderPath { testDirectory + "/" + testResultsFolder };
+    while (QDir().exists(testResultsFolderPath)) {
         messageBox.setText("Previous test results have been found");
         messageBox.setInformativeText("Delete " + testResultsFolder + " before continuing");
         messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel);
@@ -35,9 +35,12 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
         }
     }
 
+    // Create a new test results folder
+    QDir().mkdir(testResultsFolderPath);
+
     // Loop over both lists and compare each pair of images
     // Quit loop if user has aborted due to a failed test.
-    const double THRESHOLD{ 0.999 };
+    const double THRESHOLD { 0.999 };
     bool success{ true };
     bool keepOn{ true };
     for (int i = 0; keepOn && i < expectedImages.length(); ++i) {
@@ -73,7 +76,7 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
                 case USER_RESPONSE_PASS:
                     break;
                 case USE_RESPONSE_FAIL:
-                    appendTestResultsToFile(testDirectory, testFailure);
+                    appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage());
                     success = false;
                     break;
                 case USER_RESPONSE_ABORT:
@@ -90,11 +93,55 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
     return success;
 }
 
-void Test::appendTestResultsToFile(QString testDirectory, TestFailure testFailure) {
-    QFileInfo fileInfo(testResultsFileName);
-    if (!fileInfo.exists()) {
+void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
+    if (!QDir().exists(testResultsFolderPath)) {
+        messageBox.critical(0, "Internal error", "Folder " + testResultsFolderPath + " not found");
+        exit(-1);
     }
 
+    static int index = 1;
+    QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
+    if (!QDir().mkdir(failureFolderPath)) {
+        messageBox.critical(0, "Internal error", "Failed to create folder " + failureFolderPath);
+        exit(-1);
+    }
+    ++index;
+
+    QString descriptionFileName { "ReadMe.txt" };
+    QFile descriptionFile(failureFolderPath + "/" +descriptionFileName);
+    if (!descriptionFile.open(QIODevice::ReadWrite)) {
+        messageBox.critical(0, "Internal error", "Failed to create file " + descriptionFileName);
+        exit(-1);
+    }
+
+    // Create text file describing the failure
+    QTextStream stream(&descriptionFile);
+    stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/'
+    stream << "Expected image was    " << testFailure._expectedImageFilename << endl;
+    stream << "Actual image was      " << testFailure._actualImageFilename << endl;
+    stream << "Similarity index was  " << testFailure._error << endl;
+
+    descriptionFile.close();
+
+    // Copy expected and actual images, and save the difference image
+    QString sourceFile;
+    QString destinationFile;
+
+    sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
+    destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
+    if (!QFile::copy(sourceFile, destinationFile)) {
+        messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile);
+        exit(-1);
+    }
+
+    sourceFile = testFailure._pathname + testFailure._actualImageFilename;
+    destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
+    if (!QFile::copy(sourceFile, destinationFile)) {
+        messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile);
+        exit(-1);
+    }
+
+    comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
 }
 
 void Test::evaluateTests() {

tools/auto-tester/src/Test.h

@@ -35,7 +35,7 @@ public:
 
     void importTest(QTextStream& textStream, const QString& testPathname, int testNumber);
 
-    void appendTestResultsToFile(QString testDirectory, TestFailure testFailure);
+    void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
 
 private:
     const QString testFilename{ "test.js" };

tools/auto-tester/src/ui/MismatchWindow.cpp

@@ -33,11 +33,15 @@ QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultIma
             double p = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP);
             double q = R_Y * qRed(pixelQ) + G_Y * qGreen(pixelQ) + B_Y * qBlue(pixelQ);
 
-            int absDiff = (int)(fabs(p - q));
- 
-            buffer[3 * (x + y * expectedImage.width()) + 0] = absDiff;
-            buffer[3 * (x + y * expectedImage.width()) + 1] = absDiff;
-            buffer[3 * (x + y * expectedImage.width()) + 2] = absDiff;
+            // The intensity value is modified to increase the brightness of the displayed image
+            double absoluteDifference = fabs(p - q) / 255.0;
+            double modifiedDifference = pow(absoluteDifference, 0.5);
+
+            int difference = (int)(modifiedDifference * 255.0);
+
+            buffer[3 * (x + y * expectedImage.width()) + 0] = difference;
+            buffer[3 * (x + y * expectedImage.width()) + 1] = difference;
+            buffer[3 * (x + y * expectedImage.width()) + 2] = difference;
         }
     }
 
@@ -60,7 +64,7 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) {
     QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename);
     QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename);
 
-    QPixmap diffPixmap = computeDiffPixmap(
+    diffPixmap = computeDiffPixmap(
         QImage(testFailure._pathname + testFailure._expectedImageFilename), 
         QImage(testFailure._pathname + testFailure._actualImageFilename)
     );
@@ -84,3 +88,7 @@ void MismatchWindow::on_abortTestsButton_clicked() {
     _userResponse = USER_RESPONSE_ABORT;
     close();
 }
+
+QPixmap MismatchWindow::getComparisonImage() {
+    return diffPixmap;
+}

tools/auto-tester/src/ui/MismatchWindow.h

@@ -26,6 +26,7 @@ public:
     UserResponse getUserResponse() { return _userResponse; }
 
     QPixmap computeDiffPixmap(QImage expectedImage, QImage resultImage);
+    QPixmap getComparisonImage();
 
 private slots:
     void on_passTestButton_clicked();
@@ -34,6 +35,8 @@ private slots:
 
 private:
     UserResponse _userResponse{ USER_RESPONSE_INVALID };
+
+    QPixmap diffPixmap;
 };
 
 

tools/auto-tester/src/ui/MismatchWindow.ui

@@ -17,7 +17,7 @@
    <property name="geometry">
     <rect>
      <x>10</x>
-     <y>20</y>
+     <y>25</y>
      <width>800</width>
      <height>450</height>
     </rect>
@@ -30,7 +30,7 @@
    <property name="geometry">
     <rect>
      <x>900</x>
-     <y>20</y>
+     <y>25</y>
      <width>800</width>
      <height>450</height>
     </rect>
@@ -163,6 +163,32 @@
     <string>similarity</string>
    </property>
   </widget>
+  <widget class="QLabel" name="label">
+   <property name="geometry">
+    <rect>
+     <x>30</x>
+     <y>5</y>
+     <width>151</width>
+     <height>16</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Expected Image</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="label_2">
+   <property name="geometry">
+    <rect>
+     <x>930</x>
+     <y>5</y>
+     <width>151</width>
+     <height>16</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Actual Image</string>
+   </property>
+  </widget>
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <resources/>