Added option to delete old snapshots.

tools/auto-tester/CMakeLists.txt

@@ -20,28 +20,22 @@ include_directories(${Qt5Widgets_INCLUDE_DIRS})
 
 set (QT_LIBRARIES Qt5::Core Qt5::Widgets)
 
-# Find all sources files
-file (GLOB_RECURSE SOURCES src/*.cpp)
-file (GLOB_RECURSE HEADERS src/*.h)
-file (GLOB_RECURSE UIS src/ui/*.ui)
-
 if (WIN32)
     # Do not show Console
     set_property (TARGET auto-tester PROPERTY WIN32_EXECUTABLE true)
 endif()
 
-add_executable(PROJECT_NAME ${SOURCES} ${HEADERS} ${UIS})
+target_zlib()
+add_dependency_external_projects (quazip)
+find_package (QuaZip REQUIRED)
+target_include_directories( ${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS})
+target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES})
  
-target_link_libraries(PROJECT_NAME ${QT_LIBRARIES})
+package_libraries_for_deployment()
-
-# Copy required dll's.
-add_custom_command(TARGET auto-tester POST_BUILD 
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Core> $<TARGET_FILE_DIR:auto-tester>
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Gui> $<TARGET_FILE_DIR:auto-tester>
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Widgets> $<TARGET_FILE_DIR:auto-tester>
-)
 
 if (WIN32)
+    add_paths_to_fixup_libs (${QUAZIP_DLL_PATH})
+
     find_program(WINDEPLOYQT_COMMAND windeployqt PATHS ${QT_DIR}/bin NO_DEFAULT_PATH)
     
     if (NOT WINDEPLOYQT_COMMAND)

tools/auto-tester/src/ImageComparer.cpp

@@ -8,6 +8,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 #include "ImageComparer.h"
+#include "common.h"
 
 #include <cmath>
 
@@ -26,11 +27,6 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) co
     const double c1 = pow((K1 * L), 2);
     const double c2 = pow((K2 * L), 2);
 
-    // Coefficients for luminosity calculation
-    const double R_Y = 0.212655f;
-    const double G_Y = 0.715158f;
-    const double B_Y = 0.072187f;
-
     // First go over all full 8x8 blocks
     // This is done in 3 loops
     //      1) Read the pixels into a linear array (an optimization)

tools/auto-tester/src/Test.cpp

@@ -13,15 +13,59 @@
 #include <QtCore/QTextStream>
 #include <QDirIterator>
 
+#include <quazip5/quazip.h>
+#include <quazip5/JlCompress.h>
+
 Test::Test() {
-    snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg");
+    snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.*-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg");
 
     expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg");
 
     mismatchWindow.setModal(true);
 }
 
-bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages) {
+bool Test::createTestResultsFolderPathIfNeeded(QString directory) {
+    // The test results folder is located in the root of the tests (i.e. for recursive test evaluation)
+    if (testResultsFolderPath == "") {
+        testResultsFolderPath =  directory + "/" + TEST_RESULTS_FOLDER;
+        QDir testResultsFolder(testResultsFolderPath);
+
+        if (testResultsFolder.exists()) {
+            testResultsFolder.removeRecursively();
+        }
+
+        // Create a new test results folder
+        return QDir().mkdir(testResultsFolderPath);
+    } else {
+        return true;
+    }
+}
+
+void Test::zipAndDeleteTestResultsFolder() {
+    QString zippedResultsFileName { testResultsFolderPath + ".zip" };
+    QFileInfo fileInfo(zippedResultsFileName);
+    if (!fileInfo.exists()) {
+        QFile::remove(zippedResultsFileName);
+    }
+
+    QDir testResultsFolder(testResultsFolderPath);
+    if (!testResultsFolder.isEmpty()) {
+        JlCompress::compressDir(testResultsFolderPath + ".zip", testResultsFolderPath);
+    }
+
+    testResultsFolder.removeRecursively();
+
+    //In all cases, for the next evaluation
+    testResultsFolderPath = "";
+    index = 1;
+}
+
+bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar) {
+    progressBar->setMinimum(0);
+    progressBar->setMaximum(expectedImages.length() - 1);
+    progressBar->setValue(0);
+    progressBar->setVisible(true);
+
     // Loop over both lists and compare each pair of images
     // Quit loop if user has aborted due to a failed test.
     const double THRESHOLD { 0.999 };
@@ -45,19 +89,26 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
         }
 
         if (similarityIndex < THRESHOLD) {
-            mismatchWindow.setTestFailure(TestFailure{
+            TestFailure testFailure = TestFailure{
                 (float)similarityIndex,
                 expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
                 QFileInfo(expectedImages[i].toStdString().c_str()).fileName(),  // filename of expected image
                 QFileInfo(resultImages[i].toStdString().c_str()).fileName()     // filename of result image
-            });
+            };
 
+            mismatchWindow.setTestFailure(testFailure);
+
+            if (!interactiveMode) {
+                appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage());
+                success = false;
+            } else {
                 mismatchWindow.exec();
 
                 switch (mismatchWindow.getUserResponse()) {
                     case USER_RESPONSE_PASS:
                         break;
                     case USE_RESPONSE_FAIL:
+                        appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage());
                         success = false;
                         break;
                     case USER_RESPONSE_ABORT:
@@ -71,16 +122,74 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
             }
         }
 
+        progressBar->setValue(i);
+    }
+
+    progressBar->setVisible(false);
     return success;
 }
 
-void Test::evaluateTests() {
+void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
+    if (!QDir().exists(testResultsFolderPath)) {
+        messageBox.critical(0, "Internal error", "Folder " + testResultsFolderPath + " not found");
+        exit(-1);
+    }
+
+    QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
+    if (!QDir().mkdir(failureFolderPath)) {
+        messageBox.critical(0, "Internal error", "Failed to create folder " + failureFolderPath);
+        exit(-1);
+    }
+    ++index;
+
+    QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
+    if (!descriptionFile.open(QIODevice::ReadWrite)) {
+        messageBox.critical(0, "Internal error", "Failed to create file " + TEST_RESULTS_FILENAME);
+        exit(-1);
+    }
+
+    // Create text file describing the failure
+    QTextStream stream(&descriptionFile);
+    stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/'
+    stream << "Expected image was    " << testFailure._expectedImageFilename << endl;
+    stream << "Actual image was      " << testFailure._actualImageFilename << endl;
+    stream << "Similarity index was  " << testFailure._error << endl;
+
+    descriptionFile.close();
+
+    // Copy expected and actual images, and save the difference image
+    QString sourceFile;
+    QString destinationFile;
+
+    sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
+    destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
+    if (!QFile::copy(sourceFile, destinationFile)) {
+        messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile);
+        exit(-1);
+    }
+
+    sourceFile = testFailure._pathname + testFailure._actualImageFilename;
+    destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
+    if (!QFile::copy(sourceFile, destinationFile)) {
+        messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile);
+        exit(-1);
+    }
+
+    comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
+}
+
+void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) {
     // Get list of JPEG images in folder, sorted by name
     QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
     if (pathToImageDirectory == "") {
         return;
     }
 
+    // Leave if test results folder could not be created
+    if (!createTestResultsFolderPathIfNeeded(pathToImageDirectory)) {
+        return;
+    }
+
     QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
 
     // Separate images into two lists.  The first is the expected images, the second is the test results
@@ -107,36 +216,57 @@ void Test::evaluateTests() {
         exit(-1);
     }
 
-    bool success = compareImageLists(expectedImages, resultImages);
+    bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar);
 
     if (success) {
         messageBox.information(0, "Success", "All images are as expected");
     } else {
         messageBox.information(0, "Failure", "One or more images are not as expected");
     }
+
+    zipAndDeleteTestResultsFolder();
+}
+
+bool Test::isAValidDirectory(QString pathname) {
+    // Only process directories
+    QDir dir(pathname);
+    if (!dir.exists()) {
+        return false;
+    }
+
+    // Ignore '.', '..' directories
+    if (pathname[pathname.length() - 1] == '.') {
+        return false;
+    }
+
+    return true;
 }
 
 // Two criteria are used to decide if a folder contains valid test results.
 //      1) a 'test'js' file exists in the folder
 //      2) the folder has the same number of actual and expected images
-void Test::evaluateTestsRecursively() {
+void Test::evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar) {
     // Select folder to start recursing from
     QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly);
     if (topLevelDirectory == "") {
         return;
     }
 
+    // Leave if test results folder could not be created
+    if (!createTestResultsFolderPathIfNeeded(topLevelDirectory)) {
+        return;
+    }
+
     bool success{ true };
     QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
     while (it.hasNext()) {
         QString directory = it.next();
-        if (directory[directory.length() - 1] == '.') {
+
-            // ignore '.', '..' directories
+        if (!isAValidDirectory(directory)) {
             continue;
         }
 
-        // 
+        const QString testPathname{ directory + "/" + TEST_FILENAME };
-        const QString testPathname{ directory + "/" + testFilename };
         QFileInfo fileInfo(testPathname);
         if (!fileInfo.exists()) {
             // Folder does not contain 'test.js'
@@ -164,7 +294,7 @@ void Test::evaluateTestsRecursively() {
         }
 
         // Set success to false if any test has failed
-        success &= compareImageLists(expectedImages, resultImages);
+        success &= compareImageLists(expectedImages, resultImages, directory, interactiveMode, progressBar);
     }
 
     if (success) {
@@ -172,6 +302,8 @@ void Test::evaluateTestsRecursively() {
     } else {
         messageBox.information(0, "Failure", "One or more images are not as expected");
     }
+
+    zipAndDeleteTestResultsFolder();
 }
 
 void Test::importTest(QTextStream& textStream, const QString& testPathname, int testNumber) {
@@ -191,7 +323,8 @@ void Test::createRecursiveScript() {
     if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
         messageBox.critical(0,
             "Internal Error",
-            "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\"");
+            "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\""
+        );
 
         exit(-1);
     }
@@ -206,7 +339,7 @@ void Test::createRecursiveScript() {
     QVector<QString> testPathnames;
 
     // First test if top-level folder has a test.js file
-    const QString testPathname{ topLevelDirectory + "/" + testFilename };
+    const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME };
     QFileInfo fileInfo(testPathname);
     if (fileInfo.exists()) {
         // Current folder contains a test
@@ -219,12 +352,14 @@ void Test::createRecursiveScript() {
     QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
     while (it.hasNext()) {
         QString directory = it.next();
-        if (directory[directory.length() - 1] == '.') {
+
-            // ignore '.', '..' directories
+        // Only process directories
+        QDir dir(directory);
+        if (!isAValidDirectory(directory)) {
             continue;
         }
 
-        const QString testPathname{ directory + "/" + testFilename };
+        const QString testPathname{ directory + "/" + TEST_FILENAME };
         QFileInfo fileInfo(testPathname);
         if (fileInfo.exists()) {
             // Current folder contains a test
@@ -264,7 +399,7 @@ void Test::createRecursiveScript() {
         // The script produced will look as follows:
         //      if (test1HasNotStarted) {
         //          test1HasNotStarted = false;
-        //          test1.test();
+        //          test1.test("auto");
         //          print("******started test 1******");
         //      }
         //      |
@@ -287,7 +422,7 @@ void Test::createRecursiveScript() {
             textStream << tab << tab << "if (test" << i - 1 << ".complete && test" << i << "HasNotStarted) {" << endl;
         }
         textStream << tab << tab << tab << "test" << i << "HasNotStarted = false;" << endl;
-        textStream << tab << tab << tab << "test" << i << "." << testFunction << "();" << endl;
+        textStream << tab << tab << tab << "test" << i << "." << testFunction << "(\"auto\");" << endl;
         textStream << tab << tab << tab << "print(\"******started test " << i << "******\");" << endl;
 
         textStream << tab << tab << "}" << endl << endl;
@@ -366,6 +501,41 @@ void Test::createTest() {
     messageBox.information(0, "Success", "Test images have been created");
 }
 
+void Test::deleteOldSnapshots() {
+    // Select folder to start recursing from
+    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select root folder for snapshot deletion", ".", QFileDialog::ShowDirsOnly);
+    if (topLevelDirectory == "") {
+        return;
+    }
+
+    // Recurse over folders
+    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        QString directory = it.next();
+
+        // Only process directories
+        QDir dir(directory);
+        if (!isAValidDirectory(directory)) {
+            continue;
+        }
+
+        QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory);
+
+        // Delete any file that is a snapshot (NOT the Expected Images)
+        QStringList expectedImages;
+        QStringList resultImages;
+        foreach(QString currentFilename, sortedImageFilenames) {
+            QString fullCurrentFilename = directory + "/" + currentFilename;
+            if (isInSnapshotFilenameFormat(currentFilename)) {
+                if (!QFile::remove(fullCurrentFilename)) {
+                    messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nSnapshot deletion aborted");
+                    exit(-1);
+                }
+            }
+        }
+    }
+}
+
 QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) {
     imageDirectory = QDir(pathToImageDirectory);
     QStringList nameFilters;
@@ -374,6 +544,7 @@ QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirect
     return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
 }
 
+// Use regular expressions to check if files are in specific format
 bool Test::isInSnapshotFilenameFormat(QString filename) {
     return (snapshotFilenameFormat.match(filename).hasMatch());
 }

tools/auto-tester/src/Test.h

@@ -1,6 +1,5 @@
 //
 //  Test.h
-//  zone/ambientLightInheritence
 //
 //  Created by Nissim Hadar on 2 Nov 2017.
 //  Copyright 2013 High Fidelity, Inc.
@@ -15,6 +14,7 @@
 #include <QtWidgets/QFileDialog>
 #include <QtWidgets/QMessageBox>
 #include <QtCore/QRegularExpression>
+#include <QProgressBar>
 
 #include "ImageComparer.h"
 #include "ui/MismatchWindow.h"
@@ -23,10 +23,13 @@ class Test {
 public: 
     Test();
 
-    void evaluateTests();
+    void evaluateTests(bool interactiveMode, QProgressBar* progressBar);
-    void evaluateTestsRecursively();
+    void evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar);
     void createRecursiveScript();
     void createTest();
+    void deleteOldSnapshots();
+
+    bool compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar);
 
     QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory);
 
@@ -35,8 +38,17 @@ public:
 
     void importTest(QTextStream& textStream, const QString& testPathname, int testNumber);
 
+    void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
+
+    bool createTestResultsFolderPathIfNeeded(QString directory);
+    void zipAndDeleteTestResultsFolder();
+
+    bool isAValidDirectory(QString pathname);
+
 private:
-    const QString testFilename{ "test.js" };
+    const QString TEST_FILENAME { "test.js" };
+    const QString TEST_RESULTS_FOLDER { "TestResults" };
+    const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
 
     QMessageBox messageBox;
 
@@ -49,7 +61,9 @@ private:
 
     ImageComparer imageComparer;
 
-    bool compareImageLists(QStringList expectedImages, QStringList resultImages);
+
+    QString testResultsFolderPath { "" };
+    int index { 1 };
 };
 
 #endif // hifi_test_h

tools/auto-tester/src/common.h

@@ -34,4 +34,9 @@ enum UserResponse {
     USER_RESPONSE_ABORT
 };
 
+// Coefficients for luminosity calculation
+const double R_Y = 0.212655f;
+const double G_Y = 0.715158f;
+const double B_Y = 0.072187f;
+
 #endif // hifi_common_h

tools/auto-tester/src/ui/AutoTester.cpp

@@ -12,14 +12,18 @@
 
 AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
     ui.setupUi(this);
+
+    ui.checkBoxInteractiveMode->setChecked(true);
+
+    ui.progressBar->setVisible(false);
 }
 
 void AutoTester::on_evaluateTestsButton_clicked() {
-    test.evaluateTests();
+    test.evaluateTests(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
 }
 
 void AutoTester::on_evaluateTestsRecursivelyButton_clicked() {
-    test.evaluateTestsRecursively();
+    test.evaluateTestsRecursively(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
 }
 
 void AutoTester::on_createRecursiveScriptButton_clicked() {
@@ -30,6 +34,10 @@ void AutoTester::on_createTestButton_clicked() {
     test.createTest();
 }
 
+void AutoTester::on_deleteOldSnapshotsButton_clicked() {
+    test.deleteOldSnapshots();
+}
+
 void AutoTester::on_closeButton_clicked() {
     exit(0);
 }

tools/auto-tester/src/ui/AutoTester.h

@@ -1,6 +1,5 @@
 //
 //  AutoTester.h
-//  zone/ambientLightInheritence
 //
 //  Created by Nissim Hadar on 2 Nov 2017.
 //  Copyright 2013 High Fidelity, Inc.
@@ -26,6 +25,7 @@ void on_evaluateTestsButton_clicked();
     void on_evaluateTestsRecursivelyButton_clicked();
     void on_createRecursiveScriptButton_clicked();
     void on_createTestButton_clicked();
+    void on_deleteOldSnapshotsButton_clicked();
     void on_closeButton_clicked();
 
 private:

tools/auto-tester/src/ui/AutoTester.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>286</width>
+    <width>607</width>
-    <height>470</height>
+    <height>395</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -17,9 +17,9 @@
    <widget class="QPushButton" name="closeButton">
     <property name="geometry">
      <rect>
-      <x>60</x>
+      <x>190</x>
-      <y>360</y>
+      <y>300</y>
-      <width>160</width>
+      <width>220</width>
       <height>40</height>
      </rect>
     </property>
@@ -30,9 +30,9 @@
    <widget class="QPushButton" name="createTestButton">
     <property name="geometry">
      <rect>
-      <x>60</x>
+      <x>360</x>
-      <y>270</y>
+      <y>130</y>
-      <width>160</width>
+      <width>220</width>
       <height>40</height>
      </rect>
     </property>
@@ -43,9 +43,9 @@
    <widget class="QPushButton" name="evaluateTestsButton">
     <property name="geometry">
      <rect>
-      <x>60</x>
+      <x>20</x>
-      <y>20</y>
+      <y>75</y>
-      <width>160</width>
+      <width>220</width>
       <height>40</height>
      </rect>
     </property>
@@ -56,9 +56,9 @@
    <widget class="QPushButton" name="createRecursiveScriptButton">
     <property name="geometry">
      <rect>
-      <x>60</x>
+      <x>360</x>
-      <y>210</y>
+      <y>75</y>
-      <width>160</width>
+      <width>220</width>
       <height>40</height>
      </rect>
     </property>
@@ -69,9 +69,9 @@
    <widget class="QPushButton" name="evaluateTestsRecursivelyButton">
     <property name="geometry">
      <rect>
-      <x>60</x>
+      <x>20</x>
-      <y>75</y>
+      <y>130</y>
-      <width>160</width>
+      <width>220</width>
       <height>40</height>
      </rect>
     </property>
@@ -79,13 +79,55 @@
      <string>Evaluate Tests Recursively</string>
     </property>
    </widget>
+   <widget class="QCheckBox" name="checkBoxInteractiveMode">
+    <property name="geometry">
+     <rect>
+      <x>23</x>
+      <y>40</y>
+      <width>131</width>
+      <height>20</height>
+     </rect>
+    </property>
+    <property name="toolTip">
+     <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;If unchecked, will not show results during evaluation&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+    </property>
+    <property name="text">
+     <string>Interactive Mode</string>
+    </property>
+   </widget>
+   <widget class="QProgressBar" name="progressBar">
+    <property name="geometry">
+     <rect>
+      <x>20</x>
+      <y>190</y>
+      <width>255</width>
+      <height>23</height>
+     </rect>
+    </property>
+    <property name="value">
+     <number>24</number>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="deleteOldSnapshotsButton">
+    <property name="geometry">
+     <rect>
+      <x>360</x>
+      <y>240</y>
+      <width>220</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Delete Old Snapshots</string>
+    </property>
+   </widget>
   </widget>
   <widget class="QMenuBar" name="menuBar">
    <property name="geometry">
     <rect>
      <x>0</x>
      <y>0</y>
-     <width>286</width>
+     <width>607</width>
      <height>21</height>
     </rect>
    </property>

tools/auto-tester/src/ui/MismatchWindow.cpp

@@ -11,11 +11,48 @@
 
 #include <QtCore/QFileInfo>
 
+#include <cmath>
+
 MismatchWindow::MismatchWindow(QWidget *parent) : QDialog(parent) {
     setupUi(this);
 
     expectedImage->setScaledContents(true);
     resultImage->setScaledContents(true);
+    diffImage->setScaledContents(true);
+}
+
+QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultImage) {
+	// This is an optimization, as QImage.setPixel() is embarrassingly slow
+    unsigned char* buffer = new unsigned char[expectedImage.height() * expectedImage.width() * 3];
+
+	// loop over each pixel
+    for (int y = 0; y < expectedImage.height(); ++y) {
+        for (int x = 0; x < expectedImage.width(); ++x) {
+            QRgb pixelP = expectedImage.pixel(QPoint(x, y));
+            QRgb pixelQ = resultImage.pixel(QPoint(x, y));
+
+            // Convert to luminance
+            double p = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP);
+            double q = R_Y * qRed(pixelQ) + G_Y * qGreen(pixelQ) + B_Y * qBlue(pixelQ);
+
+            // The intensity value is modified to increase the brightness of the displayed image
+            double absoluteDifference = fabs(p - q) / 255.0;
+            double modifiedDifference = sqrt(absoluteDifference);
+
+            int difference = (int)(modifiedDifference * 255.0);
+
+            buffer[3 * (x + y * expectedImage.width()) + 0] = difference;
+            buffer[3 * (x + y * expectedImage.width()) + 1] = difference;
+            buffer[3 * (x + y * expectedImage.width()) + 2] = difference;
+        }
+    }
+
+    QImage diffImage(buffer, expectedImage.width(), expectedImage.height(), QImage::Format_RGB888);
+    QPixmap resultPixmap = QPixmap::fromImage(diffImage);
+
+    delete[] buffer;
+
+    return resultPixmap;
 }
 
 void MismatchWindow::setTestFailure(TestFailure testFailure) {
@@ -24,10 +61,19 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) {
     imagePath->setText("Path to test: " + testFailure._pathname);
 
     expectedFilename->setText(testFailure._expectedImageFilename);
-    expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename));
-
     resultFilename->setText(testFailure._actualImageFilename);
-    resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._actualImageFilename));
+
+    QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename);
+    QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename);
+
+    diffPixmap = computeDiffPixmap(
+        QImage(testFailure._pathname + testFailure._expectedImageFilename), 
+        QImage(testFailure._pathname + testFailure._actualImageFilename)
+    );
+
+    expectedImage->setPixmap(expectedPixmap);
+    resultImage->setPixmap(actualPixmap);
+    diffImage->setPixmap(diffPixmap);
 }
 
 void MismatchWindow::on_passTestButton_clicked() {
@@ -44,3 +90,7 @@ void MismatchWindow::on_abortTestsButton_clicked() {
     _userResponse = USER_RESPONSE_ABORT;
     close();
 }
+
+QPixmap MismatchWindow::getComparisonImage() {
+    return diffPixmap;
+}

tools/auto-tester/src/ui/MismatchWindow.h

@@ -25,6 +25,9 @@ public:
 
     UserResponse getUserResponse() { return _userResponse; }
 
+    QPixmap computeDiffPixmap(QImage expectedImage, QImage resultImage);
+    QPixmap getComparisonImage();
+
 private slots:
     void on_passTestButton_clicked();
     void on_failTestButton_clicked();
@@ -32,6 +35,8 @@ private slots:
 
 private:
     UserResponse _userResponse{ USER_RESPONSE_INVALID };
+
+    QPixmap diffPixmap;
 };
 
 

tools/auto-tester/src/ui/MismatchWindow.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>1585</width>
+    <width>1782</width>
-    <height>694</height>
+    <height>942</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -16,10 +16,10 @@
   <widget class="QLabel" name="expectedImage">
    <property name="geometry">
     <rect>
-     <x>20</x>
+     <x>10</x>
-     <y>170</y>
+     <y>25</y>
-     <width>720</width>
+     <width>800</width>
-     <height>362</height>
+     <height>450</height>
     </rect>
    </property>
    <property name="text">
@@ -29,28 +29,41 @@
   <widget class="QLabel" name="resultImage">
    <property name="geometry">
     <rect>
-     <x>760</x>
+     <x>900</x>
-     <y>170</y>
+     <y>25</y>
-     <width>720</width>
+     <width>800</width>
-     <height>362</height>
+     <height>450</height>
     </rect>
    </property>
    <property name="text">
     <string>result image</string>
    </property>
   </widget>
+  <widget class="QLabel" name="diffImage">
+   <property name="geometry">
+    <rect>
+     <x>540</x>
+     <y>480</y>
+     <width>800</width>
+     <height>450</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>diff image</string>
+   </property>
+  </widget>
   <widget class="QLabel" name="resultFilename">
    <property name="geometry">
     <rect>
-     <x>760</x>
+     <x>60</x>
-     <y>90</y>
+     <y>660</y>
-     <width>800</width>
+     <width>480</width>
      <height>28</height>
     </rect>
    </property>
    <property name="font">
     <font>
-     <pointsize>16</pointsize>
+     <pointsize>12</pointsize>
     </font>
    </property>
    <property name="text">
@@ -60,15 +73,15 @@
   <widget class="QLabel" name="expectedFilename">
    <property name="geometry">
     <rect>
-     <x>40</x>
+     <x>60</x>
-     <y>90</y>
+     <y>630</y>
-     <width>700</width>
+     <width>480</width>
      <height>28</height>
     </rect>
    </property>
    <property name="font">
     <font>
-     <pointsize>16</pointsize>
+     <pointsize>12</pointsize>
     </font>
    </property>
    <property name="text">
@@ -78,15 +91,15 @@
   <widget class="QLabel" name="imagePath">
    <property name="geometry">
     <rect>
-     <x>40</x>
+     <x>20</x>
-     <y>30</y>
+     <y>600</y>
      <width>1200</width>
      <height>28</height>
     </rect>
    </property>
    <property name="font">
     <font>
-     <pointsize>16</pointsize>
+     <pointsize>12</pointsize>
     </font>
    </property>
    <property name="text">
@@ -97,7 +110,7 @@
    <property name="geometry">
     <rect>
      <x>30</x>
-     <y>600</y>
+     <y>790</y>
      <width>75</width>
      <height>23</height>
     </rect>
@@ -109,8 +122,8 @@
   <widget class="QPushButton" name="failTestButton">
    <property name="geometry">
     <rect>
-     <x>330</x>
+     <x>120</x>
-     <y>600</y>
+     <y>790</y>
      <width>75</width>
      <height>23</height>
     </rect>
@@ -122,34 +135,60 @@
   <widget class="QPushButton" name="abortTestsButton">
    <property name="geometry">
     <rect>
-     <x>630</x>
+     <x>210</x>
-     <y>600</y>
+     <y>790</y>
-     <width>75</width>
+     <width>121</width>
      <height>23</height>
     </rect>
    </property>
    <property name="text">
-    <string>Abort Tests</string>
+    <string>Abort current test</string>
    </property>
   </widget>
   <widget class="QLabel" name="errorLabel">
    <property name="geometry">
     <rect>
-     <x>810</x>
+     <x>30</x>
-     <y>600</y>
+     <y>850</y>
-     <width>720</width>
+     <width>500</width>
      <height>28</height>
     </rect>
    </property>
    <property name="font">
     <font>
-     <pointsize>16</pointsize>
+     <pointsize>12</pointsize>
     </font>
    </property>
    <property name="text">
     <string>similarity</string>
    </property>
   </widget>
+  <widget class="QLabel" name="label">
+   <property name="geometry">
+    <rect>
+     <x>30</x>
+     <y>5</y>
+     <width>151</width>
+     <height>16</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Expected Image</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="label_2">
+   <property name="geometry">
+    <rect>
+     <x>930</x>
+     <y>5</y>
+     <width>151</width>
+     <height>16</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Actual Image</string>
+   </property>
+  </widget>
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <resources/>