Merge pull request #11879 from NissimHadar/QtImageCompare

Qt image compare

tools/CMakeLists.txt

@@ -23,4 +23,8 @@ if (BUILD_TOOLS)
 
   add_subdirectory(oven)
   set_target_properties(oven PROPERTIES FOLDER "Tools")
+
+  add_subdirectory(auto-tester)
+  set_target_properties(auto-tester PROPERTIES FOLDER "Tools")
 endif()
+

tools/auto-tester/CMakeLists.txt

@@ -0,0 +1,57 @@
+set(TARGET_NAME auto-tester)
+project(${TARGET_NAME})
+
+# Automatically run UIC and MOC.  This replaces the older WRAP macros
+SET(CMAKE_AUTOUIC ON) 
+SET(CMAKE_AUTOMOC ON) 
+
+setup_hifi_project(Core Widgets)
+link_hifi_libraries()
+
+# FIX: Qt was built with -reduce-relocations
+if (Qt5_POSITION_INDEPENDENT_CODE)
+    SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
+endif()
+
+# Qt includes
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${Qt5Core_INCLUDE_DIRS})
+include_directories(${Qt5Widgets_INCLUDE_DIRS})
+
+set(QT_LIBRARIES Qt5::Core Qt5::Widgets)
+
+# Find all sources files
+file (GLOB_RECURSE SOURCES src/*.cpp)
+file (GLOB_RECURSE HEADERS src/*.h)
+file (GLOB_RECURSE UIS src/ui/*.ui)
+
+if (WIN32)
+    # Do not show Console
+    set_property(TARGET auto-tester PROPERTY WIN32_EXECUTABLE true)
+endif()
+
+add_executable(PROJECT_NAME ${SOURCES} ${HEADERS} ${UIS})
+ 
+target_link_libraries(PROJECT_NAME ${QT_LIBRARIES})
+
+# Copy required dll's.
+add_custom_command(TARGET auto-tester POST_BUILD 
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Core> $<TARGET_FILE_DIR:auto-tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Gui> $<TARGET_FILE_DIR:auto-tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Widgets> $<TARGET_FILE_DIR:auto-tester>
+)
+
+if (WIN32)
+    find_program(WINDEPLOYQT_COMMAND windeployqt PATHS ${QT_DIR}/bin NO_DEFAULT_PATH)
+    
+    if (NOT WINDEPLOYQT_COMMAND)
+      message(FATAL_ERROR "Could not find windeployqt at ${QT_DIR}/bin. windeployqt is required.")
+    endif ()
+
+    # add a post-build command to call windeployqt to copy Qt plugins
+    add_custom_command(
+      TARGET ${TARGET_NAME}
+      POST_BUILD
+      COMMAND CMD /C "SET PATH=%PATH%;${QT_DIR}/bin && ${WINDEPLOYQT_COMMAND} ${EXTRA_DEPLOY_OPTIONS} $<$<OR:$<CONFIG:Release>,$<CONFIG:MinSizeRel>,$<CONFIG:RelWithDebInfo>>:--release> \"$<TARGET_FILE:${TARGET_NAME}>\""
+    )
+endif ()

tools/auto-tester/ReadMe.md

@@ -0,0 +1,7 @@
+After building auto-tester, it needs to be deployed to Amazon SW
+
+* In folder hifi/build/tools/auto-tester
+    * Right click on the Release folder
+    * Select 7-Zip -> Add to archive
+    * Select Option ```Create SFX archive``` to create Release.exe
+* Use Cyberduck (or any other AWS S3 client) to copy Release.exe to hifi-content/nissim/autoTester/

tools/auto-tester/src/ImageComparer.cpp

@@ -0,0 +1,119 @@
+//
+//  ImageComparer.cpp
+//
+//  Created by Nissim Hadar on 18 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "ImageComparer.h"
+
+#include <cmath>
+
+// Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity
+// The value is computed for the luminance component and the average value is returned
+double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const {
+    // Make sure the image is 8 bits per colour
+    QImage::Format format = expectedImage.format();
+    if (format != QImage::Format::Format_RGB32) {
+        throw -1;
+    }
+
+    const int L = 255; // (2^number of bits per pixel) - 1
+    const double K1{ 0.01 };
+    const double K2{ 0.03 };
+    const double c1 = pow((K1 * L), 2);
+    const double c2 = pow((K2 * L), 2);
+
+    // Coefficients for luminosity calculation
+    const double R_Y = 0.212655f;
+    const double G_Y = 0.715158f;
+    const double B_Y = 0.072187f;
+
+    // First go over all full 8x8 blocks
+    // This is done in 3 loops
+    //      1) Read the pixels into a linear array (an optimization)
+    //      2) Calculate mean
+    //      3) Calculate variance and covariance
+    //
+    // p - pixel in expected image
+    // q - pixel in result image
+    //
+    const int WIN_SIZE = 8;
+    int x{ 0 };             // column index (start of block)
+    int y{ 0 };             // row index (start of block
+    
+    // Pixels are processed in square blocks
+    double p[WIN_SIZE * WIN_SIZE];
+    double q[WIN_SIZE * WIN_SIZE];
+
+    int windowCounter{ 0 };
+    double ssim{ 0.0 };
+    while (x < expectedImage.width()) {
+        int lastX = x + WIN_SIZE - 1;
+        if (lastX > expectedImage.width() - 1) {
+            x -= (lastX - expectedImage.width());
+        }
+
+        while (y < expectedImage.height()) {
+            int lastY = y + WIN_SIZE - 1;
+            if (lastY > expectedImage.height() - 1) {
+                y -= (lastY - expectedImage.height());
+            }
+
+            // Collect pixels into linear arrays
+            int i{ 0 };
+            for (int xx = 0; xx < WIN_SIZE; ++xx) {
+                for (int yy = 0; yy < WIN_SIZE; ++yy) {
+                    // Get pixels
+                    QRgb pixelP = expectedImage.pixel(QPoint(x + xx, y + yy));
+                    QRgb pixelQ = resultImage.pixel(QPoint(x + xx, y + yy));
+
+                    // Convert to luminance
+                    p[i] = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP);
+                    q[i] = R_Y * qRed(pixelQ) + G_Y * qGreen(pixelQ) + B_Y * qBlue(pixelQ);
+
+                    ++i;
+                }
+            }
+  
+            // Calculate mean
+            double mP{ 0.0 };    // average value of expected pixel
+            double mQ{ 0.0 };    // average value of result pixel
+            for (int j = 0; j < WIN_SIZE * WIN_SIZE; ++j) {
+                mP += p[j];
+                mQ += q[j];
+            }
+            mP /= (WIN_SIZE * WIN_SIZE);
+            mQ /= (WIN_SIZE * WIN_SIZE);
+
+            // Calculate variance and covariance
+            double sigsqP{ 0.0 };
+            double sigsqQ{ 0.0 };
+            double sigPQ{ 0.0 };
+            for (int j = 0; j < WIN_SIZE * WIN_SIZE; ++j) {
+                sigsqP += pow((p[j] - mP), 2);
+                sigsqQ += pow((q[j] - mQ), 2);
+
+                sigPQ += (p[j] - mP) * (q[j] - mQ);
+            }
+            sigsqP /= (WIN_SIZE * WIN_SIZE);
+            sigsqQ /= (WIN_SIZE * WIN_SIZE);
+            sigPQ /= (WIN_SIZE * WIN_SIZE);
+
+            double numerator = (2.0 * mP * mQ + c1) * (2.0 * sigPQ + c2);
+            double denominator = (mP * mP + mQ * mQ + c1) * (sigsqP + sigsqQ + c2);
+
+            ssim += numerator / denominator;
+            ++windowCounter;
+
+            y += WIN_SIZE;
+        }
+
+        x += WIN_SIZE;
+        y = 0;
+    }
+
+    return ssim / windowCounter;
+};

tools/auto-tester/src/ImageComparer.h

@@ -0,0 +1,21 @@
+//
+//  ImageComparer.h
+//
+//  Created by Nissim Hadar on 18 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_ImageComparer_h
+#define hifi_ImageComparer_h
+
+#include <QtCore/QString>
+#include <QImage>
+
+class ImageComparer {
+public:
+    double compareImages(QImage resultImage, QImage expectedImage) const;
+};
+
+#endif // hifi_ImageComparer_h

tools/auto-tester/src/Test.cpp

@@ -0,0 +1,383 @@
+//
+//  Test.cpp
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "Test.h"
+
+#include <assert.h>
+#include <QtCore/QTextStream>
+#include <QDirIterator>
+
+Test::Test() {
+    snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg");
+
+    expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg");
+
+    mismatchWindow.setModal(true);
+}
+
+bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages) {
+    // Loop over both lists and compare each pair of images
+    // Quit loop if user has aborted due to a failed test.
+    const double THRESHOLD{ 0.999 };
+    bool success{ true };
+    bool keepOn{ true };
+    for (int i = 0; keepOn && i < expectedImages.length(); ++i) {
+        // First check that images are the same size
+        QImage resultImage(resultImages[i]);
+        QImage expectedImage(expectedImages[i]);
+        if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) {
+            messageBox.critical(0, "Internal error", "Images are not the same size");
+            exit(-1);
+        }
+
+        double similarityIndex;  // in [-1.0 .. 1.0], where 1.0 means images are identical
+        try {
+            similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
+        } catch (...) {
+            messageBox.critical(0, "Internal error", "Image not in expected format");
+            exit(-1);
+        }
+
+        if (similarityIndex < THRESHOLD) {
+            mismatchWindow.setTestFailure(TestFailure{
+                (float)similarityIndex,
+                expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
+                QFileInfo(expectedImages[i].toStdString().c_str()).fileName(),  // filename of expected image
+                QFileInfo(resultImages[i].toStdString().c_str()).fileName()     // filename of result image
+            });
+
+            mismatchWindow.exec();
+
+            switch (mismatchWindow.getUserResponse()) {
+                case USER_RESPONSE_PASS:
+                    break;
+                case USE_RESPONSE_FAIL:
+                    success = false;
+                    break;
+                case USER_RESPONSE_ABORT:
+                    keepOn = false;
+                    success = false;
+                    break;
+                default:
+                    assert(false);
+                    break;
+            }
+        }
+    }
+
+    return success;
+}
+
+void Test::evaluateTests() {
+    // Get list of JPEG images in folder, sorted by name
+    QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    if (pathToImageDirectory == "") {
+        return;
+    }
+
+    QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
+
+    // Separate images into two lists.  The first is the expected images, the second is the test results
+    // Images that are in the wrong format are ignored.
+    QStringList expectedImages;
+    QStringList resultImages;
+    foreach(QString currentFilename, sortedImageFilenames) {
+        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+        if (isInExpectedImageFilenameFormat(currentFilename)) {
+            expectedImages << fullCurrentFilename;
+        } else if (isInSnapshotFilenameFormat(currentFilename)) {
+            resultImages << fullCurrentFilename;
+        }
+    }
+
+    // The number of images in each list should be identical
+    if (expectedImages.length() != resultImages.length()) {
+        messageBox.critical(0, 
+            "Test failed", 
+            "Found " + QString::number(resultImages.length()) + " images in directory" +
+            "\nExpected to find " + QString::number(expectedImages.length()) + " images"
+        );
+
+        exit(-1);
+    }
+
+    bool success = compareImageLists(expectedImages, resultImages);
+
+    if (success) {
+        messageBox.information(0, "Success", "All images are as expected");
+    } else {
+        messageBox.information(0, "Failure", "One or more images are not as expected");
+    }
+}
+
+// Two criteria are used to decide if a folder contains valid test results.
+//      1) a 'test'js' file exists in the folder
+//      2) the folder has the same number of actual and expected images
+void Test::evaluateTestsRecursively() {
+    // Select folder to start recursing from
+    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly);
+    if (topLevelDirectory == "") {
+        return;
+    }
+
+    bool success{ true };
+    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        QString directory = it.next();
+        if (directory[directory.length() - 1] == '.') {
+            // ignore '.', '..' directories
+            continue;
+        }
+
+        // 
+        const QString testPathname{ directory + "/" + testFilename };
+        QFileInfo fileInfo(testPathname);
+        if (!fileInfo.exists()) {
+            // Folder does not contain 'test.js'
+            continue;
+        }
+
+        QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory);
+
+        // Separate images into two lists.  The first is the expected images, the second is the test results
+        // Images that are in the wrong format are ignored.
+        QStringList expectedImages;
+        QStringList resultImages;
+        foreach(QString currentFilename, sortedImageFilenames) {
+            QString fullCurrentFilename = directory + "/" + currentFilename;
+            if (isInExpectedImageFilenameFormat(currentFilename)) {
+                expectedImages << fullCurrentFilename;
+            } else if (isInSnapshotFilenameFormat(currentFilename)) {
+                resultImages << fullCurrentFilename;
+            }
+        }
+
+        if (expectedImages.length() != resultImages.length()) {
+            // Number of images doesn't match
+            continue;
+        }
+
+        // Set success to false if any test has failed
+        success &= compareImageLists(expectedImages, resultImages);
+    }
+
+    if (success) {
+        messageBox.information(0, "Success", "All images are as expected");
+    } else {
+        messageBox.information(0, "Failure", "One or more images are not as expected");
+    }
+}
+
+void Test::importTest(QTextStream& textStream, const QString& testPathname, int testNumber) {
+    textStream << "var test" << testNumber << " = Script.require(\"" << "file:///" << testPathname + "\");" << endl;
+}
+
+// Creates a single script in a user-selected folder.
+// This script will run all text.js scripts in every applicable sub-folder
+void Test::createRecursiveScript() {
+    // Select folder to start recursing from
+    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly);
+    if (topLevelDirectory == "") {
+        return;
+    }
+
+    QFile allTestsFilename(topLevelDirectory + "/" + "allTests.js");
+    if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
+        messageBox.critical(0,
+            "Internal Error",
+            "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\"");
+
+        exit(-1);
+    }
+
+    QTextStream textStream(&allTestsFilename);
+    textStream << "// This is an automatically generated file, created by auto-tester" << endl;
+
+    // The main will call each test after the previous test is completed
+    // This is implemented with an interval timer that periodically tests if a
+    // running test has increment a testNumber variable that it received as an input.
+    int testNumber = 1;
+    QVector<QString> testPathnames;
+
+    // First test if top-level folder has a test.js file
+    const QString testPathname{ topLevelDirectory + "/" + testFilename };
+    QFileInfo fileInfo(testPathname);
+    if (fileInfo.exists()) {
+        // Current folder contains a test
+        importTest(textStream, testPathname, testNumber);
+        ++testNumber;
+
+        testPathnames << testPathname;
+    }
+
+    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        QString directory = it.next();
+        if (directory[directory.length() - 1] == '.') {
+            // ignore '.', '..' directories
+            continue;
+        }
+
+        const QString testPathname{ directory + "/" + testFilename };
+        QFileInfo fileInfo(testPathname);
+        if (fileInfo.exists()) {
+            // Current folder contains a test
+            importTest(textStream, testPathname, testNumber);
+            ++testNumber;
+
+            testPathnames << testPathname;
+        }
+    }
+
+    if (testPathnames.length() <= 0) {
+        messageBox.information(0, "Failure", "No \"test.js\" files found");
+        allTestsFilename.close();
+        return;
+    }
+
+    textStream << endl;
+
+    // Define flags for each test
+    for (int i = 1; i <= testPathnames.length(); ++i) {
+        textStream << "var test" << i << "HasNotStarted = true;" << endl;
+    }
+
+    // Leave a blank line in the main
+    textStream << endl;
+
+    const int TEST_PERIOD = 1000; // in milliseconds
+    const QString tab = "    ";
+
+    textStream << "// Check every second if the current test is complete and the next test can be run" << endl;
+    textStream << "var testTimer = Script.setInterval(" << endl;
+    textStream << tab << "function() {" << endl;
+
+    const QString testFunction = "test";
+    for (int i = 1; i <= testPathnames.length(); ++i) {
+        // First test starts immediately, all other tests wait for the previous test to complete.
+        // The script produced will look as follows:
+        //      if (test1HasNotStarted) {
+        //          test1HasNotStarted = false;
+        //          test1.test();
+        //          print("******started test 1******");
+        //      }
+        //      |
+        //      |
+        //      if (test5.complete && test6HasNotStarted) {
+        //          test6HasNotStarted = false;
+        //          test7.test();
+        //          print("******started test 6******");
+        //      }
+        //      |
+        //      |
+        //      if (test12.complete) {
+        //          print("******stopping******");
+        //          Script.stop();
+        //      }
+        //
+        if (i == 1) {
+            textStream << tab << tab << "if (test1HasNotStarted) {" << endl;
+        } else {
+            textStream << tab << tab << "if (test" << i - 1 << ".complete && test" << i << "HasNotStarted) {" << endl;
+        }
+        textStream << tab << tab << tab << "test" << i << "HasNotStarted = false;" << endl;
+        textStream << tab << tab << tab << "test" << i << "." << testFunction << "();" << endl;
+        textStream << tab << tab << tab << "print(\"******started test " << i << "******\");" << endl;
+
+        textStream << tab << tab << "}" << endl << endl;
+
+    }
+
+    // Add extra step to stop the script
+    textStream << tab << tab << "if (test" << testPathnames.length() << ".complete) {" << endl;
+    textStream << tab << tab << tab << "print(\"******stopping******\");" << endl;
+    textStream << tab << tab << tab << "Script.stop();" << endl;
+    textStream << tab << tab << "}" << endl << endl;
+
+    textStream << tab << "}," << endl;
+    textStream << endl;
+    textStream << tab << TEST_PERIOD << endl;
+    textStream << ");" << endl << endl;
+
+    textStream << "// Stop the timer and clear the module cache" << endl;
+    textStream << "Script.scriptEnding.connect(" << endl;
+    textStream << tab << "function() {" << endl;
+    textStream << tab << tab << "Script.clearInterval(testTimer);" << endl;
+    textStream << tab << tab << "Script.require.cache = {};" << endl;
+    textStream << tab << "}" << endl;
+    textStream << ");" << endl;
+
+    allTestsFilename.close();
+
+    messageBox.information(0, "Success", "Script has been created");
+}
+
+void Test::createTest() {
+    // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on
+    // Any existing expected result images will be deleted
+    QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    if (pathToImageDirectory == "") {
+        return;
+    }
+
+    QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
+
+    int i = 1;
+    foreach (QString currentFilename, sortedImageFilenames) {
+        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+        if (isInExpectedImageFilenameFormat(currentFilename)) {
+            if (!QFile::remove(fullCurrentFilename)) {
+                messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
+                exit(-1);
+            }
+        } else if (isInSnapshotFilenameFormat(currentFilename)) {
+            const int MAX_IMAGES = 100000;
+            if (i >= MAX_IMAGES) {
+                messageBox.critical(0, "Error", "More than 100,000 images not supported");
+                exit(-1);
+            }
+            QString newFilename = "ExpectedImage_" + QString::number(i-1).rightJustified(5, '0') + ".jpg";
+            QString fullNewFileName = pathToImageDirectory + "/" + newFilename;
+
+            if (!imageDirectory.rename(fullCurrentFilename, newFilename)) {
+                if (!QFile::exists(fullCurrentFilename)) {
+                    messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n"
+                        + fullCurrentFilename + " not found"
+                        + "\nTest creation aborted"
+                    );
+                    exit(-1);
+                } else {
+                    messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n"
+                        + "unknown error" + "\nTest creation aborted"
+                    );
+                    exit(-1);
+                }
+            }
+            ++i;
+        }
+    }
+
+    messageBox.information(0, "Success", "Test images have been created");
+}
+
+QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) {
+    imageDirectory = QDir(pathToImageDirectory);
+    QStringList nameFilters;
+    nameFilters << "*.jpg";
+
+    return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
+}
+
+bool Test::isInSnapshotFilenameFormat(QString filename) {
+    return (snapshotFilenameFormat.match(filename).hasMatch());
+}
+
+bool Test::isInExpectedImageFilenameFormat(QString filename) {
+    return (expectedImageFilenameFormat.match(filename).hasMatch());
+}

tools/auto-tester/src/Test.h

@@ -0,0 +1,55 @@
+//
+//  Test.h
+//  zone/ambientLightInheritence
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_test_h
+#define hifi_test_h
+
+#include <QtWidgets/QFileDialog>
+#include <QtWidgets/QMessageBox>
+#include <QtCore/QRegularExpression>
+
+#include "ImageComparer.h"
+#include "ui/MismatchWindow.h"
+
+class Test {
+public: 
+    Test();
+
+    void evaluateTests();
+    void evaluateTestsRecursively();
+    void createRecursiveScript();
+    void createTest();
+
+    QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory);
+
+    bool isInSnapshotFilenameFormat(QString filename);
+    bool isInExpectedImageFilenameFormat(QString filename);
+
+    void importTest(QTextStream& textStream, const QString& testPathname, int testNumber);
+
+private:
+    const QString testFilename{ "test.js" };
+
+    QMessageBox messageBox;
+
+    QDir imageDirectory;
+
+    QRegularExpression snapshotFilenameFormat;
+    QRegularExpression expectedImageFilenameFormat;
+
+    MismatchWindow mismatchWindow;
+
+    ImageComparer imageComparer;
+
+    bool compareImageLists(QStringList expectedImages, QStringList resultImages);
+};
+
+#endif // hifi_test_h

tools/auto-tester/src/common.h

@@ -0,0 +1,37 @@
+//
+//  common.h
+//
+//  Created by Nissim Hadar on 10 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_common_h
+#define hifi_common_h
+
+#include <QtCore/QString>
+
+class TestFailure {
+public:
+    TestFailure(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) :
+        _error(error),
+        _pathname(pathname),
+        _expectedImageFilename(expectedImageFilename),
+        _actualImageFilename(actualImageFilename)
+    {}
+
+    double _error;
+    QString _pathname;
+    QString _expectedImageFilename;
+    QString _actualImageFilename;
+};
+
+enum UserResponse {
+    USER_RESPONSE_INVALID,
+    USER_RESPONSE_PASS,
+    USE_RESPONSE_FAIL,
+    USER_RESPONSE_ABORT
+};
+
+#endif // hifi_common_h

tools/auto-tester/src/main.cpp

@@ -0,0 +1,20 @@
+//
+//  main.cpp
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include <QtWidgets/QApplication>
+#include "ui/AutoTester.h"
+
+int main(int argc, char *argv[]) {
+    QApplication application(argc, argv);
+
+    AutoTester autoTester;
+    autoTester.show();
+
+    return application.exec();
+}

tools/auto-tester/src/ui/AutoTester.cpp

@@ -0,0 +1,35 @@
+//
+//  AutoTester.cpp
+//  zone/ambientLightInheritence
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "AutoTester.h"
+
+AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
+    ui.setupUi(this);
+}
+
+void AutoTester::on_evaluateTestsButton_clicked() {
+    test.evaluateTests();
+}
+
+void AutoTester::on_evaluateTestsRecursivelyButton_clicked() {
+    test.evaluateTestsRecursively();
+}
+
+void AutoTester::on_createRecursiveScriptButton_clicked() {
+    test.createRecursiveScript();
+}
+
+void AutoTester::on_createTestButton_clicked() {
+    test.createTest();
+}
+
+void AutoTester::on_closeButton_clicked() {
+    exit(0);
+}

tools/auto-tester/src/ui/AutoTester.h

@@ -0,0 +1,37 @@
+//
+//  AutoTester.h
+//  zone/ambientLightInheritence
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_AutoTester_h
+#define hifi_AutoTester_h
+
+#include <QtWidgets/QMainWindow>
+#include "ui_AutoTester.h"
+#include "../Test.h"
+
+class AutoTester : public QMainWindow {
+    Q_OBJECT
+
+public:
+    AutoTester(QWidget *parent = Q_NULLPTR);
+
+private slots:
+void on_evaluateTestsButton_clicked();
+void on_evaluateTestsRecursivelyButton_clicked();
+void on_createRecursiveScriptButton_clicked();
+    void on_createTestButton_clicked();
+    void on_closeButton_clicked();
+
+private:
+    Ui::AutoTesterClass ui;
+
+    Test test;
+};
+
+#endif // hifi_AutoTester_h

tools/auto-tester/src/ui/AutoTester.ui

@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>AutoTesterClass</class>
+ <widget class="QMainWindow" name="AutoTesterClass">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>286</width>
+    <height>470</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>AutoTester</string>
+  </property>
+  <widget class="QWidget" name="centralWidget">
+   <widget class="QPushButton" name="closeButton">
+    <property name="geometry">
+     <rect>
+      <x>60</x>
+      <y>360</y>
+      <width>160</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Close</string>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="createTestButton">
+    <property name="geometry">
+     <rect>
+      <x>60</x>
+      <y>270</y>
+      <width>160</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Create Test</string>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="evaluateTestsButton">
+    <property name="geometry">
+     <rect>
+      <x>60</x>
+      <y>20</y>
+      <width>160</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Evaluate Test</string>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="createRecursiveScriptButton">
+    <property name="geometry">
+     <rect>
+      <x>60</x>
+      <y>210</y>
+      <width>160</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Create Recursive Script</string>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="evaluateTestsRecursivelyButton">
+    <property name="geometry">
+     <rect>
+      <x>60</x>
+      <y>75</y>
+      <width>160</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Evaluate Tests Recursively</string>
+    </property>
+   </widget>
+  </widget>
+  <widget class="QMenuBar" name="menuBar">
+   <property name="geometry">
+    <rect>
+     <x>0</x>
+     <y>0</y>
+     <width>286</width>
+     <height>21</height>
+    </rect>
+   </property>
+  </widget>
+  <widget class="QToolBar" name="mainToolBar">
+   <attribute name="toolBarArea">
+    <enum>TopToolBarArea</enum>
+   </attribute>
+   <attribute name="toolBarBreak">
+    <bool>false</bool>
+   </attribute>
+  </widget>
+  <widget class="QStatusBar" name="statusBar"/>
+ </widget>
+ <layoutdefault spacing="6" margin="11"/>
+ <resources/>
+ <connections/>
+</ui>

tools/auto-tester/src/ui/MismatchWindow.cpp

@@ -0,0 +1,46 @@
+//
+//  MismatchWindow.cpp
+//
+//  Created by Nissim Hadar on 9 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "MismatchWindow.h"
+
+#include <QtCore/QFileInfo>
+
+MismatchWindow::MismatchWindow(QWidget *parent) : QDialog(parent) {
+    setupUi(this);
+
+    expectedImage->setScaledContents(true);
+    resultImage->setScaledContents(true);
+}
+
+void MismatchWindow::setTestFailure(TestFailure testFailure) {
+    errorLabel->setText("Similarity: " + QString::number(testFailure._error));
+
+    imagePath->setText("Path to test: " + testFailure._pathname);
+
+    expectedFilename->setText(testFailure._expectedImageFilename);
+    expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename));
+
+    resultFilename->setText(testFailure._actualImageFilename);
+    resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._actualImageFilename));
+}
+
+void MismatchWindow::on_passTestButton_clicked() {
+    _userResponse = USER_RESPONSE_PASS;
+    close();
+}
+
+void MismatchWindow::on_failTestButton_clicked() {
+    _userResponse = USE_RESPONSE_FAIL;
+    close();
+}
+
+void MismatchWindow::on_abortTestsButton_clicked() {
+    _userResponse = USER_RESPONSE_ABORT;
+    close();
+}

tools/auto-tester/src/ui/MismatchWindow.h

@@ -0,0 +1,38 @@
+//
+//  MismatchWindow.h
+//
+//  Created by Nissim Hadar on 9 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_MismatchWindow_h
+#define hifi_MismatchWindow_h
+
+#include "ui_MismatchWindow.h"
+
+#include "../common.h"
+
+class MismatchWindow : public QDialog, public Ui::MismatchWindow
+{
+    Q_OBJECT
+
+public:
+    MismatchWindow(QWidget *parent = Q_NULLPTR);
+
+    void setTestFailure(TestFailure testFailure);
+
+    UserResponse getUserResponse() { return _userResponse; }
+
+private slots:
+    void on_passTestButton_clicked();
+    void on_failTestButton_clicked();
+    void on_abortTestsButton_clicked();
+
+private:
+    UserResponse _userResponse{ USER_RESPONSE_INVALID };
+};
+
+
+#endif // hifi_MismatchWindow_h

tools/auto-tester/src/ui/MismatchWindow.ui

@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>MismatchWindow</class>
+ <widget class="QDialog" name="MismatchWindow">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>1585</width>
+    <height>694</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>MismatchWindow</string>
+  </property>
+  <widget class="QLabel" name="expectedImage">
+   <property name="geometry">
+    <rect>
+     <x>20</x>
+     <y>170</y>
+     <width>720</width>
+     <height>362</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>expected image</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="resultImage">
+   <property name="geometry">
+    <rect>
+     <x>760</x>
+     <y>170</y>
+     <width>720</width>
+     <height>362</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>result image</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="resultFilename">
+   <property name="geometry">
+    <rect>
+     <x>760</x>
+     <y>90</y>
+     <width>800</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>result image filename</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="expectedFilename">
+   <property name="geometry">
+    <rect>
+     <x>40</x>
+     <y>90</y>
+     <width>700</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>expected image filename</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="imagePath">
+   <property name="geometry">
+    <rect>
+     <x>40</x>
+     <y>30</y>
+     <width>1200</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>image path</string>
+   </property>
+  </widget>
+  <widget class="QPushButton" name="passTestButton">
+   <property name="geometry">
+    <rect>
+     <x>30</x>
+     <y>600</y>
+     <width>75</width>
+     <height>23</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Pass</string>
+   </property>
+  </widget>
+  <widget class="QPushButton" name="failTestButton">
+   <property name="geometry">
+    <rect>
+     <x>330</x>
+     <y>600</y>
+     <width>75</width>
+     <height>23</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Fail</string>
+   </property>
+  </widget>
+  <widget class="QPushButton" name="abortTestsButton">
+   <property name="geometry">
+    <rect>
+     <x>630</x>
+     <y>600</y>
+     <width>75</width>
+     <height>23</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Abort Tests</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="errorLabel">
+   <property name="geometry">
+    <rect>
+     <x>810</x>
+     <y>600</y>
+     <width>720</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>similarity</string>
+   </property>
+  </widget>
+ </widget>
+ <layoutdefault spacing="6" margin="11"/>
+ <resources/>
+ <connections/>
+</ui>