Merge pull request #12554 from NissimHadar/addRecursionToAutotester

Pull test images from GitHub

tools/auto-tester/CMakeLists.txt

@@ -5,7 +5,7 @@ project(${TARGET_NAME})
 SET (CMAKE_AUTOUIC ON) 
 SET (CMAKE_AUTOMOC ON) 
 
-setup_hifi_project (Core Widgets)
+setup_hifi_project (Core Widgets Network)
 link_hifi_libraries ()
 
 # FIX: Qt was built with -reduce-relocations

tools/auto-tester/src/Downloader.cpp

@@ -0,0 +1,32 @@
+//
+//  Downloader.cpp
+//
+//  Created by Nissim Hadar on 1 Mar 2018.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "Downloader.h"
+
+Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) {
+    connect(
+        &_networkAccessManager, SIGNAL (finished(QNetworkReply*)),
+        this, SLOT (fileDownloaded(QNetworkReply*))
+    );
+
+    QNetworkRequest request(imageUrl);
+    _networkAccessManager.get(request);
+}
+
+void Downloader::fileDownloaded(QNetworkReply* reply) {
+    _downloadedData = reply->readAll();
+
+    //emit a signal
+    reply->deleteLater();
+    emit downloaded();
+}
+
+QByteArray Downloader::downloadedData() const {
+    return _downloadedData;
+}

tools/auto-tester/src/Downloader.h

@@ -0,0 +1,48 @@
+//
+//  Downloader.h
+//
+//  Created by Nissim Hadar on 1 Mar 2018.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_downloader_h
+#define hifi_downloader_h
+
+#include <QObject>
+#include <QNetworkAccessManager>
+#include <QNetworkRequest>
+#include <QNetworkReply>
+#include <QUrl>
+#include <QDateTime>
+#include <QFile>
+#include <QFileInfo>
+#include <QDebug>
+
+#include <QObject>
+#include <QByteArray>
+#include <QNetworkAccessManager>
+#include <QNetworkRequest>
+#include <QNetworkReply>
+
+class Downloader : public QObject {
+Q_OBJECT
+public:
+    explicit Downloader(QUrl imageUrl, QObject *parent = 0);
+
+    QByteArray downloadedData() const;
+
+signals:
+    void downloaded();
+
+    private slots:
+    void fileDownloaded(QNetworkReply* pReply);
+
+private:
+    QNetworkAccessManager _networkAccessManager;
+    QByteArray _downloadedData;
+};
+
+#endif // hifi_downloader_h

tools/auto-tester/src/ImageComparer.cpp

@@ -17,13 +17,13 @@
 double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const {
     // Make sure the image is 8 bits per colour
     QImage::Format format = expectedImage.format();
-    if (format != QImage::Format::Format_RGB32) {
+    if (format != QImage::Format::Format_ARGB32) {
         throw -1;
     }
 
     const int L = 255; // (2^number of bits per pixel) - 1
-    const double K1{ 0.01 };
+    const double K1 { 0.01 };
-    const double K2{ 0.03 };
+    const double K2 { 0.03 };
     const double c1 = pow((K1 * L), 2);
     const double c2 = pow((K2 * L), 2);
 

tools/auto-tester/src/Test.cpp

@@ -12,33 +12,32 @@
 #include <assert.h>
 #include <QtCore/QTextStream>
 #include <QDirIterator>
+#include <QImageReader>
+#include <QImageWriter>
 
 #include <quazip5/quazip.h>
 #include <quazip5/JlCompress.h>
 
-Test::Test() {
+#include "ui/AutoTester.h"
-    snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.*-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg");
+extern AutoTester* autoTester;
 
-    expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg");
+#include <math.h>
+
+Test::Test() {
+    QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + ".png");
+    
+    expectedImageFilenameFormat = QRegularExpression(regex);
 
     mismatchWindow.setModal(true);
 }
 
-bool Test::createTestResultsFolderPathIfNeeded(QString directory) {
+bool Test::createTestResultsFolderPath(QString directory) {
-    // The test results folder is located in the root of the tests (i.e. for recursive test evaluation)
+	QDateTime now = QDateTime::currentDateTime();
-    if (testResultsFolderPath == "") {
+    testResultsFolderPath =  directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT);
-        testResultsFolderPath =  directory + "/" + TEST_RESULTS_FOLDER;
+    QDir testResultsFolder(testResultsFolderPath);
-        QDir testResultsFolder(testResultsFolderPath);
 
-        if (testResultsFolder.exists()) {
+    // Create a new test results folder
-            testResultsFolder.removeRecursively();
+    return QDir().mkdir(testResultsFolderPath);
-        }
-
-        // Create a new test results folder
-        return QDir().mkdir(testResultsFolderPath);
-    } else {
-        return true;
-    }
 }
 
 void Test::zipAndDeleteTestResultsFolder() {
@@ -60,9 +59,9 @@ void Test::zipAndDeleteTestResultsFolder() {
     index = 1;
 }
 
-bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar) {
+bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) {
     progressBar->setMinimum(0);
-    progressBar->setMaximum(expectedImages.length() - 1);
+    progressBar->setMaximum(expectedImagesFullFilenames.length() - 1);
     progressBar->setValue(0);
     progressBar->setVisible(true);
 
@@ -71,12 +70,13 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
     const double THRESHOLD { 0.999 };
     bool success{ true };
     bool keepOn{ true };
-    for (int i = 0; keepOn && i < expectedImages.length(); ++i) {
+    for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) {
         // First check that images are the same size
-        QImage resultImage(resultImages[i]);
+        QImage resultImage(resultImagesFullFilenames[i]);
-        QImage expectedImage(expectedImages[i]);
+        QImage expectedImage(expectedImagesFullFilenames[i]);
+
         if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) {
-            messageBox.critical(0, "Internal error", "Images are not the same size");
+            messageBox.critical(0, "Internal error #1", "Images are not the same size");
             exit(-1);
         }
 
@@ -84,21 +84,21 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
         try {
             similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
         } catch (...) {
-            messageBox.critical(0, "Internal error", "Image not in expected format");
+            messageBox.critical(0, "Internal error #2", "Image not in expected format");
             exit(-1);
         }
 
         if (similarityIndex < THRESHOLD) {
             TestFailure testFailure = TestFailure{
                 (float)similarityIndex,
-                expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
+                expectedImagesFullFilenames[i].left(expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
-                QFileInfo(expectedImages[i].toStdString().c_str()).fileName(),  // filename of expected image
+                QFileInfo(expectedImagesFullFilenames[i].toStdString().c_str()).fileName(),  // filename of expected image
-                QFileInfo(resultImages[i].toStdString().c_str()).fileName()     // filename of result image
+                QFileInfo(resultImagesFullFilenames[i].toStdString().c_str()).fileName()     // filename of result image
             };
 
             mismatchWindow.setTestFailure(testFailure);
 
-            if (!interactiveMode) {
+            if (!isInteractiveMode) {
                 appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage());
                 success = false;
             } else {
@@ -131,20 +131,20 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage
 
 void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
     if (!QDir().exists(testResultsFolderPath)) {
-        messageBox.critical(0, "Internal error", "Folder " + testResultsFolderPath + " not found");
+        messageBox.critical(0, "Internal error #3", "Folder " + testResultsFolderPath + " not found");
         exit(-1);
     }
 
     QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) };
     if (!QDir().mkdir(failureFolderPath)) {
-        messageBox.critical(0, "Internal error", "Failed to create folder " + failureFolderPath);
+        messageBox.critical(0, "Internal error #4", "Failed to create folder " + failureFolderPath);
         exit(-1);
     }
     ++index;
 
     QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
     if (!descriptionFile.open(QIODevice::ReadWrite)) {
-        messageBox.critical(0, "Internal error", "Failed to create file " + TEST_RESULTS_FILENAME);
+        messageBox.critical(0, "Internal error #5", "Failed to create file " + TEST_RESULTS_FILENAME);
         exit(-1);
     }
 
@@ -164,60 +164,91 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te
     sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
     destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
     if (!QFile::copy(sourceFile, destinationFile)) {
-        messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile);
+        messageBox.critical(0, "Internal error #6", "Failed to copy " + sourceFile + " to " + destinationFile);
         exit(-1);
     }
 
     sourceFile = testFailure._pathname + testFailure._actualImageFilename;
     destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
     if (!QFile::copy(sourceFile, destinationFile)) {
-        messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile);
+        messageBox.critical(0, "Internal error #7", "Failed to copy " + sourceFile + " to " + destinationFile);
         exit(-1);
     }
 
     comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
 }
 
-void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) {
+void Test::startTestsEvaluation() {
     // Get list of JPEG images in folder, sorted by name
-    QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
-    if (pathToImageDirectory == "") {
+    if (pathToTestResultsDirectory == "") {
         return;
     }
 
-    // Leave if test results folder could not be created
+    // Quit if test results folder could not be created
-    if (!createTestResultsFolderPathIfNeeded(pathToImageDirectory)) {
+    if (!createTestResultsFolderPath(pathToTestResultsDirectory)) {
         return;
     }
 
-    QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
+    // Before any processing - all images are converted to PNGs, as this is the format stored on GitHub
+    QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", pathToTestResultsDirectory);
+    foreach(QString filename, sortedSnapshotFilenames) {
+        QStringList stringParts = filename.split(".");
+        copyJPGtoPNG(
+            pathToTestResultsDirectory + "/" + stringParts[0] + ".jpg", 
+            pathToTestResultsDirectory + "/" + stringParts[0] + ".png"
+        );
 
-    // Separate images into two lists.  The first is the expected images, the second is the test results
+        QFile::remove(pathToTestResultsDirectory + "/" + stringParts[0] + ".jpg");
+    }
+
+    // Create two lists.  The first is the test results,  the second is the expected images
+    // The expected images are represented as a URL to enable download from GitHub
     // Images that are in the wrong format are ignored.
-    QStringList expectedImages;
+
-    QStringList resultImages;
+    QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", pathToTestResultsDirectory);
-    foreach(QString currentFilename, sortedImageFilenames) {
+    QStringList expectedImagesURLs;
-        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+
-        if (isInExpectedImageFilenameFormat(currentFilename)) {
+    const QString URLPrefix("https://raw.githubusercontent.com");
-            expectedImages << fullCurrentFilename;
+    const QString githubUser("NissimHadar");
-        } else if (isInSnapshotFilenameFormat(currentFilename)) {
+    const QString testsRepo("hifi_tests");
-            resultImages << fullCurrentFilename;
+    const QString branch("addRecursionToAutotester");
+
+    resultImagesFullFilenames.clear();
+    expectedImagesFilenames.clear();
+    expectedImagesFullFilenames.clear();
+
+    foreach(QString currentFilename, sortedTestResultsFilenames) {
+        QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename;
+        if (isInSnapshotFilenameFormat("png", currentFilename)) {
+            resultImagesFullFilenames << fullCurrentFilename;
+
+            QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename);
+
+            // Images are stored on GitHub as ExpectedImage_ddddd.png
+            // Extract the digits at the end of the filename (exluding the file extension)
+            QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS);
+            QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png";
+
+            QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + 
+                expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename);
+
+            expectedImagesURLs << imageURLString;
+
+            // The image retrieved from Github needs a unique name
+            QString expectedImageFilename = currentFilename.replace("/", "_").replace(".", "_EI.");
+
+            expectedImagesFilenames << expectedImageFilename;
+            expectedImagesFullFilenames << pathToTestResultsDirectory + "/" + expectedImageFilename;
         }
     }
 
-    // The number of images in each list should be identical
+    autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames);
-    if (expectedImages.length() != resultImages.length()) {
+}
-        messageBox.critical(0, 
-            "Test failed", 
-            "Found " + QString::number(resultImages.length()) + " images in directory" +
-            "\nExpected to find " + QString::number(expectedImages.length()) + " images"
-        );
-
-        exit(-1);
-    }
-
-    bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar);
 
+void Test::finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar) {
+    bool success = compareImageLists(interactiveMode, progressBar);
+    
     if (success) {
         messageBox.information(0, "Success", "All images are as expected");
     } else {
@@ -242,72 +273,25 @@ bool Test::isAValidDirectory(QString pathname) {
     return true;
 }
 
-// Two criteria are used to decide if a folder contains valid test results.
-//      1) a 'test'js' file exists in the folder
-//      2) the folder has the same number of actual and expected images
-void Test::evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar) {
-    // Select folder to start recursing from
-    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly);
-    if (topLevelDirectory == "") {
-        return;
-    }
-
-    // Leave if test results folder could not be created
-    if (!createTestResultsFolderPathIfNeeded(topLevelDirectory)) {
-        return;
-    }
-
-    bool success{ true };
-    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
-    while (it.hasNext()) {
-        QString directory = it.next();
-
-        if (!isAValidDirectory(directory)) {
-            continue;
-        }
-
-        const QString testPathname{ directory + "/" + TEST_FILENAME };
-        QFileInfo fileInfo(testPathname);
-        if (!fileInfo.exists()) {
-            // Folder does not contain 'test.js'
-            continue;
-        }
-
-        QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory);
-
-        // Separate images into two lists.  The first is the expected images, the second is the test results
-        // Images that are in the wrong format are ignored.
-        QStringList expectedImages;
-        QStringList resultImages;
-        foreach(QString currentFilename, sortedImageFilenames) {
-            QString fullCurrentFilename = directory + "/" + currentFilename;
-            if (isInExpectedImageFilenameFormat(currentFilename)) {
-                expectedImages << fullCurrentFilename;
-            } else if (isInSnapshotFilenameFormat(currentFilename)) {
-                resultImages << fullCurrentFilename;
-            }
-        }
-
-        if (expectedImages.length() != resultImages.length()) {
-            // Number of images doesn't match
-            continue;
-        }
-
-        // Set success to false if any test has failed
-        success &= compareImageLists(expectedImages, resultImages, directory, interactiveMode, progressBar);
-    }
-
-    if (success) {
-        messageBox.information(0, "Success", "All images are as expected");
-    } else {
-        messageBox.information(0, "Failure", "One or more images are not as expected");
-    }
-
-    zipAndDeleteTestResultsFolder();
-}
-
 void Test::importTest(QTextStream& textStream, const QString& testPathname) {
-    textStream << "Script.include(\"" << "file:///" << testPathname + "?raw=true\");" << endl;
+    // `testPathname` includes the full path to the test.  We need the portion below (and including) `tests`
+    QStringList filenameParts = testPathname.split('/');
+    int i{ 0 };
+    while (i < filenameParts.length() && filenameParts[i] != "tests") {
+        ++i;
+    }
+
+    if (i == filenameParts.length()) {
+        messageBox.critical(0, "Internal error #10", "Bad testPathname");
+        exit(-1);
+    }
+
+    QString filename;
+    for (int j = i; j < filenameParts.length(); ++j) {
+        filename += "/" + filenameParts[j];
+    }
+
+    textStream << "Script.include(\"" << "https://raw.githubusercontent.com/" << user << "/hifi_tests/" << branch << filename + "\");" << endl;
 }
 
 // Creates a single script in a user-selected folder.
@@ -319,11 +303,58 @@ void Test::createRecursiveScript() {
         return;
     }
 
-    QFile allTestsFilename(topLevelDirectory + "/" + "allTests.js");
+    createRecursiveScript(topLevelDirectory, true);
+}
+
+// This method creates a `testRecursive.js` script in every sub-folder.
+void Test::createRecursiveScriptsRecursively() {
+    // Select folder to start recursing from
+    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", ".", QFileDialog::ShowDirsOnly);
+    if (topLevelDirectory == "") {
+        return;
+    }
+
+    createRecursiveScript(topLevelDirectory, false);
+
+    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        QString directory = it.next();
+
+        // Only process directories
+        QDir dir;
+        if (!isAValidDirectory(directory)) {
+            continue;
+        }
+
+        // Only process directories that have sub-directories
+        bool hasNoSubDirectories{ true };
+        QDirIterator it2(directory.toStdString().c_str(), QDirIterator::Subdirectories);
+        while (it2.hasNext()) {
+            QString directory2 = it2.next();
+
+            // Only process directories
+            QDir dir;
+            if (isAValidDirectory(directory2)) {
+                hasNoSubDirectories = false;
+                break;
+            }
+        }
+
+        if (!hasNoSubDirectories) {
+            createRecursiveScript(directory, false);
+        }
+    }
+
+    messageBox.information(0, "Success", "Scripts have been created");
+}
+
+void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode) {
+    const QString recursiveTestsFilename("testRecursive.js");
+    QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename);
     if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
         messageBox.critical(0,
-            "Internal Error",
+            "Internal Error #8",
-            "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\""
+            "Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""
         );
 
         exit(-1);
@@ -332,12 +363,9 @@ void Test::createRecursiveScript() {
     QTextStream textStream(&allTestsFilename);
     textStream << "// This is an automatically generated file, created by auto-tester" << endl << endl;
 
-    textStream << "var autoTester = Script.require(\"https://github.com/highfidelity/hifi_tests/blob/master/tests/utils/autoTester.js?raw=true\");" << endl;
+    textStream << "var autoTester = Script.require(\"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + "/tests/utils/autoTester.js\");" << endl;
     textStream << "autoTester.enableRecursive();" << endl << endl;
 
-    // The main will call each test after the previous test is completed
-    // This is implemented with an interval timer that periodically tests if a
-    // running test has increment a testNumber variable that it received as an input.
     QVector<QString> testPathnames;
 
     // First test if top-level folder has a test.js file
@@ -360,7 +388,7 @@ void Test::createRecursiveScript() {
             continue;
         }
 
-        const QString testPathname{ directory + "/" + TEST_FILENAME };
+        const QString testPathname { directory + "/" + TEST_FILENAME };
         QFileInfo fileInfo(testPathname);
         if (fileInfo.exists()) {
             // Current folder contains a test
@@ -370,8 +398,8 @@ void Test::createRecursiveScript() {
         }
     }
 
-    if (testPathnames.length() <= 0) {
+    if (interactiveMode && testPathnames.length() <= 0) {
-        messageBox.information(0, "Failure", "No \"test.js\" files found");
+        messageBox.information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
         allTestsFilename.close();
         return;
     }
@@ -380,50 +408,45 @@ void Test::createRecursiveScript() {
     textStream << "autoTester.runRecursive();" << endl;
 
     allTestsFilename.close();
-    messageBox.information(0, "Success", "Script has been created");
+    
+    if (interactiveMode) {
+        messageBox.information(0, "Success", "Script has been created");
+    }
 }
 
 void Test::createTest() {
-    // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on
+    // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on
     // Any existing expected result images will be deleted
-    QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    QString imageSourceDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
-    if (pathToImageDirectory == "") {
+    if (imageSourceDirectory == "") {
         return;
     }
 
-    QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
+    QString imageDestinationDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images", ".", QFileDialog::ShowDirsOnly);
+    if (imageDestinationDirectory == "") {
+        return;
+    }
 
-    int i = 1;
+    QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", imageSourceDirectory);
+
+    int i = 1; 
+    const int maxImages = pow(10, NUM_DIGITS);
     foreach (QString currentFilename, sortedImageFilenames) {
-        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+        QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename;
-        if (isInExpectedImageFilenameFormat(currentFilename)) {
+        if (isInSnapshotFilenameFormat("jpg", currentFilename)) {
-            if (!QFile::remove(fullCurrentFilename)) {
+            if (i >= maxImages) {
+                messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
+                exit(-1);
+            }
+            QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
+            QString fullNewFileName = imageDestinationDirectory + "/" + newFilename;
+
+            try {
+                copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
+            } catch (...) {
                 messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
                 exit(-1);
             }
-        } else if (isInSnapshotFilenameFormat(currentFilename)) {
-            const int MAX_IMAGES = 100000;
-            if (i >= MAX_IMAGES) {
-                messageBox.critical(0, "Error", "More than 100,000 images not supported");
-                exit(-1);
-            }
-            QString newFilename = "ExpectedImage_" + QString::number(i-1).rightJustified(5, '0') + ".jpg";
-            QString fullNewFileName = pathToImageDirectory + "/" + newFilename;
-
-            if (!imageDirectory.rename(fullCurrentFilename, newFilename)) {
-                if (!QFile::exists(fullCurrentFilename)) {
-                    messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n"
-                        + fullCurrentFilename + " not found"
-                        + "\nTest creation aborted"
-                    );
-                    exit(-1);
-                } else {
-                    messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n"
-                        + "unknown error" + "\nTest creation aborted"
-                    );
-                    exit(-1);
-                }
-            }
             ++i;
         }
     }
@@ -431,54 +454,87 @@ void Test::createTest() {
     messageBox.information(0, "Success", "Test images have been created");
 }
 
-void Test::deleteOldSnapshots() {
+void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename) {
-    // Select folder to start recursing from
+    QFile::remove(destinationPNGFullFilename);
-    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select root folder for snapshot deletion", ".", QFileDialog::ShowDirsOnly);
-    if (topLevelDirectory == "") {
-        return;
-    }
 
-    // Recurse over folders
+    QImageReader reader;
-    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    reader.setFileName(sourceJPGFullFilename);
-    while (it.hasNext()) {
-        QString directory = it.next();
 
-        // Only process directories
+    QImage image = reader.read();
-        QDir dir(directory);
-        if (!isAValidDirectory(directory)) {
-            continue;
-        }
 
-        QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory);
+    QImageWriter writer;
-
+    writer.setFileName(destinationPNGFullFilename);
-        // Delete any file that is a snapshot (NOT the Expected Images)
+    writer.write(image);
-        QStringList expectedImages;
-        QStringList resultImages;
-        foreach(QString currentFilename, sortedImageFilenames) {
-            QString fullCurrentFilename = directory + "/" + currentFilename;
-            if (isInSnapshotFilenameFormat(currentFilename)) {
-                if (!QFile::remove(fullCurrentFilename)) {
-                    messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nSnapshot deletion aborted");
-                    exit(-1);
-                }
-            }
-        }
-    }
 }
 
-QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) {
+QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory) {
     imageDirectory = QDir(pathToImageDirectory);
     QStringList nameFilters;
-    nameFilters << "*.jpg";
+    nameFilters << "*." + imageFormat;
 
     return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
 }
 
-// Use regular expressions to check if files are in specific format
+// Snapshots are files in the following format:
-bool Test::isInSnapshotFilenameFormat(QString filename) {
+//      Filename contains no periods (excluding period before exception
-    return (snapshotFilenameFormat.match(filename).hasMatch());
+//      Filename (i.e. without extension) contains _tests_ (this is based on all test scripts being within the tests folder
+//      Last 5 characters in filename are digits
+//      Extension is jpg
+bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) {
+    QStringList filenameParts = filename.split(".");
+
+    bool filnameHasNoPeriods = (filenameParts.size() == 2);
+    bool contains_tests = filenameParts[0].contains("_tests_");
+
+    bool last5CharactersAreDigits;
+    filenameParts[0].right(5).toInt(&last5CharactersAreDigits, 10);
+
+    bool extensionIsIMAGE_FORMAT = (filenameParts[1] == imageFormat);
+
+    return (filnameHasNoPeriods && contains_tests && last5CharactersAreDigits && extensionIsIMAGE_FORMAT);
 }
 
-bool Test::isInExpectedImageFilenameFormat(QString filename) {
+// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is
-    return (expectedImageFilenameFormat.match(filename).hasMatch());
+// D:/GitHub/hifi-tests/tests/content/entity/zone/create
-}
+// This method assumes the filename is in the correct format
+QString Test::getExpectedImageDestinationDirectory(QString filename) {
+    QString filenameWithoutExtension = filename.split(".")[0];
+    QStringList filenameParts = filenameWithoutExtension.split("_");
+
+    QString result = filenameParts[0] + ":";
+
+    for (int i = 1; i < filenameParts.length() - 1; ++i) {
+        result += "/" + filenameParts[i];
+    }
+
+    return result;
+}
+
+// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the source directory on GitHub
+// is ...tests/content/entity/zone/create
+// This is used to create the full URL
+// This method assumes the filename is in the correct format
+QString Test::getExpectedImagePartialSourceDirectory(QString filename) {
+    QString filenameWithoutExtension = filename.split(".")[0];
+    QStringList filenameParts = filenameWithoutExtension.split("_");
+
+    // Note that the bottom-most "tests" folder is assumed to be the root
+    // This is required because the tests folder is named hifi_tests
+    int i { filenameParts.length() - 1 };
+    while (i >= 0 && filenameParts[i] != "tests") {
+        --i;
+    }
+
+    if (i < 0) {
+        messageBox.critical(0, "Internal error #9", "Bad filename");
+        exit(-1);
+    }
+
+    QString result = filenameParts[i];
+
+    for (int j = i + 1; j < filenameParts.length() - 1; ++j) {
+        result += "/" + filenameParts[j];
+    }
+
+    return result;
+}

tools/auto-tester/src/Test.h

@@ -23,28 +23,36 @@ class Test {
 public: 
     Test();
 
-    void evaluateTests(bool interactiveMode, QProgressBar* progressBar);
+    void startTestsEvaluation();
-    void evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar);
+    void finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar);
+
     void createRecursiveScript();
+    void createRecursiveScriptsRecursively();
+    void createRecursiveScript(QString topLevelDirectory, bool interactiveMode);
+
     void createTest();
     void deleteOldSnapshots();
 
-    bool compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar);
+    bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar);
 
-    QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory);
+    QStringList createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory);
 
-    bool isInSnapshotFilenameFormat(QString filename);
+    bool isInSnapshotFilenameFormat(QString imageFormat, QString filename);
-    bool isInExpectedImageFilenameFormat(QString filename);
 
     void importTest(QTextStream& textStream, const QString& testPathname);
 
     void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
 
-    bool createTestResultsFolderPathIfNeeded(QString directory);
+    bool createTestResultsFolderPath(QString directory);
     void zipAndDeleteTestResultsFolder();
 
     bool isAValidDirectory(QString pathname);
 
+    QString getExpectedImageDestinationDirectory(QString filename);
+    QString getExpectedImagePartialSourceDirectory(QString filename);
+
+    void copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename);
+
 private:
     const QString TEST_FILENAME { "test.js" };
     const QString TEST_RESULTS_FOLDER { "TestResults" };
@@ -54,16 +62,28 @@ private:
 
     QDir imageDirectory;
 
-    QRegularExpression snapshotFilenameFormat;
     QRegularExpression expectedImageFilenameFormat;
 
     MismatchWindow mismatchWindow;
 
     ImageComparer imageComparer;
 
-
     QString testResultsFolderPath { "" };
     int index { 1 };
+
+    // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
+    const int NUM_DIGITS { 5 };
+    const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" };
+
+    QString pathToTestResultsDirectory;
+    QStringList expectedImagesFilenames;
+    QStringList expectedImagesFullFilenames;
+    QStringList resultImagesFullFilenames;
+
+    // Used for accessing GitHub
+    const QString user { "NissimHadar" };
+    const QString branch { "addRecursionToAutotester" };
+	const QString DATETIME_FORMAT { "yyyy-MM-dd_hh-mm-ss" };
 };
 
 #endif // hifi_test_h

tools/auto-tester/src/main.cpp

@@ -10,11 +10,13 @@
 #include <QtWidgets/QApplication>
 #include "ui/AutoTester.h"
 
+AutoTester* autoTester;
+
 int main(int argc, char *argv[]) {
     QApplication application(argc, argv);
 
-    AutoTester autoTester;
+    autoTester = new AutoTester();
-    autoTester.show();
+    autoTester->show();
 
     return application.exec();
 }

tools/auto-tester/src/ui/AutoTester.cpp

@@ -12,32 +12,79 @@
 
 AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
     ui.setupUi(this);
-
     ui.checkBoxInteractiveMode->setChecked(true);
-
     ui.progressBar->setVisible(false);
+
+    test = new Test();
+
+    signalMapper = new QSignalMapper();
 }
 
 void AutoTester::on_evaluateTestsButton_clicked() {
-    test.evaluateTests(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
+    test->startTestsEvaluation();
-}
-
-void AutoTester::on_evaluateTestsRecursivelyButton_clicked() {
-    test.evaluateTestsRecursively(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
 }
 
 void AutoTester::on_createRecursiveScriptButton_clicked() {
-    test.createRecursiveScript();
+    test->createRecursiveScript();
+}
+
+void AutoTester::on_createRecursiveScriptsRecursivelyButton_clicked() {
+    test->createRecursiveScriptsRecursively();
 }
 
 void AutoTester::on_createTestButton_clicked() {
-    test.createTest();
+    test->createTest();
-}
-
-void AutoTester::on_deleteOldSnapshotsButton_clicked() {
-    test.deleteOldSnapshots();
 }
 
 void AutoTester::on_closeButton_clicked() {
     exit(0);
-}
+}
+
+void AutoTester::downloadImage(const QUrl& url) {
+    downloaders.emplace_back(new Downloader(url, this));
+    connect(downloaders[_index], SIGNAL (downloaded()), signalMapper, SLOT (map()));
+
+    signalMapper->setMapping(downloaders[_index], _index);
+
+    ++_index;
+}
+
+void AutoTester::downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames) {
+    _directoryName = directoryName;
+    _filenames = filenames;
+
+    _numberOfImagesToDownload = URLs.size();
+    _numberOfImagesDownloaded = 0;
+    _index = 0;
+
+    ui.progressBar->setMinimum(0);
+    ui.progressBar->setMaximum(_numberOfImagesToDownload - 1);
+    ui.progressBar->setValue(0);
+    ui.progressBar->setVisible(true);
+
+    for (int i = 0; i < _numberOfImagesToDownload; ++i) {
+        QUrl imageURL(URLs[i]);
+        downloadImage(imageURL);
+    }
+
+    connect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int)));
+}
+
+void AutoTester::saveImage(int index) {
+    QPixmap pixmap;
+    pixmap.loadFromData(downloaders[index]->downloadedData());
+
+    QImage image = pixmap.toImage();
+    image = image.convertToFormat(QImage::Format_ARGB32);
+
+    QString fullPathname = _directoryName + "/" + _filenames[index];
+    image.save(fullPathname, 0, 100);
+
+    ++_numberOfImagesDownloaded;
+
+    if (_numberOfImagesDownloaded == _numberOfImagesToDownload) {
+        test->finishTestsEvaluation(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
+    } else {
+        ui.progressBar->setValue(_numberOfImagesDownloaded);
+    }
+}

tools/auto-tester/src/ui/AutoTester.h

@@ -11,7 +11,10 @@
 #define hifi_AutoTester_h
 
 #include <QtWidgets/QMainWindow>
+#include <QSignalMapper>
 #include "ui_AutoTester.h"
+
+#include "../Downloader.h"
 #include "../Test.h"
 
 class AutoTester : public QMainWindow {
@@ -19,19 +22,34 @@ class AutoTester : public QMainWindow {
 
 public:
     AutoTester(QWidget *parent = Q_NULLPTR);
+    void downloadImage(const QUrl& url);
+    void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames);
 
 private slots:
     void on_evaluateTestsButton_clicked();
-    void on_evaluateTestsRecursivelyButton_clicked();
     void on_createRecursiveScriptButton_clicked();
+    void on_createRecursiveScriptsRecursivelyButton_clicked();
     void on_createTestButton_clicked();
-    void on_deleteOldSnapshotsButton_clicked();
     void on_closeButton_clicked();
 
+    void saveImage(int index);
+
 private:
     Ui::AutoTesterClass ui;
+    Test* test;
 
-    Test test;
+    std::vector<Downloader*> downloaders;
+
+    // local storage for parameters - folder to store downloaded files in, and a list of their names
+    QString _directoryName;
+    QStringList _filenames;
+
+    // Used to enable passing a parameter to slots
+    QSignalMapper* signalMapper;
+
+    int _numberOfImagesToDownload;
+    int _numberOfImagesDownloaded;
+    int _index;
 };
 
 #endif // hifi_AutoTester_h

tools/auto-tester/src/ui/AutoTester.ui

@@ -17,7 +17,7 @@
    <widget class="QPushButton" name="closeButton">
     <property name="geometry">
      <rect>
-      <x>190</x>
+      <x>20</x>
       <y>300</y>
       <width>220</width>
       <height>40</height>
@@ -30,8 +30,8 @@
    <widget class="QPushButton" name="createTestButton">
     <property name="geometry">
      <rect>
-      <x>360</x>
+      <x>20</x>
-      <y>130</y>
+      <y>30</y>
       <width>220</width>
       <height>40</height>
      </rect>
@@ -44,7 +44,7 @@
     <property name="geometry">
      <rect>
       <x>20</x>
-      <y>75</y>
+      <y>135</y>
       <width>220</width>
       <height>40</height>
      </rect>
@@ -66,24 +66,11 @@
      <string>Create Recursive Script</string>
     </property>
    </widget>
-   <widget class="QPushButton" name="evaluateTestsRecursivelyButton">
-    <property name="geometry">
-     <rect>
-      <x>20</x>
-      <y>130</y>
-      <width>220</width>
-      <height>40</height>
-     </rect>
-    </property>
-    <property name="text">
-     <string>Evaluate Tests Recursively</string>
-    </property>
-   </widget>
    <widget class="QCheckBox" name="checkBoxInteractiveMode">
     <property name="geometry">
      <rect>
       <x>23</x>
-      <y>40</y>
+      <y>100</y>
       <width>131</width>
       <height>20</height>
      </rect>
@@ -108,17 +95,17 @@
      <number>24</number>
     </property>
    </widget>
-   <widget class="QPushButton" name="deleteOldSnapshotsButton">
+   <widget class="QPushButton" name="createRecursiveScriptsRecursivelyButton">
     <property name="geometry">
      <rect>
       <x>360</x>
-      <y>240</y>
+      <y>140</y>
       <width>220</width>
       <height>40</height>
      </rect>
     </property>
     <property name="text">
-     <string>Delete Old Snapshots</string>
+     <string>Create Recursive Scripts Recursively</string>
     </property>
    </widget>
   </widget>
@@ -145,4 +132,4 @@
  <layoutdefault spacing="6" margin="11"/>
  <resources/>
  <connections/>
-</ui>
+</ui>