Changed stored failure images from JPG to PNG

Now uses snapshot name to find correct test folder - allows creating multiple tests
Reversed order of recursive scripts, so execution will be in alphabetical order.

tools/auto-tester/src/ImageComparer.cpp

@@ -15,12 +15,6 @@
 // Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity
 // The value is computed for the luminance component and the average value is returned
 double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const {
-    // Make sure the image is 8 bits per colour
-    QImage::Format format = expectedImage.format();
-    if (format != QImage::Format::Format_ARGB32) {
-        throw -1;
-    }
-
     const int L = 255; // (2^number of bits per pixel) - 1
     const double K1 { 0.01 };
     const double K2 { 0.03 };

tools/auto-tester/src/Test.cpp

@@ -158,20 +158,20 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
     QString destinationFile;
 
     sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
-    destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
+    destinationFile = failureFolderPath + "/" + "Expected Image.png";
     if (!QFile::copy(sourceFile, destinationFile)) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
         exit(-1);
     }
 
     sourceFile = testFailure._pathname + testFailure._actualImageFilename;
-    destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
+    destinationFile = failureFolderPath + "/" + "Actual Image.png";
     if (!QFile::copy(sourceFile, destinationFile)) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
         exit(-1);
     }
 
-    comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
+    comparisonImage.save(failureFolderPath + "/" + "Difference Image.png");
 }
 
 void Test::startTestsEvaluation(const QString& testFolder) {
@@ -295,7 +295,7 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) {
     return partialPath;
 }
 
-void Test::importTest(QTextStream& textStream, const QString& testPathname) {
+void Test::includeTest(QTextStream& textStream, const QString& testPathname) {
     QString partialPath = extractPathFromTestsDown(testPathname);
     textStream << "Script.include(\""
                << "https://github.com/" << GIT_HUB_USER << "/hifi_tests/blob/" << GIT_HUB_BRANCH
@@ -329,24 +329,24 @@ void Test::createRecursiveScript() {
 // This method creates a `testRecursive.js` script in every sub-folder.
 void Test::createAllRecursiveScripts() {
     // Select folder to start recursing from
-    QString previousSelection = testDirectory;
+    QString previousSelection = testsRootDirectory;
     QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
     if (!parent.isNull() && parent.right(1) != "/") {
         parent += "/";
     }
 
-    testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
+    testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
                                                       parent, QFileDialog::ShowDirsOnly);
 
     // If user cancelled then restore previous selection and return
-    if (testDirectory == "") {
+    if (testsRootDirectory == "") {
-        testDirectory = previousSelection;
+        testsRootDirectory = previousSelection;
         return;
     }
 
-    createRecursiveScript(testDirectory, false);
+    createRecursiveScript(testsRootDirectory, false);
 
-    QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
     while (it.hasNext()) {
         QString directory = it.next();
 
@@ -395,22 +395,31 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
     const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
     textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
 
+    textStream << "user = \"" + GIT_HUB_USER + "/\"" << endl;
+    textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\"" << endl;
+    textStream << "branch = \"" + GIT_HUB_BRANCH + "/\"" << endl << endl;
+
     textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/" 
         + GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;
 
     textStream << "autoTester.enableRecursive();" << endl;
     textStream << "autoTester.enableAuto();" << endl << endl;
 
-    QVector<QString> testPathnames;
+    // This is used to verify that the recursive test contains at least one test
+    bool testFound{ false };
+
+    // Directories are included in reverse order.  The autoTester scripts use a stack mechanism,
+    // so this ensures that the tests run in alphabetical order (a convenience when debugging)
+    QStringList directories;
 
     // First test if top-level folder has a test.js file
     const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME };
     QFileInfo fileInfo(testPathname);
     if (fileInfo.exists()) {
         // Current folder contains a test
-        importTest(textStream, testPathname);
+        directories.push_front(testPathname);
 
-        testPathnames << testPathname;
+        testFound = true;
     }
 
     QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
@@ -427,18 +436,23 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
         QFileInfo fileInfo(testPathname);
         if (fileInfo.exists()) {
             // Current folder contains a test
-            importTest(textStream, testPathname);
+            directories.push_front(testPathname);
 
-            testPathnames << testPathname;
+            testFound = true;
         }
     }
 
-    if (interactiveMode && testPathnames.length() <= 0) {
+    if (interactiveMode && !testFound) {
         QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
         allTestsFilename.close();
         return;
     }
 
+    // Now include the test scripts
+    for (int i = 0; i < directories.length(); ++i) {
+        includeTest(textStream, directories.at(i));
+    }
+
     textStream << endl;
     textStream << "autoTester.runRecursive();" << endl;
 
@@ -449,7 +463,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
     }
 }
 
-void Test::createTest() {
+void Test::createTests() {
     // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on
     // Any existing expected result images will be deleted
     QString previousSelection = snapshotDirectory;
@@ -467,18 +481,18 @@ void Test::createTest() {
         return;
     }
 
-    previousSelection = testDirectory;
+    previousSelection = testsRootDirectory;
     parent = previousSelection.left(previousSelection.lastIndexOf('/'));
     if (!parent.isNull() && parent.right(1) != "/") {
         parent += "/";
     }
 
-    testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images", parent,
+    testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent,
                                                               QFileDialog::ShowDirsOnly);
 
     // If user cancelled then restore previous selection and return
-    if (testDirectory == "") {
+    if (testsRootDirectory == "") {
-        testDirectory = previousSelection;
+        testsRootDirectory = previousSelection;
         return;
     }
 
@@ -493,8 +507,23 @@ void Test::createTest() {
                 QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
                 exit(-1);
             }
-            QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
+
-            QString fullNewFileName = testDirectory + "/" + newFilename;
+            // Path to test is extracted from the file name
+            // Example:
+            //      filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg
+            //      path is <testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd
+            //
+            //      Note: we don't use the first part and the last 2 parts of the filename at this stage
+            //
+            QStringList pathParts = currentFilename.split(".");
+            QString fullNewFileName = testsRootDirectory;
+            for (int j = 1; j < pathParts.size() - 2; ++j) {
+                fullNewFileName += "/" + pathParts[j];
+            }
+
+            // The image index is the penultimate component of the path parts (the last being the file extension)
+            QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png";
+            fullNewFileName += "/" + newFilename;
 
             try {
                 copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
@@ -602,29 +631,29 @@ void Test::createMDFile() {
 
 void Test::createAllMDFiles() {
     // Select folder to start recursing from
-    QString previousSelection = testDirectory;
+    QString previousSelection = testsRootDirectory;
     QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
     if (!parent.isNull() && parent.right(1) != "/") {
         parent += "/";
     }
 
-    testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent,
+    testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent,
                                                       QFileDialog::ShowDirsOnly);
 
     // If user cancelled then restore previous selection and return
-    if (testDirectory == "") {
+    if (testsRootDirectory == "") {
-        testDirectory = previousSelection;
+        testsRootDirectory = previousSelection;
         return;
     }
 
     // First test if top-level folder has a test.js file
-    const QString testPathname { testDirectory + "/" + TEST_FILENAME };
+    const QString testPathname{ testsRootDirectory + "/" + TEST_FILENAME };
     QFileInfo fileInfo(testPathname);
     if (fileInfo.exists()) {
-        createMDFile(testDirectory);
+        createMDFile(testsRootDirectory);
     }
 
-    QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
     while (it.hasNext()) {
         QString directory = it.next();
 

tools/auto-tester/src/Test.h

@@ -44,7 +44,7 @@ public:
     void createAllRecursiveScripts();
     void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
 
-    void createTest();
+    void createTests();
     void createMDFile();
     void createAllMDFiles();
     void createMDFile(const QString& topLevelDirectory);
@@ -57,7 +57,7 @@ public:
 
     bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename);
 
-    void importTest(QTextStream& textStream, const QString& testPathname);
+    void includeTest(QTextStream& textStream, const QString& testPathname);
 
     void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
 
@@ -91,8 +91,10 @@ private:
 
     // We have two directories to work with.
     // The first is the directory containing the test we are working with
-    // The second contains the snapshots taken for test runs that need to be evaluated
+    // The second is the root directory of all tests
+    // The third contains the snapshots taken for test runs that need to be evaluated
     QString testDirectory;
+    QString testsRootDirectory;
     QString snapshotDirectory;
 
     QStringList expectedImagesFilenames;
@@ -100,9 +102,11 @@ private:
     QStringList resultImagesFullFilenames;
 
     // Used for accessing GitHub
-    const QString GIT_HUB_USER{ "highfidelity" };
+    const QString GIT_HUB_USER{ "NissimHadar" };
-    const QString GIT_HUB_BRANCH { "master" };
+    const QString GIT_HUB_REPOSITORY{ "hifi_tests" };
-	const QString DATETIME_FORMAT { "yyyy-MM-dd_hh-mm-ss" };
+    const QString GIT_HUB_BRANCH{ "newAvatar" };
+
+    const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" };
 
 	ExtractedText getTestScriptLines(QString testFileName);
 

tools/auto-tester/src/ui/AutoTester.cpp

@@ -40,8 +40,8 @@ void AutoTester::on_createAllRecursiveScriptsButton_clicked() {
     test->createAllRecursiveScripts();
 }
 
-void AutoTester::on_createTestButton_clicked() {
+void AutoTester::on_createTestsButton_clicked() {
-	test->createTest();
+	test->createTests();
 }
 
 void AutoTester::on_createMDFileButton_clicked() {
@@ -95,7 +95,7 @@ void AutoTester::saveImage(int index) {
     pixmap.loadFromData(downloaders[index]->downloadedData());
 
     QImage image = pixmap.toImage();
-    image = image.convertToFormat(QImage::Format_ARGB32);
+    image = image.convertToFormat(QImage::Format_RGB32);
 
     QString fullPathname = _directoryName + "/" + _filenames[index];
     if (!image.save(fullPathname, 0, 100)) {

tools/auto-tester/src/ui/AutoTester.h

@@ -32,7 +32,7 @@ private slots:
     void on_evaluateTestsButton_clicked();
     void on_createRecursiveScriptButton_clicked();
     void on_createAllRecursiveScriptsButton_clicked();
-	void on_createTestButton_clicked();
+	void on_createTestsButton_clicked();
     void on_createMDFileButton_clicked();
     void on_createAllMDFilesButton_clicked();
     void on_createTestsOutlineButton_clicked();

tools/auto-tester/src/ui/AutoTester.ui

@@ -27,7 +27,7 @@
      <string>Close</string>
     </property>
    </widget>
-   <widget class="QPushButton" name="createTestButton">
+   <widget class="QPushButton" name="createTestsButton">
     <property name="geometry">
      <rect>
       <x>20</x>
@@ -37,7 +37,7 @@
      </rect>
     </property>
     <property name="text">
-     <string>Create Test</string>
+     <string>Create Tests</string>
     </property>
    </widget>
    <widget class="QPushButton" name="evaluateTestsButton">