Recursive test evaluations.

tools/auto-tester/src/ImageComparer.cpp

@@ -26,9 +26,9 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) co
     const double c2 = (K2 * L) * (K2 * L);
 
     // Coefficients for luminosity calculation
-    const double RED_COEFFICIENT = 0.212655f;
+    const double RED_COEFFICIENT = 0.212655;
-    const double GREEN_COEFFICIENT = 0.715158f;
+    const double GREEN_COEFFICIENT = 0.715158;
-    const double BLUE_COEFFICIENT = 0.072187f;
+    const double BLUE_COEFFICIENT = 0.072187;
 
     // First go over all full 8x8 blocks
     // This is done in 3 loops

tools/auto-tester/src/Test.cpp

@@ -21,34 +21,8 @@ Test::Test() {
     mismatchWindow.setModal(true);
 }
 
-void Test::evaluateTests() {
+bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages) {
-    createListOfAllJPEGimagesInDirectory();
+    // Loop over both lists and compare each pair of images
-
-    // Separate images into two lists.  The first is the expected images, the second is the test results
-    // Images that are in the wrong format are ignored.
-    QStringList expectedImages;
-    QStringList resultImages;
-    foreach(QString currentFilename, sortedImageFilenames) {
-        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
-        if (isInExpectedImageFilenameFormat(currentFilename)) {
-            expectedImages << fullCurrentFilename;
-        } else if (isInSnapshotFilenameFormat(currentFilename)) {
-            resultImages << fullCurrentFilename;
-        }
-    }
-
-    // The number of images in each list should be identical
-    if (expectedImages.length() != resultImages.length()) {
-        messageBox.critical(0, 
-            "Test failed", 
-            "Found " + QString::number(resultImages.length()) + " images in directory" +
-            "\nExpected to find " + QString::number(expectedImages.length()) + " images"
-        );
-
-        exit(-1);
-    }
-
-    // Now loop over both lists and compare each pair of images
     // Quit loop if user has aborted due to a failed test.
     const double THRESHOLD{ 0.99999 };
     bool success{ true };
@@ -65,7 +39,8 @@ void Test::evaluateTests() {
         double similarityIndex;  // in [-1.0 .. 1.0], where 1.0 means images are identical
         try {
             similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
-        } catch (...) {
+        }
+        catch (...) {
             messageBox.critical(0, "Internal error", "Image not in expected format");
             exit(-1);
         }
@@ -96,10 +71,98 @@ void Test::evaluateTests() {
         }
     }
 
+    return success;
+}
+
+void Test::evaluateTests() {
+    // Get list of JPEG images in folder, sorted by name
+    QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
+
+    // Separate images into two lists.  The first is the expected images, the second is the test results
+    // Images that are in the wrong format are ignored.
+    QStringList expectedImages;
+    QStringList resultImages;
+    foreach(QString currentFilename, sortedImageFilenames) {
+        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+        if (isInExpectedImageFilenameFormat(currentFilename)) {
+            expectedImages << fullCurrentFilename;
+        } else if (isInSnapshotFilenameFormat(currentFilename)) {
+            resultImages << fullCurrentFilename;
+        }
+    }
+
+    // The number of images in each list should be identical
+    if (expectedImages.length() != resultImages.length()) {
+        messageBox.critical(0, 
+            "Test failed", 
+            "Found " + QString::number(resultImages.length()) + " images in directory" +
+            "\nExpected to find " + QString::number(expectedImages.length()) + " images"
+        );
+
+        exit(-1);
+    }
+
+    bool success = compareImageLists(expectedImages, resultImages);
+
     if (success) {
         messageBox.information(0, "Success", "All images are as expected");
+    } else {
+        messageBox.information(0, "Failure", "One or more images are not as expected");
     }
-    else {
+}
+
+// Two criteria are used to decide if a folder contains valid test results.
+//      1) a 'test'js' file exists in the folder
+//      2) the folder has the same number of anual and expected images
+void Test::evaluateTestsRecursively() {
+    // Select folder to start recursing from
+    QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly);
+
+    bool success{ true };
+    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        QString directory = it.next();
+        if (directory[directory.length() - 1] == '.') {
+            // ignore '.', '..' directories
+            continue;
+        }
+
+        // 
+        const QString testPathname{ directory + "/" + testFilename };
+        QFileInfo fileInfo(testPathname);
+        if (!fileInfo.exists()) {
+            // Folder does not contain 'test.js'
+            continue;
+        }
+
+        QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory);
+
+        // Separate images into two lists.  The first is the expected images, the second is the test results
+        // Images that are in the wrong format are ignored.
+        QStringList expectedImages;
+        QStringList resultImages;
+        foreach(QString currentFilename, sortedImageFilenames) {
+            QString fullCurrentFilename = directory + "/" + currentFilename;
+            if (isInExpectedImageFilenameFormat(currentFilename)) {
+                expectedImages << fullCurrentFilename;
+            } else if (isInSnapshotFilenameFormat(currentFilename)) {
+                resultImages << fullCurrentFilename;
+            }
+        }
+
+        if (expectedImages.length() != resultImages.length()) {
+            // Number of images doesn't match
+            continue;
+        }
+
+        // Set success to false if any test has failed
+        success &= compareImageLists(expectedImages, resultImages);
+    }
+
+    if (success) {
+        messageBox.information(0, "Success", "All images are as expected");
+    } else {
         messageBox.information(0, "Failure", "One or more images are not as expected");
     }
 }
@@ -113,7 +176,6 @@ void Test::importTest(QTextStream& textStream, const QString& testPathname, int
 void Test::createRecursiveScript() {
     // Select folder to start recursing from
     QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly);
-    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
 
     QFile allTestsFilename(topLevelDirectory + "/" + "allTests.js");
     if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) {
@@ -132,7 +194,6 @@ void Test::createRecursiveScript() {
     // running test has increment a testNumber variable that it received as an input.
     int testNumber = 1;
     QVector<QString> testPathnames;
-    const QString testFilename{ "test.js" };
 
     // First test if top-level folder has a test.js file
     const QString testPathname{ topLevelDirectory + "/" + testFilename };
@@ -145,6 +206,7 @@ void Test::createRecursiveScript() {
         testPathnames << testPathname;
     }
 
+    QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
     while (it.hasNext()) {
         QString directory = it.next();
         if (directory[directory.length() - 1] == '.') {
@@ -249,7 +311,8 @@ void Test::createRecursiveScript() {
 void Test::createTest() {
     // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on
     // Any existing expected result images will be deleted
-    createListOfAllJPEGimagesInDirectory();
+    QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+    QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory);
 
     int i = 1;
     foreach (QString currentFilename, sortedImageFilenames) {
@@ -271,15 +334,12 @@ void Test::createTest() {
     messageBox.information(0, "Success", "Test images have been created");
 }
 
-void Test::createListOfAllJPEGimagesInDirectory() {
+QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) {
-    // Get list of JPEG images in folder, sorted by name
-    pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
-
     imageDirectory = QDir(pathToImageDirectory);
     QStringList nameFilters;
     nameFilters << "*.jpg";
 
-    sortedImageFilenames = imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
+    return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
 }
 
 bool Test::isInSnapshotFilenameFormat(QString filename) {

tools/auto-tester/src/Test.h

@@ -24,10 +24,11 @@ public:
     Test();
 
     void evaluateTests();
+    void evaluateTestsRecursively();
     void createRecursiveScript();
     void createTest();
 
-    void createListOfAllJPEGimagesInDirectory();
+    QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory);
 
     bool isInSnapshotFilenameFormat(QString filename);
     bool isInExpectedImageFilenameFormat(QString filename);
@@ -35,11 +36,11 @@ public:
     void importTest(QTextStream& textStream, const QString& testPathname, int testNumber);
 
 private:
+    const QString testFilename{ "test.js" };
+
     QMessageBox messageBox;
 
-    QString pathToImageDirectory;
     QDir imageDirectory;
-    QStringList sortedImageFilenames;
 
     QRegularExpression snapshotFilenameFormat;
     QRegularExpression expectedImageFilenameFormat;
@@ -47,6 +48,8 @@ private:
     MismatchWindow mismatchWindow;
 
     ImageComparer imageComparer;
+
+    bool compareImageLists(QStringList expectedImages, QStringList resultImages);
 };
 
 #endif // hifi_test_h

tools/auto-tester/src/ui/AutoTester.cpp

@@ -16,12 +16,16 @@ AutoTester::AutoTester(QWidget *parent)
     ui.setupUi(this);
 }
 
-
 void AutoTester::on_evaluateTestsButton_clicked()
 {
     test.evaluateTests();
 }
 
+void AutoTester::on_evaluateTestsRecursivelyButton_clicked()
+{
+    test.evaluateTestsRecursively();
+}
+
 void AutoTester::on_createRecursiveScriptButton_clicked()
 {
     test.createRecursiveScript();

tools/auto-tester/src/ui/AutoTester.h

@@ -23,8 +23,9 @@ public:
     AutoTester(QWidget *parent = Q_NULLPTR);
 
 private slots:
-    void on_evaluateTestsButton_clicked();
+void on_evaluateTestsButton_clicked();
-    void on_createRecursiveScriptButton_clicked();
+void on_evaluateTestsRecursivelyButton_clicked();
+void on_createRecursiveScriptButton_clicked();
     void on_createTestButton_clicked();
     void on_closeButton_clicked();
 

tools/auto-tester/src/ui/AutoTester.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>277</width>
+    <width>286</width>
-    <height>351</height>
+    <height>470</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -18,8 +18,8 @@
     <property name="geometry">
      <rect>
       <x>60</x>
-      <y>240</y>
+      <y>360</y>
-      <width>159</width>
+      <width>160</width>
       <height>40</height>
      </rect>
     </property>
@@ -31,7 +31,7 @@
     <property name="geometry">
      <rect>
       <x>60</x>
-      <y>180</y>
+      <y>270</y>
       <width>160</width>
       <height>40</height>
      </rect>
@@ -50,14 +50,14 @@
      </rect>
     </property>
     <property name="text">
-     <string>Evaulate Tests</string>
+     <string>Evaluate Test</string>
     </property>
    </widget>
    <widget class="QPushButton" name="createRecursiveScriptButton">
     <property name="geometry">
      <rect>
       <x>60</x>
-      <y>120</y>
+      <y>210</y>
       <width>160</width>
       <height>40</height>
      </rect>
@@ -66,13 +66,26 @@
      <string>Create Recursive Script</string>
     </property>
    </widget>
+   <widget class="QPushButton" name="evaluateTestsRecursivelyButton">
+    <property name="geometry">
+     <rect>
+      <x>60</x>
+      <y>75</y>
+      <width>160</width>
+      <height>40</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Evaluate Tests Recursively</string>
+    </property>
+   </widget>
   </widget>
   <widget class="QMenuBar" name="menuBar">
    <property name="geometry">
     <rect>
      <x>0</x>
      <y>0</y>
-     <width>277</width>
+     <width>286</width>
      <height>21</height>
     </rect>
    </property>

tools/auto-tester/src/ui/MismatchWindow.cpp

@@ -21,7 +21,7 @@ MismatchWindow::MismatchWindow(QWidget *parent)
 }
 
 void MismatchWindow::setTestFailure(TestFailure testFailure) {
-    errorLabel->setText("Error: " + QString::number((int)testFailure._error));
+    errorLabel->setText("Similarity: " + QString::number(testFailure._error));
 
     imagePath->setText("Path to test: " + testFailure._pathname);