WIP.

tools/auto-tester/CMakeLists.txt

@@ -10,4 +10,4 @@ source_group("UI Files" FILES ${QT_UI_FILES})
 qt5_wrap_ui(QT_UI_HEADERS "${QT_UI_FILES}")
 
 # add them to the interface source files
-set(INTERFACE_SRCS ${INTERFACE_SRCS} "${QT_UI_HEADERS}")
+##set(INTERFACE_SRCS ${INTERFACE_SRCS} "${QT_UI_HEADERS}")

tools/auto-tester/src/AutoTester.cpp

@@ -0,0 +1,32 @@
+//
+//  autoTester.cpp
+//  zone/ambientLightInheritence
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "AutoTester.h"
+
+AutoTester::AutoTester(QWidget *parent)
+    : QMainWindow(parent)
+{
+    ui.setupUi(this);
+}
+
+void AutoTester::on_closeButton_clicked()
+{
+    exit(0);
+}
+
+void AutoTester::on_evaluateTestsButton_clicked()
+{
+    test.evaluateTests();
+}
+
+void AutoTester::on_createTestButton_clicked()
+{
+    test.createTest();
+}

tools/auto-tester/src/MismatchWindow.cpp

@@ -0,0 +1,51 @@
+//
+//  MismatchWindow.cpp
+//
+//  Created by Nissim Hadar on 9 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "MismatchWindow.h"
+
+#include <QFileInfo>
+
+MismatchWindow::MismatchWindow(QWidget *parent)
+    : QDialog(parent)
+{
+    setupUi(this);
+
+    expectedImage->setScaledContents(true);
+    resultImage->setScaledContents(true);
+}
+
+void MismatchWindow::setTestFailure(TestFailure testFailure) {
+    errorLabel->setText("Error: " + QString::number((int)testFailure._error));
+
+    imagePath->setText("Path to test: " + testFailure._pathname);
+
+    expectedFilename->setText(testFailure._expectedImageFilename);
+    expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename));
+
+    resultFilename->setText(testFailure._resultImageFilename);
+    resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._resultImageFilename));
+}
+
+void MismatchWindow::on_passTestButton_clicked()
+{
+    _userResponse = USER_RESPONSE_PASS;
+    close();
+}
+
+void MismatchWindow::on_failTestButton_clicked()
+{
+    _userResponse = USE_RESPONSE_FAIL;
+    close();
+}
+
+void MismatchWindow::on_abortTestsButton_clicked()
+{
+    _userResponse = USER_RESPONSE_ABORT;
+    close();
+}

tools/auto-tester/src/MismatchWindow.h

@@ -0,0 +1,38 @@
+//
+//  MismatchWindow.cpp
+//
+//  Created by Nissim Hadar on 9 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_MismatchWindow_h
+#define hifi_MismatchWindow_h
+
+#include "ui_MismatchWindow.h"
+
+#include "common.h"
+
+class MismatchWindow : public QDialog, public Ui::MismatchWindow
+{
+    Q_OBJECT
+
+public:
+    MismatchWindow(QWidget *parent = Q_NULLPTR);
+
+    void setTestFailure(TestFailure testFailure);
+
+    UserResponse getUserResponse() { return _userResponse; }
+
+private slots:
+    void on_passTestButton_clicked();
+    void on_failTestButton_clicked();
+    void on_abortTestsButton_clicked();
+
+private:
+    UserResponse _userResponse{ USER_RESPONSE_INVALID };
+};
+
+
+#endif // hifi_MismatchWindow_h

tools/auto-tester/src/Test.cpp

@@ -0,0 +1,163 @@
+//
+//  Test.cpp
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+// Images are compared using the ImagMagick command line tool magick.exe
+// A number of comparison metrics are available, including:
+//      AE      Absolute error                  count of the number of different pixels (0=equal)
+//
+//      DSSIM   Stuctural dissimilarity index
+//
+//      PAE     Peak Absolute error             of any one pixel
+//
+//      PSNR    Peak Signal to Noise Ratio      The ratio of mean square difference to the maximum mean square
+//                                              that can exist between any two images, expressed as a decibel value.
+//                                              The higher the PSNR the closer the closer the images are, with
+//                                              a maximum difference occurring at 1.  A PSNR of 20 means
+//                                              differences are 1 / 100 of maximum.
+//
+//      MAE     Mean Absolute Error             average channel error distance
+//
+//      MSE     Mean Squared Error              average squared error distance
+//
+//      RMSE    squareRoot Mean Error           sqrt(MSE)
+//
+#include "Test.h"
+
+#include <assert.h>
+#include <QTextStream>
+
+Test::Test() {
+    snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg");
+
+    expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg");
+
+    mismatchWindow.setModal(true);
+}
+
+void Test::evaluateTests() {
+    createListOfAllJPEGimagesInDirectory();
+
+    // Separate images into two lists.  The first is the expected images, the second is the test results
+    // Images that are in the wrong format are ignored.
+    QStringList expectedImages;
+    QStringList resultImages;
+    foreach(QString currentFilename, sortedImageFilenames) {
+        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+        if (isInExpectedImageFilenameFormat(currentFilename)) {
+            expectedImages << fullCurrentFilename;
+        } else if (isInSnapshotFilenameFormat(currentFilename)) {
+            resultImages << fullCurrentFilename;
+        }
+    }
+
+    // The number of images in each list should be identical
+    if (expectedImages.length() != resultImages.length()) {
+        messageBox.critical(0, 
+            "Test failed", 
+            "Found " + QString::number(resultImages.length()) + " images in directory" + 
+            "\nExpected to find " + QString::number(expectedImages.length()) + " images");
+
+        exit(-1);
+    }
+
+    // Now loop over both lists and compare each pair of images
+    // Quit loop if user has aborted due to a failed test.
+    const float THRESHOLD{ 10.0f };
+    bool success{ true };
+    bool keepOn{ true };
+    for (int i = 0; keepOn && i < expectedImages.length(); ++i) {
+        QString diffFilename = "hifi_autoTest_diff.txt";
+        QString command = "magick.exe compare -metric MAE " + expectedImages[i] + " " + resultImages[i] + " null: 2>" + diffFilename;
+        system(command.toStdString().c_str());
+
+        QFile file(diffFilename);
+        if (!file.open(QIODevice::ReadOnly)) {
+            messageBox.critical(0, "error", file.errorString());
+        }
+
+        // First value on line is the comparison result
+        QTextStream in(&file);
+        QString line = in.readLine();
+        QStringList tokens = line.split(' ');
+        float error = tokens[0].toFloat();
+
+        if (error > THRESHOLD) {
+            mismatchWindow.setTestFailure(TestFailure{
+                error,                                                          // value of the error (float)
+                expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /
+                QFileInfo(expectedImages[i].toStdString().c_str()).fileName(),  // filename of expected image
+                QFileInfo(resultImages[i].toStdString().c_str()).fileName()     // filename of result image
+            });
+            
+            mismatchWindow.exec();
+
+            switch (mismatchWindow.getUserResponse()) {
+                case USER_RESPONSE_PASS:
+                    break;
+                case USE_RESPONSE_FAIL:
+                    success = false;
+                    break;
+                case USER_RESPONSE_ABORT:
+                    keepOn = false;
+                    success = false;
+                    break;
+                default:
+                    assert(false);
+            }
+        }
+    }
+
+    if (success) {
+        messageBox.information(0, "Success", "All images are as expected");
+    } else {
+        messageBox.information(0, "Failure", "One or more images are not as expected");
+    }
+}
+    
+void Test::createTest() {
+    // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on
+    // Any existing expected result images will be deleted
+    createListOfAllJPEGimagesInDirectory();
+
+    int i = 1;
+    foreach (QString currentFilename, sortedImageFilenames) {
+        QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
+        if (isInExpectedImageFilenameFormat(currentFilename)) {
+            if (!QFile::remove(fullCurrentFilename)) {
+                messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
+                exit(-1);
+            }
+        } else if (isInSnapshotFilenameFormat(currentFilename)) {
+            QString newFilename = "ExpectedImage_" + QString::number(i) + ".jpg";
+            QString fullNewFileName = pathToImageDirectory + "/" + newFilename;
+
+            imageDirectory.rename(fullCurrentFilename, newFilename);
+            ++i;
+        }
+    }
+}
+
+void Test::createListOfAllJPEGimagesInDirectory() {
+    // get list of JPEG images in folder, sorted by name
+    pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
+
+    imageDirectory = QDir(pathToImageDirectory);
+    QStringList nameFilters;
+    nameFilters << "*.jpg";
+
+    sortedImageFilenames = imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
+}
+
+bool Test::isInSnapshotFilenameFormat(QString filename) {
+    return (snapshotFilenameFormat.match(filename).hasMatch());
+}
+
+bool Test::isInExpectedImageFilenameFormat(QString filename) {
+    return (expectedImageFilenameFormat.match(filename).hasMatch());
+}

tools/auto-tester/src/Test.h

@@ -0,0 +1,46 @@
+//
+//  Test.h
+//  zone/ambientLightInheritence
+//
+//  Created by Nissim Hadar on 2 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_test_h
+#define hifi_test_h
+
+#include <QFileDialog>
+#include <QMessagebox>
+#include <QRegularExpression>
+
+#include "MismatchWindow.h"
+
+class Test {
+public: 
+    Test();
+
+    void evaluateTests();
+    void createTest();
+
+    void createListOfAllJPEGimagesInDirectory();
+
+    bool isInSnapshotFilenameFormat(QString filename);
+    bool isInExpectedImageFilenameFormat(QString filename);
+
+private:
+    QMessageBox messageBox;
+
+    QString pathToImageDirectory;
+    QDir imageDirectory;
+    QStringList sortedImageFilenames;
+
+    QRegularExpression snapshotFilenameFormat;
+    QRegularExpression expectedImageFilenameFormat;
+
+    MismatchWindow mismatchWindow;
+};
+
+#endif // hifi_test_h

tools/auto-tester/src/autoTester.h

@@ -13,24 +13,24 @@
 
 #include <QtWidgets/QMainWindow>
 #include "ui_autoTester.h"
-////#include "Test.h"
-////
-////class AutoTester : public QMainWindow
-////{
-////    Q_OBJECT
-////
-////public:
-////    AutoTester(QWidget *parent = Q_NULLPTR);
-////
-////private slots:
-////    void on_evaluateTestsButton_clicked();
-////    void on_createTestButton_clicked();
-////    void on_closeButton_clicked();
-////
-////private:
-////    Ui::AutoTesterClass ui;
-////
-////    Test test;
-////};
+#include "Test.h"
+
+class AutoTester : public QMainWindow
+{
+    Q_OBJECT
+
+public:
+    AutoTester(QWidget *parent = Q_NULLPTR);
+
+private slots:
+    void on_evaluateTestsButton_clicked();
+    void on_createTestButton_clicked();
+    void on_closeButton_clicked();
+
+private:
+    Ui::AutoTesterClass ui;
+
+    Test test;
+};
 
 #endif // hifi_AutoTester_h

tools/auto-tester/src/common.h

@@ -0,0 +1,37 @@
+//
+//  common.h
+//
+//  Created by Nissim Hadar on 10 Nov 2017.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#ifndef hifi_common_h
+#define hifi_common_h
+
+#include <QString>
+
+class TestFailure {
+public:
+    TestFailure(float error, QString pathname, QString expectedImageFilename, QString resultImageFilename) {
+        _error = error;
+         _pathname = pathname;
+         _expectedImageFilename = expectedImageFilename;
+         _resultImageFilename = resultImageFilename;
+    }
+
+    float _error;
+    QString _pathname;
+    QString _expectedImageFilename;
+    QString _resultImageFilename;
+};
+
+enum UserResponse {
+    USER_RESPONSE_INVALID,
+    USER_RESPONSE_PASS,
+    USE_RESPONSE_FAIL,
+    USER_RESPONSE_ABORT
+};
+
+#endif // hifi_common_h

tools/auto-tester/src/main.cpp

@@ -14,8 +14,8 @@ int main(int argc, char *argv[])
 {
     QApplication application(argc, argv);
 
-////    AutoTester autoTester;
-////    autoTester.show();
+    AutoTester autoTester;
+    autoTester.show();
 
-////    return application.exec();
+    return application.exec();
 }

tools/auto-tester/src/ui/MismatchWindow.ui

@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>MismatchWindow</class>
+ <widget class="QDialog" name="MismatchWindow">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>1585</width>
+    <height>694</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>MismatchWindow</string>
+  </property>
+  <widget class="QLabel" name="expectedImage">
+   <property name="geometry">
+    <rect>
+     <x>20</x>
+     <y>170</y>
+     <width>720</width>
+     <height>362</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>expected image</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="resultImage">
+   <property name="geometry">
+    <rect>
+     <x>760</x>
+     <y>170</y>
+     <width>720</width>
+     <height>362</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>result image</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="resultFilename">
+   <property name="geometry">
+    <rect>
+     <x>760</x>
+     <y>90</y>
+     <width>800</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>result image filename</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="expectedFilename">
+   <property name="geometry">
+    <rect>
+     <x>40</x>
+     <y>90</y>
+     <width>700</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>expected image filename</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="imagePath">
+   <property name="geometry">
+    <rect>
+     <x>40</x>
+     <y>30</y>
+     <width>1200</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>image path</string>
+   </property>
+  </widget>
+  <widget class="QPushButton" name="passTestButton">
+   <property name="geometry">
+    <rect>
+     <x>30</x>
+     <y>600</y>
+     <width>75</width>
+     <height>23</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Pass</string>
+   </property>
+  </widget>
+  <widget class="QPushButton" name="failTestButton">
+   <property name="geometry">
+    <rect>
+     <x>330</x>
+     <y>600</y>
+     <width>75</width>
+     <height>23</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Fail</string>
+   </property>
+  </widget>
+  <widget class="QPushButton" name="abortTestsButton">
+   <property name="geometry">
+    <rect>
+     <x>630</x>
+     <y>600</y>
+     <width>75</width>
+     <height>23</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Abort Tests</string>
+   </property>
+  </widget>
+  <widget class="QLabel" name="errorLabel">
+   <property name="geometry">
+    <rect>
+     <x>960</x>
+     <y>600</y>
+     <width>181</width>
+     <height>28</height>
+    </rect>
+   </property>
+   <property name="font">
+    <font>
+     <pointsize>16</pointsize>
+    </font>
+   </property>
+   <property name="text">
+    <string>error</string>
+   </property>
+  </widget>
+ </widget>
+ <layoutdefault spacing="6" margin="11"/>
+ <resources/>
+ <connections/>
+</ui>