From 7a0d188149d92c969fde559df7bca583c2a434cc Mon Sep 17 00:00:00 2001 From: Nissim Hadar Date: Sat, 11 Nov 2017 18:10:47 -0800 Subject: [PATCH] WIP. --- tools/auto-tester/CMakeLists.txt | 2 +- tools/auto-tester/src/AutoTester.cpp | 32 ++++ tools/auto-tester/src/MismatchWindow.cpp | 51 +++++++ tools/auto-tester/src/MismatchWindow.h | 38 +++++ tools/auto-tester/src/Test.cpp | 163 +++++++++++++++++++++ tools/auto-tester/src/Test.h | 46 ++++++ tools/auto-tester/src/autoTester.h | 38 ++--- tools/auto-tester/src/common.h | 37 +++++ tools/auto-tester/src/main.cpp | 6 +- tools/auto-tester/src/ui/MismatchWindow.ui | 157 ++++++++++++++++++++ 10 files changed, 547 insertions(+), 23 deletions(-) create mode 100644 tools/auto-tester/src/AutoTester.cpp create mode 100644 tools/auto-tester/src/MismatchWindow.cpp create mode 100644 tools/auto-tester/src/MismatchWindow.h create mode 100644 tools/auto-tester/src/Test.cpp create mode 100644 tools/auto-tester/src/Test.h create mode 100644 tools/auto-tester/src/common.h create mode 100644 tools/auto-tester/src/ui/MismatchWindow.ui diff --git a/tools/auto-tester/CMakeLists.txt b/tools/auto-tester/CMakeLists.txt index 832133c284..e59cb192b1 100644 --- a/tools/auto-tester/CMakeLists.txt +++ b/tools/auto-tester/CMakeLists.txt @@ -10,4 +10,4 @@ source_group("UI Files" FILES ${QT_UI_FILES}) qt5_wrap_ui(QT_UI_HEADERS "${QT_UI_FILES}") # add them to the interface source files -set(INTERFACE_SRCS ${INTERFACE_SRCS} "${QT_UI_HEADERS}") +##set(INTERFACE_SRCS ${INTERFACE_SRCS} "${QT_UI_HEADERS}") diff --git a/tools/auto-tester/src/AutoTester.cpp b/tools/auto-tester/src/AutoTester.cpp new file mode 100644 index 0000000000..5117f78ce5 --- /dev/null +++ b/tools/auto-tester/src/AutoTester.cpp @@ -0,0 +1,32 @@ +// +// autoTester.cpp +// zone/ambientLightInheritence +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "AutoTester.h" + +AutoTester::AutoTester(QWidget *parent) + : QMainWindow(parent) +{ + ui.setupUi(this); +} + +void AutoTester::on_closeButton_clicked() +{ + exit(0); +} + +void AutoTester::on_evaluateTestsButton_clicked() +{ + test.evaluateTests(); +} + +void AutoTester::on_createTestButton_clicked() +{ + test.createTest(); +} diff --git a/tools/auto-tester/src/MismatchWindow.cpp b/tools/auto-tester/src/MismatchWindow.cpp new file mode 100644 index 0000000000..43cf1c2c30 --- /dev/null +++ b/tools/auto-tester/src/MismatchWindow.cpp @@ -0,0 +1,51 @@ +// +// MismatchWindow.cpp +// +// Created by Nissim Hadar on 9 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "MismatchWindow.h" + +#include + +MismatchWindow::MismatchWindow(QWidget *parent) + : QDialog(parent) +{ + setupUi(this); + + expectedImage->setScaledContents(true); + resultImage->setScaledContents(true); +} + +void MismatchWindow::setTestFailure(TestFailure testFailure) { + errorLabel->setText("Error: " + QString::number((int)testFailure._error)); + + imagePath->setText("Path to test: " + testFailure._pathname); + + expectedFilename->setText(testFailure._expectedImageFilename); + expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename)); + + resultFilename->setText(testFailure._resultImageFilename); + resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._resultImageFilename)); +} + +void MismatchWindow::on_passTestButton_clicked() +{ + _userResponse = USER_RESPONSE_PASS; + close(); +} + +void MismatchWindow::on_failTestButton_clicked() +{ + _userResponse = USE_RESPONSE_FAIL; + close(); +} + +void MismatchWindow::on_abortTestsButton_clicked() +{ + _userResponse = USER_RESPONSE_ABORT; + close(); +} diff --git a/tools/auto-tester/src/MismatchWindow.h b/tools/auto-tester/src/MismatchWindow.h new file mode 100644 index 0000000000..7aa173cc36 --- /dev/null +++ b/tools/auto-tester/src/MismatchWindow.h @@ -0,0 +1,38 @@ +// +// MismatchWindow.cpp +// +// Created by Nissim Hadar on 9 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_MismatchWindow_h +#define hifi_MismatchWindow_h + +#include "ui_MismatchWindow.h" + +#include "common.h" + +class MismatchWindow : public QDialog, public Ui::MismatchWindow +{ + Q_OBJECT + +public: + MismatchWindow(QWidget *parent = Q_NULLPTR); + + void setTestFailure(TestFailure testFailure); + + UserResponse getUserResponse() { return _userResponse; } + +private slots: + void on_passTestButton_clicked(); + void on_failTestButton_clicked(); + void on_abortTestsButton_clicked(); + +private: + UserResponse _userResponse{ USER_RESPONSE_INVALID }; +}; + + +#endif // hifi_MismatchWindow_h diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp new file mode 100644 index 0000000000..e695ca37c3 --- /dev/null +++ b/tools/auto-tester/src/Test.cpp @@ -0,0 +1,163 @@ +// +// Test.cpp +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +// Images are compared using the ImagMagick command line tool magick.exe +// A number of comparison metrics are available, including: +// AE Absolute error count of the number of different pixels (0=equal) +// +// DSSIM Stuctural dissimilarity index +// +// PAE Peak Absolute error of any one pixel +// +// PSNR Peak Signal to Noise Ratio The ratio of mean square difference to the maximum mean square +// that can exist between any two images, expressed as a decibel value. +// The higher the PSNR the closer the closer the images are, with +// a maximum difference occurring at 1. A PSNR of 20 means +// differences are 1 / 100 of maximum. +// +// MAE Mean Absolute Error average channel error distance +// +// MSE Mean Squared Error average squared error distance +// +// RMSE squareRoot Mean Error sqrt(MSE) +// +#include "Test.h" + +#include +#include + +Test::Test() { + snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg"); + + expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg"); + + mismatchWindow.setModal(true); +} + +void Test::evaluateTests() { + createListOfAllJPEGimagesInDirectory(); + + // Separate images into two lists. The first is the expected images, the second is the test results + // Images that are in the wrong format are ignored. + QStringList expectedImages; + QStringList resultImages; + foreach(QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + expectedImages << fullCurrentFilename; + } else if (isInSnapshotFilenameFormat(currentFilename)) { + resultImages << fullCurrentFilename; + } + } + + // The number of images in each list should be identical + if (expectedImages.length() != resultImages.length()) { + messageBox.critical(0, + "Test failed", + "Found " + QString::number(resultImages.length()) + " images in directory" + + "\nExpected to find " + QString::number(expectedImages.length()) + " images"); + + exit(-1); + } + + // Now loop over both lists and compare each pair of images + // Quit loop if user has aborted due to a failed test. + const float THRESHOLD{ 10.0f }; + bool success{ true }; + bool keepOn{ true }; + for (int i = 0; keepOn && i < expectedImages.length(); ++i) { + QString diffFilename = "hifi_autoTest_diff.txt"; + QString command = "magick.exe compare -metric MAE " + expectedImages[i] + " " + resultImages[i] + " null: 2>" + diffFilename; + system(command.toStdString().c_str()); + + QFile file(diffFilename); + if (!file.open(QIODevice::ReadOnly)) { + messageBox.critical(0, "error", file.errorString()); + } + + // First value on line is the comparison result + QTextStream in(&file); + QString line = in.readLine(); + QStringList tokens = line.split(' '); + float error = tokens[0].toFloat(); + + if (error > THRESHOLD) { + mismatchWindow.setTestFailure(TestFailure{ + error, // value of the error (float) + expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing / + QFileInfo(expectedImages[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(resultImages[i].toStdString().c_str()).fileName() // filename of result image + }); + + mismatchWindow.exec(); + + switch (mismatchWindow.getUserResponse()) { + case USER_RESPONSE_PASS: + break; + case USE_RESPONSE_FAIL: + success = false; + break; + case USER_RESPONSE_ABORT: + keepOn = false; + success = false; + break; + default: + assert(false); + } + } + } + + if (success) { + messageBox.information(0, "Success", "All images are as expected"); + } else { + messageBox.information(0, "Failure", "One or more images are not as expected"); + } +} + +void Test::createTest() { + // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on + // Any existing expected result images will be deleted + createListOfAllJPEGimagesInDirectory(); + + int i = 1; + foreach (QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + if (!QFile::remove(fullCurrentFilename)) { + messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); + exit(-1); + } + } else if (isInSnapshotFilenameFormat(currentFilename)) { + QString newFilename = "ExpectedImage_" + QString::number(i) + ".jpg"; + QString fullNewFileName = pathToImageDirectory + "/" + newFilename; + + imageDirectory.rename(fullCurrentFilename, newFilename); + ++i; + } + } +} + +void Test::createListOfAllJPEGimagesInDirectory() { + // get list of JPEG images in folder, sorted by name + pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + + imageDirectory = QDir(pathToImageDirectory); + QStringList nameFilters; + nameFilters << "*.jpg"; + + sortedImageFilenames = imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); +} + +bool Test::isInSnapshotFilenameFormat(QString filename) { + return (snapshotFilenameFormat.match(filename).hasMatch()); +} + +bool Test::isInExpectedImageFilenameFormat(QString filename) { + return (expectedImageFilenameFormat.match(filename).hasMatch()); +} \ No newline at end of file diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h new file mode 100644 index 0000000000..5f63c71399 --- /dev/null +++ b/tools/auto-tester/src/Test.h @@ -0,0 +1,46 @@ +// +// Test.h +// zone/ambientLightInheritence +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_test_h +#define hifi_test_h + +#include +#include +#include + +#include "MismatchWindow.h" + +class Test { +public: + Test(); + + void evaluateTests(); + void createTest(); + + void createListOfAllJPEGimagesInDirectory(); + + bool isInSnapshotFilenameFormat(QString filename); + bool isInExpectedImageFilenameFormat(QString filename); + +private: + QMessageBox messageBox; + + QString pathToImageDirectory; + QDir imageDirectory; + QStringList sortedImageFilenames; + + QRegularExpression snapshotFilenameFormat; + QRegularExpression expectedImageFilenameFormat; + + MismatchWindow mismatchWindow; +}; + +#endif // hifi_test_h diff --git a/tools/auto-tester/src/autoTester.h b/tools/auto-tester/src/autoTester.h index a03aa3e053..245937a5c7 100644 --- a/tools/auto-tester/src/autoTester.h +++ b/tools/auto-tester/src/autoTester.h @@ -13,24 +13,24 @@ #include #include "ui_autoTester.h" -////#include "Test.h" -//// -////class AutoTester : public QMainWindow -////{ -//// Q_OBJECT -//// -////public: -//// AutoTester(QWidget *parent = Q_NULLPTR); -//// -////private slots: -//// void on_evaluateTestsButton_clicked(); -//// void on_createTestButton_clicked(); -//// void on_closeButton_clicked(); -//// -////private: -//// Ui::AutoTesterClass ui; -//// -//// Test test; -////}; +#include "Test.h" + +class AutoTester : public QMainWindow +{ + Q_OBJECT + +public: + AutoTester(QWidget *parent = Q_NULLPTR); + +private slots: + void on_evaluateTestsButton_clicked(); + void on_createTestButton_clicked(); + void on_closeButton_clicked(); + +private: + Ui::AutoTesterClass ui; + + Test test; +}; #endif // hifi_AutoTester_h \ No newline at end of file diff --git a/tools/auto-tester/src/common.h b/tools/auto-tester/src/common.h new file mode 100644 index 0000000000..cb2a023c37 --- /dev/null +++ b/tools/auto-tester/src/common.h @@ -0,0 +1,37 @@ +// +// common.h +// +// Created by Nissim Hadar on 10 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_common_h +#define hifi_common_h + +#include + +class TestFailure { +public: + TestFailure(float error, QString pathname, QString expectedImageFilename, QString resultImageFilename) { + _error = error; + _pathname = pathname; + _expectedImageFilename = expectedImageFilename; + _resultImageFilename = resultImageFilename; + } + + float _error; + QString _pathname; + QString _expectedImageFilename; + QString _resultImageFilename; +}; + +enum UserResponse { + USER_RESPONSE_INVALID, + USER_RESPONSE_PASS, + USE_RESPONSE_FAIL, + USER_RESPONSE_ABORT +}; + +#endif // hifi_common_h diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index 774ae0a99a..d8b23f5342 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -14,8 +14,8 @@ int main(int argc, char *argv[]) { QApplication application(argc, argv); -//// AutoTester autoTester; -//// autoTester.show(); + AutoTester autoTester; + autoTester.show(); -//// return application.exec(); + return application.exec(); } diff --git a/tools/auto-tester/src/ui/MismatchWindow.ui b/tools/auto-tester/src/ui/MismatchWindow.ui new file mode 100644 index 0000000000..ab3a8d57c8 --- /dev/null +++ b/tools/auto-tester/src/ui/MismatchWindow.ui @@ -0,0 +1,157 @@ + + + MismatchWindow + + + + 0 + 0 + 1585 + 694 + + + + MismatchWindow + + + + + 20 + 170 + 720 + 362 + + + + expected image + + + + + + 760 + 170 + 720 + 362 + + + + result image + + + + + + 760 + 90 + 800 + 28 + + + + + 16 + + + + result image filename + + + + + + 40 + 90 + 700 + 28 + + + + + 16 + + + + expected image filename + + + + + + 40 + 30 + 1200 + 28 + + + + + 16 + + + + image path + + + + + + 30 + 600 + 75 + 23 + + + + Pass + + + + + + 330 + 600 + 75 + 23 + + + + Fail + + + + + + 630 + 600 + 75 + 23 + + + + Abort Tests + + + + + + 960 + 600 + 181 + 28 + + + + + 16 + + + + error + + + + + + +