From 0f4206d330d47add73fe93c6296f39ff1eb097ef Mon Sep 17 00:00:00 2001 From: Nissim Hadar Date: Wed, 10 Jan 2018 18:21:34 -0800 Subject: [PATCH] Added option to delete old snapshots. --- tools/auto-tester/CMakeLists.txt | 44 ++-- tools/auto-tester/src/ImageComparer.cpp | 8 +- tools/auto-tester/src/Test.cpp | 239 +++++++++++++++++--- tools/auto-tester/src/Test.h | 26 ++- tools/auto-tester/src/common.h | 7 +- tools/auto-tester/src/main.cpp | 2 +- tools/auto-tester/src/ui/AutoTester.cpp | 12 +- tools/auto-tester/src/ui/AutoTester.h | 8 +- tools/auto-tester/src/ui/AutoTester.ui | 80 +++++-- tools/auto-tester/src/ui/MismatchWindow.cpp | 56 ++++- tools/auto-tester/src/ui/MismatchWindow.h | 7 +- tools/auto-tester/src/ui/MismatchWindow.ui | 105 ++++++--- 12 files changed, 459 insertions(+), 135 deletions(-) diff --git a/tools/auto-tester/CMakeLists.txt b/tools/auto-tester/CMakeLists.txt index e5f2c1fb97..a875f5676a 100644 --- a/tools/auto-tester/CMakeLists.txt +++ b/tools/auto-tester/CMakeLists.txt @@ -1,47 +1,41 @@ -set(TARGET_NAME auto-tester) +set (TARGET_NAME auto-tester) project(${TARGET_NAME}) # Automatically run UIC and MOC. This replaces the older WRAP macros -SET(CMAKE_AUTOUIC ON) -SET(CMAKE_AUTOMOC ON) +SET (CMAKE_AUTOUIC ON) +SET (CMAKE_AUTOMOC ON) -setup_hifi_project(Core Widgets) -link_hifi_libraries() +setup_hifi_project (Core Widgets) +link_hifi_libraries () # FIX: Qt was built with -reduce-relocations if (Qt5_POSITION_INDEPENDENT_CODE) - SET(CMAKE_POSITION_INDEPENDENT_CODE ON) + SET (CMAKE_POSITION_INDEPENDENT_CODE ON) endif() # Qt includes -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${Qt5Core_INCLUDE_DIRS}) -include_directories(${Qt5Widgets_INCLUDE_DIRS}) +include_directories (${CMAKE_CURRENT_SOURCE_DIR}) +include_directories (${Qt5Core_INCLUDE_DIRS}) +include_directories (${Qt5Widgets_INCLUDE_DIRS}) -set(QT_LIBRARIES Qt5::Core Qt5::Widgets) - -# Find all sources files -file (GLOB_RECURSE SOURCES src/*.cpp) -file (GLOB_RECURSE HEADERS src/*.h) -file (GLOB_RECURSE UIS src/ui/*.ui) +set (QT_LIBRARIES Qt5::Core Qt5::Widgets) if (WIN32) # Do not show Console - set_property(TARGET auto-tester PROPERTY WIN32_EXECUTABLE true) + set_property (TARGET auto-tester PROPERTY WIN32_EXECUTABLE true) endif() -add_executable(PROJECT_NAME ${SOURCES} ${HEADERS} ${UIS}) +target_zlib() +add_dependency_external_projects (quazip) +find_package (QuaZip REQUIRED) +target_include_directories( ${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS}) +target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES}) -target_link_libraries(PROJECT_NAME ${QT_LIBRARIES}) - -# Copy required dll's. -add_custom_command(TARGET auto-tester POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $ - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $ - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $ -) +package_libraries_for_deployment() if (WIN32) + add_paths_to_fixup_libs (${QUAZIP_DLL_PATH}) + find_program(WINDEPLOYQT_COMMAND windeployqt PATHS ${QT_DIR}/bin NO_DEFAULT_PATH) if (NOT WINDEPLOYQT_COMMAND) diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 121c98e16e..94b95a5ab6 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -8,6 +8,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include "ImageComparer.h" +#include "common.h" #include @@ -26,11 +27,6 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) co const double c1 = pow((K1 * L), 2); const double c2 = pow((K2 * L), 2); - // Coefficients for luminosity calculation - const double R_Y = 0.212655f; - const double G_Y = 0.715158f; - const double B_Y = 0.072187f; - // First go over all full 8x8 blocks // This is done in 3 loops // 1) Read the pixels into a linear array (an optimization) @@ -116,4 +112,4 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) co } return ssim / windowCounter; -}; +}; \ No newline at end of file diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 8cb36fcfca..6c637ab404 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -13,18 +13,62 @@ #include #include +#include +#include + Test::Test() { - snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg"); + snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.*-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg"); expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg"); mismatchWindow.setModal(true); } -bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages) { +bool Test::createTestResultsFolderPathIfNeeded(QString directory) { + // The test results folder is located in the root of the tests (i.e. for recursive test evaluation) + if (testResultsFolderPath == "") { + testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER; + QDir testResultsFolder(testResultsFolderPath); + + if (testResultsFolder.exists()) { + testResultsFolder.removeRecursively(); + } + + // Create a new test results folder + return QDir().mkdir(testResultsFolderPath); + } else { + return true; + } +} + +void Test::zipAndDeleteTestResultsFolder() { + QString zippedResultsFileName { testResultsFolderPath + ".zip" }; + QFileInfo fileInfo(zippedResultsFileName); + if (!fileInfo.exists()) { + QFile::remove(zippedResultsFileName); + } + + QDir testResultsFolder(testResultsFolderPath); + if (!testResultsFolder.isEmpty()) { + JlCompress::compressDir(testResultsFolderPath + ".zip", testResultsFolderPath); + } + + testResultsFolder.removeRecursively(); + + //In all cases, for the next evaluation + testResultsFolderPath = ""; + index = 1; +} + +bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar) { + progressBar->setMinimum(0); + progressBar->setMaximum(expectedImages.length() - 1); + progressBar->setValue(0); + progressBar->setVisible(true); + // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. - const double THRESHOLD{ 0.999 }; + const double THRESHOLD { 0.999 }; bool success{ true }; bool keepOn{ true }; for (int i = 0; keepOn && i < expectedImages.length(); ++i) { @@ -45,42 +89,107 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage } if (similarityIndex < THRESHOLD) { - mismatchWindow.setTestFailure(TestFailure{ + TestFailure testFailure = TestFailure{ (float)similarityIndex, expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /) QFileInfo(expectedImages[i].toStdString().c_str()).fileName(), // filename of expected image QFileInfo(resultImages[i].toStdString().c_str()).fileName() // filename of result image - }); + }; - mismatchWindow.exec(); + mismatchWindow.setTestFailure(testFailure); - switch (mismatchWindow.getUserResponse()) { - case USER_RESPONSE_PASS: - break; - case USE_RESPONSE_FAIL: - success = false; - break; - case USER_RESPONSE_ABORT: - keepOn = false; - success = false; - break; - default: - assert(false); - break; + if (!interactiveMode) { + appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); + success = false; + } else { + mismatchWindow.exec(); + + switch (mismatchWindow.getUserResponse()) { + case USER_RESPONSE_PASS: + break; + case USE_RESPONSE_FAIL: + appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); + success = false; + break; + case USER_RESPONSE_ABORT: + keepOn = false; + success = false; + break; + default: + assert(false); + break; + } } } + + progressBar->setValue(i); } + progressBar->setVisible(false); return success; } -void Test::evaluateTests() { +void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { + if (!QDir().exists(testResultsFolderPath)) { + messageBox.critical(0, "Internal error", "Folder " + testResultsFolderPath + " not found"); + exit(-1); + } + + QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) }; + if (!QDir().mkdir(failureFolderPath)) { + messageBox.critical(0, "Internal error", "Failed to create folder " + failureFolderPath); + exit(-1); + } + ++index; + + QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME); + if (!descriptionFile.open(QIODevice::ReadWrite)) { + messageBox.critical(0, "Internal error", "Failed to create file " + TEST_RESULTS_FILENAME); + exit(-1); + } + + // Create text file describing the failure + QTextStream stream(&descriptionFile); + stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/' + stream << "Expected image was " << testFailure._expectedImageFilename << endl; + stream << "Actual image was " << testFailure._actualImageFilename << endl; + stream << "Similarity index was " << testFailure._error << endl; + + descriptionFile.close(); + + // Copy expected and actual images, and save the difference image + QString sourceFile; + QString destinationFile; + + sourceFile = testFailure._pathname + testFailure._expectedImageFilename; + destinationFile = failureFolderPath + "/" + "Expected Image.jpg"; + if (!QFile::copy(sourceFile, destinationFile)) { + messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile); + exit(-1); + } + + sourceFile = testFailure._pathname + testFailure._actualImageFilename; + destinationFile = failureFolderPath + "/" + "Actual Image.jpg"; + if (!QFile::copy(sourceFile, destinationFile)) { + messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile); + exit(-1); + } + + comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg"); +} + +void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { // Get list of JPEG images in folder, sorted by name QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); if (pathToImageDirectory == "") { return; } + // Leave if test results folder could not be created + if (!createTestResultsFolderPathIfNeeded(pathToImageDirectory)) { + return; + } + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); // Separate images into two lists. The first is the expected images, the second is the test results @@ -107,36 +216,57 @@ void Test::evaluateTests() { exit(-1); } - bool success = compareImageLists(expectedImages, resultImages); + bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar); if (success) { messageBox.information(0, "Success", "All images are as expected"); } else { messageBox.information(0, "Failure", "One or more images are not as expected"); } + + zipAndDeleteTestResultsFolder(); +} + +bool Test::isAValidDirectory(QString pathname) { + // Only process directories + QDir dir(pathname); + if (!dir.exists()) { + return false; + } + + // Ignore '.', '..' directories + if (pathname[pathname.length() - 1] == '.') { + return false; + } + + return true; } // Two criteria are used to decide if a folder contains valid test results. // 1) a 'test'js' file exists in the folder // 2) the folder has the same number of actual and expected images -void Test::evaluateTestsRecursively() { +void Test::evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar) { // Select folder to start recursing from QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); if (topLevelDirectory == "") { return; } + // Leave if test results folder could not be created + if (!createTestResultsFolderPathIfNeeded(topLevelDirectory)) { + return; + } + bool success{ true }; QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); - if (directory[directory.length() - 1] == '.') { - // ignore '.', '..' directories + + if (!isAValidDirectory(directory)) { continue; } - // - const QString testPathname{ directory + "/" + testFilename }; + const QString testPathname{ directory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (!fileInfo.exists()) { // Folder does not contain 'test.js' @@ -164,7 +294,7 @@ void Test::evaluateTestsRecursively() { } // Set success to false if any test has failed - success &= compareImageLists(expectedImages, resultImages); + success &= compareImageLists(expectedImages, resultImages, directory, interactiveMode, progressBar); } if (success) { @@ -172,6 +302,8 @@ void Test::evaluateTestsRecursively() { } else { messageBox.information(0, "Failure", "One or more images are not as expected"); } + + zipAndDeleteTestResultsFolder(); } void Test::importTest(QTextStream& textStream, const QString& testPathname, int testNumber) { @@ -191,7 +323,8 @@ void Test::createRecursiveScript() { if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { messageBox.critical(0, "Internal Error", - "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\""); + "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\"" + ); exit(-1); } @@ -206,7 +339,7 @@ void Test::createRecursiveScript() { QVector testPathnames; // First test if top-level folder has a test.js file - const QString testPathname{ topLevelDirectory + "/" + testFilename }; + const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { // Current folder contains a test @@ -219,12 +352,14 @@ void Test::createRecursiveScript() { QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); - if (directory[directory.length() - 1] == '.') { - // ignore '.', '..' directories + + // Only process directories + QDir dir(directory); + if (!isAValidDirectory(directory)) { continue; } - const QString testPathname{ directory + "/" + testFilename }; + const QString testPathname{ directory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { // Current folder contains a test @@ -264,7 +399,7 @@ void Test::createRecursiveScript() { // The script produced will look as follows: // if (test1HasNotStarted) { // test1HasNotStarted = false; - // test1.test(); + // test1.test("auto"); // print("******started test 1******"); // } // | @@ -287,7 +422,7 @@ void Test::createRecursiveScript() { textStream << tab << tab << "if (test" << i - 1 << ".complete && test" << i << "HasNotStarted) {" << endl; } textStream << tab << tab << tab << "test" << i << "HasNotStarted = false;" << endl; - textStream << tab << tab << tab << "test" << i << "." << testFunction << "();" << endl; + textStream << tab << tab << tab << "test" << i << "." << testFunction << "(\"auto\");" << endl; textStream << tab << tab << tab << "print(\"******started test " << i << "******\");" << endl; textStream << tab << tab << "}" << endl << endl; @@ -366,6 +501,41 @@ void Test::createTest() { messageBox.information(0, "Success", "Test images have been created"); } +void Test::deleteOldSnapshots() { + // Select folder to start recursing from + QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select root folder for snapshot deletion", ".", QFileDialog::ShowDirsOnly); + if (topLevelDirectory == "") { + return; + } + + // Recurse over folders + QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + while (it.hasNext()) { + QString directory = it.next(); + + // Only process directories + QDir dir(directory); + if (!isAValidDirectory(directory)) { + continue; + } + + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory); + + // Delete any file that is a snapshot (NOT the Expected Images) + QStringList expectedImages; + QStringList resultImages; + foreach(QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = directory + "/" + currentFilename; + if (isInSnapshotFilenameFormat(currentFilename)) { + if (!QFile::remove(fullCurrentFilename)) { + messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nSnapshot deletion aborted"); + exit(-1); + } + } + } + } +} + QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) { imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; @@ -374,6 +544,7 @@ QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirect return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } +// Use regular expressions to check if files are in specific format bool Test::isInSnapshotFilenameFormat(QString filename) { return (snapshotFilenameFormat.match(filename).hasMatch()); } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 1f7b1e92a7..aa1346fa2a 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -1,6 +1,5 @@ // // Test.h -// zone/ambientLightInheritence // // Created by Nissim Hadar on 2 Nov 2017. // Copyright 2013 High Fidelity, Inc. @@ -15,6 +14,7 @@ #include #include #include +#include #include "ImageComparer.h" #include "ui/MismatchWindow.h" @@ -23,10 +23,13 @@ class Test { public: Test(); - void evaluateTests(); - void evaluateTestsRecursively(); + void evaluateTests(bool interactiveMode, QProgressBar* progressBar); + void evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar); void createRecursiveScript(); void createTest(); + void deleteOldSnapshots(); + + bool compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar); QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory); @@ -35,8 +38,17 @@ public: void importTest(QTextStream& textStream, const QString& testPathname, int testNumber); + void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); + + bool createTestResultsFolderPathIfNeeded(QString directory); + void zipAndDeleteTestResultsFolder(); + + bool isAValidDirectory(QString pathname); + private: - const QString testFilename{ "test.js" }; + const QString TEST_FILENAME { "test.js" }; + const QString TEST_RESULTS_FOLDER { "TestResults" }; + const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; QMessageBox messageBox; @@ -49,7 +61,9 @@ private: ImageComparer imageComparer; - bool compareImageLists(QStringList expectedImages, QStringList resultImages); + + QString testResultsFolderPath { "" }; + int index { 1 }; }; -#endif // hifi_test_h +#endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/common.h b/tools/auto-tester/src/common.h index 126177358f..939814df62 100644 --- a/tools/auto-tester/src/common.h +++ b/tools/auto-tester/src/common.h @@ -34,4 +34,9 @@ enum UserResponse { USER_RESPONSE_ABORT }; -#endif // hifi_common_h +// Coefficients for luminosity calculation +const double R_Y = 0.212655f; +const double G_Y = 0.715158f; +const double B_Y = 0.072187f; + +#endif // hifi_common_h \ No newline at end of file diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index 6e5e06b732..45a3743482 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -17,4 +17,4 @@ int main(int argc, char *argv[]) { autoTester.show(); return application.exec(); -} +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 105baddb92..2834ff81e0 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -12,14 +12,18 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { ui.setupUi(this); + + ui.checkBoxInteractiveMode->setChecked(true); + + ui.progressBar->setVisible(false); } void AutoTester::on_evaluateTestsButton_clicked() { - test.evaluateTests(); + test.evaluateTests(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); } void AutoTester::on_evaluateTestsRecursivelyButton_clicked() { - test.evaluateTestsRecursively(); + test.evaluateTestsRecursively(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); } void AutoTester::on_createRecursiveScriptButton_clicked() { @@ -30,6 +34,10 @@ void AutoTester::on_createTestButton_clicked() { test.createTest(); } +void AutoTester::on_deleteOldSnapshotsButton_clicked() { + test.deleteOldSnapshots(); +} + void AutoTester::on_closeButton_clicked() { exit(0); } \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index acfea32ba1..35f609a89d 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -1,6 +1,5 @@ // // AutoTester.h -// zone/ambientLightInheritence // // Created by Nissim Hadar on 2 Nov 2017. // Copyright 2013 High Fidelity, Inc. @@ -22,10 +21,11 @@ public: AutoTester(QWidget *parent = Q_NULLPTR); private slots: -void on_evaluateTestsButton_clicked(); -void on_evaluateTestsRecursivelyButton_clicked(); -void on_createRecursiveScriptButton_clicked(); + void on_evaluateTestsButton_clicked(); + void on_evaluateTestsRecursivelyButton_clicked(); + void on_createRecursiveScriptButton_clicked(); void on_createTestButton_clicked(); + void on_deleteOldSnapshotsButton_clicked(); void on_closeButton_clicked(); private: diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 7032ef9710..d06255acf6 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 286 - 470 + 607 + 395 @@ -17,9 +17,9 @@ - 60 - 360 - 160 + 190 + 300 + 220 40 @@ -30,9 +30,9 @@ - 60 - 270 - 160 + 360 + 130 + 220 40 @@ -43,9 +43,9 @@ - 60 - 20 - 160 + 20 + 75 + 220 40 @@ -56,9 +56,9 @@ - 60 - 210 - 160 + 360 + 75 + 220 40 @@ -69,9 +69,9 @@ - 60 - 75 - 160 + 20 + 130 + 220 40 @@ -79,13 +79,55 @@ Evaluate Tests Recursively + + + + 23 + 40 + 131 + 20 + + + + <html><head/><body><p>If unchecked, will not show results during evaluation</p></body></html> + + + Interactive Mode + + + + + + 20 + 190 + 255 + 23 + + + + 24 + + + + + + 360 + 240 + 220 + 40 + + + + Delete Old Snapshots + + 0 0 - 286 + 607 21 @@ -103,4 +145,4 @@ - + \ No newline at end of file diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp index 07664a1667..d880a1abdc 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.cpp +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -11,11 +11,48 @@ #include +#include + MismatchWindow::MismatchWindow(QWidget *parent) : QDialog(parent) { setupUi(this); expectedImage->setScaledContents(true); resultImage->setScaledContents(true); + diffImage->setScaledContents(true); +} + +QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultImage) { + // This is an optimization, as QImage.setPixel() is embarrassingly slow + unsigned char* buffer = new unsigned char[expectedImage.height() * expectedImage.width() * 3]; + + // loop over each pixel + for (int y = 0; y < expectedImage.height(); ++y) { + for (int x = 0; x < expectedImage.width(); ++x) { + QRgb pixelP = expectedImage.pixel(QPoint(x, y)); + QRgb pixelQ = resultImage.pixel(QPoint(x, y)); + + // Convert to luminance + double p = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP); + double q = R_Y * qRed(pixelQ) + G_Y * qGreen(pixelQ) + B_Y * qBlue(pixelQ); + + // The intensity value is modified to increase the brightness of the displayed image + double absoluteDifference = fabs(p - q) / 255.0; + double modifiedDifference = sqrt(absoluteDifference); + + int difference = (int)(modifiedDifference * 255.0); + + buffer[3 * (x + y * expectedImage.width()) + 0] = difference; + buffer[3 * (x + y * expectedImage.width()) + 1] = difference; + buffer[3 * (x + y * expectedImage.width()) + 2] = difference; + } + } + + QImage diffImage(buffer, expectedImage.width(), expectedImage.height(), QImage::Format_RGB888); + QPixmap resultPixmap = QPixmap::fromImage(diffImage); + + delete[] buffer; + + return resultPixmap; } void MismatchWindow::setTestFailure(TestFailure testFailure) { @@ -24,10 +61,19 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) { imagePath->setText("Path to test: " + testFailure._pathname); expectedFilename->setText(testFailure._expectedImageFilename); - expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename)); - resultFilename->setText(testFailure._actualImageFilename); - resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._actualImageFilename)); + + QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename); + QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename); + + diffPixmap = computeDiffPixmap( + QImage(testFailure._pathname + testFailure._expectedImageFilename), + QImage(testFailure._pathname + testFailure._actualImageFilename) + ); + + expectedImage->setPixmap(expectedPixmap); + resultImage->setPixmap(actualPixmap); + diffImage->setPixmap(diffPixmap); } void MismatchWindow::on_passTestButton_clicked() { @@ -44,3 +90,7 @@ void MismatchWindow::on_abortTestsButton_clicked() { _userResponse = USER_RESPONSE_ABORT; close(); } + +QPixmap MismatchWindow::getComparisonImage() { + return diffPixmap; +} diff --git a/tools/auto-tester/src/ui/MismatchWindow.h b/tools/auto-tester/src/ui/MismatchWindow.h index 7c72b7b0b7..cdbdcb4098 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.h +++ b/tools/auto-tester/src/ui/MismatchWindow.h @@ -25,6 +25,9 @@ public: UserResponse getUserResponse() { return _userResponse; } + QPixmap computeDiffPixmap(QImage expectedImage, QImage resultImage); + QPixmap getComparisonImage(); + private slots: void on_passTestButton_clicked(); void on_failTestButton_clicked(); @@ -32,7 +35,9 @@ private slots: private: UserResponse _userResponse{ USER_RESPONSE_INVALID }; + + QPixmap diffPixmap; }; -#endif // hifi_MismatchWindow_h +#endif // hifi_MismatchWindow_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/MismatchWindow.ui b/tools/auto-tester/src/ui/MismatchWindow.ui index cab6c61e1c..72f86261ab 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.ui +++ b/tools/auto-tester/src/ui/MismatchWindow.ui @@ -6,8 +6,8 @@ 0 0 - 1585 - 694 + 1782 + 942 @@ -16,10 +16,10 @@ - 20 - 170 - 720 - 362 + 10 + 25 + 800 + 450 @@ -29,28 +29,41 @@ - 760 - 170 - 720 - 362 + 900 + 25 + 800 + 450 result image + + + + 540 + 480 + 800 + 450 + + + + diff image + + - 760 - 90 - 800 + 60 + 660 + 480 28 - 16 + 12 @@ -60,15 +73,15 @@ - 40 - 90 - 700 + 60 + 630 + 480 28 - 16 + 12 @@ -78,15 +91,15 @@ - 40 - 30 + 20 + 600 1200 28 - 16 + 12 @@ -97,7 +110,7 @@ 30 - 600 + 790 75 23 @@ -109,8 +122,8 @@ - 330 - 600 + 120 + 790 75 23 @@ -122,36 +135,62 @@ - 630 - 600 - 75 + 210 + 790 + 121 23 - Abort Tests + Abort current test - 810 - 600 - 720 + 30 + 850 + 500 28 - 16 + 12 similarity + + + + 30 + 5 + 151 + 16 + + + + Expected Image + + + + + + 930 + 5 + 151 + 16 + + + + Actual Image + + - + \ No newline at end of file