From 2e8fe40da8e5827b6ee3c82de00ac089820582bc Mon Sep 17 00:00:00 2001 From: Nissim Hadar Date: Thu, 30 Nov 2017 13:07:54 -0800 Subject: [PATCH] Recursive test evaluations. --- tools/auto-tester/src/ImageComparer.cpp | 6 +- tools/auto-tester/src/Test.cpp | 136 ++++++++++++++------ tools/auto-tester/src/Test.h | 9 +- tools/auto-tester/src/ui/AutoTester.cpp | 6 +- tools/auto-tester/src/ui/AutoTester.h | 5 +- tools/auto-tester/src/ui/AutoTester.ui | 29 +++-- tools/auto-tester/src/ui/MismatchWindow.cpp | 2 +- 7 files changed, 137 insertions(+), 56 deletions(-) diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 2616f7f1ea..9151255d96 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -26,9 +26,9 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) co const double c2 = (K2 * L) * (K2 * L); // Coefficients for luminosity calculation - const double RED_COEFFICIENT = 0.212655f; - const double GREEN_COEFFICIENT = 0.715158f; - const double BLUE_COEFFICIENT = 0.072187f; + const double RED_COEFFICIENT = 0.212655; + const double GREEN_COEFFICIENT = 0.715158; + const double BLUE_COEFFICIENT = 0.072187; // First go over all full 8x8 blocks // This is done in 3 loops diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 06771adb23..ae5fd33dd5 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -21,34 +21,8 @@ Test::Test() { mismatchWindow.setModal(true); } -void Test::evaluateTests() { - createListOfAllJPEGimagesInDirectory(); - - // Separate images into two lists. The first is the expected images, the second is the test results - // Images that are in the wrong format are ignored. - QStringList expectedImages; - QStringList resultImages; - foreach(QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; - if (isInExpectedImageFilenameFormat(currentFilename)) { - expectedImages << fullCurrentFilename; - } else if (isInSnapshotFilenameFormat(currentFilename)) { - resultImages << fullCurrentFilename; - } - } - - // The number of images in each list should be identical - if (expectedImages.length() != resultImages.length()) { - messageBox.critical(0, - "Test failed", - "Found " + QString::number(resultImages.length()) + " images in directory" + - "\nExpected to find " + QString::number(expectedImages.length()) + " images" - ); - - exit(-1); - } - - // Now loop over both lists and compare each pair of images +bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages) { + // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. const double THRESHOLD{ 0.99999 }; bool success{ true }; @@ -65,7 +39,8 @@ void Test::evaluateTests() { double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical try { similarityIndex = imageComparer.compareImages(resultImage, expectedImage); - } catch (...) { + } + catch (...) { messageBox.critical(0, "Internal error", "Image not in expected format"); exit(-1); } @@ -96,10 +71,98 @@ void Test::evaluateTests() { } } + return success; +} + +void Test::evaluateTests() { + // Get list of JPEG images in folder, sorted by name + QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); + + // Separate images into two lists. The first is the expected images, the second is the test results + // Images that are in the wrong format are ignored. + QStringList expectedImages; + QStringList resultImages; + foreach(QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + expectedImages << fullCurrentFilename; + } else if (isInSnapshotFilenameFormat(currentFilename)) { + resultImages << fullCurrentFilename; + } + } + + // The number of images in each list should be identical + if (expectedImages.length() != resultImages.length()) { + messageBox.critical(0, + "Test failed", + "Found " + QString::number(resultImages.length()) + " images in directory" + + "\nExpected to find " + QString::number(expectedImages.length()) + " images" + ); + + exit(-1); + } + + bool success = compareImageLists(expectedImages, resultImages); + if (success) { messageBox.information(0, "Success", "All images are as expected"); + } else { + messageBox.information(0, "Failure", "One or more images are not as expected"); } - else { +} + +// Two criteria are used to decide if a folder contains valid test results. +// 1) a 'test'js' file exists in the folder +// 2) the folder has the same number of anual and expected images +void Test::evaluateTestsRecursively() { + // Select folder to start recursing from + QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); + + bool success{ true }; + QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + while (it.hasNext()) { + QString directory = it.next(); + if (directory[directory.length() - 1] == '.') { + // ignore '.', '..' directories + continue; + } + + // + const QString testPathname{ directory + "/" + testFilename }; + QFileInfo fileInfo(testPathname); + if (!fileInfo.exists()) { + // Folder does not contain 'test.js' + continue; + } + + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory); + + // Separate images into two lists. The first is the expected images, the second is the test results + // Images that are in the wrong format are ignored. + QStringList expectedImages; + QStringList resultImages; + foreach(QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = directory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + expectedImages << fullCurrentFilename; + } else if (isInSnapshotFilenameFormat(currentFilename)) { + resultImages << fullCurrentFilename; + } + } + + if (expectedImages.length() != resultImages.length()) { + // Number of images doesn't match + continue; + } + + // Set success to false if any test has failed + success &= compareImageLists(expectedImages, resultImages); + } + + if (success) { + messageBox.information(0, "Success", "All images are as expected"); + } else { messageBox.information(0, "Failure", "One or more images are not as expected"); } } @@ -113,7 +176,6 @@ void Test::importTest(QTextStream& textStream, const QString& testPathname, int void Test::createRecursiveScript() { // Select folder to start recursing from QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); - QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); QFile allTestsFilename(topLevelDirectory + "/" + "allTests.js"); if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { @@ -132,7 +194,6 @@ void Test::createRecursiveScript() { // running test has increment a testNumber variable that it received as an input. int testNumber = 1; QVector testPathnames; - const QString testFilename{ "test.js" }; // First test if top-level folder has a test.js file const QString testPathname{ topLevelDirectory + "/" + testFilename }; @@ -145,6 +206,7 @@ void Test::createRecursiveScript() { testPathnames << testPathname; } + QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); if (directory[directory.length() - 1] == '.') { @@ -249,7 +311,8 @@ void Test::createRecursiveScript() { void Test::createTest() { // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on // Any existing expected result images will be deleted - createListOfAllJPEGimagesInDirectory(); + QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); int i = 1; foreach (QString currentFilename, sortedImageFilenames) { @@ -271,15 +334,12 @@ void Test::createTest() { messageBox.information(0, "Success", "Test images have been created"); } -void Test::createListOfAllJPEGimagesInDirectory() { - // Get list of JPEG images in folder, sorted by name - pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); - +QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) { imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; nameFilters << "*.jpg"; - sortedImageFilenames = imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); + return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } bool Test::isInSnapshotFilenameFormat(QString filename) { diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index a7fc412bff..1f7b1e92a7 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -24,10 +24,11 @@ public: Test(); void evaluateTests(); + void evaluateTestsRecursively(); void createRecursiveScript(); void createTest(); - void createListOfAllJPEGimagesInDirectory(); + QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory); bool isInSnapshotFilenameFormat(QString filename); bool isInExpectedImageFilenameFormat(QString filename); @@ -35,11 +36,11 @@ public: void importTest(QTextStream& textStream, const QString& testPathname, int testNumber); private: + const QString testFilename{ "test.js" }; + QMessageBox messageBox; - QString pathToImageDirectory; QDir imageDirectory; - QStringList sortedImageFilenames; QRegularExpression snapshotFilenameFormat; QRegularExpression expectedImageFilenameFormat; @@ -47,6 +48,8 @@ private: MismatchWindow mismatchWindow; ImageComparer imageComparer; + + bool compareImageLists(QStringList expectedImages, QStringList resultImages); }; #endif // hifi_test_h diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index aa18dc4378..46f50f40f7 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -16,12 +16,16 @@ AutoTester::AutoTester(QWidget *parent) ui.setupUi(this); } - void AutoTester::on_evaluateTestsButton_clicked() { test.evaluateTests(); } +void AutoTester::on_evaluateTestsRecursivelyButton_clicked() +{ + test.evaluateTestsRecursively(); +} + void AutoTester::on_createRecursiveScriptButton_clicked() { test.createRecursiveScript(); diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 93eb22af7d..31c26e0ef2 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -23,8 +23,9 @@ public: AutoTester(QWidget *parent = Q_NULLPTR); private slots: - void on_evaluateTestsButton_clicked(); - void on_createRecursiveScriptButton_clicked(); +void on_evaluateTestsButton_clicked(); +void on_evaluateTestsRecursivelyButton_clicked(); +void on_createRecursiveScriptButton_clicked(); void on_createTestButton_clicked(); void on_closeButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index d6c16beeb4..7032ef9710 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 277 - 351 + 286 + 470 @@ -18,8 +18,8 @@ 60 - 240 - 159 + 360 + 160 40 @@ -31,7 +31,7 @@ 60 - 180 + 270 160 40 @@ -50,14 +50,14 @@ - Evaulate Tests + Evaluate Test 60 - 120 + 210 160 40 @@ -66,13 +66,26 @@ Create Recursive Script + + + + 60 + 75 + 160 + 40 + + + + Evaluate Tests Recursively + + 0 0 - 277 + 286 21 diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp index c3fb5023e7..09ad97a596 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.cpp +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -21,7 +21,7 @@ MismatchWindow::MismatchWindow(QWidget *parent) } void MismatchWindow::setTestFailure(TestFailure testFailure) { - errorLabel->setText("Error: " + QString::number((int)testFailure._error)); + errorLabel->setText("Similarity: " + QString::number(testFailure._error)); imagePath->setText("Path to test: " + testFailure._pathname);