diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index d9269e3391..25477b3b91 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -22,10 +22,10 @@ Test::Test() { } bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory) { - // Delete any previous test results, if user agrees - QString s = testDirectory + "/" + testResultsFolder; - QFileInfo fileInfo(testDirectory + "/" + testResultsFolder); - while (fileInfo.exists()) { + // If a previous test results folder is found then wait for the user to delete it, or cancel + // (e.g. the user may want to move the folder elsewhere) + QString testResultsFolderPath { testDirectory + "/" + testResultsFolder }; + while (QDir().exists(testResultsFolderPath)) { messageBox.setText("Previous test results have been found"); messageBox.setInformativeText("Delete " + testResultsFolder + " before continuing"); messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel); @@ -35,9 +35,12 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage } } + // Create a new test results folder + QDir().mkdir(testResultsFolderPath); + // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. - const double THRESHOLD{ 0.999 }; + const double THRESHOLD { 0.999 }; bool success{ true }; bool keepOn{ true }; for (int i = 0; keepOn && i < expectedImages.length(); ++i) { @@ -73,7 +76,7 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: - appendTestResultsToFile(testDirectory, testFailure); + appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); success = false; break; case USER_RESPONSE_ABORT: @@ -90,11 +93,55 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage return success; } -void Test::appendTestResultsToFile(QString testDirectory, TestFailure testFailure) { - QFileInfo fileInfo(testResultsFileName); - if (!fileInfo.exists()) { +void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { + if (!QDir().exists(testResultsFolderPath)) { + messageBox.critical(0, "Internal error", "Folder " + testResultsFolderPath + " not found"); + exit(-1); } + static int index = 1; + QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) }; + if (!QDir().mkdir(failureFolderPath)) { + messageBox.critical(0, "Internal error", "Failed to create folder " + failureFolderPath); + exit(-1); + } + ++index; + + QString descriptionFileName { "ReadMe.txt" }; + QFile descriptionFile(failureFolderPath + "/" +descriptionFileName); + if (!descriptionFile.open(QIODevice::ReadWrite)) { + messageBox.critical(0, "Internal error", "Failed to create file " + descriptionFileName); + exit(-1); + } + + // Create text file describing the failure + QTextStream stream(&descriptionFile); + stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/' + stream << "Expected image was " << testFailure._expectedImageFilename << endl; + stream << "Actual image was " << testFailure._actualImageFilename << endl; + stream << "Similarity index was " << testFailure._error << endl; + + descriptionFile.close(); + + // Copy expected and actual images, and save the difference image + QString sourceFile; + QString destinationFile; + + sourceFile = testFailure._pathname + testFailure._expectedImageFilename; + destinationFile = failureFolderPath + "/" + "Expected Image.jpg"; + if (!QFile::copy(sourceFile, destinationFile)) { + messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile); + exit(-1); + } + + sourceFile = testFailure._pathname + testFailure._actualImageFilename; + destinationFile = failureFolderPath + "/" + "Actual Image.jpg"; + if (!QFile::copy(sourceFile, destinationFile)) { + messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile); + exit(-1); + } + + comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg"); } void Test::evaluateTests() { diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index aa71aa5ba4..73814c2311 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -35,7 +35,7 @@ public: void importTest(QTextStream& textStream, const QString& testPathname, int testNumber); - void appendTestResultsToFile(QString testDirectory, TestFailure testFailure); + void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); private: const QString testFilename{ "test.js" }; diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp index ec6dd9ac82..711ead4fbe 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.cpp +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -33,11 +33,15 @@ QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultIma double p = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP); double q = R_Y * qRed(pixelQ) + G_Y * qGreen(pixelQ) + B_Y * qBlue(pixelQ); - int absDiff = (int)(fabs(p - q)); - - buffer[3 * (x + y * expectedImage.width()) + 0] = absDiff; - buffer[3 * (x + y * expectedImage.width()) + 1] = absDiff; - buffer[3 * (x + y * expectedImage.width()) + 2] = absDiff; + // The intensity value is modified to increase the brightness of the displayed image + double absoluteDifference = fabs(p - q) / 255.0; + double modifiedDifference = pow(absoluteDifference, 0.5); + + int difference = (int)(modifiedDifference * 255.0); + + buffer[3 * (x + y * expectedImage.width()) + 0] = difference; + buffer[3 * (x + y * expectedImage.width()) + 1] = difference; + buffer[3 * (x + y * expectedImage.width()) + 2] = difference; } } @@ -60,7 +64,7 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) { QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename); QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename); - QPixmap diffPixmap = computeDiffPixmap( + diffPixmap = computeDiffPixmap( QImage(testFailure._pathname + testFailure._expectedImageFilename), QImage(testFailure._pathname + testFailure._actualImageFilename) ); @@ -84,3 +88,7 @@ void MismatchWindow::on_abortTestsButton_clicked() { _userResponse = USER_RESPONSE_ABORT; close(); } + +QPixmap MismatchWindow::getComparisonImage() { + return diffPixmap; +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/MismatchWindow.h b/tools/auto-tester/src/ui/MismatchWindow.h index af18832f2a..ad8be16580 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.h +++ b/tools/auto-tester/src/ui/MismatchWindow.h @@ -26,6 +26,7 @@ public: UserResponse getUserResponse() { return _userResponse; } QPixmap computeDiffPixmap(QImage expectedImage, QImage resultImage); + QPixmap getComparisonImage(); private slots: void on_passTestButton_clicked(); @@ -34,6 +35,8 @@ private slots: private: UserResponse _userResponse{ USER_RESPONSE_INVALID }; + + QPixmap diffPixmap; }; diff --git a/tools/auto-tester/src/ui/MismatchWindow.ui b/tools/auto-tester/src/ui/MismatchWindow.ui index 090121c277..392bc1774b 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.ui +++ b/tools/auto-tester/src/ui/MismatchWindow.ui @@ -17,7 +17,7 @@ 10 - 20 + 25 800 450 @@ -30,7 +30,7 @@ 900 - 20 + 25 800 450 @@ -163,6 +163,32 @@ similarity + + + + 30 + 5 + 151 + 16 + + + + Expected Image + + + + + + 930 + 5 + 151 + 16 + + + + Actual Image + +