diff --git a/tools/auto-tester/src/AWSInterface.cpp b/tools/auto-tester/src/AWSInterface.cpp index 9fdebb1f2a..a87a8beb41 100644 --- a/tools/auto-tester/src/AWSInterface.cpp +++ b/tools/auto-tester/src/AWSInterface.cpp @@ -231,13 +231,13 @@ void AWSInterface::closeTable(QTextStream& stream) { stream << "\t\t\n"; } -void AWSInterface::createEntry(int index, const QString& testFailure, QTextStream& stream) { +void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream) { stream << "\t\t\t\n"; stream << "\t\t\t\t

" << QString::number(index) << "

\n"; // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000` // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000` - QStringList failureNameComponents = testFailure.split('/'); + QStringList failureNameComponents = testResult.split('/'); QString failureName = failureNameComponents[failureNameComponents.length() - 1]; stream << "\t\t\t\t\n"; diff --git a/tools/auto-tester/src/AWSInterface.h b/tools/auto-tester/src/AWSInterface.h index b0e23066a2..08df326000 100644 --- a/tools/auto-tester/src/AWSInterface.h +++ b/tools/auto-tester/src/AWSInterface.h @@ -43,7 +43,7 @@ public: void openTable(QTextStream& stream); void closeTable(QTextStream& stream); - void createEntry(int index, const QString& testFailure, QTextStream& stream); + void createEntry(int index, const QString& testResult, QTextStream& stream); void updateAWS(); diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index a9682b1536..0e787d1dbf 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -58,7 +58,8 @@ QString Test::zipAndDeleteTestResultsFolder() { //In all cases, for the next evaluation _testResultsFolderPath = ""; - _index = 1; + _failureIndex = 1; + _successIndex = 1; return zippedResultsFileName; } @@ -90,19 +91,20 @@ int Test::compareImageLists() { similarityIndex = _imageComparer.compareImages(resultImage, expectedImage); } + TestResult testResult = TestResult{ + (float)similarityIndex, + _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) + QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image + }; + if (similarityIndex < THRESHOLD) { ++numberOfFailures; - TestFailure testFailure = TestFailure{ - (float)similarityIndex, - _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) - QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image - QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image - }; - _mismatchWindow.setTestFailure(testFailure); + _mismatchWindow.setTestResult(testResult); if (!isInteractiveMode) { - appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); + appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true); } else { _mismatchWindow.exec(); @@ -110,7 +112,7 @@ int Test::compareImageLists() { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: - appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); + appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true); break; case USER_RESPONSE_ABORT: keepOn = false; @@ -120,6 +122,8 @@ int Test::compareImageLists() { break; } } + } else { + appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), false); } _progressBar->setValue(i); @@ -129,20 +133,32 @@ int Test::compareImageLists() { return numberOfFailures; } -void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { +void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed) { if (!QDir().exists(_testResultsFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } - QString failureFolderPath { _testResultsFolderPath + "/Failure_" + QString::number(_index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) }; - if (!QDir().mkdir(failureFolderPath)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath); + QString resultFolderPath; + if (hasFailed) { + resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + + testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); + + ++_failureIndex; + } else { + resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + + testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4); + + ++_successIndex; + } + + if (!QDir().mkdir(resultFolderPath)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to create folder " + resultFolderPath); exit(-1); } - ++_index; - QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME); + QFile descriptionFile(resultFolderPath + "/" + TEST_RESULTS_FILENAME); if (!descriptionFile.open(QIODevice::ReadWrite)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME); exit(-1); @@ -150,10 +166,10 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa // Create text file describing the failure QTextStream stream(&descriptionFile); - stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/' - stream << "Expected image was " << testFailure._expectedImageFilename << endl; - stream << "Actual image was " << testFailure._actualImageFilename << endl; - stream << "Similarity index was " << testFailure._error << endl; + stream << "Test failed in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/' + stream << "Expected image was " << testResult._expectedImageFilename << endl; + stream << "Actual image was " << testResult._actualImageFilename << endl; + stream << "Similarity index was " << testResult._error << endl; descriptionFile.close(); @@ -161,21 +177,21 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa QString sourceFile; QString destinationFile; - sourceFile = testFailure._pathname + testFailure._expectedImageFilename; - destinationFile = failureFolderPath + "/" + "Expected Image.png"; + sourceFile = testResult._pathname + testResult._expectedImageFilename; + destinationFile = resultFolderPath + "/" + "Expected Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } - sourceFile = testFailure._pathname + testFailure._actualImageFilename; - destinationFile = failureFolderPath + "/" + "Actual Image.png"; + sourceFile = testResult._pathname + testResult._actualImageFilename; + destinationFile = resultFolderPath + "/" + "Actual Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } - comparisonImage.save(failureFolderPath + "/" + "Difference Image.png"); + comparisonImage.save(resultFolderPath + "/" + "Difference Image.png"); } void Test::startTestsEvaluation(const bool isRunningFromCommandLine, diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 28bc7ccad0..f653a91782 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -84,7 +84,7 @@ public: void includeTest(QTextStream& textStream, const QString& testPathname); - void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); + void appendTestResultsToFile(const QString& testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed); bool createTestResultsFolderPath(const QString& directory); QString zipAndDeleteTestResultsFolder(); @@ -120,7 +120,8 @@ private: ImageComparer _imageComparer; QString _testResultsFolderPath; - int _index { 1 }; + int _failureIndex{ 1 }; + int _successIndex{ 1 }; // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) const int NUM_DIGITS { 5 }; diff --git a/tools/auto-tester/src/common.h b/tools/auto-tester/src/common.h index 939814df62..5df4e9c921 100644 --- a/tools/auto-tester/src/common.h +++ b/tools/auto-tester/src/common.h @@ -12,9 +12,9 @@ #include -class TestFailure { +class TestResult { public: - TestFailure(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) : + TestResult(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) : _error(error), _pathname(pathname), _expectedImageFilename(expectedImageFilename), diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 4b3762a3b0..99c1ce8e52 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -36,7 +36,7 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) { _ui.statusLabel->setText(""); _ui.plainTextEdit->setReadOnly(true); - setWindowTitle("Auto Tester - v6.5"); + setWindowTitle("Auto Tester - v6.6"); // Coming soon to an auto-tester near you... //// _helpWindow.textBrowser->setText() diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp index ef6e35ba8a..58189b4795 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.cpp +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -60,20 +60,20 @@ QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultIma return resultPixmap; } -void MismatchWindow::setTestFailure(TestFailure testFailure) { - errorLabel->setText("Similarity: " + QString::number(testFailure._error)); +void MismatchWindow::setTestResult(TestResult testResult) { + errorLabel->setText("Similarity: " + QString::number(testResult._error)); - imagePath->setText("Path to test: " + testFailure._pathname); + imagePath->setText("Path to test: " + testResult._pathname); - expectedFilename->setText(testFailure._expectedImageFilename); - resultFilename->setText(testFailure._actualImageFilename); + expectedFilename->setText(testResult._expectedImageFilename); + resultFilename->setText(testResult._actualImageFilename); - QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename); - QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename); + QPixmap expectedPixmap = QPixmap(testResult._pathname + testResult._expectedImageFilename); + QPixmap actualPixmap = QPixmap(testResult._pathname + testResult._actualImageFilename); _diffPixmap = computeDiffPixmap( - QImage(testFailure._pathname + testFailure._expectedImageFilename), - QImage(testFailure._pathname + testFailure._actualImageFilename) + QImage(testResult._pathname + testResult._expectedImageFilename), + QImage(testResult._pathname + testResult._actualImageFilename) ); expectedImage->setPixmap(expectedPixmap); diff --git a/tools/auto-tester/src/ui/MismatchWindow.h b/tools/auto-tester/src/ui/MismatchWindow.h index f203a2be6a..30c29076b3 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.h +++ b/tools/auto-tester/src/ui/MismatchWindow.h @@ -20,7 +20,7 @@ class MismatchWindow : public QDialog, public Ui::MismatchWindow { public: MismatchWindow(QWidget *parent = Q_NULLPTR); - void setTestFailure(TestFailure testFailure); + void setTestResult(TestResult testResult); UserResponse getUserResponse() { return _userResponse; }