Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-06 20:58:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

TesftFailure (etal.) changed to TestResult.

Browse source 
Successes are also added to the zipped results folder.
This commit is contained in:
NissimHadar 2018-10-09 16:54:38 -07:00
parent 9b816321af
commit 697e678612
8 changed files with 60 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
tools/auto-tester/src/AWSInterface.cpp
View file

@ -231,13 +231,13 @@ void AWSInterface::closeTable(QTextStream& stream) {
stream << "\t\t</table>\n"; stream << "\t\t</table>\n";
} }
void AWSInterface::createEntry(int index, const QString& testFailure, QTextStream& stream) { void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream) {
stream << "\t\t\t<tr>\n"; stream << "\t\t\t<tr>\n";
stream << "\t\t\t\t<td><h1>" << QString::number(index) << "</h1></td>\n"; stream << "\t\t\t\t<td><h1>" << QString::number(index) << "</h1></td>\n";
// For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000` // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000`
// we need `Failure_1--tests.engine.interaction.pick.collision.many.00000` // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000`
QStringList failureNameComponents = testFailure.split('/'); QStringList failureNameComponents = testResult.split('/');
QString failureName = failureNameComponents[failureNameComponents.length() - 1]; QString failureName = failureNameComponents[failureNameComponents.length() - 1];
stream << "\t\t\t\t<td><img src=\"./" << FAILURE_FOLDER << "/" << failureName << "/Actual Image.png\" width = \"576\" height = \"324\" ></td>\n"; stream << "\t\t\t\t<td><img src=\"./" << FAILURE_FOLDER << "/" << failureName << "/Actual Image.png\" width = \"576\" height = \"324\" ></td>\n";

2
tools/auto-tester/src/AWSInterface.h
View file

@ -43,7 +43,7 @@ public:
void openTable(QTextStream& stream); void openTable(QTextStream& stream);
void closeTable(QTextStream& stream); void closeTable(QTextStream& stream);
void createEntry(int index, const QString& testFailure, QTextStream& stream); void createEntry(int index, const QString& testResult, QTextStream& stream);
void updateAWS(); void updateAWS();

66
tools/auto-tester/src/Test.cpp
View file

@ -58,7 +58,8 @@ QString Test::zipAndDeleteTestResultsFolder() {
//In all cases, for the next evaluation //In all cases, for the next evaluation
_testResultsFolderPath = ""; _testResultsFolderPath = "";
_index = 1; _failureIndex = 1;
_successIndex = 1;
return zippedResultsFileName; return zippedResultsFileName;
} }
@ -90,19 +91,20 @@ int Test::compareImageLists() {
similarityIndex = _imageComparer.compareImages(resultImage, expectedImage); similarityIndex = _imageComparer.compareImages(resultImage, expectedImage);
} }
TestResult testResult = TestResult{
(float)similarityIndex,
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
};
if (similarityIndex < THRESHOLD) { if (similarityIndex < THRESHOLD) {
++numberOfFailures; ++numberOfFailures;
TestFailure testFailure = TestFailure{
(float)similarityIndex,
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
};
_mismatchWindow.setTestFailure(testFailure); _mismatchWindow.setTestResult(testResult);
if (!isInteractiveMode) { if (!isInteractiveMode) {
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true);
} else { } else {
_mismatchWindow.exec(); _mismatchWindow.exec();
@ -110,7 +112,7 @@ int Test::compareImageLists() {
case USER_RESPONSE_PASS: case USER_RESPONSE_PASS:
break; break;
case USE_RESPONSE_FAIL: case USE_RESPONSE_FAIL:
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true);
break; break;
case USER_RESPONSE_ABORT: case USER_RESPONSE_ABORT:
keepOn = false; keepOn = false;
@ -120,6 +122,8 @@ int Test::compareImageLists() {
break; break;
} }
} }
} else {
appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), false);
} }
_progressBar->setValue(i); _progressBar->setValue(i);
@ -129,20 +133,32 @@ int Test::compareImageLists() {
return numberOfFailures; return numberOfFailures;
} }
void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed) {
if (!QDir().exists(_testResultsFolderPath)) { if (!QDir().exists(_testResultsFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found");
exit(-1); exit(-1);
} }
QString failureFolderPath { _testResultsFolderPath + "/Failure_" + QString::number(_index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) }; QString resultFolderPath;
if (!QDir().mkdir(failureFolderPath)) { if (hasFailed) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath); resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" +
testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4);
++_failureIndex;
} else {
resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" +
testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4);
++_successIndex;
}
if (!QDir().mkdir(resultFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to create folder " + resultFolderPath);
exit(-1); exit(-1);
} }
++_index;
QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME); QFile descriptionFile(resultFolderPath + "/" + TEST_RESULTS_FILENAME);
if (!descriptionFile.open(QIODevice::ReadWrite)) { if (!descriptionFile.open(QIODevice::ReadWrite)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME); QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
exit(-1); exit(-1);
@ -150,10 +166,10 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa
// Create text file describing the failure // Create text file describing the failure
QTextStream stream(&descriptionFile); QTextStream stream(&descriptionFile);
stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/' stream << "Test failed in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/'
stream << "Expected image was " << testFailure._expectedImageFilename << endl; stream << "Expected image was " << testResult._expectedImageFilename << endl;
stream << "Actual image was " << testFailure._actualImageFilename << endl; stream << "Actual image was " << testResult._actualImageFilename << endl;
stream << "Similarity index was " << testFailure._error << endl; stream << "Similarity index was " << testResult._error << endl;
descriptionFile.close(); descriptionFile.close();
@ -161,21 +177,21 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa
QString sourceFile; QString sourceFile;
QString destinationFile; QString destinationFile;
sourceFile = testFailure._pathname + testFailure._expectedImageFilename; sourceFile = testResult._pathname + testResult._expectedImageFilename;
destinationFile = failureFolderPath + "/" + "Expected Image.png"; destinationFile = resultFolderPath + "/" + "Expected Image.png";
if (!QFile::copy(sourceFile, destinationFile)) { if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1); exit(-1);
} }
sourceFile = testFailure._pathname + testFailure._actualImageFilename; sourceFile = testResult._pathname + testResult._actualImageFilename;
destinationFile = failureFolderPath + "/" + "Actual Image.png"; destinationFile = resultFolderPath + "/" + "Actual Image.png";
if (!QFile::copy(sourceFile, destinationFile)) { if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1); exit(-1);
} }
comparisonImage.save(failureFolderPath + "/" + "Difference Image.png"); comparisonImage.save(resultFolderPath + "/" + "Difference Image.png");
} }
void Test::startTestsEvaluation(const bool isRunningFromCommandLine, void Test::startTestsEvaluation(const bool isRunningFromCommandLine,

5
tools/auto-tester/src/Test.h
View file

@ -84,7 +84,7 @@ public:
void includeTest(QTextStream& textStream, const QString& testPathname); void includeTest(QTextStream& textStream, const QString& testPathname);
void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); void appendTestResultsToFile(const QString& testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed);
bool createTestResultsFolderPath(const QString& directory); bool createTestResultsFolderPath(const QString& directory);
QString zipAndDeleteTestResultsFolder(); QString zipAndDeleteTestResultsFolder();
@ -120,7 +120,8 @@ private:
ImageComparer _imageComparer; ImageComparer _imageComparer;
QString _testResultsFolderPath; QString _testResultsFolderPath;
int _index { 1 }; int _failureIndex{ 1 };
int _successIndex{ 1 };
// Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
const int NUM_DIGITS { 5 }; const int NUM_DIGITS { 5 };

4
tools/auto-tester/src/common.h
View file

@ -12,9 +12,9 @@
#include <QtCore/QString> #include <QtCore/QString>
class TestFailure { class TestResult {
public: public:
TestFailure(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) : TestResult(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) :
_error(error), _error(error),
_pathname(pathname), _pathname(pathname),
_expectedImageFilename(expectedImageFilename), _expectedImageFilename(expectedImageFilename),

2
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -36,7 +36,7 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) {
_ui.statusLabel->setText(""); _ui.statusLabel->setText("");
_ui.plainTextEdit->setReadOnly(true); _ui.plainTextEdit->setReadOnly(true);
setWindowTitle("Auto Tester - v6.5"); setWindowTitle("Auto Tester - v6.6");
// Coming soon to an auto-tester near you... // Coming soon to an auto-tester near you...
//// _helpWindow.textBrowser->setText() //// _helpWindow.textBrowser->setText()

18
tools/auto-tester/src/ui/MismatchWindow.cpp
View file

@ -60,20 +60,20 @@ QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultIma
return resultPixmap; return resultPixmap;
} }
void MismatchWindow::setTestFailure(TestFailure testFailure) { void MismatchWindow::setTestResult(TestResult testResult) {
errorLabel->setText("Similarity: " + QString::number(testFailure._error)); errorLabel->setText("Similarity: " + QString::number(testResult._error));
imagePath->setText("Path to test: " + testFailure._pathname); imagePath->setText("Path to test: " + testResult._pathname);
expectedFilename->setText(testFailure._expectedImageFilename); expectedFilename->setText(testResult._expectedImageFilename);
resultFilename->setText(testFailure._actualImageFilename); resultFilename->setText(testResult._actualImageFilename);
QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename); QPixmap expectedPixmap = QPixmap(testResult._pathname + testResult._expectedImageFilename);
QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename); QPixmap actualPixmap = QPixmap(testResult._pathname + testResult._actualImageFilename);
_diffPixmap = computeDiffPixmap( _diffPixmap = computeDiffPixmap(
QImage(testFailure._pathname + testFailure._expectedImageFilename), QImage(testResult._pathname + testResult._expectedImageFilename),
QImage(testFailure._pathname + testFailure._actualImageFilename) QImage(testResult._pathname + testResult._actualImageFilename)
); );
expectedImage->setPixmap(expectedPixmap); expectedImage->setPixmap(expectedPixmap);

2
tools/auto-tester/src/ui/MismatchWindow.h
View file

@ -20,7 +20,7 @@ class MismatchWindow : public QDialog, public Ui::MismatchWindow {
public: public:
MismatchWindow(QWidget *parent = Q_NULLPTR); MismatchWindow(QWidget *parent = Q_NULLPTR);
void setTestFailure(TestFailure testFailure); void setTestResult(TestResult testResult);
UserResponse getUserResponse() { return _userResponse; } UserResponse getUserResponse() { return _userResponse; }