Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-24 03:53:52 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

TesftFailure (etal.) changed to TestResult.

Browse source 
Successes are also added to the zipped results folder.
This commit is contained in:
NissimHadar 2018-10-09 16:54:38 -07:00
parent 9b816321af
commit 697e678612
8 changed files with 60 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
tools/auto-tester/src/AWSInterface.cpp
View file

@ -231,13 +231,13 @@ void AWSInterface::closeTable(QTextStream& stream) {
stream << "\t\t</table>\n";
}
void AWSInterface::createEntry(int index, const QString& testFailure, QTextStream& stream) {
void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream) {
stream << "\t\t\t<tr>\n";
stream << "\t\t\t\t<td><h1>" << QString::number(index) << "</h1></td>\n";
// For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000`
// we need `Failure_1--tests.engine.interaction.pick.collision.many.00000`
QStringList failureNameComponents = testFailure.split('/');
QStringList failureNameComponents = testResult.split('/');
QString failureName = failureNameComponents[failureNameComponents.length() - 1];
stream << "\t\t\t\t<td><img src=\"./" << FAILURE_FOLDER << "/" << failureName << "/Actual Image.png\" width = \"576\" height = \"324\" ></td>\n";

2
tools/auto-tester/src/AWSInterface.h
View file

@ -43,7 +43,7 @@ public:
void openTable(QTextStream& stream);
void closeTable(QTextStream& stream);
void createEntry(int index, const QString& testFailure, QTextStream& stream);
void createEntry(int index, const QString& testResult, QTextStream& stream);
void updateAWS();

66
tools/auto-tester/src/Test.cpp
View file

@ -58,7 +58,8 @@ QString Test::zipAndDeleteTestResultsFolder() {
//In all cases, for the next evaluation
_testResultsFolderPath = "";
_index = 1;
_failureIndex = 1;
_successIndex = 1;
return zippedResultsFileName;
}
@ -90,19 +91,20 @@ int Test::compareImageLists() {
similarityIndex = _imageComparer.compareImages(resultImage, expectedImage);
}
TestResult testResult = TestResult{
(float)similarityIndex,
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
};
if (similarityIndex < THRESHOLD) {
++numberOfFailures;
TestFailure testFailure = TestFailure{
(float)similarityIndex,
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
};
_mismatchWindow.setTestFailure(testFailure);
_mismatchWindow.setTestResult(testResult);
if (!isInteractiveMode) {
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true);
} else {
_mismatchWindow.exec();
@ -110,7 +112,7 @@ int Test::compareImageLists() {
case USER_RESPONSE_PASS:
break;
case USE_RESPONSE_FAIL:
appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage());
appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true);
break;
case USER_RESPONSE_ABORT:
keepOn = false;
@ -120,6 +122,8 @@ int Test::compareImageLists() {
break;
}
}
} else {
appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), false);
}
_progressBar->setValue(i);
@ -129,20 +133,32 @@ int Test::compareImageLists() {
return numberOfFailures;
}
void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) {
void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed) {
if (!QDir().exists(_testResultsFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found");
exit(-1);
}
QString failureFolderPath { _testResultsFolderPath + "/Failure_" + QString::number(_index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) };
if (!QDir().mkdir(failureFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath);
QString resultFolderPath;
if (hasFailed) {
resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" +
testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4);
++_failureIndex;
} else {
resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" +
testResult._actualImageFilename.left(testResult._actualImageFilename.length() - 4);
++_successIndex;
}
if (!QDir().mkdir(resultFolderPath)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
"Failed to create folder " + resultFolderPath);
exit(-1);
}
++_index;
QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME);
QFile descriptionFile(resultFolderPath + "/" + TEST_RESULTS_FILENAME);
if (!descriptionFile.open(QIODevice::ReadWrite)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + TEST_RESULTS_FILENAME);
exit(-1);
@ -150,10 +166,10 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa
// Create text file describing the failure
QTextStream stream(&descriptionFile);
stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/'
stream << "Expected image was " << testFailure._expectedImageFilename << endl;
stream << "Actual image was " << testFailure._actualImageFilename << endl;
stream << "Similarity index was " << testFailure._error << endl;
stream << "Test failed in folder " << testResult._pathname.left(testResult._pathname.length() - 1) << endl; // remove trailing '/'
stream << "Expected image was " << testResult._expectedImageFilename << endl;
stream << "Actual image was " << testResult._actualImageFilename << endl;
stream << "Similarity index was " << testResult._error << endl;
descriptionFile.close();
@ -161,21 +177,21 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFa
QString sourceFile;
QString destinationFile;
sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
destinationFile = failureFolderPath + "/" + "Expected Image.png";
sourceFile = testResult._pathname + testResult._expectedImageFilename;
destinationFile = resultFolderPath + "/" + "Expected Image.png";
if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
sourceFile = testFailure._pathname + testFailure._actualImageFilename;
destinationFile = failureFolderPath + "/" + "Actual Image.png";
sourceFile = testResult._pathname + testResult._actualImageFilename;
destinationFile = resultFolderPath + "/" + "Actual Image.png";
if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
comparisonImage.save(failureFolderPath + "/" + "Difference Image.png");
comparisonImage.save(resultFolderPath + "/" + "Difference Image.png");
}
void Test::startTestsEvaluation(const bool isRunningFromCommandLine,

5
tools/auto-tester/src/Test.h
View file

@ -84,7 +84,7 @@ public:
void includeTest(QTextStream& textStream, const QString& testPathname);
void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
void appendTestResultsToFile(const QString& testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed);
bool createTestResultsFolderPath(const QString& directory);
QString zipAndDeleteTestResultsFolder();
@ -120,7 +120,8 @@ private:
ImageComparer _imageComparer;
QString _testResultsFolderPath;
int _index { 1 };
int _failureIndex{ 1 };
int _successIndex{ 1 };
// Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit)
const int NUM_DIGITS { 5 };

4
tools/auto-tester/src/common.h
View file

@ -12,9 +12,9 @@
#include <QtCore/QString>
class TestFailure {
class TestResult {
public:
TestFailure(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) :
TestResult(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) :
_error(error),
_pathname(pathname),
_expectedImageFilename(expectedImageFilename),

2
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -36,7 +36,7 @@ AutoTester::AutoTester(QWidget* parent) : QMainWindow(parent) {
_ui.statusLabel->setText("");
_ui.plainTextEdit->setReadOnly(true);
setWindowTitle("Auto Tester - v6.5");
setWindowTitle("Auto Tester - v6.6");
// Coming soon to an auto-tester near you...
//// _helpWindow.textBrowser->setText()

18
tools/auto-tester/src/ui/MismatchWindow.cpp
View file

@ -60,20 +60,20 @@ QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultIma
return resultPixmap;
}
void MismatchWindow::setTestFailure(TestFailure testFailure) {
errorLabel->setText("Similarity: " + QString::number(testFailure._error));
void MismatchWindow::setTestResult(TestResult testResult) {
errorLabel->setText("Similarity: " + QString::number(testResult._error));
imagePath->setText("Path to test: " + testFailure._pathname);
imagePath->setText("Path to test: " + testResult._pathname);
expectedFilename->setText(testFailure._expectedImageFilename);
resultFilename->setText(testFailure._actualImageFilename);
expectedFilename->setText(testResult._expectedImageFilename);
resultFilename->setText(testResult._actualImageFilename);
QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename);
QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename);
QPixmap expectedPixmap = QPixmap(testResult._pathname + testResult._expectedImageFilename);
QPixmap actualPixmap = QPixmap(testResult._pathname + testResult._actualImageFilename);
_diffPixmap = computeDiffPixmap(
QImage(testFailure._pathname + testFailure._expectedImageFilename),
QImage(testFailure._pathname + testFailure._actualImageFilename)
QImage(testResult._pathname + testResult._expectedImageFilename),
QImage(testResult._pathname + testResult._actualImageFilename)
);
expectedImage->setPixmap(expectedPixmap);

2
tools/auto-tester/src/ui/MismatchWindow.h
View file

@ -20,7 +20,7 @@ class MismatchWindow : public QDialog, public Ui::MismatchWindow {
public:
MismatchWindow(QWidget *parent = Q_NULLPTR);
void setTestFailure(TestFailure testFailure);
void setTestResult(TestResult testResult);
UserResponse getUserResponse() { return _userResponse; }