Mirrors/overte-thingvellir
3
0
Fork 0
mirror of https://thingvellir.net/git/overte synced 2025-03-27 23:52:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Working.

Browse source
This commit is contained in:
NissimHadar 2019-02-28 20:33:21 -08:00
parent 15989e3a89
commit d4fa1e25fb
12 changed files with 175 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
tools/nitpick/src/AWSInterface.cpp
View file

@ -27,7 +27,10 @@ AWSInterface::AWSInterface(QObject* parent) : QObject(parent) {
void AWSInterface::createWebPageFromResults(const QString& testResults,
const QString& workingDirectory,
QCheckBox* updateAWSCheckBox,
QLineEdit* urlLineEdit) {
QRadioButton* diffImageRadioButton,
QRadioButton* ssimImageRadionButton,
QLineEdit* urlLineEdit
) {
_workingDirectory = workingDirectory;
// Verify filename is in correct format
@ -52,6 +55,13 @@ void AWSInterface::createWebPageFromResults(const QString& testResults,
QString zipFilenameWithoutExtension = zipFilename.split('.')[0];
extractTestFailuresFromZippedFolder(_workingDirectory + "/" + zipFilenameWithoutExtension);
if (diffImageRadioButton->isChecked()) {
_comparisonImageFilename = "Difference Image.png";
} else {
_comparisonImageFilename = "SSIM Image.png";
}
createHTMLFile();
if (updateAWSCheckBox->isChecked()) {
@ -353,7 +363,7 @@ void AWSInterface::openTable(QTextStream& stream, const QString& testResult, con
stream << "\t\t\t\t<th><h1>Test</h1></th>\n";
stream << "\t\t\t\t<th><h1>Actual Image</h1></th>\n";
stream << "\t\t\t\t<th><h1>Expected Image</h1></th>\n";
stream << "\t\t\t\t<th><h1>Difference Image</h1></th>\n";
stream << "\t\t\t\t<th><h1>Comparison Image</h1></th>\n";
stream << "\t\t\t</tr>\n";
}
}
@ -378,12 +388,13 @@ void AWSInterface::createEntry(const int index, const QString& testResult, QText
QString folder;
bool differenceFileFound;
if (isFailure) {
folder = FAILURES_FOLDER;
differenceFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Difference Image.png");
differenceFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/" + _comparisonImageFilename);
} else {
folder = SUCCESSES_FOLDER;
differenceFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Difference Image.png");
differenceFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/" + _comparisonImageFilename);
}
if (textResultsFileFound) {
@ -450,7 +461,7 @@ void AWSInterface::createEntry(const int index, const QString& testResult, QText
stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Expected Image.png\" width = \"576\" height = \"324\" ></td>\n";
if (differenceFileFound) {
stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/Difference Image.png\" width = \"576\" height = \"324\" ></td>\n";
stream << "\t\t\t\t<td><img src=\"./" << folder << "/" << resultName << "/" << _comparisonImageFilename << "\" width = \"576\" height = \"324\" ></td>\n";
} else {
stream << "\t\t\t\t<td><h2>No Image Found</h2>\n";
}
@ -512,12 +523,12 @@ void AWSInterface::updateAWS() {
stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Expected Image.png" << "', Body=data)\n\n";
if (QFile::exists(_htmlFailuresFolder + "/" + parts[parts.length() - 1] + "/Difference Image.png")) {
if (QFile::exists(_htmlFailuresFolder + "/" + parts[parts.length() - 1] + "/" + _comparisonImageFilename)) {
stream << "data = open('" << _workingDirectory << "/" << filename << "/"
<< "Difference Image.png"
<< _comparisonImageFilename
<< "', 'rb')\n";
stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Difference Image.png" << "', Body=data)\n\n";
stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << _comparisonImageFilename << "', Body=data)\n\n";
}
}
}
@ -555,12 +566,12 @@ void AWSInterface::updateAWS() {
stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Expected Image.png" << "', Body=data)\n\n";
if (QFile::exists(_htmlSuccessesFolder + "/" + parts[parts.length() - 1] + "/Difference Image.png")) {
if (QFile::exists(_htmlSuccessesFolder + "/" + parts[parts.length() - 1] + "/" + _comparisonImageFilename)) {
stream << "data = open('" << _workingDirectory << "/" << filename << "/"
<< "Difference Image.png"
<< _comparisonImageFilename
<< "', 'rb')\n";
stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Difference Image.png" << "', Body=data)\n\n";
stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << _comparisonImageFilename << "', Body=data)\n\n";
}
}
}

6
tools/nitpick/src/AWSInterface.h
View file

@ -14,6 +14,7 @@
#include <QCheckBox>
#include <QLineEdit>
#include <QObject>
#include <QRadioButton>
#include <QTextStream>
#include "BusyWindow.h"
@ -28,6 +29,8 @@ public:
void createWebPageFromResults(const QString& testResults,
const QString& workingDirectory,
QCheckBox* updateAWSCheckBox,
QRadioButton* diffImageRadioButton,
QRadioButton* ssimImageRadionButton,
QLineEdit* urlLineEdit);
void extractTestFailuresFromZippedFolder(const QString& folderName);
@ -67,6 +70,9 @@ private:
QString AWS_BUCKET{ "hifi-qa" };
QLineEdit* _urlLineEdit;
QString _comparisonImageFilename;
};
#endif // hifi_AWSInterface_h

39
tools/nitpick/src/ImageComparer.cpp
View file

@ -12,17 +12,9 @@
#include <cmath>
ImageComparer::ImageComparer() {
_ssimResults = new SSIMResults();
}
ImageComparer::~ImageComparer() {
delete _ssimResults;
}
// Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity
// The value is computed for the luminance component and the average value is returned
double ImageComparer::compareImages(const QImage& resultImage, const QImage& expectedImage) const {
void ImageComparer::compareImages(const QImage& resultImage, const QImage& expectedImage) {
const int L = 255; // (2^number of bits per pixel) - 1
const double K1 { 0.01 };
@ -47,8 +39,13 @@ double ImageComparer::compareImages(const QImage& resultImage, const QImage& exp
double p[WIN_SIZE * WIN_SIZE];
double q[WIN_SIZE * WIN_SIZE];
_ssimResults.results.clear();
int windowCounter{ 0 };
double ssim{ 0.0 };
double min { 1.0 };
double max { -1.0 };
while (x < expectedImage.width()) {
int lastX = x + WIN_SIZE - 1;
if (lastX > expectedImage.width() - 1) {
@ -104,8 +101,13 @@ double ImageComparer::compareImages(const QImage& resultImage, const QImage& exp
double numerator = (2.0 * mP * mQ + c1) * (2.0 * sigPQ + c2);
double denominator = (mP * mP + mQ * mQ + c1) * (sigsqP + sigsqQ + c2);
_ssimResults->results.push_back(numerator / denominator);
ssim += numerator / denominator;
double value { numerator / denominator };
_ssimResults.results.push_back(value);
ssim += value;
if (value < min) min = value;
if (value > max) max = value;
++windowCounter;
y += WIN_SIZE;
@ -115,12 +117,17 @@ double ImageComparer::compareImages(const QImage& resultImage, const QImage& exp
y = 0;
}
_ssimResults->width = (int)(expectedImage.width() / WIN_SIZE);
_ssimResults->height = (int)(expectedImage.height() / WIN_SIZE);
return ssim / windowCounter;
_ssimResults.width = (int)(expectedImage.width() / WIN_SIZE);
_ssimResults.height = (int)(expectedImage.height() / WIN_SIZE);
_ssimResults.min = min;
_ssimResults.max = max;
_ssimResults.ssim = ssim / windowCounter;
};
SSIMResults* ImageComparer::getSSIMResults() {
double ImageComparer::getSSIMValue() {
return _ssimResults.ssim;
}
SSIMResults ImageComparer::getSSIMResults() {
return _ssimResults;
}

9
tools/nitpick/src/ImageComparer.h
View file

@ -17,14 +17,13 @@
class ImageComparer {
public:
ImageComparer();
~ImageComparer();
void compareImages(const QImage& resultImage, const QImage& expectedImage);
double getSSIMValue();
double compareImages(const QImage& resultImage, const QImage& expectedImage) const;
SSIMResults* getSSIMResults();
SSIMResults getSSIMResults();
private:
SSIMResults* _ssimResults;
SSIMResults _ssimResults;
};
#endif // hifi_ImageComparer_h

51
tools/nitpick/src/MismatchWindow.cpp
View file

@ -21,7 +21,7 @@ MismatchWindow::MismatchWindow(QWidget *parent) : QDialog(parent) {
diffImage->setScaledContents(true);
}
QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultImage) {
QPixmap MismatchWindow::computeDiffPixmap(const QImage& expectedImage, const QImage& resultImage) {
// Create an empty difference image if the images differ in size
if (expectedImage.height() != resultImage.height() || expectedImage.width() != resultImage.width()) {
return QPixmap();
@ -60,7 +60,7 @@ QPixmap MismatchWindow::computeDiffPixmap(QImage expectedImage, QImage resultIma
return resultPixmap;
}
void MismatchWindow::setTestResult(TestResult testResult) {
void MismatchWindow::setTestResult(const TestResult& testResult) {
errorLabel->setText("Similarity: " + QString::number(testResult._error));
imagePath->setText("Path to test: " + testResult._pathname);
@ -100,34 +100,35 @@ QPixmap MismatchWindow::getComparisonImage() {
return _diffPixmap;
}
QPixmap MismatchWindow::getSSIMResultsImage(SSIMResults* ssimResults) {
QPixmap MismatchWindow::getSSIMResultsImage(const SSIMResults& ssimResults) {
// This is an optimization, as QImage.setPixel() is embarrassingly slow
const int ELEMENT_SIZE { 8 };
unsigned char* buffer = new unsigned char[(ssimResults->height * ELEMENT_SIZE) * (ssimResults->width * ELEMENT_SIZE ) * 3];
const int WIDTH{ ssimResults.width * ELEMENT_SIZE };
const int HEIGHT{ ssimResults.height * ELEMENT_SIZE };
unsigned char* buffer = new unsigned char[WIDTH * HEIGHT * 3];
// loop over each SSIM result (a double in [-1.0 .. 1.0]
int i { 0 };
for (int y = 0; y < ssimResults->height; ++y) {
for (int x = 0; x < ssimResults->width; ++x) {
////QRgb pixelP = expectedImage.pixel(QPoint(x, y));
////// Convert to luminance
////double p = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP);
////// The intensity value is modified to increase the brightness of the displayed image
////double absoluteDifference = fabs(p - q) / 255.0;
////double modifiedDifference = sqrt(absoluteDifference);
////int difference = (int)(modifiedDifference * 255.0);
////buffer[3 * (x + y * expectedImage.width()) + 0] = difference;
////buffer[3 * (x + y * expectedImage.width()) + 1] = difference;
////buffer[3 * (x + y * expectedImage.width()) + 2] = difference;
++i;
// loop over each SSIM result
for (int y = 0; y < ssimResults.height; ++y) {
for (int x = 0; x < ssimResults.width; ++x) {
double scaledResult = (ssimResults.results[x * ssimResults.height + y] + 1.0) / (2.0);
//double scaledResult = (ssimResults.results[x * ssimResults.height + y] - ssimResults.min) / (ssimResults.max - ssimResults.min);
// Create a square
for (int yy = 0; yy < ELEMENT_SIZE; ++yy) {
for (int xx = 0; xx < ELEMENT_SIZE; ++xx) {
buffer[(xx + yy * WIDTH + x * ELEMENT_SIZE + y * WIDTH * ELEMENT_SIZE) * 3 + 0] = 255 * (1.0 - scaledResult); // R
buffer[(xx + yy * WIDTH + x * ELEMENT_SIZE + y * WIDTH * ELEMENT_SIZE) * 3 + 1] = 255 * scaledResult; // G
buffer[(xx + yy * WIDTH + x * ELEMENT_SIZE + y * WIDTH * ELEMENT_SIZE) * 3 + 2] = 0; // B
}
}
}
}
return QPixmap();
QImage image(buffer, WIDTH, HEIGHT, QImage::Format_RGB888);
QPixmap pixmap = QPixmap::fromImage(image);
delete[] buffer;
return pixmap;
}

6
tools/nitpick/src/MismatchWindow.h
View file

@ -20,14 +20,14 @@ class MismatchWindow : public QDialog, public Ui::MismatchWindow {
public:
MismatchWindow(QWidget *parent = Q_NULLPTR);
void setTestResult(TestResult testResult);
void setTestResult(const TestResult& testResult);
UserResponse getUserResponse() { return _userResponse; }
QPixmap computeDiffPixmap(QImage expectedImage, QImage resultImage);
QPixmap computeDiffPixmap(const QImage& expectedImage, const QImage& resultImage);
QPixmap getComparisonImage();
QPixmap getSSIMResultsImage(SSIMResults* ssimResults);
QPixmap getSSIMResultsImage(const SSIMResults& ssimResults);
private slots:
void on_passTestButton_clicked();

10
tools/nitpick/src/Nitpick.cpp
View file

@ -148,10 +148,6 @@ void Nitpick::on_tabWidget_currentChanged(int index) {
}
}
void Nitpick::on_evaluateTestsPushbutton_clicked() {
_test->startTestsEvaluation(false, false);
}
void Nitpick::on_createRecursiveScriptPushbutton_clicked() {
_test->createRecursiveScript();
}
@ -242,6 +238,10 @@ void Nitpick::on_showTaskbarPushbutton_clicked() {
#endif
}
void Nitpick::on_evaluateTestsPushbutton_clicked() {
_test->startTestsEvaluation(false, false);
}
void Nitpick::on_closePushbutton_clicked() {
exit(0);
}
@ -255,7 +255,7 @@ void Nitpick::on_createXMLScriptRadioButton_clicked() {
}
void Nitpick::on_createWebPagePushbutton_clicked() {
_test->createWebPage(_ui.updateAWSCheckBox, _ui.awsURLLineEdit);
_test->createWebPage(_ui.updateAWSCheckBox, _ui.diffImageRadioButton, _ui.ssimImageRadioButton, _ui.awsURLLineEdit);
}
void Nitpick::downloadFile(const QUrl& url) {

3
tools/nitpick/src/Nitpick.h
View file

@ -56,7 +56,6 @@ private slots:
void on_tabWidget_currentChanged(int index);
void on_evaluateTestsPushbutton_clicked();
void on_createRecursiveScriptPushbutton_clicked();
void on_createAllRecursiveScriptsPushbutton_clicked();
void on_createTestsPushbutton_clicked();
@ -82,6 +81,8 @@ private slots:
void on_hideTaskbarPushbutton_clicked();
void on_showTaskbarPushbutton_clicked();
void on_evaluateTestsPushbutton_clicked();
void on_createPythonScriptRadioButton_clicked();
void on_createXMLScriptRadioButton_clicked();

41
tools/nitpick/src/Test.cpp
View file

@ -89,25 +89,25 @@ int Test::compareImageLists() {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
similarityIndex = -100.0;
} else {
similarityIndex = _imageComparer.compareImages(resultImage, expectedImage);
_imageComparer.compareImages(resultImage, expectedImage);
similarityIndex = _imageComparer.getSSIMValue();
}
TestResult testResult = TestResult{
(float)similarityIndex,
_expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /)
QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image
QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of result image
_imageComparer.getSSIMResults() // results of SSIM algoritm
};
_mismatchWindow.setTestResult(testResult);
QPixmap ssimResultsPixMap = _mismatchWindow.getSSIMResultsImage(_imageComparer.getSSIMResults());
if (similarityIndex < THRESHOLD) {
++numberOfFailures;
if (!isInteractiveMode) {
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), ssimResultsPixMap, true);
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true);
} else {
_mismatchWindow.exec();
@ -115,7 +115,7 @@ int Test::compareImageLists() {
case USER_RESPONSE_PASS:
break;
case USE_RESPONSE_FAIL:
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), ssimResultsPixMap, true);
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), true);
break;
case USER_RESPONSE_ABORT:
keepOn = false;
@ -126,7 +126,7 @@ int Test::compareImageLists() {
}
}
} else {
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), ssimResultsPixMap, false);
appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), _mismatchWindow.getSSIMResultsImage(testResult._ssimResults), false);
}
_progressBar->setValue(i);
@ -218,15 +218,10 @@ void Test::appendTestResultsToFile(const TestResult& testResult, const QPixmap&
exit(-1);
}
// Create the SSIM results image
sourceFile = testResult._pathname + testResult._actualImageFilename;
destinationFile = resultFolderPath + "/" + "SSIM results.png";
if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
comparisonImage.save(resultFolderPath + "/" + "Difference Image.png");
// Save the SSIM results image
ssimResultsImage.save(resultFolderPath + "/" + "SSIM Image.png");
}
void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) {
@ -1105,7 +1100,12 @@ void Test::setTestRailCreateMode(TestRailCreateMode testRailCreateMode) {
_testRailCreateMode = testRailCreateMode;
}
void Test::createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) {
void Test::createWebPage(
QCheckBox* updateAWSCheckBox,
QRadioButton* diffImageRadioButton,
QRadioButton* ssimImageRadionButton,
QLineEdit* urlLineEdit
) {
QString testResults = QFileDialog::getOpenFileName(nullptr, "Please select the zipped test results to update from", nullptr,
"Zipped Test Results (TestResults--*.zip)");
if (testResults.isNull()) {
@ -1122,5 +1122,12 @@ void Test::createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) {
_awsInterface = new AWSInterface;
}
_awsInterface->createWebPageFromResults(testResults, workingDirectory, updateAWSCheckBox, urlLineEdit);
_awsInterface->createWebPageFromResults(
testResults,
workingDirectory,
updateAWSCheckBox,
diffImageRadioButton,
ssimImageRadionButton,
urlLineEdit
);
}

6
tools/nitpick/src/Test.h
View file

@ -102,7 +102,11 @@ public:
void setTestRailCreateMode(TestRailCreateMode testRailCreateMode);
void createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit);
void createWebPage(
QCheckBox* updateAWSCheckBox,
QRadioButton* diffImageRadioButton,
QRadioButton* ssimImageRadionButton,
QLineEdit* urlLineEdit);
private:
QProgressBar* _progressBar;

27
tools/nitpick/src/common.h
View file

@ -13,19 +13,35 @@
#include <vector>
#include <QtCore/QString>
class SSIMResults {
public:
int width;
int height;
std::vector<double> results;
double ssim;
// Used for scaling
double min;
double max;
};
class TestResult {
public:
TestResult(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) :
TestResult(float error, const QString& pathname, const QString& expectedImageFilename, const QString& actualImageFilename, const SSIMResults& ssimResults) :
_error(error),
_pathname(pathname),
_expectedImageFilename(expectedImageFilename),
_actualImageFilename(actualImageFilename)
_actualImageFilename(actualImageFilename),
_ssimResults(ssimResults)
{}
double _error;
QString _pathname;
QString _expectedImageFilename;
QString _actualImageFilename;
SSIMResults _ssimResults;
};
enum UserResponse {
@ -40,11 +56,4 @@ const double R_Y = 0.212655f;
const double G_Y = 0.715158f;
const double B_Y = 0.072187f;
class SSIMResults {
public:
int width;
int height;
std::vector<double> results;
};
#endif // hifi_common_h

42
tools/nitpick/ui/Nitpick.ui
View file

@ -43,7 +43,7 @@
</rect>
</property>
<property name="currentIndex">
<number>0</number>
<number>5</number>
</property>
<widget class="QWidget" name="tab_1">
<attribute name="title">
@ -760,7 +760,7 @@
<property name="geometry">
<rect>
<x>190</x>
<y>180</y>
<y>200</y>
<width>131</width>
<height>20</height>
</rect>
@ -776,7 +776,7 @@
<property name="geometry">
<rect>
<x>330</x>
<y>170</y>
<y>190</y>
<width>181</width>
<height>51</height>
</rect>
@ -889,8 +889,8 @@
</property>
<property name="geometry">
<rect>
<x>270</x>
<y>30</y>
<x>370</x>
<y>20</y>
<width>160</width>
<height>51</height>
</rect>
@ -922,6 +922,38 @@
</rect>
</property>
</widget>
<widget class="QRadioButton" name="diffImageRadioButton">
<property name="geometry">
<rect>
<x>260</x>
<y>50</y>
<width>95</width>
<height>20</height>
</rect>
</property>
<property name="text">
<string>Diff Image</string>
</property>
<property name="checked">
<bool>false</bool>
</property>
</widget>
<widget class="QRadioButton" name="ssimImageRadioButton">
<property name="geometry">
<rect>
<x>260</x>
<y>30</y>
<width>95</width>
<height>20</height>
</rect>
</property>
<property name="text">
<string>SSIM Image</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
</widget>
<zorder>groupBox</zorder>
<zorder>updateTestRailRunResultsPushbutton</zorder>