Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-23 13:33:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #13460 from NissimHadar/autoTesterUpdate

Browse source 
Auto tester update
This commit is contained in:
NissimHadar 2018-06-26 11:26:25 -07:00 committed by GitHub
commit 337719f45e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 234 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
tools/auto-tester/src/ImageComparer.cpp
View file

@ -15,13 +15,8 @@
// Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity
// The value is computed for the luminance component and the average value is returned
double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const {
// Make sure the image is 8 bits per colour
QImage::Format format = expectedImage.format();
if (format != QImage::Format::Format_ARGB32) {
throw -1;
}
const int L = 255; // (2^number of bits per pixel) - 1
const double K1 { 0.01 };
const double K2 { 0.03 };
const double c1 = pow((K1 * L), 2);

123
tools/auto-tester/src/Test.cpp
View file

@ -25,6 +25,11 @@ extern AutoTester* autoTester;
Test::Test() {
mismatchWindow.setModal(true);
if (autoTester) {
autoTester->setUserText("highfidelity");
autoTester->setBranchText("master");
}
}
bool Test::createTestResultsFolderPath(const QString& directory) {
@ -63,7 +68,6 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
// Loop over both lists and compare each pair of images
// Quit loop if user has aborted due to a failed test.
const double THRESHOLD { 0.9995 };
bool success{ true };
bool keepOn{ true };
for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) {
@ -71,17 +75,14 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar)
QImage resultImage(resultImagesFullFilenames[i]);
QImage expectedImage(expectedImagesFullFilenames[i]);
double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical
// similarityIndex is set to -100.0 to indicate images are not the same size
if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size");
exit(-1);
}
double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical
try {
similarityIndex = -100.0;
} else {
similarityIndex = imageComparer.compareImages(resultImage, expectedImage);
} catch (...) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format");
exit(-1);
}
if (similarityIndex < THRESHOLD) {
@ -176,20 +177,25 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
comparisonImage.save(failureFolderPath + "/" + "Difference Image.png");
}
void Test::startTestsEvaluation(const QString& testFolder) {
// Get list of JPEG images in folder, sorted by name
QString previousSelection = snapshotDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
QFileDialog::ShowDirsOnly);
void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine, const QString& userFromCommandLine) {
if (testFolder.isNull()) {
// Get list of JPEG images in folder, sorted by name
QString previousSelection = snapshotDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (snapshotDirectory == "") {
snapshotDirectory = previousSelection;
return;
// If user cancelled then restore previous selection and return
if (snapshotDirectory == "") {
snapshotDirectory = previousSelection;
return;
}
} else {
snapshotDirectory = testFolder;
exitWhenComplete = true;
}
// Quit if test results folder could not be created
@ -197,17 +203,6 @@ void Test::startTestsEvaluation(const QString& testFolder) {
return;
}
// Before any processing - all images are converted to PNGs, as this is the format stored on GitHub
QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory);
foreach(QString filename, sortedSnapshotFilenames) {
QString filenameWithoutExtension = filename.left(filename.length() - 4);
copyJPGtoPNG(snapshotDirectory + "/" + filenameWithoutExtension + ".jpg",
snapshotDirectory + "/" + filenameWithoutExtension + ".png"
);
QFile::remove(snapshotDirectory + "/" + filenameWithoutExtension + ".jpg");
}
// Create two lists. The first is the test results, the second is the expected images
// The expected images are represented as a URL to enable download from GitHub
// Images that are in the wrong format are ignored.
@ -219,6 +214,9 @@ void Test::startTestsEvaluation(const QString& testFolder) {
expectedImagesFilenames.clear();
expectedImagesFullFilenames.clear();
QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine;
QString user = (userFromCommandLine.isNull()) ? autoTester->getSelectedUser() : userFromCommandLine;
foreach(QString currentFilename, sortedTestResultsFilenames) {
QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("png", currentFilename)) {
@ -231,7 +229,7 @@ void Test::startTestsEvaluation(const QString& testFolder) {
QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS);
QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png";
QString imageURLString("https://raw.githubusercontent.com/" + GIT_HUB_USER + "/hifi_tests/" + GIT_HUB_BRANCH + "/" +
QString imageURLString("https://raw.githubusercontent.com/" + user + "/" + GIT_HUB_REPOSITORY + "/" + branch + "/" +
expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename);
expectedImagesURLs << imageURLString;
@ -259,6 +257,10 @@ void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactive
}
zipAndDeleteTestResultsFolder();
if (exitWhenComplete) {
exit(0);
}
}
bool Test::isAValidDirectory(const QString& pathname) {
@ -299,10 +301,9 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) {
void Test::includeTest(QTextStream& textStream, const QString& testPathname) {
QString partialPath = extractPathFromTestsDown(testPathname);
textStream << "Script.include(\""
<< "https://github.com/" << GIT_HUB_USER << "/hifi_tests/blob/" << GIT_HUB_BRANCH
<< partialPath + "?raw=true\");"
<< endl;
QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7);
textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << endl;
}
// Creates a single script in a user-selected folder.
@ -397,12 +398,20 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
textStream << "user = \"" + GIT_HUB_USER + "/\";" << endl;
textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl;
textStream << "branch = \"" + GIT_HUB_BRANCH + "/\";" << endl << endl;
// Include 'autoTest.js'
QString branch = autoTester->getSelectedBranch();
QString user = autoTester->getSelectedUser();
textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/"
+ GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;
textStream << "PATH_TO_THE_REPO_PATH_UTILS_FILE = \"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + "/tests/utils/branchUtils.js\";" << endl;
textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl;
textStream << "var autoTester = createAutoTester(Script.resolvePath(\".\"));" << endl << endl;
textStream << "var testsRootPath = autoTester.getTestsRootPath();" << endl << endl;
// Wait 10 seconds before starting
textStream << "if (typeof Test !== 'undefined') {" << endl;
textStream << " Test.wait(10000);" << endl;
textStream << "};" << endl << endl;
textStream << "autoTester.enableRecursive();" << endl;
textStream << "autoTester.enableAuto();" << endl << endl;
@ -498,13 +507,13 @@ void Test::createTests() {
return;
}
QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory);
QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory);
int i = 1;
const int maxImages = pow(10, NUM_DIGITS);
foreach (QString currentFilename, sortedImageFilenames) {
QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename;
if (isInSnapshotFilenameFormat("jpg", currentFilename)) {
if (isInSnapshotFilenameFormat("png", currentFilename)) {
if (i >= maxImages) {
QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
exit(-1);
@ -528,9 +537,12 @@ void Test::createTests() {
fullNewFileName += "/" + newFilename;
try {
copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
if (QFile::exists(fullNewFileName)) {
QFile::remove(fullNewFileName);
}
QFile::copy(fullCurrentFilename, fullNewFileName);
} catch (...) {
QMessageBox::critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
QMessageBox::critical(0, "Error", "Could not copy file: " + fullCurrentFilename + " to " + fullNewFileName + "\n");
exit(-1);
}
++i;
@ -560,9 +572,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) {
const QString ws("\\h*"); //white-space character
const QString functionPerformName(ws + "autoTester" + ws + "\\." + ws + "perform");
const QString quotedString("\\\".+\\\"");
const QString ownPath("Script" + ws + "\\." + ws + "resolvePath" + ws + "\\(" + ws + "\\\"\\.\\\"" + ws + "\\)");
const QString functionParameter("function" + ws + "\\(testType" + ws + "\\)");
QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString + "\\," + ws + ownPath + "\\," + ws + functionParameter + ws + "{" + ".*");
QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString);
QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle);
@ -811,19 +821,6 @@ void Test::createTestsOutline() {
QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created");
}
void Test::copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename) {
QFile::remove(destinationPNGFullFilename);
QImageReader reader;
reader.setFileName(sourceJPGFullFilename);
QImage image = reader.read();
QImageWriter writer;
writer.setFileName(destinationPNGFullFilename);
writer.write(image);
}
QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) {
imageDirectory = QDir(pathToImageDirectory);
QStringList nameFilters;

10
tools/auto-tester/src/Test.h
View file

@ -37,7 +37,7 @@ class Test {
public:
Test();
void startTestsEvaluation(const QString& testFolder = QString());
void startTestsEvaluation(const QString& testFolder = QString(), const QString& branchFromCommandLine = QString(), const QString& userFromCommandLine = QString());
void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar);
void createRecursiveScript();
@ -69,13 +69,13 @@ public:
QString getExpectedImageDestinationDirectory(const QString& filename);
QString getExpectedImagePartialSourceDirectory(const QString& filename);
void copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename);
private:
const QString TEST_FILENAME { "test.js" };
const QString TEST_RESULTS_FOLDER { "TestResults" };
const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
const double THRESHOLD{ 0.96 };
QDir imageDirectory;
MismatchWindow mismatchWindow;
@ -102,9 +102,7 @@ private:
QStringList resultImagesFullFilenames;
// Used for accessing GitHub
const QString GIT_HUB_USER{ "highfidelity" };
const QString GIT_HUB_REPOSITORY{ "hifi_tests" };
const QString GIT_HUB_BRANCH{ "master" };
const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" };
@ -115,6 +113,8 @@ private:
// var pathSeparator = ".";
const QString ADVANCE_KEY{ "n" };
const QString PATH_SEPARATOR{ "." };
bool exitWhenComplete{ false };
};
#endif // hifi_test_h

50
tools/auto-tester/src/main.cpp
View file

@ -10,23 +10,63 @@
#include <QtWidgets/QApplication>
#include "ui/AutoTester.h"
#include <iostream>
AutoTester* autoTester;
int main(int argc, char *argv[]) {
// Only parameter is "--testFolder"
// If no parameters then run in interactive mode
// Parameter --testFolder <folder containing the test images>
// Parameter --branch <branch on GitHub>
// default is "master"
// Parameter --user <GitHub user>
// default is "highfidelity"
// Parameter --repository <repository on GitHub>
// default is "highfidelity"
QString testFolder;
if (argc == 3) {
if (QString(argv[1]) == "--testFolder") {
testFolder = QString(argv[2]);
QString branch{ "master" };
QString user{ "highfidelity" };
for (int i = 1; i < argc - 1; ++i) {
if (QString(argv[i]) == "--testFolder") {
++i;
if (i < argc) {
testFolder = QString(argv[i]);
} else {
std::cout << "Missing parameter after --testFolder" << std::endl;
exit(-1);
}
} else if (QString(argv[i]) == "--branch") {
++i;
if (i < argc) {
branch = QString(argv[i]);
} else {
std::cout << "Missing parameter after --branch" << std::endl;
exit(-1);
}
} else if (QString(argv[i]) == "--user") {
++i;
if (i < argc) {
user = QString(argv[i]);
} else {
std::cout << "Missing parameter after --user" << std::endl;
exit(-1);
}
} else {
std::cout << "Unknown parameter" << std::endl;
exit(-1);
}
}
QApplication application(argc, argv);
autoTester = new AutoTester();
autoTester->setup();
if (!testFolder.isNull()) {
autoTester->runFromCommandLine(testFolder);
autoTester->runFromCommandLine(testFolder, branch, user);
} else {
autoTester->show();
}

39
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -29,13 +29,15 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) {
ui.hideTaskbarButton->setVisible(false);
ui.showTaskbarButton->setVisible(false);
#endif
}
void AutoTester::setup() {
test = new Test();
}
void AutoTester::runFromCommandLine(const QString& testFolder) {
void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) {
isRunningFromCommandline = true;
test->startTestsEvaluation(testFolder);
test->startTestsEvaluation(testFolder, branch, user);
}
void AutoTester::on_evaluateTestsButton_clicked() {
@ -116,6 +118,7 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director
ui.progressBar->setValue(0);
ui.progressBar->setVisible(true);
downloaders.clear();
for (int i = 0; i < _numberOfImagesToDownload; ++i) {
QUrl imageURL(URLs[i]);
downloadImage(imageURL);
@ -125,14 +128,12 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director
}
void AutoTester::saveImage(int index) {
QPixmap pixmap;
pixmap.loadFromData(downloaders[index]->downloadedData());
QImage image = pixmap.toImage();
image = image.convertToFormat(QImage::Format_ARGB32);
QString fullPathname = _directoryName + "/" + _filenames[index];
if (!image.save(fullPathname, 0, 100)) {
try {
QFile file(_directoryName + "/" + _filenames[index]);
file.open(QIODevice::WriteOnly);
file.write(downloaders[index]->downloadedData());
file.close();
} catch (...) {
QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]);
ui.progressBar->setVisible(false);
return;
@ -141,6 +142,7 @@ void AutoTester::saveImage(int index) {
++_numberOfImagesDownloaded;
if (_numberOfImagesDownloaded == _numberOfImagesToDownload) {
disconnect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int)));
test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar);
} else {
ui.progressBar->setValue(_numberOfImagesDownloaded);
@ -150,3 +152,20 @@ void AutoTester::saveImage(int index) {
void AutoTester::about() {
QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__);
}
void AutoTester::setUserText(const QString& user) {
ui.userTextEdit->setText(user);
}
QString AutoTester::getSelectedUser()
{
return ui.userTextEdit->toPlainText();
}
void AutoTester::setBranchText(const QString& branch) {
ui.branchTextEdit->setText(branch);
}
QString AutoTester::getSelectedBranch() {
return ui.branchTextEdit->toPlainText();
}

11
tools/auto-tester/src/ui/AutoTester.h
View file

@ -12,6 +12,7 @@
#include <QtWidgets/QMainWindow>
#include <QSignalMapper>
#include <QTextEdit>
#include "ui_AutoTester.h"
#include "../Downloader.h"
@ -23,11 +24,19 @@ class AutoTester : public QMainWindow {
public:
AutoTester(QWidget *parent = Q_NULLPTR);
void runFromCommandLine(const QString& testFolder);
void setup();
void runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user);
void downloadImage(const QUrl& url);
void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames);
void setUserText(const QString& user);
QString getSelectedUser();
void setBranchText(const QString& branch);
QString getSelectedBranch();
private slots:
void on_evaluateTestsButton_clicked();
void on_createRecursiveScriptButton_clicked();

112
tools/auto-tester/src/ui/AutoTester.ui
View file

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>607</width>
<height>514</height>
<width>612</width>
<height>537</height>
</rect>
</property>
<property name="windowTitle">
@ -17,9 +17,9 @@
<widget class="QPushButton" name="closeButton">
<property name="geometry">
<rect>
<x>360</x>
<y>400</y>
<width>220</width>
<x>380</x>
<y>430</y>
<width>101</width>
<height>40</height>
</rect>
</property>
@ -43,9 +43,9 @@
<widget class="QPushButton" name="evaluateTestsButton">
<property name="geometry">
<rect>
<x>20</x>
<y>285</y>
<width>220</width>
<x>430</x>
<y>270</y>
<width>101</width>
<height>40</height>
</rect>
</property>
@ -56,8 +56,8 @@
<widget class="QPushButton" name="createRecursiveScriptButton">
<property name="geometry">
<rect>
<x>360</x>
<y>35</y>
<x>330</x>
<y>110</y>
<width>220</width>
<height>40</height>
</rect>
@ -69,8 +69,8 @@
<widget class="QCheckBox" name="checkBoxInteractiveMode">
<property name="geometry">
<rect>
<x>23</x>
<y>250</y>
<x>320</x>
<y>280</y>
<width>131</width>
<height>20</height>
</rect>
@ -85,8 +85,8 @@
<widget class="QProgressBar" name="progressBar">
<property name="geometry">
<rect>
<x>20</x>
<y>340</y>
<x>320</x>
<y>330</y>
<width>255</width>
<height>23</height>
</rect>
@ -98,8 +98,8 @@
<widget class="QPushButton" name="createAllRecursiveScriptsButton">
<property name="geometry">
<rect>
<x>360</x>
<y>100</y>
<x>330</x>
<y>170</y>
<width>220</width>
<height>40</height>
</rect>
@ -112,7 +112,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>80</y>
<y>110</y>
<width>220</width>
<height>40</height>
</rect>
@ -125,7 +125,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>130</y>
<y>160</y>
<width>220</width>
<height>40</height>
</rect>
@ -138,7 +138,7 @@
<property name="geometry">
<rect>
<x>20</x>
<y>180</y>
<y>250</y>
<width>220</width>
<height>40</height>
</rect>
@ -150,27 +150,83 @@
<widget class="QPushButton" name="showTaskbarButton">
<property name="geometry">
<rect>
<x>490</x>
<y>280</y>
<width>91</width>
<x>10</x>
<y>440</y>
<width>211</width>
<height>40</height>
</rect>
</property>
<property name="text">
<string>Show Taskbar</string>
<string>Show Windows Taskbar</string>
</property>
</widget>
<widget class="QPushButton" name="hideTaskbarButton">
<property name="geometry">
<rect>
<x>490</x>
<y>230</y>
<width>91</width>
<x>10</x>
<y>390</y>
<width>211</width>
<height>40</height>
</rect>
</property>
<property name="text">
<string>Hide Taskbar</string>
<string>Hide Windows Taskbar</string>
</property>
</widget>
<widget class="QLabel" name="label">
<property name="geometry">
<rect>
<x>330</x>
<y>55</y>
<width>81</width>
<height>16</height>
</rect>
</property>
<property name="font">
<font>
<pointsize>10</pointsize>
</font>
</property>
<property name="text">
<string>GitHub Branch</string>
</property>
</widget>
<widget class="QLabel" name="label_2">
<property name="geometry">
<rect>
<x>330</x>
<y>15</y>
<width>81</width>
<height>16</height>
</rect>
</property>
<property name="font">
<font>
<pointsize>10</pointsize>
</font>
</property>
<property name="text">
<string>GitHub User</string>
</property>
</widget>
<widget class="QTextEdit" name="userTextEdit">
<property name="geometry">
<rect>
<x>420</x>
<y>12</y>
<width>140</width>
<height>24</height>
</rect>
</property>
</widget>
<widget class="QTextEdit" name="branchTextEdit">
<property name="geometry">
<rect>
<x>420</x>
<y>50</y>
<width>140</width>
<height>24</height>
</rect>
</property>
</widget>
</widget>
@ -179,7 +235,7 @@
<rect>
<x>0</x>
<y>0</y>
<width>607</width>
<width>612</width>
<height>21</height>
</rect>
</property>