Mirrors/overte-thingvellir
3
0
Fork 0
mirror of https://thingvellir.net/git/overte synced 2025-03-27 23:52:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Changed stored failure images from JPG to PNG

Browse source 
Now uses snapshot name to find correct test folder - allows creating multiple tests
Reversed order of recursive scripts, so execution will be in alphabetical order.
This commit is contained in:
NissimHadar 2018-05-22 16:34:32 -07:00
parent 87989268bf
commit f8b69ef1f1
6 changed files with 75 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
tools/auto-tester/src/ImageComparer.cpp
View file

@ -15,12 +15,6 @@
// Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity
// The value is computed for the luminance component and the average value is returned
double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const {
// Make sure the image is 8 bits per colour
QImage::Format format = expectedImage.format();
if (format != QImage::Format::Format_ARGB32) {
throw -1;
}
const int L = 255; // (2^number of bits per pixel) - 1
const double K1 { 0.01 };
const double K2 { 0.03 };

89
tools/auto-tester/src/Test.cpp
View file

@ -158,20 +158,20 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai
QString destinationFile;
sourceFile = testFailure._pathname + testFailure._expectedImageFilename;
destinationFile = failureFolderPath + "/" + "Expected Image.jpg";
destinationFile = failureFolderPath + "/" + "Expected Image.png";
if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
sourceFile = testFailure._pathname + testFailure._actualImageFilename;
destinationFile = failureFolderPath + "/" + "Actual Image.jpg";
destinationFile = failureFolderPath + "/" + "Actual Image.png";
if (!QFile::copy(sourceFile, destinationFile)) {
QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile);
exit(-1);
}
comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg");
comparisonImage.save(failureFolderPath + "/" + "Difference Image.png");
}
void Test::startTestsEvaluation(const QString& testFolder) {
@ -295,7 +295,7 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) {
return partialPath;
}
void Test::importTest(QTextStream& textStream, const QString& testPathname) {
void Test::includeTest(QTextStream& textStream, const QString& testPathname) {
QString partialPath = extractPathFromTestsDown(testPathname);
textStream << "Script.include(\""
<< "https://github.com/" << GIT_HUB_USER << "/hifi_tests/blob/" << GIT_HUB_BRANCH
@ -329,24 +329,24 @@ void Test::createRecursiveScript() {
// This method creates a `testRecursive.js` script in every sub-folder.
void Test::createAllRecursiveScripts() {
// Select folder to start recursing from
QString previousSelection = testDirectory;
QString previousSelection = testsRootDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts",
parent, QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (testsRootDirectory == "") {
testsRootDirectory = previousSelection;
return;
}
createRecursiveScript(testDirectory, false);
createRecursiveScript(testsRootDirectory, false);
QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) {
QString directory = it.next();
@ -395,22 +395,31 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm");
textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl;
textStream << "user = \"" + GIT_HUB_USER + "/\"" << endl;
textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\"" << endl;
textStream << "branch = \"" + GIT_HUB_BRANCH + "/\"" << endl << endl;
textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/"
+ GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl;
textStream << "autoTester.enableRecursive();" << endl;
textStream << "autoTester.enableAuto();" << endl << endl;
QVector<QString> testPathnames;
// This is used to verify that the recursive test contains at least one test
bool testFound{ false };
// Directories are included in reverse order. The autoTester scripts use a stack mechanism,
// so this ensures that the tests run in alphabetical order (a convenience when debugging)
QStringList directories;
// First test if top-level folder has a test.js file
const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME };
QFileInfo fileInfo(testPathname);
if (fileInfo.exists()) {
// Current folder contains a test
importTest(textStream, testPathname);
directories.push_front(testPathname);
testPathnames << testPathname;
testFound = true;
}
QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
@ -427,18 +436,23 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
QFileInfo fileInfo(testPathname);
if (fileInfo.exists()) {
// Current folder contains a test
importTest(textStream, testPathname);
directories.push_front(testPathname);
testPathnames << testPathname;
testFound = true;
}
}
if (interactiveMode && testPathnames.length() <= 0) {
if (interactiveMode && !testFound) {
QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
allTestsFilename.close();
return;
}
// Now include the test scripts
for (int i = 0; i < directories.length(); ++i) {
includeTest(textStream, directories.at(i));
}
textStream << endl;
textStream << "autoTester.runRecursive();" << endl;
@ -449,7 +463,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
}
}
void Test::createTest() {
void Test::createTests() {
// Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on
// Any existing expected result images will be deleted
QString previousSelection = snapshotDirectory;
@ -467,18 +481,18 @@ void Test::createTest() {
return;
}
previousSelection = testDirectory;
previousSelection = testsRootDirectory;
parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images", parent,
testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (testsRootDirectory == "") {
testsRootDirectory = previousSelection;
return;
}
@ -493,8 +507,23 @@ void Test::createTest() {
QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported");
exit(-1);
}
QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png";
QString fullNewFileName = testDirectory + "/" + newFilename;
// Path to test is extracted from the file name
// Example:
// filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg
// path is <testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd
//
// Note: we don't use the first part and the last 2 parts of the filename at this stage
//
QStringList pathParts = currentFilename.split(".");
QString fullNewFileName = testsRootDirectory;
for (int j = 1; j < pathParts.size() - 2; ++j) {
fullNewFileName += "/" + pathParts[j];
}
// The image index is the penultimate component of the path parts (the last being the file extension)
QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png";
fullNewFileName += "/" + newFilename;
try {
copyJPGtoPNG(fullCurrentFilename, fullNewFileName);
@ -602,29 +631,29 @@ void Test::createMDFile() {
void Test::createAllMDFiles() {
// Select folder to start recursing from
QString previousSelection = testDirectory;
QString previousSelection = testsRootDirectory;
QString parent = previousSelection.left(previousSelection.lastIndexOf('/'));
if (!parent.isNull() && parent.right(1) != "/") {
parent += "/";
}
testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent,
testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent,
QFileDialog::ShowDirsOnly);
// If user cancelled then restore previous selection and return
if (testDirectory == "") {
testDirectory = previousSelection;
if (testsRootDirectory == "") {
testsRootDirectory = previousSelection;
return;
}
// First test if top-level folder has a test.js file
const QString testPathname { testDirectory + "/" + TEST_FILENAME };
const QString testPathname{ testsRootDirectory + "/" + TEST_FILENAME };
QFileInfo fileInfo(testPathname);
if (fileInfo.exists()) {
createMDFile(testDirectory);
createMDFile(testsRootDirectory);
}
QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories);
while (it.hasNext()) {
QString directory = it.next();

16
tools/auto-tester/src/Test.h
View file

@ -44,7 +44,7 @@ public:
void createAllRecursiveScripts();
void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
void createTest();
void createTests();
void createMDFile();
void createAllMDFiles();
void createMDFile(const QString& topLevelDirectory);
@ -57,7 +57,7 @@ public:
bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename);
void importTest(QTextStream& textStream, const QString& testPathname);
void includeTest(QTextStream& textStream, const QString& testPathname);
void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage);
@ -91,8 +91,10 @@ private:
// We have two directories to work with.
// The first is the directory containing the test we are working with
// The second contains the snapshots taken for test runs that need to be evaluated
// The second is the root directory of all tests
// The third contains the snapshots taken for test runs that need to be evaluated
QString testDirectory;
QString testsRootDirectory;
QString snapshotDirectory;
QStringList expectedImagesFilenames;
@ -100,9 +102,11 @@ private:
QStringList resultImagesFullFilenames;
// Used for accessing GitHub
const QString GIT_HUB_USER{ "highfidelity" };
const QString GIT_HUB_BRANCH { "master" };
const QString DATETIME_FORMAT { "yyyy-MM-dd_hh-mm-ss" };
const QString GIT_HUB_USER{ "NissimHadar" };
const QString GIT_HUB_REPOSITORY{ "hifi_tests" };
const QString GIT_HUB_BRANCH{ "newAvatar" };
const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" };
ExtractedText getTestScriptLines(QString testFileName);

6
tools/auto-tester/src/ui/AutoTester.cpp
View file

@ -40,8 +40,8 @@ void AutoTester::on_createAllRecursiveScriptsButton_clicked() {
test->createAllRecursiveScripts();
}
void AutoTester::on_createTestButton_clicked() {
test->createTest();
void AutoTester::on_createTestsButton_clicked() {
test->createTests();
}
void AutoTester::on_createMDFileButton_clicked() {
@ -95,7 +95,7 @@ void AutoTester::saveImage(int index) {
pixmap.loadFromData(downloaders[index]->downloadedData());
QImage image = pixmap.toImage();
image = image.convertToFormat(QImage::Format_ARGB32);
image = image.convertToFormat(QImage::Format_RGB32);
QString fullPathname = _directoryName + "/" + _filenames[index];
if (!image.save(fullPathname, 0, 100)) {

2
tools/auto-tester/src/ui/AutoTester.h
View file

@ -32,7 +32,7 @@ private slots:
void on_evaluateTestsButton_clicked();
void on_createRecursiveScriptButton_clicked();
void on_createAllRecursiveScriptsButton_clicked();
void on_createTestButton_clicked();
void on_createTestsButton_clicked();
void on_createMDFileButton_clicked();
void on_createAllMDFilesButton_clicked();
void on_createTestsOutlineButton_clicked();

4
tools/auto-tester/src/ui/AutoTester.ui
View file

@ -27,7 +27,7 @@
<string>Close</string>
</property>
</widget>
<widget class="QPushButton" name="createTestButton">
<widget class="QPushButton" name="createTestsButton">
<property name="geometry">
<rect>
<x>20</x>
@ -37,7 +37,7 @@
</rect>
</property>
<property name="text">
<string>Create Test</string>
<string>Create Tests</string>
</property>
</widget>
<widget class="QPushButton" name="evaluateTestsButton">