Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-08-07 17:41:12 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'autoTester' of https://github.com/NissimHadar/hifi into autoTester

Browse source
This commit is contained in:
nissim.hadar 2017-11-13 13:44:59 -08:00
commit a4b5690150
14 changed files with 728 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
tools/CMakeLists.txt
View file

@ -23,4 +23,7 @@ if (BUILD_TOOLS)
add_subdirectory(oven) add_subdirectory(oven)
set_target_properties(oven PROPERTIES FOLDER "Tools") set_target_properties(oven PROPERTIES FOLDER "Tools")
add_subdirectory(auto-tester)
set_target_properties(auto-tester PROPERTIES FOLDER "Tools")
endif() endif()

39
tools/auto-tester/CMakeLists.txt Normal file
View file

@ -0,0 +1,39 @@
set(TARGET_NAME auto-tester)
project(${TARGET_NAME})
# Automatically run UIC and MOC. This replaces the older WRAP macros
SET(CMAKE_AUTOUIC ON)
SET(CMAKE_AUTOMOC ON)
setup_hifi_project(Widgets)
link_hifi_libraries()
set_property(TARGET auto-tester PROPERTY WIN32_EXECUTABLE true)
# This is preferred to GLOB_RECURSE, as per CMake documentation
set(SOURCES src/main.cpp src/Test.cpp src/ui/autoTester.cpp src/ui/mismatchWindow.cpp)
set(MOC_HEADERS src/autoTester.h src/mismatchWindow.h)
set(UIS src/ui/autoTester.ui src/ui/mismatchWindow.ui)
if (WIN32)
add_executable(PROJECT_NAME WIN32 ${SOURCES} ${MOC_SRCS} ${UI_HEADERS})
else()
add_executable(PROJECT_NAME ${SOURCES} ${MOC_SRCS} ${RES_SOURCES} ${UI_HEADERS})
endif()
target_link_libraries(PROJECT_NAME ${QT_LIBRARIES})
# Copy required dll's.
# Note that the two ImageMagick files are copied twice. This is to allow the tester to run from VS as well as
# directly from the executable.
add_custom_command(
TARGET auto-tester
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Core> $<TARGET_FILE_DIR:auto-tester>
COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Gui> $<TARGET_FILE_DIR:auto-tester>
COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Qt5::Widgets> $<TARGET_FILE_DIR:auto-tester>
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/ImageMagick/magick.exe $<TARGET_FILE_DIR:auto-tester>
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/ImageMagick/magic.xml $<TARGET_FILE_DIR:auto-tester>
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/ImageMagick/magick.exe $<TARGET_FILE_DIR:auto-tester>/..
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/ImageMagick/magic.xml $<TARGET_FILE_DIR:auto-tester>/..
)

23
tools/auto-tester/ImageMagick/magic.xml Normal file
View file

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE magicmap [
<!ELEMENT magicmap (magic)+>
<!ELEMENT magic (#PCDATA)>
<!ATTLIST magic name CDATA #REQUIRED>
<!ATTLIST magic offset CDATA "0">
<!ATTLIST magic target CDATA #REQUIRED>
]>
<!--
Associate an image format with a unique identifier.
Many image formats have identifiers that uniquely identify a particular
image format. For example, the GIF image format always begins with GIF8
as the first 4 characters of the image. ImageMagick uses this information
to quickly determine the type of image it is dealing with when it reads
an image.
-->
<magicmap>
<!-- <magic name="GIF" offset="0" target="GIF8"/> -->
<!-- <magic name="JPEG" offset="0" target="\377\330\377"/> -->
<!-- <magic name="PNG" offset="0" target="\211PNG\r\n\032\n"/> -->
<!-- <magic name="TIFF" offset="0" target="\115\115\000\052"/> -->
</magicmap>

BIN
tools/auto-tester/ImageMagick/magick.exe Normal file
View file

Binary file not shown.

163
tools/auto-tester/src/Test.cpp Normal file
View file

@ -0,0 +1,163 @@
//
// Test.cpp
//
// Created by Nissim Hadar on 2 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
// Images are compared using the ImagMagick command line tool magick.exe
// A number of comparison metrics are available, including:
// AE Absolute error count of the number of different pixels (0=equal)
//
// DSSIM Stuctural dissimilarity index
//
// PAE Peak Absolute error of any one pixel
//
// PSNR Peak Signal to Noise Ratio The ratio of mean square difference to the maximum mean square
// that can exist between any two images, expressed as a decibel value.
// The higher the PSNR the closer the closer the images are, with
// a maximum difference occurring at 1. A PSNR of 20 means
// differences are 1 / 100 of maximum.
//
// MAE Mean Absolute Error average channel error distance
//
// MSE Mean Squared Error average squared error distance
//
// RMSE squareRoot Mean Error sqrt(MSE)
//
#include "Test.h"
#include <assert.h>
#include <QTextStream>
Test::Test() {
snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg");
expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg");
mismatchWindow.setModal(true);
}
void Test::evaluateTests() {
createListOfAllJPEGimagesInDirectory();
// Separate images into two lists. The first is the expected images, the second is the test results
// Images that are in the wrong format are ignored.
QStringList expectedImages;
QStringList resultImages;
foreach(QString currentFilename, sortedImageFilenames) {
QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
if (isInExpectedImageFilenameFormat(currentFilename)) {
expectedImages << fullCurrentFilename;
} else if (isInSnapshotFilenameFormat(currentFilename)) {
resultImages << fullCurrentFilename;
}
}
// The number of images in each list should be identical
if (expectedImages.length() != resultImages.length()) {
messageBox.critical(0,
"Test failed",
"Found " + QString::number(resultImages.length()) + " images in directory" +
"\nExpected to find " + QString::number(expectedImages.length()) + " images");
exit(-1);
}
// Now loop over both lists and compare each pair of images
// Quit loop if user has aborted due to a failed test.
const float THRESHOLD{ 10.0f };
bool success{ true };
bool keepOn{ true };
for (int i = 0; keepOn && i < expectedImages.length(); ++i) {
QString diffFilename = "HIFI_AutoTest_diff.txt";
QString command = "magick.exe compare -metric MAE " + expectedImages[i] + " " + resultImages[i] + " null: 2>" + diffFilename;
system(command.toStdString().c_str());
QFile file(diffFilename);
if (!file.open(QIODevice::ReadOnly)) {
messageBox.critical(0, "error", file.errorString());
}
// First value on line is the comparison result
QTextStream in(&file);
QString line = in.readLine();
QStringList tokens = line.split(' ');
float error = tokens[0].toFloat();
if (error > THRESHOLD) {
mismatchWindow.setTestFailure(TestFailure{
error, // value of the error (float)
expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /
QFileInfo(expectedImages[i].toStdString().c_str()).fileName(), // filename of expected image
QFileInfo(resultImages[i].toStdString().c_str()).fileName() // filename of result image
});
mismatchWindow.exec();
switch (mismatchWindow.getUserResponse()) {
case USER_RESPONSE_PASS:
break;
case USE_RESPONSE_FAIL:
success = false;
break;
case USER_RESPONSE_ABORT:
keepOn = false;
success = false;
break;
default:
assert(false);
}
}
}
if (success) {
messageBox.information(0, "Success", "All images are as expected");
} else {
messageBox.information(0, "Failure", "One or more images are not as expected");
}
}
void Test::createTest() {
// Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on
// Any existing expected result images will be deleted
createListOfAllJPEGimagesInDirectory();
int i = 1;
foreach (QString currentFilename, sortedImageFilenames) {
QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename;
if (isInExpectedImageFilenameFormat(currentFilename)) {
if (!QFile::remove(fullCurrentFilename)) {
messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted");
exit(-1);
}
} else if (isInSnapshotFilenameFormat(currentFilename)) {
QString newFilename = "ExpectedImage_" + QString::number(i) + ".jpg";
QString fullNewFileName = pathToImageDirectory + "/" + newFilename;
imageDirectory.rename(fullCurrentFilename, newFilename);
++i;
}
}
}
void Test::createListOfAllJPEGimagesInDirectory() {
// get list of JPEG images in folder, sorted by name
pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly);
imageDirectory = QDir(pathToImageDirectory);
QStringList nameFilters;
nameFilters << "*.jpg";
sortedImageFilenames = imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name);
}
bool Test::isInSnapshotFilenameFormat(QString filename) {
return (snapshotFilenameFormat.match(filename).hasMatch());
}
bool Test::isInExpectedImageFilenameFormat(QString filename) {
return (expectedImageFilenameFormat.match(filename).hasMatch());
}

46
tools/auto-tester/src/Test.h Normal file
View file

@ -0,0 +1,46 @@
//
// Test.h
// zone/ambientLightInheritence
//
// Created by Nissim Hadar on 2 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_test_h
#define hifi_test_h
#include <QFileDialog>
#include <QMessagebox>
#include <QRegularExpression>
#include "ui/MismatchWindow.h"
class Test {
public:
Test();
void evaluateTests();
void createTest();
void createListOfAllJPEGimagesInDirectory();
bool isInSnapshotFilenameFormat(QString filename);
bool isInExpectedImageFilenameFormat(QString filename);
private:
QMessageBox messageBox;
QString pathToImageDirectory;
QDir imageDirectory;
QStringList sortedImageFilenames;
QRegularExpression snapshotFilenameFormat;
QRegularExpression expectedImageFilenameFormat;
MismatchWindow mismatchWindow;
};
#endif // hifi_test_h

37
tools/auto-tester/src/common.h Normal file
View file

@ -0,0 +1,37 @@
//
// common.h
//
// Created by Nissim Hadar on 10 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_common_h
#define hifi_common_h
#include <QString>
class TestFailure {
public:
TestFailure(float error, QString pathname, QString expectedImageFilename, QString resultImageFilename) {
_error = error;
_pathname = pathname;
_expectedImageFilename = expectedImageFilename;
_resultImageFilename = resultImageFilename;
}
float _error;
QString _pathname;
QString _expectedImageFilename;
QString _resultImageFilename;
};
enum UserResponse {
USER_RESPONSE_INVALID,
USER_RESPONSE_PASS,
USE_RESPONSE_FAIL,
USER_RESPONSE_ABORT
};
#endif // hifi_common_h

21
tools/auto-tester/src/main.cpp Normal file
View file

@ -0,0 +1,21 @@
//
// Test.cpp
//
// Created by Nissim Hadar on 2 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <QtWidgets/QApplication>
#include "ui/AutoTester.h"
int main(int argc, char *argv[])
{
QApplication application(argc, argv);
AutoTester autoTester;
autoTester.show();
return application.exec();
}

32
tools/auto-tester/src/ui/AutoTester.cpp Normal file
View file

@ -0,0 +1,32 @@
//
// AutoTester.cpp
// zone/ambientLightInheritence
//
// Created by Nissim Hadar on 2 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "AutoTester.h"
AutoTester::AutoTester(QWidget *parent)
: QMainWindow(parent)
{
ui.setupUi(this);
}
void AutoTester::on_closeButton_clicked()
{
exit(0);
}
void AutoTester::on_evaluateTestsButton_clicked()
{
test.evaluateTests();
}
void AutoTester::on_createTestButton_clicked()
{
test.createTest();
}

36
tools/auto-tester/src/ui/AutoTester.h Normal file
View file

@ -0,0 +1,36 @@
//
// AutoTester.h
// zone/ambientLightInheritence
//
// Created by Nissim Hadar on 2 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AutoTester_h
#define hifi_AutoTester_h
#include <QtWidgets/QMainWindow>
#include "ui_AutoTester.h"
#include "../Test.h"
class AutoTester : public QMainWindow
{
Q_OBJECT
public:
AutoTester(QWidget *parent = Q_NULLPTR);
private slots:
void on_evaluateTestsButton_clicked();
void on_createTestButton_clicked();
void on_closeButton_clicked();
private:
Ui::AutoTesterClass ui;
Test test;
};
#endif // hifi_AutoTester_h

82
tools/auto-tester/src/ui/AutoTester.ui Normal file
View file

@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>AutoTesterClass</class>
<widget class="QMainWindow" name="AutoTesterClass">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>261</width>
<height>330</height>
</rect>
</property>
<property name="windowTitle">
<string>AutoTester</string>
</property>
<widget class="QWidget" name="centralWidget">
<widget class="QPushButton" name="closeButton">
<property name="geometry">
<rect>
<x>80</x>
<y>210</y>
<width>100</width>
<height>23</height>
</rect>
</property>
<property name="text">
<string>Close</string>
</property>
</widget>
<widget class="QPushButton" name="createTestButton">
<property name="geometry">
<rect>
<x>80</x>
<y>160</y>
<width>100</width>
<height>23</height>
</rect>
</property>
<property name="text">
<string>Create Test</string>
</property>
</widget>
<widget class="QPushButton" name="evaluateTestsButton">
<property name="geometry">
<rect>
<x>80</x>
<y>20</y>
<width>100</width>
<height>23</height>
</rect>
</property>
<property name="text">
<string>Evaulate Tests</string>
</property>
</widget>
</widget>
<widget class="QMenuBar" name="menuBar">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>261</width>
<height>21</height>
</rect>
</property>
</widget>
<widget class="QToolBar" name="mainToolBar">
<attribute name="toolBarArea">
<enum>TopToolBarArea</enum>
</attribute>
<attribute name="toolBarBreak">
<bool>false</bool>
</attribute>
</widget>
<widget class="QStatusBar" name="statusBar"/>
</widget>
<layoutdefault spacing="6" margin="11"/>
<resources>
<include location="../src/AutoTester.qrc"/>
</resources>
<connections/>
</ui>

51
tools/auto-tester/src/ui/MismatchWindow.cpp Normal file
View file

@ -0,0 +1,51 @@
//
// MismatchWindow.cpp
//
// Created by Nissim Hadar on 9 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "MismatchWindow.h"
#include <QFileInfo>
MismatchWindow::MismatchWindow(QWidget *parent)
: QDialog(parent)
{
setupUi(this);
expectedImage->setScaledContents(true);
resultImage->setScaledContents(true);
}
void MismatchWindow::setTestFailure(TestFailure testFailure) {
errorLabel->setText("Error: " + QString::number((int)testFailure._error));
imagePath->setText("Path to test: " + testFailure._pathname);
expectedFilename->setText(testFailure._expectedImageFilename);
expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename));
resultFilename->setText(testFailure._resultImageFilename);
resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._resultImageFilename));
}
void MismatchWindow::on_passTestButton_clicked()
{
_userResponse = USER_RESPONSE_PASS;
close();
}
void MismatchWindow::on_failTestButton_clicked()
{
_userResponse = USE_RESPONSE_FAIL;
close();
}
void MismatchWindow::on_abortTestsButton_clicked()
{
_userResponse = USER_RESPONSE_ABORT;
close();
}

38
tools/auto-tester/src/ui/MismatchWindow.h Normal file
View file

@ -0,0 +1,38 @@
//
// MismatchWindow.cpp
//
// Created by Nissim Hadar on 9 Nov 2017.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_MismatchWindow_h
#define hifi_MismatchWindow_h
#include "ui_MismatchWindow.h"
#include "../common.h"
class MismatchWindow : public QDialog, public Ui::MismatchWindow
{
Q_OBJECT
public:
MismatchWindow(QWidget *parent = Q_NULLPTR);
void setTestFailure(TestFailure testFailure);
UserResponse getUserResponse() { return _userResponse; }
private slots:
void on_passTestButton_clicked();
void on_failTestButton_clicked();
void on_abortTestsButton_clicked();
private:
UserResponse _userResponse{ USER_RESPONSE_INVALID };
};
#endif // hifi_MismatchWindow_h

157
tools/auto-tester/src/ui/MismatchWindow.ui Normal file
View file

@ -0,0 +1,157 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>MismatchWindow</class>
<widget class="QDialog" name="MismatchWindow">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>1585</width>
<height>694</height>
</rect>
</property>
<property name="windowTitle">
<string>MismatchWindow</string>
</property>
<widget class="QLabel" name="expectedImage">
<property name="geometry">
<rect>
<x>20</x>
<y>170</y>
<width>720</width>
<height>362</height>
</rect>
</property>
<property name="text">
<string>expected image</string>
</property>
</widget>
<widget class="QLabel" name="resultImage">
<property name="geometry">
<rect>
<x>760</x>
<y>170</y>
<width>720</width>
<height>362</height>
</rect>
</property>
<property name="text">
<string>result image</string>
</property>
</widget>
<widget class="QLabel" name="resultFilename">
<property name="geometry">
<rect>
<x>760</x>
<y>90</y>
<width>800</width>
<height>28</height>
</rect>
</property>
<property name="font">
<font>
<pointsize>16</pointsize>
</font>
</property>
<property name="text">
<string>result image filename</string>
</property>
</widget>
<widget class="QLabel" name="expectedFilename">
<property name="geometry">
<rect>
<x>40</x>
<y>90</y>
<width>700</width>
<height>28</height>
</rect>
</property>
<property name="font">
<font>
<pointsize>16</pointsize>
</font>
</property>
<property name="text">
<string>expected image filename</string>
</property>
</widget>
<widget class="QLabel" name="imagePath">
<property name="geometry">
<rect>
<x>40</x>
<y>30</y>
<width>1200</width>
<height>28</height>
</rect>
</property>
<property name="font">
<font>
<pointsize>16</pointsize>
</font>
</property>
<property name="text">
<string>image path</string>
</property>
</widget>
<widget class="QPushButton" name="passTestButton">
<property name="geometry">
<rect>
<x>30</x>
<y>600</y>
<width>75</width>
<height>23</height>
</rect>
</property>
<property name="text">
<string>Pass</string>
</property>
</widget>
<widget class="QPushButton" name="failTestButton">
<property name="geometry">
<rect>
<x>330</x>
<y>600</y>
<width>75</width>
<height>23</height>
</rect>
</property>
<property name="text">
<string>Fail</string>
</property>
</widget>
<widget class="QPushButton" name="abortTestsButton">
<property name="geometry">
<rect>
<x>630</x>
<y>600</y>
<width>75</width>
<height>23</height>
</rect>
</property>
<property name="text">
<string>Abort Tests</string>
</property>
</widget>
<widget class="QLabel" name="errorLabel">
<property name="geometry">
<rect>
<x>960</x>
<y>600</y>
<width>181</width>
<height>28</height>
</rect>
</property>
<property name="font">
<font>
<pointsize>16</pointsize>
</font>
</property>
<property name="text">
<string>error</string>
</property>
</widget>
</widget>
<layoutdefault spacing="6" margin="11"/>
<resources/>
<connections/>
</ui>