diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index cf11ef9e7a..16446c5071 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -23,4 +23,8 @@ if (BUILD_TOOLS) add_subdirectory(oven) set_target_properties(oven PROPERTIES FOLDER "Tools") + + add_subdirectory(auto-tester) + set_target_properties(auto-tester PROPERTIES FOLDER "Tools") endif() + diff --git a/tools/auto-tester/CMakeLists.txt b/tools/auto-tester/CMakeLists.txt new file mode 100644 index 0000000000..e5f2c1fb97 --- /dev/null +++ b/tools/auto-tester/CMakeLists.txt @@ -0,0 +1,57 @@ +set(TARGET_NAME auto-tester) +project(${TARGET_NAME}) + +# Automatically run UIC and MOC. This replaces the older WRAP macros +SET(CMAKE_AUTOUIC ON) +SET(CMAKE_AUTOMOC ON) + +setup_hifi_project(Core Widgets) +link_hifi_libraries() + +# FIX: Qt was built with -reduce-relocations +if (Qt5_POSITION_INDEPENDENT_CODE) + SET(CMAKE_POSITION_INDEPENDENT_CODE ON) +endif() + +# Qt includes +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${Qt5Core_INCLUDE_DIRS}) +include_directories(${Qt5Widgets_INCLUDE_DIRS}) + +set(QT_LIBRARIES Qt5::Core Qt5::Widgets) + +# Find all sources files +file (GLOB_RECURSE SOURCES src/*.cpp) +file (GLOB_RECURSE HEADERS src/*.h) +file (GLOB_RECURSE UIS src/ui/*.ui) + +if (WIN32) + # Do not show Console + set_property(TARGET auto-tester PROPERTY WIN32_EXECUTABLE true) +endif() + +add_executable(PROJECT_NAME ${SOURCES} ${HEADERS} ${UIS}) + +target_link_libraries(PROJECT_NAME ${QT_LIBRARIES}) + +# Copy required dll's. +add_custom_command(TARGET auto-tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $ +) + +if (WIN32) + find_program(WINDEPLOYQT_COMMAND windeployqt PATHS ${QT_DIR}/bin NO_DEFAULT_PATH) + + if (NOT WINDEPLOYQT_COMMAND) + message(FATAL_ERROR "Could not find windeployqt at ${QT_DIR}/bin. windeployqt is required.") + endif () + + # add a post-build command to call windeployqt to copy Qt plugins + add_custom_command( + TARGET ${TARGET_NAME} + POST_BUILD + COMMAND CMD /C "SET PATH=%PATH%;${QT_DIR}/bin && ${WINDEPLOYQT_COMMAND} ${EXTRA_DEPLOY_OPTIONS} $<$,$,$>:--release> \"$\"" + ) +endif () \ No newline at end of file diff --git a/tools/auto-tester/ReadMe.md b/tools/auto-tester/ReadMe.md new file mode 100644 index 0000000000..57ec7ea623 --- /dev/null +++ b/tools/auto-tester/ReadMe.md @@ -0,0 +1,7 @@ +After building auto-tester, it needs to be deployed to Amazon SW + +* In folder hifi/build/tools/auto-tester + * Right click on the Release folder + * Select 7-Zip -> Add to archive + * Select Option ```Create SFX archive``` to create Release.exe +* Use Cyberduck (or any other AWS S3 client) to copy Release.exe to hifi-content/nissim/autoTester/ \ No newline at end of file diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp new file mode 100644 index 0000000000..121c98e16e --- /dev/null +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -0,0 +1,119 @@ +// +// ImageComparer.cpp +// +// Created by Nissim Hadar on 18 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "ImageComparer.h" + +#include + +// Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity +// The value is computed for the luminance component and the average value is returned +double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const { + // Make sure the image is 8 bits per colour + QImage::Format format = expectedImage.format(); + if (format != QImage::Format::Format_RGB32) { + throw -1; + } + + const int L = 255; // (2^number of bits per pixel) - 1 + const double K1{ 0.01 }; + const double K2{ 0.03 }; + const double c1 = pow((K1 * L), 2); + const double c2 = pow((K2 * L), 2); + + // Coefficients for luminosity calculation + const double R_Y = 0.212655f; + const double G_Y = 0.715158f; + const double B_Y = 0.072187f; + + // First go over all full 8x8 blocks + // This is done in 3 loops + // 1) Read the pixels into a linear array (an optimization) + // 2) Calculate mean + // 3) Calculate variance and covariance + // + // p - pixel in expected image + // q - pixel in result image + // + const int WIN_SIZE = 8; + int x{ 0 }; // column index (start of block) + int y{ 0 }; // row index (start of block + + // Pixels are processed in square blocks + double p[WIN_SIZE * WIN_SIZE]; + double q[WIN_SIZE * WIN_SIZE]; + + int windowCounter{ 0 }; + double ssim{ 0.0 }; + while (x < expectedImage.width()) { + int lastX = x + WIN_SIZE - 1; + if (lastX > expectedImage.width() - 1) { + x -= (lastX - expectedImage.width()); + } + + while (y < expectedImage.height()) { + int lastY = y + WIN_SIZE - 1; + if (lastY > expectedImage.height() - 1) { + y -= (lastY - expectedImage.height()); + } + + // Collect pixels into linear arrays + int i{ 0 }; + for (int xx = 0; xx < WIN_SIZE; ++xx) { + for (int yy = 0; yy < WIN_SIZE; ++yy) { + // Get pixels + QRgb pixelP = expectedImage.pixel(QPoint(x + xx, y + yy)); + QRgb pixelQ = resultImage.pixel(QPoint(x + xx, y + yy)); + + // Convert to luminance + p[i] = R_Y * qRed(pixelP) + G_Y * qGreen(pixelP) + B_Y * qBlue(pixelP); + q[i] = R_Y * qRed(pixelQ) + G_Y * qGreen(pixelQ) + B_Y * qBlue(pixelQ); + + ++i; + } + } + + // Calculate mean + double mP{ 0.0 }; // average value of expected pixel + double mQ{ 0.0 }; // average value of result pixel + for (int j = 0; j < WIN_SIZE * WIN_SIZE; ++j) { + mP += p[j]; + mQ += q[j]; + } + mP /= (WIN_SIZE * WIN_SIZE); + mQ /= (WIN_SIZE * WIN_SIZE); + + // Calculate variance and covariance + double sigsqP{ 0.0 }; + double sigsqQ{ 0.0 }; + double sigPQ{ 0.0 }; + for (int j = 0; j < WIN_SIZE * WIN_SIZE; ++j) { + sigsqP += pow((p[j] - mP), 2); + sigsqQ += pow((q[j] - mQ), 2); + + sigPQ += (p[j] - mP) * (q[j] - mQ); + } + sigsqP /= (WIN_SIZE * WIN_SIZE); + sigsqQ /= (WIN_SIZE * WIN_SIZE); + sigPQ /= (WIN_SIZE * WIN_SIZE); + + double numerator = (2.0 * mP * mQ + c1) * (2.0 * sigPQ + c2); + double denominator = (mP * mP + mQ * mQ + c1) * (sigsqP + sigsqQ + c2); + + ssim += numerator / denominator; + ++windowCounter; + + y += WIN_SIZE; + } + + x += WIN_SIZE; + y = 0; + } + + return ssim / windowCounter; +}; diff --git a/tools/auto-tester/src/ImageComparer.h b/tools/auto-tester/src/ImageComparer.h new file mode 100644 index 0000000000..7b7b8b0b74 --- /dev/null +++ b/tools/auto-tester/src/ImageComparer.h @@ -0,0 +1,21 @@ +// +// ImageComparer.h +// +// Created by Nissim Hadar on 18 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_ImageComparer_h +#define hifi_ImageComparer_h + +#include +#include + +class ImageComparer { +public: + double compareImages(QImage resultImage, QImage expectedImage) const; +}; + +#endif // hifi_ImageComparer_h diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp new file mode 100644 index 0000000000..8cb36fcfca --- /dev/null +++ b/tools/auto-tester/src/Test.cpp @@ -0,0 +1,383 @@ +// +// Test.cpp +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "Test.h" + +#include +#include +#include + +Test::Test() { + snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.+-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg"); + + expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg"); + + mismatchWindow.setModal(true); +} + +bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages) { + // Loop over both lists and compare each pair of images + // Quit loop if user has aborted due to a failed test. + const double THRESHOLD{ 0.999 }; + bool success{ true }; + bool keepOn{ true }; + for (int i = 0; keepOn && i < expectedImages.length(); ++i) { + // First check that images are the same size + QImage resultImage(resultImages[i]); + QImage expectedImage(expectedImages[i]); + if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) { + messageBox.critical(0, "Internal error", "Images are not the same size"); + exit(-1); + } + + double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical + try { + similarityIndex = imageComparer.compareImages(resultImage, expectedImage); + } catch (...) { + messageBox.critical(0, "Internal error", "Image not in expected format"); + exit(-1); + } + + if (similarityIndex < THRESHOLD) { + mismatchWindow.setTestFailure(TestFailure{ + (float)similarityIndex, + expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /) + QFileInfo(expectedImages[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(resultImages[i].toStdString().c_str()).fileName() // filename of result image + }); + + mismatchWindow.exec(); + + switch (mismatchWindow.getUserResponse()) { + case USER_RESPONSE_PASS: + break; + case USE_RESPONSE_FAIL: + success = false; + break; + case USER_RESPONSE_ABORT: + keepOn = false; + success = false; + break; + default: + assert(false); + break; + } + } + } + + return success; +} + +void Test::evaluateTests() { + // Get list of JPEG images in folder, sorted by name + QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + if (pathToImageDirectory == "") { + return; + } + + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); + + // Separate images into two lists. The first is the expected images, the second is the test results + // Images that are in the wrong format are ignored. + QStringList expectedImages; + QStringList resultImages; + foreach(QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + expectedImages << fullCurrentFilename; + } else if (isInSnapshotFilenameFormat(currentFilename)) { + resultImages << fullCurrentFilename; + } + } + + // The number of images in each list should be identical + if (expectedImages.length() != resultImages.length()) { + messageBox.critical(0, + "Test failed", + "Found " + QString::number(resultImages.length()) + " images in directory" + + "\nExpected to find " + QString::number(expectedImages.length()) + " images" + ); + + exit(-1); + } + + bool success = compareImageLists(expectedImages, resultImages); + + if (success) { + messageBox.information(0, "Success", "All images are as expected"); + } else { + messageBox.information(0, "Failure", "One or more images are not as expected"); + } +} + +// Two criteria are used to decide if a folder contains valid test results. +// 1) a 'test'js' file exists in the folder +// 2) the folder has the same number of actual and expected images +void Test::evaluateTestsRecursively() { + // Select folder to start recursing from + QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); + if (topLevelDirectory == "") { + return; + } + + bool success{ true }; + QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + while (it.hasNext()) { + QString directory = it.next(); + if (directory[directory.length() - 1] == '.') { + // ignore '.', '..' directories + continue; + } + + // + const QString testPathname{ directory + "/" + testFilename }; + QFileInfo fileInfo(testPathname); + if (!fileInfo.exists()) { + // Folder does not contain 'test.js' + continue; + } + + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory); + + // Separate images into two lists. The first is the expected images, the second is the test results + // Images that are in the wrong format are ignored. + QStringList expectedImages; + QStringList resultImages; + foreach(QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = directory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + expectedImages << fullCurrentFilename; + } else if (isInSnapshotFilenameFormat(currentFilename)) { + resultImages << fullCurrentFilename; + } + } + + if (expectedImages.length() != resultImages.length()) { + // Number of images doesn't match + continue; + } + + // Set success to false if any test has failed + success &= compareImageLists(expectedImages, resultImages); + } + + if (success) { + messageBox.information(0, "Success", "All images are as expected"); + } else { + messageBox.information(0, "Failure", "One or more images are not as expected"); + } +} + +void Test::importTest(QTextStream& textStream, const QString& testPathname, int testNumber) { + textStream << "var test" << testNumber << " = Script.require(\"" << "file:///" << testPathname + "\");" << endl; +} + +// Creates a single script in a user-selected folder. +// This script will run all text.js scripts in every applicable sub-folder +void Test::createRecursiveScript() { + // Select folder to start recursing from + QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); + if (topLevelDirectory == "") { + return; + } + + QFile allTestsFilename(topLevelDirectory + "/" + "allTests.js"); + if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { + messageBox.critical(0, + "Internal Error", + "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\""); + + exit(-1); + } + + QTextStream textStream(&allTestsFilename); + textStream << "// This is an automatically generated file, created by auto-tester" << endl; + + // The main will call each test after the previous test is completed + // This is implemented with an interval timer that periodically tests if a + // running test has increment a testNumber variable that it received as an input. + int testNumber = 1; + QVector testPathnames; + + // First test if top-level folder has a test.js file + const QString testPathname{ topLevelDirectory + "/" + testFilename }; + QFileInfo fileInfo(testPathname); + if (fileInfo.exists()) { + // Current folder contains a test + importTest(textStream, testPathname, testNumber); + ++testNumber; + + testPathnames << testPathname; + } + + QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + while (it.hasNext()) { + QString directory = it.next(); + if (directory[directory.length() - 1] == '.') { + // ignore '.', '..' directories + continue; + } + + const QString testPathname{ directory + "/" + testFilename }; + QFileInfo fileInfo(testPathname); + if (fileInfo.exists()) { + // Current folder contains a test + importTest(textStream, testPathname, testNumber); + ++testNumber; + + testPathnames << testPathname; + } + } + + if (testPathnames.length() <= 0) { + messageBox.information(0, "Failure", "No \"test.js\" files found"); + allTestsFilename.close(); + return; + } + + textStream << endl; + + // Define flags for each test + for (int i = 1; i <= testPathnames.length(); ++i) { + textStream << "var test" << i << "HasNotStarted = true;" << endl; + } + + // Leave a blank line in the main + textStream << endl; + + const int TEST_PERIOD = 1000; // in milliseconds + const QString tab = " "; + + textStream << "// Check every second if the current test is complete and the next test can be run" << endl; + textStream << "var testTimer = Script.setInterval(" << endl; + textStream << tab << "function() {" << endl; + + const QString testFunction = "test"; + for (int i = 1; i <= testPathnames.length(); ++i) { + // First test starts immediately, all other tests wait for the previous test to complete. + // The script produced will look as follows: + // if (test1HasNotStarted) { + // test1HasNotStarted = false; + // test1.test(); + // print("******started test 1******"); + // } + // | + // | + // if (test5.complete && test6HasNotStarted) { + // test6HasNotStarted = false; + // test7.test(); + // print("******started test 6******"); + // } + // | + // | + // if (test12.complete) { + // print("******stopping******"); + // Script.stop(); + // } + // + if (i == 1) { + textStream << tab << tab << "if (test1HasNotStarted) {" << endl; + } else { + textStream << tab << tab << "if (test" << i - 1 << ".complete && test" << i << "HasNotStarted) {" << endl; + } + textStream << tab << tab << tab << "test" << i << "HasNotStarted = false;" << endl; + textStream << tab << tab << tab << "test" << i << "." << testFunction << "();" << endl; + textStream << tab << tab << tab << "print(\"******started test " << i << "******\");" << endl; + + textStream << tab << tab << "}" << endl << endl; + + } + + // Add extra step to stop the script + textStream << tab << tab << "if (test" << testPathnames.length() << ".complete) {" << endl; + textStream << tab << tab << tab << "print(\"******stopping******\");" << endl; + textStream << tab << tab << tab << "Script.stop();" << endl; + textStream << tab << tab << "}" << endl << endl; + + textStream << tab << "}," << endl; + textStream << endl; + textStream << tab << TEST_PERIOD << endl; + textStream << ");" << endl << endl; + + textStream << "// Stop the timer and clear the module cache" << endl; + textStream << "Script.scriptEnding.connect(" << endl; + textStream << tab << "function() {" << endl; + textStream << tab << tab << "Script.clearInterval(testTimer);" << endl; + textStream << tab << tab << "Script.require.cache = {};" << endl; + textStream << tab << "}" << endl; + textStream << ");" << endl; + + allTestsFilename.close(); + + messageBox.information(0, "Success", "Script has been created"); +} + +void Test::createTest() { + // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on + // Any existing expected result images will be deleted + QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + if (pathToImageDirectory == "") { + return; + } + + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); + + int i = 1; + foreach (QString currentFilename, sortedImageFilenames) { + QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; + if (isInExpectedImageFilenameFormat(currentFilename)) { + if (!QFile::remove(fullCurrentFilename)) { + messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); + exit(-1); + } + } else if (isInSnapshotFilenameFormat(currentFilename)) { + const int MAX_IMAGES = 100000; + if (i >= MAX_IMAGES) { + messageBox.critical(0, "Error", "More than 100,000 images not supported"); + exit(-1); + } + QString newFilename = "ExpectedImage_" + QString::number(i-1).rightJustified(5, '0') + ".jpg"; + QString fullNewFileName = pathToImageDirectory + "/" + newFilename; + + if (!imageDirectory.rename(fullCurrentFilename, newFilename)) { + if (!QFile::exists(fullCurrentFilename)) { + messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n" + + fullCurrentFilename + " not found" + + "\nTest creation aborted" + ); + exit(-1); + } else { + messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n" + + "unknown error" + "\nTest creation aborted" + ); + exit(-1); + } + } + ++i; + } + } + + messageBox.information(0, "Success", "Test images have been created"); +} + +QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) { + imageDirectory = QDir(pathToImageDirectory); + QStringList nameFilters; + nameFilters << "*.jpg"; + + return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); +} + +bool Test::isInSnapshotFilenameFormat(QString filename) { + return (snapshotFilenameFormat.match(filename).hasMatch()); +} + +bool Test::isInExpectedImageFilenameFormat(QString filename) { + return (expectedImageFilenameFormat.match(filename).hasMatch()); +} \ No newline at end of file diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h new file mode 100644 index 0000000000..1f7b1e92a7 --- /dev/null +++ b/tools/auto-tester/src/Test.h @@ -0,0 +1,55 @@ +// +// Test.h +// zone/ambientLightInheritence +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_test_h +#define hifi_test_h + +#include +#include +#include + +#include "ImageComparer.h" +#include "ui/MismatchWindow.h" + +class Test { +public: + Test(); + + void evaluateTests(); + void evaluateTestsRecursively(); + void createRecursiveScript(); + void createTest(); + + QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory); + + bool isInSnapshotFilenameFormat(QString filename); + bool isInExpectedImageFilenameFormat(QString filename); + + void importTest(QTextStream& textStream, const QString& testPathname, int testNumber); + +private: + const QString testFilename{ "test.js" }; + + QMessageBox messageBox; + + QDir imageDirectory; + + QRegularExpression snapshotFilenameFormat; + QRegularExpression expectedImageFilenameFormat; + + MismatchWindow mismatchWindow; + + ImageComparer imageComparer; + + bool compareImageLists(QStringList expectedImages, QStringList resultImages); +}; + +#endif // hifi_test_h diff --git a/tools/auto-tester/src/common.h b/tools/auto-tester/src/common.h new file mode 100644 index 0000000000..126177358f --- /dev/null +++ b/tools/auto-tester/src/common.h @@ -0,0 +1,37 @@ +// +// common.h +// +// Created by Nissim Hadar on 10 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_common_h +#define hifi_common_h + +#include + +class TestFailure { +public: + TestFailure(float error, QString pathname, QString expectedImageFilename, QString actualImageFilename) : + _error(error), + _pathname(pathname), + _expectedImageFilename(expectedImageFilename), + _actualImageFilename(actualImageFilename) + {} + + double _error; + QString _pathname; + QString _expectedImageFilename; + QString _actualImageFilename; +}; + +enum UserResponse { + USER_RESPONSE_INVALID, + USER_RESPONSE_PASS, + USE_RESPONSE_FAIL, + USER_RESPONSE_ABORT +}; + +#endif // hifi_common_h diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp new file mode 100644 index 0000000000..6e5e06b732 --- /dev/null +++ b/tools/auto-tester/src/main.cpp @@ -0,0 +1,20 @@ +// +// main.cpp +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include +#include "ui/AutoTester.h" + +int main(int argc, char *argv[]) { + QApplication application(argc, argv); + + AutoTester autoTester; + autoTester.show(); + + return application.exec(); +} diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp new file mode 100644 index 0000000000..105baddb92 --- /dev/null +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -0,0 +1,35 @@ +// +// AutoTester.cpp +// zone/ambientLightInheritence +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "AutoTester.h" + +AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { + ui.setupUi(this); +} + +void AutoTester::on_evaluateTestsButton_clicked() { + test.evaluateTests(); +} + +void AutoTester::on_evaluateTestsRecursivelyButton_clicked() { + test.evaluateTestsRecursively(); +} + +void AutoTester::on_createRecursiveScriptButton_clicked() { + test.createRecursiveScript(); +} + +void AutoTester::on_createTestButton_clicked() { + test.createTest(); +} + +void AutoTester::on_closeButton_clicked() { + exit(0); +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h new file mode 100644 index 0000000000..acfea32ba1 --- /dev/null +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -0,0 +1,37 @@ +// +// AutoTester.h +// zone/ambientLightInheritence +// +// Created by Nissim Hadar on 2 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_AutoTester_h +#define hifi_AutoTester_h + +#include +#include "ui_AutoTester.h" +#include "../Test.h" + +class AutoTester : public QMainWindow { + Q_OBJECT + +public: + AutoTester(QWidget *parent = Q_NULLPTR); + +private slots: +void on_evaluateTestsButton_clicked(); +void on_evaluateTestsRecursivelyButton_clicked(); +void on_createRecursiveScriptButton_clicked(); + void on_createTestButton_clicked(); + void on_closeButton_clicked(); + +private: + Ui::AutoTesterClass ui; + + Test test; +}; + +#endif // hifi_AutoTester_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui new file mode 100644 index 0000000000..7032ef9710 --- /dev/null +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -0,0 +1,106 @@ + + + AutoTesterClass + + + + 0 + 0 + 286 + 470 + + + + AutoTester + + + + + + 60 + 360 + 160 + 40 + + + + Close + + + + + + 60 + 270 + 160 + 40 + + + + Create Test + + + + + + 60 + 20 + 160 + 40 + + + + Evaluate Test + + + + + + 60 + 210 + 160 + 40 + + + + Create Recursive Script + + + + + + 60 + 75 + 160 + 40 + + + + Evaluate Tests Recursively + + + + + + + 0 + 0 + 286 + 21 + + + + + + TopToolBarArea + + + false + + + + + + + + diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp new file mode 100644 index 0000000000..07664a1667 --- /dev/null +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -0,0 +1,46 @@ +// +// MismatchWindow.cpp +// +// Created by Nissim Hadar on 9 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "MismatchWindow.h" + +#include + +MismatchWindow::MismatchWindow(QWidget *parent) : QDialog(parent) { + setupUi(this); + + expectedImage->setScaledContents(true); + resultImage->setScaledContents(true); +} + +void MismatchWindow::setTestFailure(TestFailure testFailure) { + errorLabel->setText("Similarity: " + QString::number(testFailure._error)); + + imagePath->setText("Path to test: " + testFailure._pathname); + + expectedFilename->setText(testFailure._expectedImageFilename); + expectedImage->setPixmap(QPixmap(testFailure._pathname + testFailure._expectedImageFilename)); + + resultFilename->setText(testFailure._actualImageFilename); + resultImage->setPixmap(QPixmap(testFailure._pathname + testFailure._actualImageFilename)); +} + +void MismatchWindow::on_passTestButton_clicked() { + _userResponse = USER_RESPONSE_PASS; + close(); +} + +void MismatchWindow::on_failTestButton_clicked() { + _userResponse = USE_RESPONSE_FAIL; + close(); +} + +void MismatchWindow::on_abortTestsButton_clicked() { + _userResponse = USER_RESPONSE_ABORT; + close(); +} diff --git a/tools/auto-tester/src/ui/MismatchWindow.h b/tools/auto-tester/src/ui/MismatchWindow.h new file mode 100644 index 0000000000..7c72b7b0b7 --- /dev/null +++ b/tools/auto-tester/src/ui/MismatchWindow.h @@ -0,0 +1,38 @@ +// +// MismatchWindow.h +// +// Created by Nissim Hadar on 9 Nov 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_MismatchWindow_h +#define hifi_MismatchWindow_h + +#include "ui_MismatchWindow.h" + +#include "../common.h" + +class MismatchWindow : public QDialog, public Ui::MismatchWindow +{ + Q_OBJECT + +public: + MismatchWindow(QWidget *parent = Q_NULLPTR); + + void setTestFailure(TestFailure testFailure); + + UserResponse getUserResponse() { return _userResponse; } + +private slots: + void on_passTestButton_clicked(); + void on_failTestButton_clicked(); + void on_abortTestsButton_clicked(); + +private: + UserResponse _userResponse{ USER_RESPONSE_INVALID }; +}; + + +#endif // hifi_MismatchWindow_h diff --git a/tools/auto-tester/src/ui/MismatchWindow.ui b/tools/auto-tester/src/ui/MismatchWindow.ui new file mode 100644 index 0000000000..cab6c61e1c --- /dev/null +++ b/tools/auto-tester/src/ui/MismatchWindow.ui @@ -0,0 +1,157 @@ + + + MismatchWindow + + + + 0 + 0 + 1585 + 694 + + + + MismatchWindow + + + + + 20 + 170 + 720 + 362 + + + + expected image + + + + + + 760 + 170 + 720 + 362 + + + + result image + + + + + + 760 + 90 + 800 + 28 + + + + + 16 + + + + result image filename + + + + + + 40 + 90 + 700 + 28 + + + + + 16 + + + + expected image filename + + + + + + 40 + 30 + 1200 + 28 + + + + + 16 + + + + image path + + + + + + 30 + 600 + 75 + 23 + + + + Pass + + + + + + 330 + 600 + 75 + 23 + + + + Fail + + + + + + 630 + 600 + 75 + 23 + + + + Abort Tests + + + + + + 810 + 600 + 720 + 28 + + + + + 16 + + + + similarity + + + + + + +