diff --git a/tools/auto-tester/CMakeLists.txt b/tools/auto-tester/CMakeLists.txt index 1546a35f4c..940de5d7bb 100644 --- a/tools/auto-tester/CMakeLists.txt +++ b/tools/auto-tester/CMakeLists.txt @@ -5,7 +5,7 @@ project(${TARGET_NAME}) SET (CMAKE_AUTOUIC ON) SET (CMAKE_AUTOMOC ON) -setup_hifi_project (Core Widgets Network Xml) +setup_hifi_project (Core Widgets Network Xml WebEngineWidgets) link_hifi_libraries () # FIX: Qt was built with -reduce-relocations diff --git a/tools/auto-tester/README.md b/tools/auto-tester/README.md new file mode 100644 index 0000000000..aa2cea9352 --- /dev/null +++ b/tools/auto-tester/README.md @@ -0,0 +1,83 @@ +# Auto Tester + +The auto-tester is a stand alone application that provides a mechanism for regression testing. The general idea is simple: +* Each test folder has a script that produces a set of snapshots. +* The snapshots are compared to a 'canonical' set of images that have been produced beforehand. +* The result, if any test failed, is a zipped folder describing the failure. + +Auto-tester has 4 functions, separated into 4 tabs: +1. Creating tests, MD files and recursive scripts +2. Evaluating the results of running tests +3. TestRail interface +4. Windows task bar utility (Windows only) + + +## Evaluation - general +A checkbox is provided for de-selecting interactive mode. If this checkbox ix selected then each error will be displayed to the user; otherwise, the evaluation will procede till completion. In both cases, all errors are logged to file. +For each failed test, a zipped folder is created within the test folder, named *TestResults*. This folder contains a folder for each failed test named *Failure_n*, numbered sequentially. Each folder contains 4 files: +1. Expected Image +2. Actual Image +3. Difference Image +4. TestResults - a text file providing details of the test folder, the original image names, and the size of the mismatch. +### Evaluate Test +Evaluating a test is performed after running a **test.js** script to create new snapshots. After selecting the folder, the images are compared in lexical order. If the similarity between any image pair does not pass a fixed threshold, the image pair is displayed and the user can select to accept the difference, fail this specific test case, or abort testing. +![](./autoTesterMismatchExample.png) +### Evaluate Tests Recursively +This is a recursive version of the previous function. Auto-tester will recurse through all folders from the selected folder. A test will be evaluated if the following is true: +* The folder contains a **test.js** script +* The number of actual and expected snapshots is the same (see Create Test for an explanation) +### Create a recursive test script +Auto-tester will create a script named **allTests.js** that will call all **test.js** scripts found in the folder, and any subfolders. An example of the script created is: +``` +// This is an automatically generated file, created by auto-tester +var test1 = Script.require("file:///D:/GitHub/hifi-tests/tests/content/entity/zone/ambientLightInheritance/test.js"); +var test2 = Script.require("file:///D:/GitHub/hifi-tests/tests/content/entity/zone/create/test.js"); +var test3 = Script.require("file:///D:/GitHub/hifi-tests/tests/content/entity/zone/createMultipleZones/test.js"); + +var test1HasNotStarted = true; +var test2HasNotStarted = true; +var test3HasNotStarted = true; + +// Check every second if the current test is complete and the next test can be run +var testTimer = Script.setInterval( + function() { + if (test1HasNotStarted) { + test1HasNotStarted = false; + test1.test(); + print("******started test 1******"); + } + + if (test1.complete && test2HasNotStarted) { + test2HasNotStarted = false; + test2.test(); + print("******started test 2******"); + } + + if (test2.complete && test3HasNotStarted) { + test3HasNotStarted = false; + test3.test(); + print("******started test 3******"); + } + + if (test3.complete) { + print("******stopping******"); + Script.stop(); + } + + }, + + 1000 +); + +// Stop the timer and clear the module cache +Script.scriptEnding.connect( + function() { + Script.clearInterval(testTimer); + Script.require.cache = {}; + } +); +``` +### Create a Test Case +A test case is created after running the test script. Running the script produces a series of snapshots, named **hifi-snap-by-**_user name_**-on-YYYY-MM-DD_HH-MM-SS.jpg**. This function simply renames these files to **ExpectedImage_1.jpg**, **ExpectedImage_2.jpg** and so on. These files can be added to version control as they are a fixed asset of the test. +### Delete old snapshots +This button recursively deletes all snapshots created for testign (not the Expected Images). diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 1cf5526ee0..60fb28583e 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -26,6 +26,7 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { connect(_ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked); connect(_ui.actionAbout, &QAction::triggered, this, &AutoTester::about); + connect(_ui.actionContent, &QAction::triggered, this, &AutoTester::content); #ifndef Q_OS_WIN _ui.tabWidget->setTabEnabled(3, false); @@ -184,6 +185,11 @@ void AutoTester::about() { QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__); } +void AutoTester::content() { + // BrowserWindow browserWindow; + // browserWindow.show(); +} + void AutoTester::setUserText(const QString& user) { _ui.userTextEdit->setText(user); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 1acde4b118..75acff0de0 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -65,6 +65,7 @@ private slots: void saveImage(int index); void about(); + void content(); private: Ui::AutoTesterClass _ui; diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index ab9ebcdc25..e37c7c86be 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -351,6 +351,7 @@ Help + @@ -374,6 +375,11 @@ About + + + Content + + diff --git a/tools/auto-tester/src/ui/BusyWindow.cpp b/tools/auto-tester/src/ui/BusyWindow.cpp index 904dc47d75..8066d576f8 100644 --- a/tools/auto-tester/src/ui/BusyWindow.cpp +++ b/tools/auto-tester/src/ui/BusyWindow.cpp @@ -9,10 +9,6 @@ // #include "BusyWindow.h" -#include - -#include - BusyWindow::BusyWindow(QWidget *parent) { setupUi(this); }