From b81f69d97a7aabaf2f6dc50e573dad7cb065fd45 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 23 Feb 2018 18:40:16 -0800 Subject: [PATCH 01/19] WIP - recursive tests. --- tools/auto-tester/src/Test.cpp | 13 +++++++------ tools/auto-tester/src/Test.h | 1 + tools/auto-tester/src/ui/AutoTester.cpp | 4 ++++ tools/auto-tester/src/ui/AutoTester.h | 1 + tools/auto-tester/src/ui/AutoTester.ui | 21 +++++++++++++++++---- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 816eac7fbd..931a940760 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -319,11 +319,12 @@ void Test::createRecursiveScript() { return; } - QFile allTestsFilename(topLevelDirectory + "/" + "allTests.js"); + const QString recursiveTestsFilename("testRecursive.js"); + QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename); if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { messageBox.critical(0, "Internal Error", - "Failed to create \"allTests.js\" in directory \"" + topLevelDirectory + "\"" + "Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\"" ); exit(-1); @@ -335,9 +336,6 @@ void Test::createRecursiveScript() { textStream << "var autoTester = Script.require(\"https://github.com/highfidelity/hifi_tests/blob/master/tests/utils/autoTester.js?raw=true\");" << endl; textStream << "autoTester.enableRecursive();" << endl << endl; - // The main will call each test after the previous test is completed - // This is implemented with an interval timer that periodically tests if a - // running test has increment a testNumber variable that it received as an input. QVector testPathnames; // First test if top-level folder has a test.js file @@ -371,7 +369,7 @@ void Test::createRecursiveScript() { } if (testPathnames.length() <= 0) { - messageBox.information(0, "Failure", "No \"test.js\" files found"); + messageBox.information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found"); allTestsFilename.close(); return; } @@ -383,6 +381,9 @@ void Test::createRecursiveScript() { messageBox.information(0, "Success", "Script has been created"); } +void Test::createRecursiveScriptsRecursively() { +} + void Test::createTest() { // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on // Any existing expected result images will be deleted diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 3177df4d47..3da5badb6d 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -26,6 +26,7 @@ public: void evaluateTests(bool interactiveMode, QProgressBar* progressBar); void evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar); void createRecursiveScript(); + void createRecursiveScriptsRecursively(); void createTest(); void deleteOldSnapshots(); diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 2834ff81e0..18eef13e6e 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -30,6 +30,10 @@ void AutoTester::on_createRecursiveScriptButton_clicked() { test.createRecursiveScript(); } +void AutoTester::on_createRecursiveScriptsRecursivelyButton_clicked() { + test.createRecursiveScriptsRecursively(); +} + void AutoTester::on_createTestButton_clicked() { test.createTest(); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 35f609a89d..d25969589b 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -24,6 +24,7 @@ private slots: void on_evaluateTestsButton_clicked(); void on_evaluateTestsRecursivelyButton_clicked(); void on_createRecursiveScriptButton_clicked(); + void on_createRecursiveScriptsRecursivelyButton_clicked(); void on_createTestButton_clicked(); void on_deleteOldSnapshotsButton_clicked(); void on_closeButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index d06255acf6..0d142ec43e 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -17,7 +17,7 @@ - 190 + 20 300 220 40 @@ -31,7 +31,7 @@ 360 - 130 + 210 220 40 @@ -112,7 +112,7 @@ 360 - 240 + 270 220 40 @@ -121,6 +121,19 @@ Delete Old Snapshots + + + + 360 + 140 + 220 + 40 + + + + Create Recursive Scripts Recursively + + @@ -145,4 +158,4 @@ - \ No newline at end of file + From 2a1c22e7a5eb53548d0f901cfb3e9143e1c6c7ef Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Thu, 1 Mar 2018 11:10:45 -0800 Subject: [PATCH 02/19] "Create Test" copies (and renames) images from snapshot folder to the corresponding test folder. --- tools/auto-tester/src/Test.cpp | 68 ++++++++++++++++++++++------------ tools/auto-tester/src/Test.h | 4 +- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 931a940760..357fc7b296 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -17,8 +17,6 @@ #include Test::Test() { - snapshotFilenameFormat = QRegularExpression("hifi-snap-by-.*-on-\\d\\d\\d\\d-\\d\\d-\\d\\d_\\d\\d-\\d\\d-\\d\\d.jpg"); - expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg"); mismatchWindow.setModal(true); @@ -387,16 +385,16 @@ void Test::createRecursiveScriptsRecursively() { void Test::createTest() { // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on // Any existing expected result images will be deleted - QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); - if (pathToImageDirectory == "") { + QString imageSourceDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + if (imageSourceDirectory == "") { return; } - QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); + QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(imageSourceDirectory); int i = 1; foreach (QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; + QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename; if (isInExpectedImageFilenameFormat(currentFilename)) { if (!QFile::remove(fullCurrentFilename)) { messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); @@ -408,22 +406,15 @@ void Test::createTest() { messageBox.critical(0, "Error", "More than 100,000 images not supported"); exit(-1); } - QString newFilename = "ExpectedImage_" + QString::number(i-1).rightJustified(5, '0') + ".jpg"; - QString fullNewFileName = pathToImageDirectory + "/" + newFilename; + QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".jpg"; + QString imageDestinationDirectory = getImageDestinationDirectory(currentFilename); + QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; - if (!imageDirectory.rename(fullCurrentFilename, newFilename)) { - if (!QFile::exists(fullCurrentFilename)) { - messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n" - + fullCurrentFilename + " not found" - + "\nTest creation aborted" - ); - exit(-1); - } else { - messageBox.critical(0, "Error", "Could not rename file: " + fullCurrentFilename + " to: " + newFilename + "\n" - + "unknown error" + "\nTest creation aborted" - ); - exit(-1); - } + try { + QFile::copy(fullCurrentFilename, fullNewFileName); + } catch (...) { + messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); + exit(-1); } ++i; } @@ -475,9 +466,40 @@ QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirect return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } -// Use regular expressions to check if files are in specific format +// Snapshots are files in the following format: +// Filename contains no periods (excluding period before exception +// Filename (i.e. without extension) contains _tests_ (this is based on all test scripts being within the tests folder +// Last 5 characters in filename are digits +// Extension is jpg bool Test::isInSnapshotFilenameFormat(QString filename) { - return (snapshotFilenameFormat.match(filename).hasMatch()); + QStringList filenameParts = filename.split("."); + + bool filnameHasNoPeriods = (filenameParts.size() == 2); + bool contains_tests = filenameParts[0].contains("_tests_"); + + bool last5CharactersAreDigits; + filenameParts[0].right(5).toInt(&last5CharactersAreDigits, 10); + + bool extensionIsJPG = filenameParts[1] == "jpg"; + + return (filnameHasNoPeriods && contains_tests && last5CharactersAreDigits && extensionIsJPG); +} + +// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is +// D:/GitHub/hifi-tests/tests/content/entity/zone/create +// This method assumes the filename is in the correct format +// The final part of the filename is the image number. This is checked for sanity +QString Test::getImageDestinationDirectory(QString filename) { + QString filenameWithoutExtension = filename.split(".")[0]; + QStringList filenameParts = filenameWithoutExtension.split("_"); + + QString result = filenameParts[0] + ":"; + + for (int i = 1; i < filenameParts.length() - 1; ++i) { + result += "/" + filenameParts[i]; + } + + return result; } bool Test::isInExpectedImageFilenameFormat(QString filename) { diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 3da5badb6d..55fb7229d1 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -46,6 +46,8 @@ public: bool isAValidDirectory(QString pathname); + QString getImageDestinationDirectory(QString filename); + private: const QString TEST_FILENAME { "test.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; @@ -55,14 +57,12 @@ private: QDir imageDirectory; - QRegularExpression snapshotFilenameFormat; QRegularExpression expectedImageFilenameFormat; MismatchWindow mismatchWindow; ImageComparer imageComparer; - QString testResultsFolderPath { "" }; int index { 1 }; }; From 5bb4023fb5d62869923a2a0f9a1b47383d05c3df Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Thu, 1 Mar 2018 19:23:17 -0800 Subject: [PATCH 03/19] First version - can download fixed URL to fixed location. --- tools/auto-tester/CMakeLists.txt | 2 +- tools/auto-tester/src/Test.cpp | 63 +++++++++++++------------ tools/auto-tester/src/Test.h | 7 ++- tools/auto-tester/src/main.cpp | 6 ++- tools/auto-tester/src/ui/AutoTester.cpp | 35 ++++++++++---- tools/auto-tester/src/ui/AutoTester.h | 8 +++- 6 files changed, 76 insertions(+), 45 deletions(-) diff --git a/tools/auto-tester/CMakeLists.txt b/tools/auto-tester/CMakeLists.txt index a875f5676a..a2589bb760 100644 --- a/tools/auto-tester/CMakeLists.txt +++ b/tools/auto-tester/CMakeLists.txt @@ -5,7 +5,7 @@ project(${TARGET_NAME}) SET (CMAKE_AUTOUIC ON) SET (CMAKE_AUTOMOC ON) -setup_hifi_project (Core Widgets) +setup_hifi_project (Core Widgets Network) link_hifi_libraries () # FIX: Qt was built with -reduce-relocations diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 357fc7b296..458ce0838b 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -16,8 +16,13 @@ #include #include +#include "ui/AutoTester.h" +extern AutoTester* autoTester; + Test::Test() { - expectedImageFilenameFormat = QRegularExpression("ExpectedImage_\\d+.jpg"); + QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + EXPECTED_IMAGE_TYPE); + + expectedImageFilenameFormat = QRegularExpression(regex); mismatchWindow.setModal(true); } @@ -178,49 +183,45 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { // Get list of JPEG images in folder, sorted by name - QString pathToImageDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); - if (pathToImageDirectory == "") { + QString pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + if (pathToTestResultsDirectory == "") { return; } // Leave if test results folder could not be created - if (!createTestResultsFolderPathIfNeeded(pathToImageDirectory)) { + if (!createTestResultsFolderPathIfNeeded(pathToTestResultsDirectory)) { return; } - QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(pathToImageDirectory); - - // Separate images into two lists. The first is the expected images, the second is the test results + // Create two lists. The first is the test results, the second is the expected images // Images that are in the wrong format are ignored. + + QStringList sortedTestResultsFilenames = createListOfAllJPEGimagesInDirectory(pathToTestResultsDirectory); QStringList expectedImages; QStringList resultImages; - foreach(QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = pathToImageDirectory + "/" + currentFilename; - if (isInExpectedImageFilenameFormat(currentFilename)) { - expectedImages << fullCurrentFilename; - } else if (isInSnapshotFilenameFormat(currentFilename)) { + foreach(QString currentFilename, sortedTestResultsFilenames) { + QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename; + if (isInSnapshotFilenameFormat(currentFilename)) { resultImages << fullCurrentFilename; + + QString expectedImageDirectory = getExpectedImageDestinationDirectory(currentFilename); + + // extract the digits at the end of the filename (exluding the file extension) + QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); + + QString expectedImageFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + EXPECTED_IMAGE_TYPE; + expectedImages << (expectedImageDirectory + "/" + expectedImageFilename); } } - // The number of images in each list should be identical - if (expectedImages.length() != resultImages.length()) { - messageBox.critical(0, - "Test failed", - "Found " + QString::number(resultImages.length()) + " images in directory" + - "\nExpected to find " + QString::number(expectedImages.length()) + " images" - ); + autoTester->downloadImage(QUrl("http://ribafreixo.com/wp-content/uploads/2017/03/Order-Now-Button-300x113.png")); + ////bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar); - exit(-1); - } - - bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar); - - if (success) { - messageBox.information(0, "Success", "All images are as expected"); - } else { - messageBox.information(0, "Failure", "One or more images are not as expected"); - } + ////if (success) { + //// messageBox.information(0, "Success", "All images are as expected"); + ////} else { + //// messageBox.information(0, "Failure", "One or more images are not as expected"); + ////} zipAndDeleteTestResultsFolder(); } @@ -407,7 +408,7 @@ void Test::createTest() { exit(-1); } QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".jpg"; - QString imageDestinationDirectory = getImageDestinationDirectory(currentFilename); + QString imageDestinationDirectory = getExpectedImageDestinationDirectory(currentFilename); QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; try { @@ -489,7 +490,7 @@ bool Test::isInSnapshotFilenameFormat(QString filename) { // D:/GitHub/hifi-tests/tests/content/entity/zone/create // This method assumes the filename is in the correct format // The final part of the filename is the image number. This is checked for sanity -QString Test::getImageDestinationDirectory(QString filename) { +QString Test::getExpectedImageDestinationDirectory(QString filename) { QString filenameWithoutExtension = filename.split(".")[0]; QStringList filenameParts = filenameWithoutExtension.split("_"); diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 55fb7229d1..7168a8ef69 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -46,7 +46,7 @@ public: bool isAValidDirectory(QString pathname); - QString getImageDestinationDirectory(QString filename); + QString getExpectedImageDestinationDirectory(QString filename); private: const QString TEST_FILENAME { "test.js" }; @@ -65,6 +65,11 @@ private: QString testResultsFolderPath { "" }; int index { 1 }; + + // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) + const int NUM_DIGITS { 5 }; + const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" }; + const QString EXPECTED_IMAGE_TYPE { ".jpg" }; }; #endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index 6e5e06b732..cd0ce22b13 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -10,11 +10,13 @@ #include #include "ui/AutoTester.h" +AutoTester* autoTester; + int main(int argc, char *argv[]) { QApplication application(argc, argv); - AutoTester autoTester; - autoTester.show(); + autoTester = new AutoTester(); + autoTester->show(); return application.exec(); } diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 18eef13e6e..8d0b0dc6a5 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -12,36 +12,53 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { ui.setupUi(this); - ui.checkBoxInteractiveMode->setChecked(true); - ui.progressBar->setVisible(false); + + test = new Test(); } void AutoTester::on_evaluateTestsButton_clicked() { - test.evaluateTests(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); + ////QUrl imageUrl("http://ribafreixo.com/wp-content/uploads/2017/03/Order-Now-Button-300x113.png"); + ////downloader = new Downloader(imageUrl, this); + ////connect(downloader, SIGNAL (downloaded()), this, SLOT (saveImage())); + test->evaluateTests(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); } void AutoTester::on_evaluateTestsRecursivelyButton_clicked() { - test.evaluateTestsRecursively(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); + test->evaluateTestsRecursively(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); } void AutoTester::on_createRecursiveScriptButton_clicked() { - test.createRecursiveScript(); + test->createRecursiveScript(); } void AutoTester::on_createRecursiveScriptsRecursivelyButton_clicked() { - test.createRecursiveScriptsRecursively(); + test->createRecursiveScriptsRecursively(); } void AutoTester::on_createTestButton_clicked() { - test.createTest(); + test->createTest(); } void AutoTester::on_deleteOldSnapshotsButton_clicked() { - test.deleteOldSnapshots(); + test->deleteOldSnapshots(); } void AutoTester::on_closeButton_clicked() { exit(0); -} \ No newline at end of file +} + +void AutoTester::downloadImage(QUrl url) { + downloader = new Downloader(url, this); + + connect(downloader, SIGNAL (downloaded()), this, SLOT (saveImage())); +} + +void AutoTester::saveImage() { + QPixmap image; + image.loadFromData(downloader->downloadedData()); + int er = image.width(); + int df = image.height(); + image.save("D:/Dicom/lll.png"); +} diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index d25969589b..2f6a9bca1e 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -12,6 +12,8 @@ #include #include "ui_AutoTester.h" + +#include "../Downloader.h" #include "../Test.h" class AutoTester : public QMainWindow { @@ -19,6 +21,7 @@ class AutoTester : public QMainWindow { public: AutoTester(QWidget *parent = Q_NULLPTR); + void downloadImage(QUrl url); private slots: void on_evaluateTestsButton_clicked(); @@ -29,10 +32,13 @@ private slots: void on_deleteOldSnapshotsButton_clicked(); void on_closeButton_clicked(); + void saveImage(); + private: Ui::AutoTesterClass ui; - Test test; + Test* test; + Downloader* downloader; }; #endif // hifi_AutoTester_h \ No newline at end of file From 07267f9fe6bd8618b4cf1e6d20b98677693a1946 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 2 Mar 2018 09:13:07 -0800 Subject: [PATCH 04/19] WIP. --- tools/auto-tester/src/Downloader.cpp | 32 +++++++++++++++++ tools/auto-tester/src/Downloader.h | 48 +++++++++++++++++++++++++ tools/auto-tester/src/Test.cpp | 4 ++- tools/auto-tester/src/ui/AutoTester.cpp | 2 +- 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 tools/auto-tester/src/Downloader.cpp create mode 100644 tools/auto-tester/src/Downloader.h diff --git a/tools/auto-tester/src/Downloader.cpp b/tools/auto-tester/src/Downloader.cpp new file mode 100644 index 0000000000..030aa95a19 --- /dev/null +++ b/tools/auto-tester/src/Downloader.cpp @@ -0,0 +1,32 @@ +// +// Downloader.cpp +// +// Created by Nissim Hadar on 1 Mar 2018. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "Downloader.h" + +Downloader::Downloader(QUrl imageUrl, QObject *parent) : QObject(parent) { + connect( + &_networkAccessManager, SIGNAL (finished(QNetworkReply*)), + this, SLOT (fileDownloaded(QNetworkReply*)) + ); + + QNetworkRequest request(imageUrl); + _networkAccessManager.get(request); +} + +void Downloader::fileDownloaded(QNetworkReply* reply) { + _downloadedData = reply->readAll(); + + //emit a signal + reply->deleteLater(); + emit downloaded(); +} + +QByteArray Downloader::downloadedData() const { + return _downloadedData; +} \ No newline at end of file diff --git a/tools/auto-tester/src/Downloader.h b/tools/auto-tester/src/Downloader.h new file mode 100644 index 0000000000..b0ad58fac5 --- /dev/null +++ b/tools/auto-tester/src/Downloader.h @@ -0,0 +1,48 @@ +// +// Downloader.h +// +// Created by Nissim Hadar on 1 Mar 2018. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_downloader_h +#define hifi_downloader_h + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +class Downloader : public QObject { +Q_OBJECT +public: + explicit Downloader(QUrl imageUrl, QObject *parent = 0); + + QByteArray downloadedData() const; + +signals: + void downloaded(); + + private slots: + void fileDownloaded(QNetworkReply* pReply); + +private: + QNetworkAccessManager _networkAccessManager; + QByteArray _downloadedData; +}; + +#endif // hifi_downloader_h \ No newline at end of file diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 458ce0838b..ee792c0ea4 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -214,7 +214,9 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { } } - autoTester->downloadImage(QUrl("http://ribafreixo.com/wp-content/uploads/2017/03/Order-Now-Button-300x113.png")); + ////autoTester->downloadImage(QUrl("http://ribafreixo.com/wp-content/uploads/2017/03/Order-Now-Button-300x113.png")); + ////autoTester->downloadImage(QUrl("https://github.com/NissimHadar/hifi_tests/blob/addRecursionToAutotester/tests/content/entity/zone/ambientLightInheritance/ExpectedImage_00000.jpg?raw=true")); + autoTester->downloadImage(QUrl("https://hifi-content.s3.amazonaws.com/nissim/autoTester/resources/ColourBox.jpg")); ////bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar); ////if (success) { diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 8d0b0dc6a5..5a666aa5eb 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -60,5 +60,5 @@ void AutoTester::saveImage() { image.loadFromData(downloader->downloadedData()); int er = image.width(); int df = image.height(); - image.save("D:/Dicom/lll.png"); + image.save("D:/Dicom/lll.jpg"); } From d29adcd000dd5dffc33849c9763008a1f62e78d1 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 2 Mar 2018 16:08:14 -0800 Subject: [PATCH 05/19] Can download multiple images from GitHub. --- tools/auto-tester/src/Test.cpp | 54 +++++++++++++++++++------ tools/auto-tester/src/Test.h | 1 + tools/auto-tester/src/ui/AutoTester.cpp | 36 +++++++++++++---- tools/auto-tester/src/ui/AutoTester.h | 12 +++++- 4 files changed, 81 insertions(+), 22 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index ee792c0ea4..ac10d1ab33 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -188,36 +188,43 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { return; } - // Leave if test results folder could not be created + // Quit if test results folder could not be created if (!createTestResultsFolderPathIfNeeded(pathToTestResultsDirectory)) { return; } // Create two lists. The first is the test results, the second is the expected images + // The expected images are represented as a URL to enabel download from GitHub // Images that are in the wrong format are ignored. QStringList sortedTestResultsFilenames = createListOfAllJPEGimagesInDirectory(pathToTestResultsDirectory); QStringList expectedImages; QStringList resultImages; + + const QString URLPrefix("https://raw.githubusercontent.com"); + const QString githubUser("NissimHadar"); + const QString testsRepo("hifi_tests"); + const QString branch("addRecursionToAutotester"); + foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat(currentFilename)) { resultImages << fullCurrentFilename; - QString expectedImageDirectory = getExpectedImageDestinationDirectory(currentFilename); + QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); // extract the digits at the end of the filename (exluding the file extension) QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); - QString expectedImageFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + EXPECTED_IMAGE_TYPE; - expectedImages << (expectedImageDirectory + "/" + expectedImageFilename); + + QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageFilename); + expectedImages << imageURLString; } } - ////autoTester->downloadImage(QUrl("http://ribafreixo.com/wp-content/uploads/2017/03/Order-Now-Button-300x113.png")); - ////autoTester->downloadImage(QUrl("https://github.com/NissimHadar/hifi_tests/blob/addRecursionToAutotester/tests/content/entity/zone/ambientLightInheritance/ExpectedImage_00000.jpg?raw=true")); - autoTester->downloadImage(QUrl("https://hifi-content.s3.amazonaws.com/nissim/autoTester/resources/ColourBox.jpg")); - ////bool success = compareImageLists(expectedImages, resultImages, pathToImageDirectory, interactiveMode, progressBar); + //autoTester->downloadImage(QUrl("https://raw.githubusercontent.com/NissimHadar/hifi_tests/addRecursionToAutotester/tests/content/entity/zone/ambientLightInheritance/ExpectedImage_00000.jpg")); + + autoTester->downloadImages(expectedImages); ////if (success) { //// messageBox.information(0, "Success", "All images are as expected"); @@ -386,7 +393,7 @@ void Test::createRecursiveScriptsRecursively() { } void Test::createTest() { - // Rename files sequentially, as ExpectedResult_1.jpeg, ExpectedResult_2.jpg and so on + // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on // Any existing expected result images will be deleted QString imageSourceDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); if (imageSourceDirectory == "") { @@ -404,9 +411,9 @@ void Test::createTest() { exit(-1); } } else if (isInSnapshotFilenameFormat(currentFilename)) { - const int MAX_IMAGES = 100000; - if (i >= MAX_IMAGES) { - messageBox.critical(0, "Error", "More than 100,000 images not supported"); + const int maxImages = pow(10, NUM_DIGITS); + if (i >= maxImages) { + messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); } QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".jpg"; @@ -491,7 +498,6 @@ bool Test::isInSnapshotFilenameFormat(QString filename) { // For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is // D:/GitHub/hifi-tests/tests/content/entity/zone/create // This method assumes the filename is in the correct format -// The final part of the filename is the image number. This is checked for sanity QString Test::getExpectedImageDestinationDirectory(QString filename) { QString filenameWithoutExtension = filename.split(".")[0]; QStringList filenameParts = filenameWithoutExtension.split("_"); @@ -505,6 +511,28 @@ QString Test::getExpectedImageDestinationDirectory(QString filename) { return result; } +// For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the source directory on GitHub +// is ...tests/content/entity/zone/create +// This is used to create the full URL +// This method assumes the filename is in the correct format +QString Test::getExpectedImagePartialSourceDirectory(QString filename) { + QString filenameWithoutExtension = filename.split(".")[0]; + QStringList filenameParts = filenameWithoutExtension.split("_"); + + int i { 0 }; + while (filenameParts[i] != "tests") { + ++i; + } + + QString result = filenameParts[i]; + + for (int j = i + 1; j < filenameParts.length() - 1; ++j) { + result += "/" + filenameParts[j]; + } + + return result; +} + bool Test::isInExpectedImageFilenameFormat(QString filename) { return (expectedImageFilenameFormat.match(filename).hasMatch()); } \ No newline at end of file diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 7168a8ef69..026dbf25f6 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -47,6 +47,7 @@ public: bool isAValidDirectory(QString pathname); QString getExpectedImageDestinationDirectory(QString filename); + QString getExpectedImagePartialSourceDirectory(QString filename); private: const QString TEST_FILENAME { "test.js" }; diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 5a666aa5eb..c3ab88b7f4 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -16,6 +16,8 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { ui.progressBar->setVisible(false); test = new Test(); + + signalMapper = new QSignalMapper(); } void AutoTester::on_evaluateTestsButton_clicked() { @@ -50,15 +52,35 @@ void AutoTester::on_closeButton_clicked() { } void AutoTester::downloadImage(QUrl url) { - downloader = new Downloader(url, this); + downloaders.emplace_back(new Downloader(url, this)); + connect(downloaders[_index], SIGNAL (downloaded()), signalMapper, SLOT (map())); - connect(downloader, SIGNAL (downloaded()), this, SLOT (saveImage())); + signalMapper->setMapping(downloaders[_index], _index); + + ++_index; } -void AutoTester::saveImage() { +void AutoTester::downloadImages(QStringList listOfURLs) { + _numberOfImagesToDownload = listOfURLs.size(); + _numberOfImagesDownloaded = 0; + _index = 0; + + for (int i = 0; i < _numberOfImagesToDownload; ++i) { + QUrl imageURL(listOfURLs[i]); + downloadImage(imageURL); + } + + connect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); +} + +void AutoTester::saveImage(int index) { QPixmap image; - image.loadFromData(downloader->downloadedData()); - int er = image.width(); - int df = image.height(); - image.save("D:/Dicom/lll.jpg"); + image.loadFromData(downloaders[index]->downloadedData()); + + int w = image.width(); + int h = image.height(); + + ++_numberOfImagesDownloaded; + + image.save("D:/Dicom/lll_" + QString::number(index) + ".jpg"); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 2f6a9bca1e..5c1f217421 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -11,6 +11,7 @@ #define hifi_AutoTester_h #include +#include #include "ui_AutoTester.h" #include "../Downloader.h" @@ -22,6 +23,7 @@ class AutoTester : public QMainWindow { public: AutoTester(QWidget *parent = Q_NULLPTR); void downloadImage(QUrl url); + void downloadImages(QStringList listOfURLs); private slots: void on_evaluateTestsButton_clicked(); @@ -32,13 +34,19 @@ private slots: void on_deleteOldSnapshotsButton_clicked(); void on_closeButton_clicked(); - void saveImage(); + void saveImage(int index); private: Ui::AutoTesterClass ui; Test* test; - Downloader* downloader; + std::vector downloaders; + + QSignalMapper* signalMapper; + + int _numberOfImagesToDownload; + int _numberOfImagesDownloaded; + int _index; }; #endif // hifi_AutoTester_h \ No newline at end of file From 4b836a2ca9ef0194af2c6f1c03ff96ee7f409fe6 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 2 Mar 2018 16:49:09 -0800 Subject: [PATCH 06/19] Stores downloaded images in correct folder and with correct filenames. --- tools/auto-tester/src/Test.cpp | 11 +++++++---- tools/auto-tester/src/ui/AutoTester.cpp | 17 +++++++++-------- tools/auto-tester/src/ui/AutoTester.h | 11 ++++++++--- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index ac10d1ab33..d30349b395 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -198,7 +198,8 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { // Images that are in the wrong format are ignored. QStringList sortedTestResultsFilenames = createListOfAllJPEGimagesInDirectory(pathToTestResultsDirectory); - QStringList expectedImages; + QStringList expectedImagesURLs; + QStringList expectedImagesFilenames; QStringList resultImages; const QString URLPrefix("https://raw.githubusercontent.com"); @@ -217,14 +218,16 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); QString expectedImageFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + EXPECTED_IMAGE_TYPE; + expectedImagesFilenames << expectedImageFilename; + QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageFilename); - expectedImages << imageURLString; + expectedImagesURLs << imageURLString; } } - //autoTester->downloadImage(QUrl("https://raw.githubusercontent.com/NissimHadar/hifi_tests/addRecursionToAutotester/tests/content/entity/zone/ambientLightInheritance/ExpectedImage_00000.jpg")); + autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames); - autoTester->downloadImages(expectedImages); + // Wait for do ////if (success) { //// messageBox.information(0, "Success", "All images are as expected"); diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index c3ab88b7f4..5fd726ad72 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -51,7 +51,7 @@ void AutoTester::on_closeButton_clicked() { exit(0); } -void AutoTester::downloadImage(QUrl url) { +void AutoTester::downloadImage(const QUrl& url) { downloaders.emplace_back(new Downloader(url, this)); connect(downloaders[_index], SIGNAL (downloaded()), signalMapper, SLOT (map())); @@ -60,13 +60,16 @@ void AutoTester::downloadImage(QUrl url) { ++_index; } -void AutoTester::downloadImages(QStringList listOfURLs) { - _numberOfImagesToDownload = listOfURLs.size(); +void AutoTester::downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames) { + _directoryName = directoryName; + _filenames = filenames; + + _numberOfImagesToDownload = URLs.size(); _numberOfImagesDownloaded = 0; _index = 0; for (int i = 0; i < _numberOfImagesToDownload; ++i) { - QUrl imageURL(listOfURLs[i]); + QUrl imageURL(URLs[i]); downloadImage(imageURL); } @@ -77,10 +80,8 @@ void AutoTester::saveImage(int index) { QPixmap image; image.loadFromData(downloaders[index]->downloadedData()); - int w = image.width(); - int h = image.height(); + + image.save(_directoryName + "/" + _filenames[index]); ++_numberOfImagesDownloaded; - - image.save("D:/Dicom/lll_" + QString::number(index) + ".jpg"); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 5c1f217421..d5bab6d8a4 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -22,8 +22,8 @@ class AutoTester : public QMainWindow { public: AutoTester(QWidget *parent = Q_NULLPTR); - void downloadImage(QUrl url); - void downloadImages(QStringList listOfURLs); + void downloadImage(const QUrl& url); + void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames); private slots: void on_evaluateTestsButton_clicked(); @@ -38,10 +38,15 @@ private slots: private: Ui::AutoTesterClass ui; - Test* test; + std::vector downloaders; + // local storage for parameters - folder to store downloaded files in, and a list of their names + QString _directoryName; + QStringList _filenames; + + // Used to enable passing a parameter to slots QSignalMapper* signalMapper; int _numberOfImagesToDownload; From 361b5331bafba0d7862c52c8527a3c3550a91dc8 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 2 Mar 2018 18:09:32 -0800 Subject: [PATCH 07/19] File sizes are wrong, and so is format. --- tools/auto-tester/src/Test.cpp | 101 ++++-------------------- tools/auto-tester/src/Test.h | 11 ++- tools/auto-tester/src/ui/AutoTester.cpp | 23 +++--- tools/auto-tester/src/ui/AutoTester.h | 1 - tools/auto-tester/src/ui/AutoTester.ui | 13 --- 5 files changed, 36 insertions(+), 113 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index d30349b395..8fed094b07 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -63,9 +63,9 @@ void Test::zipAndDeleteTestResultsFolder() { index = 1; } -bool Test::compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar) { +bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) { progressBar->setMinimum(0); - progressBar->setMaximum(expectedImages.length() - 1); + progressBar->setMaximum(expectedImagesFilenames.length() - 1); progressBar->setValue(0); progressBar->setVisible(true); @@ -74,10 +74,10 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage const double THRESHOLD { 0.999 }; bool success{ true }; bool keepOn{ true }; - for (int i = 0; keepOn && i < expectedImages.length(); ++i) { + for (int i = 0; keepOn && i < expectedImagesFilenames.length(); ++i) { // First check that images are the same size - QImage resultImage(resultImages[i]); - QImage expectedImage(expectedImages[i]); + QImage resultImage(resultImagesFilenames[i]); + QImage expectedImage(expectedImagesFilenames[i]); if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) { messageBox.critical(0, "Internal error", "Images are not the same size"); exit(-1); @@ -94,14 +94,14 @@ bool Test::compareImageLists(QStringList expectedImages, QStringList resultImage if (similarityIndex < THRESHOLD) { TestFailure testFailure = TestFailure{ (float)similarityIndex, - expectedImages[i].left(expectedImages[i].lastIndexOf("/") + 1), // path to the test (including trailing /) - QFileInfo(expectedImages[i].toStdString().c_str()).fileName(), // filename of expected image - QFileInfo(resultImages[i].toStdString().c_str()).fileName() // filename of result image + expectedImagesFilenames[i].left(expectedImagesFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) + QFileInfo(expectedImagesFilenames[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(resultImagesFilenames[i].toStdString().c_str()).fileName() // filename of result image }; mismatchWindow.setTestFailure(testFailure); - if (!interactiveMode) { + if (!isInteractiveMode) { appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); success = false; } else { @@ -181,9 +181,9 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg"); } -void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { +void Test::startTestsEvaluation() { // Get list of JPEG images in folder, sorted by name - QString pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); + pathToTestResultsDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", ".", QFileDialog::ShowDirsOnly); if (pathToTestResultsDirectory == "") { return; } @@ -199,8 +199,6 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { QStringList sortedTestResultsFilenames = createListOfAllJPEGimagesInDirectory(pathToTestResultsDirectory); QStringList expectedImagesURLs; - QStringList expectedImagesFilenames; - QStringList resultImages; const QString URLPrefix("https://raw.githubusercontent.com"); const QString githubUser("NissimHadar"); @@ -210,7 +208,7 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat(currentFilename)) { - resultImages << fullCurrentFilename; + resultImagesFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); @@ -226,15 +224,10 @@ void Test::evaluateTests(bool interactiveMode, QProgressBar* progressBar) { } autoTester->downloadImages(expectedImagesURLs, pathToTestResultsDirectory, expectedImagesFilenames); +} - // Wait for do - - ////if (success) { - //// messageBox.information(0, "Success", "All images are as expected"); - ////} else { - //// messageBox.information(0, "Failure", "One or more images are not as expected"); - ////} - +void Test::finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar) { + bool success = compareImageLists(interactiveMode, progressBar); zipAndDeleteTestResultsFolder(); } @@ -253,70 +246,6 @@ bool Test::isAValidDirectory(QString pathname) { return true; } -// Two criteria are used to decide if a folder contains valid test results. -// 1) a 'test'js' file exists in the folder -// 2) the folder has the same number of actual and expected images -void Test::evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar) { - // Select folder to start recursing from - QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", ".", QFileDialog::ShowDirsOnly); - if (topLevelDirectory == "") { - return; - } - - // Leave if test results folder could not be created - if (!createTestResultsFolderPathIfNeeded(topLevelDirectory)) { - return; - } - - bool success{ true }; - QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); - - if (!isAValidDirectory(directory)) { - continue; - } - - const QString testPathname{ directory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (!fileInfo.exists()) { - // Folder does not contain 'test.js' - continue; - } - - QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory); - - // Separate images into two lists. The first is the expected images, the second is the test results - // Images that are in the wrong format are ignored. - QStringList expectedImages; - QStringList resultImages; - foreach(QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = directory + "/" + currentFilename; - if (isInExpectedImageFilenameFormat(currentFilename)) { - expectedImages << fullCurrentFilename; - } else if (isInSnapshotFilenameFormat(currentFilename)) { - resultImages << fullCurrentFilename; - } - } - - if (expectedImages.length() != resultImages.length()) { - // Number of images doesn't match - continue; - } - - // Set success to false if any test has failed - success &= compareImageLists(expectedImages, resultImages, directory, interactiveMode, progressBar); - } - - if (success) { - messageBox.information(0, "Success", "All images are as expected"); - } else { - messageBox.information(0, "Failure", "One or more images are not as expected"); - } - - zipAndDeleteTestResultsFolder(); -} - void Test::importTest(QTextStream& textStream, const QString& testPathname) { textStream << "Script.include(\"" << "file:///" << testPathname + "?raw=true\");" << endl; } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 026dbf25f6..bb7831da5e 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -23,14 +23,15 @@ class Test { public: Test(); - void evaluateTests(bool interactiveMode, QProgressBar* progressBar); - void evaluateTestsRecursively(bool interactiveMode, QProgressBar* progressBar); + void startTestsEvaluation(); + void finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar); + void createRecursiveScript(); void createRecursiveScriptsRecursively(); void createTest(); void deleteOldSnapshots(); - bool compareImageLists(QStringList expectedImages, QStringList resultImages, QString testDirectory, bool interactiveMode, QProgressBar* progressBar); + bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar); QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory); @@ -71,6 +72,10 @@ private: const int NUM_DIGITS { 5 }; const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" }; const QString EXPECTED_IMAGE_TYPE { ".jpg" }; + + QString pathToTestResultsDirectory; + QStringList expectedImagesFilenames; + QStringList resultImagesFilenames; }; #endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 5fd726ad72..52499cd448 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -21,14 +21,7 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { } void AutoTester::on_evaluateTestsButton_clicked() { - ////QUrl imageUrl("http://ribafreixo.com/wp-content/uploads/2017/03/Order-Now-Button-300x113.png"); - ////downloader = new Downloader(imageUrl, this); - ////connect(downloader, SIGNAL (downloaded()), this, SLOT (saveImage())); - test->evaluateTests(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); -} - -void AutoTester::on_evaluateTestsRecursivelyButton_clicked() { - test->evaluateTestsRecursively(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); + test->startTestsEvaluation(); } void AutoTester::on_createRecursiveScriptButton_clicked() { @@ -68,6 +61,11 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director _numberOfImagesDownloaded = 0; _index = 0; + ui.progressBar->setMinimum(0); + ui.progressBar->setMaximum(_numberOfImagesToDownload - 1); + ui.progressBar->setValue(0); + ui.progressBar->setVisible(true); + for (int i = 0; i < _numberOfImagesToDownload; ++i) { QUrl imageURL(URLs[i]); downloadImage(imageURL); @@ -80,8 +78,13 @@ void AutoTester::saveImage(int index) { QPixmap image; image.loadFromData(downloaders[index]->downloadedData()); - - image.save(_directoryName + "/" + _filenames[index]); + image.save(_directoryName + "/" + _filenames[index], 0, 100); ++_numberOfImagesDownloaded; + + if (_numberOfImagesDownloaded == _numberOfImagesToDownload) { + test->finishTestsEvaluation(ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); + } else { + ui.progressBar->setValue(_numberOfImagesDownloaded); + } } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index d5bab6d8a4..acf34c60a3 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -27,7 +27,6 @@ public: private slots: void on_evaluateTestsButton_clicked(); - void on_evaluateTestsRecursivelyButton_clicked(); void on_createRecursiveScriptButton_clicked(); void on_createRecursiveScriptsRecursivelyButton_clicked(); void on_createTestButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 0d142ec43e..70060940d1 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -66,19 +66,6 @@ Create Recursive Script - - - - 20 - 130 - 220 - 40 - - - - Evaluate Tests Recursively - - From 0a2610a9e0b8335e89f6c725e117d24dacb2707e Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sat, 3 Mar 2018 09:58:30 -0800 Subject: [PATCH 08/19] Evaluation seems to work. --- tools/auto-tester/src/ImageComparer.cpp | 6 ++-- tools/auto-tester/src/Test.cpp | 47 ++++++++++++++----------- tools/auto-tester/src/Test.h | 7 ++-- tools/auto-tester/src/ui/AutoTester.cpp | 7 ++-- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 94b95a5ab6..99f7d22c5f 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -17,13 +17,13 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const { // Make sure the image is 8 bits per colour QImage::Format format = expectedImage.format(); - if (format != QImage::Format::Format_RGB32) { + if (format != QImage::Format::Format_ARGB32) { throw -1; } const int L = 255; // (2^number of bits per pixel) - 1 - const double K1{ 0.01 }; - const double K2{ 0.03 }; + const double K1 { 0.01 }; + const double K2 { 0.03 }; const double c1 = pow((K1 * L), 2); const double c2 = pow((K2 * L), 2); diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 8fed094b07..48fa530384 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -20,7 +20,7 @@ extern AutoTester* autoTester; Test::Test() { - QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + EXPECTED_IMAGE_TYPE); + QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + IMAGE_FORMAT); expectedImageFilenameFormat = QRegularExpression(regex); @@ -65,7 +65,7 @@ void Test::zipAndDeleteTestResultsFolder() { bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) { progressBar->setMinimum(0); - progressBar->setMaximum(expectedImagesFilenames.length() - 1); + progressBar->setMaximum(expectedImagesFullFilenames.length() - 1); progressBar->setValue(0); progressBar->setVisible(true); @@ -74,10 +74,11 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) const double THRESHOLD { 0.999 }; bool success{ true }; bool keepOn{ true }; - for (int i = 0; keepOn && i < expectedImagesFilenames.length(); ++i) { + for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size - QImage resultImage(resultImagesFilenames[i]); - QImage expectedImage(expectedImagesFilenames[i]); + QImage resultImage(resultImagesFullFilenames[i]); + QImage expectedImage(expectedImagesFullFilenames[i]); + if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) { messageBox.critical(0, "Internal error", "Images are not the same size"); exit(-1); @@ -94,9 +95,9 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) if (similarityIndex < THRESHOLD) { TestFailure testFailure = TestFailure{ (float)similarityIndex, - expectedImagesFilenames[i].left(expectedImagesFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) - QFileInfo(expectedImagesFilenames[i].toStdString().c_str()).fileName(), // filename of expected image - QFileInfo(resultImagesFilenames[i].toStdString().c_str()).fileName() // filename of result image + expectedImagesFullFilenames[i].left(expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) + QFileInfo(expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image }; mismatchWindow.setTestFailure(testFailure); @@ -197,7 +198,7 @@ void Test::startTestsEvaluation() { // The expected images are represented as a URL to enabel download from GitHub // Images that are in the wrong format are ignored. - QStringList sortedTestResultsFilenames = createListOfAllJPEGimagesInDirectory(pathToTestResultsDirectory); + QStringList sortedTestResultsFilenames = createListOfAll_IMAGE_FORMAT_imagesInDirectory(pathToTestResultsDirectory); QStringList expectedImagesURLs; const QString URLPrefix("https://raw.githubusercontent.com"); @@ -205,18 +206,23 @@ void Test::startTestsEvaluation() { const QString testsRepo("hifi_tests"); const QString branch("addRecursionToAutotester"); + resultImagesFullFilenames.clear(); + expectedImagesFilenames.clear(); + expectedImagesFullFilenames.clear(); + foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat(currentFilename)) { - resultImagesFilenames << fullCurrentFilename; + resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); // extract the digits at the end of the filename (exluding the file extension) QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); - QString expectedImageFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + EXPECTED_IMAGE_TYPE; + QString expectedImageFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + IMAGE_FORMAT; expectedImagesFilenames << expectedImageFilename; + expectedImagesFullFilenames << pathToTestResultsDirectory + "/" + expectedImageFilename; QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageFilename); expectedImagesURLs << imageURLString; @@ -332,9 +338,10 @@ void Test::createTest() { return; } - QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(imageSourceDirectory); + QStringList sortedImageFilenames = createListOfAll_IMAGE_FORMAT_imagesInDirectory(imageSourceDirectory); - int i = 1; + int i = 1; + const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename; if (isInExpectedImageFilenameFormat(currentFilename)) { @@ -343,12 +350,12 @@ void Test::createTest() { exit(-1); } } else if (isInSnapshotFilenameFormat(currentFilename)) { - const int maxImages = pow(10, NUM_DIGITS); + if (i >= maxImages) { messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); } - QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".jpg"; + QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + IMAGE_FORMAT; QString imageDestinationDirectory = getExpectedImageDestinationDirectory(currentFilename); QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; @@ -383,7 +390,7 @@ void Test::deleteOldSnapshots() { continue; } - QStringList sortedImageFilenames = createListOfAllJPEGimagesInDirectory(directory); + QStringList sortedImageFilenames = createListOfAll_IMAGE_FORMAT_imagesInDirectory(directory); // Delete any file that is a snapshot (NOT the Expected Images) QStringList expectedImages; @@ -400,10 +407,10 @@ void Test::deleteOldSnapshots() { } } -QStringList Test::createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory) { +QStringList Test::createListOfAll_IMAGE_FORMAT_imagesInDirectory(QString pathToImageDirectory) { imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; - nameFilters << "*.jpg"; + nameFilters << "*" + IMAGE_FORMAT; return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } @@ -422,9 +429,9 @@ bool Test::isInSnapshotFilenameFormat(QString filename) { bool last5CharactersAreDigits; filenameParts[0].right(5).toInt(&last5CharactersAreDigits, 10); - bool extensionIsJPG = filenameParts[1] == "jpg"; + bool extensionIsIMAGE_FORMAT = filenameParts[1] == IMAGE_FORMAT.right(3); // without the period - return (filnameHasNoPeriods && contains_tests && last5CharactersAreDigits && extensionIsJPG); + return (filnameHasNoPeriods && contains_tests && last5CharactersAreDigits && extensionIsIMAGE_FORMAT); } // For a file named "D_GitHub_hifi-tests_tests_content_entity_zone_create_0.jpg", the test directory is diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index bb7831da5e..56d924b82a 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -33,7 +33,7 @@ public: bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar); - QStringList createListOfAllJPEGimagesInDirectory(QString pathToImageDirectory); + QStringList createListOfAll_IMAGE_FORMAT_imagesInDirectory(QString pathToImageDirectory); bool isInSnapshotFilenameFormat(QString filename); bool isInExpectedImageFilenameFormat(QString filename); @@ -71,11 +71,12 @@ private: // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) const int NUM_DIGITS { 5 }; const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" }; - const QString EXPECTED_IMAGE_TYPE { ".jpg" }; + const QString IMAGE_FORMAT { ".png" }; QString pathToTestResultsDirectory; QStringList expectedImagesFilenames; - QStringList resultImagesFilenames; + QStringList expectedImagesFullFilenames; + QStringList resultImagesFullFilenames; }; #endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 52499cd448..ade6b0514e 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -75,8 +75,11 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director } void AutoTester::saveImage(int index) { - QPixmap image; - image.loadFromData(downloaders[index]->downloadedData()); + QPixmap pixmap; + pixmap.loadFromData(downloaders[index]->downloadedData()); + + QImage image = pixmap.toImage(); + image = image.convertToFormat(QImage::Format_ARGB32); image.save(_directoryName + "/" + _filenames[index], 0, 100); From b32ad2f69c7120cfcd76d5464bd6591141160aca Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sat, 3 Mar 2018 18:22:51 -0800 Subject: [PATCH 09/19] createTests fixed. --- tools/auto-tester/src/Test.cpp | 13 ++----------- tools/auto-tester/src/Test.h | 1 - 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 48fa530384..dbaa3a05a5 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -344,13 +344,7 @@ void Test::createTest() { const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename; - if (isInExpectedImageFilenameFormat(currentFilename)) { - if (!QFile::remove(fullCurrentFilename)) { - messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); - exit(-1); - } - } else if (isInSnapshotFilenameFormat(currentFilename)) { - + if (isInSnapshotFilenameFormat(currentFilename)) { if (i >= maxImages) { messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); @@ -360,6 +354,7 @@ void Test::createTest() { QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; try { + QFile::remove(fullNewFileName); QFile::copy(fullCurrentFilename, fullNewFileName); } catch (...) { messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); @@ -471,7 +466,3 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) { return result; } - -bool Test::isInExpectedImageFilenameFormat(QString filename) { - return (expectedImageFilenameFormat.match(filename).hasMatch()); -} \ No newline at end of file diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 56d924b82a..29e1b135ed 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -36,7 +36,6 @@ public: QStringList createListOfAll_IMAGE_FORMAT_imagesInDirectory(QString pathToImageDirectory); bool isInSnapshotFilenameFormat(QString filename); - bool isInExpectedImageFilenameFormat(QString filename); void importTest(QTextStream& textStream, const QString& testPathname); From 913ad569c0585769e53bc2d1e926a0afef47c572 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sun, 4 Mar 2018 17:36:20 -0800 Subject: [PATCH 10/19] Added completion message. --- tools/auto-tester/src/Test.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index dbaa3a05a5..7b6293dcbc 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -234,6 +234,13 @@ void Test::startTestsEvaluation() { void Test::finishTestsEvaluation(bool interactiveMode, QProgressBar* progressBar) { bool success = compareImageLists(interactiveMode, progressBar); + + if (success) { + messageBox.information(0, "Success", "All images are as expected"); + } else { + messageBox.information(0, "Failure", "One or more images are not as expected"); + } + zipAndDeleteTestResultsFolder(); } From 5a79f272f3e9ef29ed87977c586b2d7c34edc852 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sun, 4 Mar 2018 20:56:57 -0800 Subject: [PATCH 11/19] Allow multiple evaluations. --- tools/auto-tester/src/Test.cpp | 18 ++++++++++++------ tools/auto-tester/src/ui/AutoTester.cpp | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 7b6293dcbc..fbd30e7307 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -216,16 +216,22 @@ void Test::startTestsEvaluation() { resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); - - // extract the digits at the end of the filename (exluding the file extension) + + // Images are stored on GitHub as ExpectedImage_ddddd.png + // Extract the digits at the end of the filename (exluding the file extension) QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); - QString expectedImageFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + IMAGE_FORMAT; + QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + IMAGE_FORMAT; + + QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); + + expectedImagesURLs << imageURLString; + + // The image retrieved from Github needs a unique name + QString expectedImageFilename = currentFilename.replace("/", "_").replace(".", "_EI."); expectedImagesFilenames << expectedImageFilename; expectedImagesFullFilenames << pathToTestResultsDirectory + "/" + expectedImageFilename; - - QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageFilename); - expectedImagesURLs << imageURLString; } } diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index ade6b0514e..5ac7be3dd8 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -81,7 +81,8 @@ void AutoTester::saveImage(int index) { QImage image = pixmap.toImage(); image = image.convertToFormat(QImage::Format_ARGB32); - image.save(_directoryName + "/" + _filenames[index], 0, 100); + QString fullPathname = _directoryName + "/" + _filenames[index]; + image.save(fullPathname, 0, 100); ++_numberOfImagesDownloaded; From 84815fadaeb6bf2476074466cc0d0da28dfb549d Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 5 Mar 2018 17:45:51 -0800 Subject: [PATCH 12/19] Use JPG->PNG trick to keep file sizes down. --- tools/auto-tester/src/Test.cpp | 79 +++++++++++-------------- tools/auto-tester/src/Test.h | 7 ++- tools/auto-tester/src/ui/AutoTester.cpp | 4 -- tools/auto-tester/src/ui/AutoTester.h | 1 - tools/auto-tester/src/ui/AutoTester.ui | 21 ++----- 5 files changed, 43 insertions(+), 69 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index fbd30e7307..53687fdeda 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -20,7 +22,7 @@ extern AutoTester* autoTester; Test::Test() { - QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + IMAGE_FORMAT); + QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + ".png"); expectedImageFilenameFormat = QRegularExpression(regex); @@ -194,11 +196,23 @@ void Test::startTestsEvaluation() { return; } + // Before any processing - all images are converted to PNGs, as this is the format stored on GitHub + QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", pathToTestResultsDirectory); + foreach(QString filename, sortedSnapshotFilenames) { + QStringList stringParts = filename.split("."); + copyJPGtoPNG( + pathToTestResultsDirectory + "/" + stringParts[0] + ".jpg", + pathToTestResultsDirectory + "/" + stringParts[0] + ".png" + ); + + QFile::remove(pathToTestResultsDirectory + "/" + stringParts[0] + ".jpg"); + } + // Create two lists. The first is the test results, the second is the expected images - // The expected images are represented as a URL to enabel download from GitHub + // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. - QStringList sortedTestResultsFilenames = createListOfAll_IMAGE_FORMAT_imagesInDirectory(pathToTestResultsDirectory); + QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", pathToTestResultsDirectory); QStringList expectedImagesURLs; const QString URLPrefix("https://raw.githubusercontent.com"); @@ -212,7 +226,7 @@ void Test::startTestsEvaluation() { foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = pathToTestResultsDirectory + "/" + currentFilename; - if (isInSnapshotFilenameFormat(currentFilename)) { + if (isInSnapshotFilenameFormat("png", currentFilename)) { resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); @@ -220,7 +234,7 @@ void Test::startTestsEvaluation() { // Images are stored on GitHub as ExpectedImage_ddddd.png // Extract the digits at the end of the filename (exluding the file extension) QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); - QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + IMAGE_FORMAT; + QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; QString imageURLString(URLPrefix + "/" + githubUser + "/" + testsRepo + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); @@ -351,24 +365,23 @@ void Test::createTest() { return; } - QStringList sortedImageFilenames = createListOfAll_IMAGE_FORMAT_imagesInDirectory(imageSourceDirectory); + QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", imageSourceDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = imageSourceDirectory + "/" + currentFilename; - if (isInSnapshotFilenameFormat(currentFilename)) { + if (isInSnapshotFilenameFormat("jpg", currentFilename)) { if (i >= maxImages) { messageBox.critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); } - QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + IMAGE_FORMAT; + QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png"; QString imageDestinationDirectory = getExpectedImageDestinationDirectory(currentFilename); QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; try { - QFile::remove(fullNewFileName); - QFile::copy(fullCurrentFilename, fullNewFileName); + copyJPGtoPNG(fullCurrentFilename, fullNewFileName); } catch (...) { messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); exit(-1); @@ -380,45 +393,23 @@ void Test::createTest() { messageBox.information(0, "Success", "Test images have been created"); } -void Test::deleteOldSnapshots() { - // Select folder to start recursing from - QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select root folder for snapshot deletion", ".", QFileDialog::ShowDirsOnly); - if (topLevelDirectory == "") { - return; - } +void Test::copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename) { + QFile::remove(destinationPNGFullFilename); - // Recurse over folders - QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); + QImageReader reader; + reader.setFileName(sourceJPGFullFilename); - // Only process directories - QDir dir(directory); - if (!isAValidDirectory(directory)) { - continue; - } + QImage image = reader.read(); - QStringList sortedImageFilenames = createListOfAll_IMAGE_FORMAT_imagesInDirectory(directory); - - // Delete any file that is a snapshot (NOT the Expected Images) - QStringList expectedImages; - QStringList resultImages; - foreach(QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = directory + "/" + currentFilename; - if (isInSnapshotFilenameFormat(currentFilename)) { - if (!QFile::remove(fullCurrentFilename)) { - messageBox.critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nSnapshot deletion aborted"); - exit(-1); - } - } - } - } + QImageWriter writer; + writer.setFileName(destinationPNGFullFilename); + writer.write(image); } -QStringList Test::createListOfAll_IMAGE_FORMAT_imagesInDirectory(QString pathToImageDirectory) { +QStringList Test::createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory) { imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; - nameFilters << "*" + IMAGE_FORMAT; + nameFilters << "*." + imageFormat; return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } @@ -428,7 +419,7 @@ QStringList Test::createListOfAll_IMAGE_FORMAT_imagesInDirectory(QString pathToI // Filename (i.e. without extension) contains _tests_ (this is based on all test scripts being within the tests folder // Last 5 characters in filename are digits // Extension is jpg -bool Test::isInSnapshotFilenameFormat(QString filename) { +bool Test::isInSnapshotFilenameFormat(QString imageFormat, QString filename) { QStringList filenameParts = filename.split("."); bool filnameHasNoPeriods = (filenameParts.size() == 2); @@ -437,7 +428,7 @@ bool Test::isInSnapshotFilenameFormat(QString filename) { bool last5CharactersAreDigits; filenameParts[0].right(5).toInt(&last5CharactersAreDigits, 10); - bool extensionIsIMAGE_FORMAT = filenameParts[1] == IMAGE_FORMAT.right(3); // without the period + bool extensionIsIMAGE_FORMAT = (filenameParts[1] == imageFormat); return (filnameHasNoPeriods && contains_tests && last5CharactersAreDigits && extensionIsIMAGE_FORMAT); } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 29e1b135ed..b849ab577f 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -33,9 +33,9 @@ public: bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar); - QStringList createListOfAll_IMAGE_FORMAT_imagesInDirectory(QString pathToImageDirectory); + QStringList createListOfAll_imagesInDirectory(QString imageFormat, QString pathToImageDirectory); - bool isInSnapshotFilenameFormat(QString filename); + bool isInSnapshotFilenameFormat(QString imageFormat, QString filename); void importTest(QTextStream& textStream, const QString& testPathname); @@ -49,6 +49,8 @@ public: QString getExpectedImageDestinationDirectory(QString filename); QString getExpectedImagePartialSourceDirectory(QString filename); + void copyJPGtoPNG(QString sourceJPGFullFilename, QString destinationPNGFullFilename); + private: const QString TEST_FILENAME { "test.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; @@ -70,7 +72,6 @@ private: // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) const int NUM_DIGITS { 5 }; const QString EXPECTED_IMAGE_PREFIX { "ExpectedImage_" }; - const QString IMAGE_FORMAT { ".png" }; QString pathToTestResultsDirectory; QStringList expectedImagesFilenames; diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 5ac7be3dd8..a5e13331dd 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -36,10 +36,6 @@ void AutoTester::on_createTestButton_clicked() { test->createTest(); } -void AutoTester::on_deleteOldSnapshotsButton_clicked() { - test->deleteOldSnapshots(); -} - void AutoTester::on_closeButton_clicked() { exit(0); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index acf34c60a3..938e7ca2d2 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -30,7 +30,6 @@ private slots: void on_createRecursiveScriptButton_clicked(); void on_createRecursiveScriptsRecursivelyButton_clicked(); void on_createTestButton_clicked(); - void on_deleteOldSnapshotsButton_clicked(); void on_closeButton_clicked(); void saveImage(int index); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 70060940d1..55c3897e58 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -30,8 +30,8 @@ - 360 - 210 + 20 + 30 220 40 @@ -44,7 +44,7 @@ 20 - 75 + 135 220 40 @@ -70,7 +70,7 @@ 23 - 40 + 100 131 20 @@ -95,19 +95,6 @@ 24 - - - - 360 - 270 - 220 - 40 - - - - Delete Old Snapshots - - From 4b3d63eea4952c46749593fdb49d1e79699172d2 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 6 Mar 2018 16:37:05 -0800 Subject: [PATCH 13/19] Added math.h to includes (MacOS build error). --- tools/auto-tester/src/Test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 53687fdeda..2b3d121638 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -21,6 +21,8 @@ #include "ui/AutoTester.h" extern AutoTester* autoTester; +#include + Test::Test() { QString regex(EXPECTED_IMAGE_PREFIX + QString("\\\\d").repeated(NUM_DIGITS) + ".png"); From da414b94fdd88344cb0e8f06c72dbfc575f6a18f Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Wed, 7 Mar 2018 10:47:37 -0800 Subject: [PATCH 14/19] WIP - adding the creation of recursive scripts. --- tools/auto-tester/src/Test.cpp | 45 +++++++++++++++++++++++++--------- tools/auto-tester/src/Test.h | 4 +++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 2b3d121638..f514696ad3 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -84,7 +84,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) QImage expectedImage(expectedImagesFullFilenames[i]); if (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height()) { - messageBox.critical(0, "Internal error", "Images are not the same size"); + messageBox.critical(0, "Internal error #1", "Images are not the same size"); exit(-1); } @@ -92,7 +92,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) try { similarityIndex = imageComparer.compareImages(resultImage, expectedImage); } catch (...) { - messageBox.critical(0, "Internal error", "Image not in expected format"); + messageBox.critical(0, "Internal error #2", "Image not in expected format"); exit(-1); } @@ -139,20 +139,20 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { if (!QDir().exists(testResultsFolderPath)) { - messageBox.critical(0, "Internal error", "Folder " + testResultsFolderPath + " not found"); + messageBox.critical(0, "Internal error #3", "Folder " + testResultsFolderPath + " not found"); exit(-1); } QString failureFolderPath { testResultsFolderPath + "/" + "Failure_" + QString::number(index) }; if (!QDir().mkdir(failureFolderPath)) { - messageBox.critical(0, "Internal error", "Failed to create folder " + failureFolderPath); + messageBox.critical(0, "Internal error #4", "Failed to create folder " + failureFolderPath); exit(-1); } ++index; QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME); if (!descriptionFile.open(QIODevice::ReadWrite)) { - messageBox.critical(0, "Internal error", "Failed to create file " + TEST_RESULTS_FILENAME); + messageBox.critical(0, "Internal error #5", "Failed to create file " + TEST_RESULTS_FILENAME); exit(-1); } @@ -172,14 +172,14 @@ void Test::appendTestResultsToFile(QString testResultsFolderPath, TestFailure te sourceFile = testFailure._pathname + testFailure._expectedImageFilename; destinationFile = failureFolderPath + "/" + "Expected Image.jpg"; if (!QFile::copy(sourceFile, destinationFile)) { - messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile); + messageBox.critical(0, "Internal error #6", "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } sourceFile = testFailure._pathname + testFailure._actualImageFilename; destinationFile = failureFolderPath + "/" + "Actual Image.jpg"; if (!QFile::copy(sourceFile, destinationFile)) { - messageBox.critical(0, "Internal error", "Failed to copy " + sourceFile + " to " + destinationFile); + messageBox.critical(0, "Internal error #7", "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } @@ -282,7 +282,24 @@ bool Test::isAValidDirectory(QString pathname) { } void Test::importTest(QTextStream& textStream, const QString& testPathname) { - textStream << "Script.include(\"" << "file:///" << testPathname + "?raw=true\");" << endl; + // `testPathname` includes the full path to the test. We need the portion below (and including) `tests` + QStringList filenameParts = testPathname.split('/'); + int i{ 0 }; + while (i < filenameParts.length() && filenameParts[i] != "tests") { + ++i; + } + + if (i == filenameParts.length()) { + messageBox.critical(0, "Internal error #10", "Bad testPathname"); + exit(-1); + } + + QString filename; + for (int j = i; j < filenameParts.length(); ++j) { + filename += "/" + filenameParts[j]; + } + + textStream << "Script.include(\"" << "https://raw.githubusercontent.com/" << user << "/hifi_tests/" << branch << filename + "\");" << endl; } // Creates a single script in a user-selected folder. @@ -298,7 +315,7 @@ void Test::createRecursiveScript() { QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename); if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { messageBox.critical(0, - "Internal Error", + "Internal Error #8", "Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\"" ); @@ -308,7 +325,7 @@ void Test::createRecursiveScript() { QTextStream textStream(&allTestsFilename); textStream << "// This is an automatically generated file, created by auto-tester" << endl << endl; - textStream << "var autoTester = Script.require(\"https://github.com/highfidelity/hifi_tests/blob/master/tests/utils/autoTester.js?raw=true\");" << endl; + textStream << "var autoTester = Script.require(\"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + "/tests/utils/autoTester.js\");" << endl; textStream << "autoTester.enableRecursive();" << endl << endl; QVector testPathnames; @@ -459,11 +476,17 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) { QString filenameWithoutExtension = filename.split(".")[0]; QStringList filenameParts = filenameWithoutExtension.split("_"); + // Note that the topmost "tests" folder is assumed to be the root int i { 0 }; - while (filenameParts[i] != "tests") { + while (i < filenameParts.length() && filenameParts[i] != "tests") { ++i; } + if (i == filenameParts.length()) { + messageBox.critical(0, "Internal error #9", "Bad filename"); + exit(-1); + } + QString result = filenameParts[i]; for (int j = i + 1; j < filenameParts.length() - 1; ++j) { diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index b849ab577f..2a26e042b9 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -77,6 +77,10 @@ private: QStringList expectedImagesFilenames; QStringList expectedImagesFullFilenames; QStringList resultImagesFullFilenames; + + // Used for accessing GitHub + const QString user{ "NissimHadar" }; + const QString branch{ "addRecursionToAutotester" }; }; #endif // hifi_test_h \ No newline at end of file From 9bd7f31ff32b7a68bcc333e71217604194c5034f Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Wed, 7 Mar 2018 11:30:30 -0800 Subject: [PATCH 15/19] Now search for `tests` folder bottom-up. --- tools/auto-tester/src/Test.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index f514696ad3..8267435fad 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -476,13 +476,14 @@ QString Test::getExpectedImagePartialSourceDirectory(QString filename) { QString filenameWithoutExtension = filename.split(".")[0]; QStringList filenameParts = filenameWithoutExtension.split("_"); - // Note that the topmost "tests" folder is assumed to be the root - int i { 0 }; - while (i < filenameParts.length() && filenameParts[i] != "tests") { - ++i; + // Note that the bottom-most "tests" folder is assumed to be the root + // This is required because the tests folder is named hifi_tests + int i { filenameParts.length() - 1 }; + while (i >= 0 && filenameParts[i] != "tests") { + --i; } - if (i == filenameParts.length()) { + if (i < 0) { messageBox.critical(0, "Internal error #9", "Bad filename"); exit(-1); } From b47b512ab9a767039b629e26daa75b2d2528b025 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Wed, 7 Mar 2018 13:25:40 -0800 Subject: [PATCH 16/19] Completed recursive creation of recursive scripts. --- tools/auto-tester/src/Test.cpp | 58 ++++++++++++++++++++++++++++++---- tools/auto-tester/src/Test.h | 2 ++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 8267435fad..1078ce1bc4 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -311,6 +311,52 @@ void Test::createRecursiveScript() { return; } + createRecursiveScript(topLevelDirectory, true); +} + +// This method creates a `testRecursive.js` script in every sub-folder. +void Test::createRecursiveScriptsRecursively() { + // Select folder to start recursing from + QString topLevelDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", ".", QFileDialog::ShowDirsOnly); + if (topLevelDirectory == "") { + return; + } + + createRecursiveScript(topLevelDirectory, false); + + QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + while (it.hasNext()) { + QString directory = it.next(); + + // Only process directories + QDir dir(); + if (!isAValidDirectory(directory)) { + continue; + } + + // Only process directories that have sub-directories + bool hasNoSubDirectories{ true }; + QDirIterator it2(directory.toStdString().c_str(), QDirIterator::Subdirectories); + while (it2.hasNext()) { + QString directory2 = it2.next(); + + // Only process directories + QDir dir(); + if (isAValidDirectory(directory2)) { + hasNoSubDirectories = false; + break; + } + } + + if (!hasNoSubDirectories) { + createRecursiveScript(directory, false); + } + } + + messageBox.information(0, "Success", "Scripts have been created"); +} + +void Test::createRecursiveScript(QString topLevelDirectory, bool interactiveMode) { const QString recursiveTestsFilename("testRecursive.js"); QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename); if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { @@ -350,7 +396,7 @@ void Test::createRecursiveScript() { continue; } - const QString testPathname{ directory + "/" + TEST_FILENAME }; + const QString testPathname { directory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { // Current folder contains a test @@ -360,7 +406,7 @@ void Test::createRecursiveScript() { } } - if (testPathnames.length() <= 0) { + if (interactiveMode && testPathnames.length() <= 0) { messageBox.information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found"); allTestsFilename.close(); return; @@ -370,10 +416,10 @@ void Test::createRecursiveScript() { textStream << "autoTester.runRecursive();" << endl; allTestsFilename.close(); - messageBox.information(0, "Success", "Script has been created"); -} - -void Test::createRecursiveScriptsRecursively() { + + if (interactiveMode) { + messageBox.information(0, "Success", "Script has been created"); + } } void Test::createTest() { diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 2a26e042b9..411f27eae6 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -28,6 +28,8 @@ public: void createRecursiveScript(); void createRecursiveScriptsRecursively(); + void createRecursiveScript(QString topLevelDirectory, bool interactiveMode); + void createTest(); void deleteOldSnapshots(); From a4854839a4c3148e91d1a103f2b5b634f581fe14 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Wed, 7 Mar 2018 17:45:10 -0800 Subject: [PATCH 17/19] Enable user selection of destination folder when creating a test. --- tools/auto-tester/src/Test.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 1078ce1bc4..c14c6b9437 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -430,6 +430,11 @@ void Test::createTest() { return; } + QString imageDestinationDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images", ".", QFileDialog::ShowDirsOnly); + if (imageDestinationDirectory == "") { + return; + } + QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", imageSourceDirectory); int i = 1; @@ -442,7 +447,6 @@ void Test::createTest() { exit(-1); } QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png"; - QString imageDestinationDirectory = getExpectedImageDestinationDirectory(currentFilename); QString fullNewFileName = imageDestinationDirectory + "/" + newFilename; try { From 77901e5089b38ae91b810d23d0b4c80f1c8c89dc Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Thu, 8 Mar 2018 20:11:35 -0800 Subject: [PATCH 18/19] Corrected MacOS warnings. --- tools/auto-tester/src/Test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index c14c6b9437..9bced3d1fc 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -329,7 +329,7 @@ void Test::createRecursiveScriptsRecursively() { QString directory = it.next(); // Only process directories - QDir dir(); + QDir dir; if (!isAValidDirectory(directory)) { continue; } @@ -341,7 +341,7 @@ void Test::createRecursiveScriptsRecursively() { QString directory2 = it2.next(); // Only process directories - QDir dir(); + QDir dir; if (isAValidDirectory(directory2)) { hasNoSubDirectories = false; break; From 84cc4c18a40c7025da738137da87ee4bc91628bc Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Fri, 9 Mar 2018 18:13:26 -0800 Subject: [PATCH 19/19] Added date/time to testResultsFolder. --- tools/auto-tester/src/Test.cpp | 22 +++++++--------------- tools/auto-tester/src/Test.h | 7 ++++--- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 9bced3d1fc..347cfd90dc 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -31,21 +31,13 @@ Test::Test() { mismatchWindow.setModal(true); } -bool Test::createTestResultsFolderPathIfNeeded(QString directory) { - // The test results folder is located in the root of the tests (i.e. for recursive test evaluation) - if (testResultsFolderPath == "") { - testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER; - QDir testResultsFolder(testResultsFolderPath); +bool Test::createTestResultsFolderPath(QString directory) { + QDateTime now = QDateTime::currentDateTime(); + testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT); + QDir testResultsFolder(testResultsFolderPath); - if (testResultsFolder.exists()) { - testResultsFolder.removeRecursively(); - } - - // Create a new test results folder - return QDir().mkdir(testResultsFolderPath); - } else { - return true; - } + // Create a new test results folder + return QDir().mkdir(testResultsFolderPath); } void Test::zipAndDeleteTestResultsFolder() { @@ -194,7 +186,7 @@ void Test::startTestsEvaluation() { } // Quit if test results folder could not be created - if (!createTestResultsFolderPathIfNeeded(pathToTestResultsDirectory)) { + if (!createTestResultsFolderPath(pathToTestResultsDirectory)) { return; } diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 411f27eae6..cd5075002a 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -43,7 +43,7 @@ public: void appendTestResultsToFile(QString testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); - bool createTestResultsFolderPathIfNeeded(QString directory); + bool createTestResultsFolderPath(QString directory); void zipAndDeleteTestResultsFolder(); bool isAValidDirectory(QString pathname); @@ -81,8 +81,9 @@ private: QStringList resultImagesFullFilenames; // Used for accessing GitHub - const QString user{ "NissimHadar" }; - const QString branch{ "addRecursionToAutotester" }; + const QString user { "NissimHadar" }; + const QString branch { "addRecursionToAutotester" }; + const QString DATETIME_FORMAT { "yyyy-MM-dd_hh-mm-ss" }; }; #endif // hifi_test_h \ No newline at end of file