diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 3afdeaec0a..fa73f97887 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -15,13 +15,8 @@ // Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity // The value is computed for the luminance component and the average value is returned double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const { - // Make sure the image is 8 bits per colour - QImage::Format format = expectedImage.format(); - if (format != QImage::Format::Format_ARGB32) { - throw -1; - } - const int L = 255; // (2^number of bits per pixel) - 1 + const double K1 { 0.01 }; const double K2 { 0.03 }; const double c1 = pow((K1 * L), 2); diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 1a25cc5c0e..4f02544c12 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -25,6 +25,11 @@ extern AutoTester* autoTester; Test::Test() { mismatchWindow.setModal(true); + + if (autoTester) { + autoTester->setUserText("highfidelity"); + autoTester->setBranchText("master"); + } } bool Test::createTestResultsFolderPath(const QString& directory) { @@ -63,7 +68,6 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. - const double THRESHOLD { 0.9995 }; bool success{ true }; bool keepOn{ true }; for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) { @@ -71,17 +75,14 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) QImage resultImage(resultImagesFullFilenames[i]); QImage expectedImage(expectedImagesFullFilenames[i]); + double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical + + // similarityIndex is set to -100.0 to indicate images are not the same size if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); - exit(-1); - } - - double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical - try { + similarityIndex = -100.0; + } else { similarityIndex = imageComparer.compareImages(resultImage, expectedImage); - } catch (...) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format"); - exit(-1); } if (similarityIndex < THRESHOLD) { @@ -176,20 +177,25 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai comparisonImage.save(failureFolderPath + "/" + "Difference Image.png"); } -void Test::startTestsEvaluation(const QString& testFolder) { - // Get list of JPEG images in folder, sorted by name - QString previousSelection = snapshotDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, - QFileDialog::ShowDirsOnly); +void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine, const QString& userFromCommandLine) { + if (testFolder.isNull()) { + // Get list of JPEG images in folder, sorted by name + QString previousSelection = snapshotDirectory; + QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); + if (!parent.isNull() && parent.right(1) != "/") { + parent += "/"; + } + snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + QFileDialog::ShowDirsOnly); - // If user cancelled then restore previous selection and return - if (snapshotDirectory == "") { - snapshotDirectory = previousSelection; - return; + // If user cancelled then restore previous selection and return + if (snapshotDirectory == "") { + snapshotDirectory = previousSelection; + return; + } + } else { + snapshotDirectory = testFolder; + exitWhenComplete = true; } // Quit if test results folder could not be created @@ -197,17 +203,6 @@ void Test::startTestsEvaluation(const QString& testFolder) { return; } - // Before any processing - all images are converted to PNGs, as this is the format stored on GitHub - QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory); - foreach(QString filename, sortedSnapshotFilenames) { - QString filenameWithoutExtension = filename.left(filename.length() - 4); - copyJPGtoPNG(snapshotDirectory + "/" + filenameWithoutExtension + ".jpg", - snapshotDirectory + "/" + filenameWithoutExtension + ".png" - ); - - QFile::remove(snapshotDirectory + "/" + filenameWithoutExtension + ".jpg"); - } - // Create two lists. The first is the test results, the second is the expected images // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. @@ -219,6 +214,9 @@ void Test::startTestsEvaluation(const QString& testFolder) { expectedImagesFilenames.clear(); expectedImagesFullFilenames.clear(); + QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine; + QString user = (userFromCommandLine.isNull()) ? autoTester->getSelectedUser() : userFromCommandLine; + foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { @@ -231,7 +229,7 @@ void Test::startTestsEvaluation(const QString& testFolder) { QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; - QString imageURLString("https://raw.githubusercontent.com/" + GIT_HUB_USER + "/hifi_tests/" + GIT_HUB_BRANCH + "/" + + QString imageURLString("https://raw.githubusercontent.com/" + user + "/" + GIT_HUB_REPOSITORY + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); expectedImagesURLs << imageURLString; @@ -259,6 +257,10 @@ void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactive } zipAndDeleteTestResultsFolder(); + + if (exitWhenComplete) { + exit(0); + } } bool Test::isAValidDirectory(const QString& pathname) { @@ -299,10 +301,9 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) { void Test::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); - textStream << "Script.include(\"" - << "https://github.com/" << GIT_HUB_USER << "/hifi_tests/blob/" << GIT_HUB_BRANCH - << partialPath + "?raw=true\");" - << endl; + QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); + + textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << endl; } // Creates a single script in a user-selected folder. @@ -397,12 +398,20 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm"); textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl; - textStream << "user = \"" + GIT_HUB_USER + "/\";" << endl; - textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl; - textStream << "branch = \"" + GIT_HUB_BRANCH + "/\";" << endl << endl; + // Include 'autoTest.js' + QString branch = autoTester->getSelectedBranch(); + QString user = autoTester->getSelectedUser(); - textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/" - + GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl; + textStream << "PATH_TO_THE_REPO_PATH_UTILS_FILE = \"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + "/tests/utils/branchUtils.js\";" << endl; + textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl; + textStream << "var autoTester = createAutoTester(Script.resolvePath(\".\"));" << endl << endl; + + textStream << "var testsRootPath = autoTester.getTestsRootPath();" << endl << endl; + + // Wait 10 seconds before starting + textStream << "if (typeof Test !== 'undefined') {" << endl; + textStream << " Test.wait(10000);" << endl; + textStream << "};" << endl << endl; textStream << "autoTester.enableRecursive();" << endl; textStream << "autoTester.enableAuto();" << endl << endl; @@ -498,13 +507,13 @@ void Test::createTests() { return; } - QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory); + QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; - if (isInSnapshotFilenameFormat("jpg", currentFilename)) { + if (isInSnapshotFilenameFormat("png", currentFilename)) { if (i >= maxImages) { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); @@ -528,9 +537,12 @@ void Test::createTests() { fullNewFileName += "/" + newFilename; try { - copyJPGtoPNG(fullCurrentFilename, fullNewFileName); + if (QFile::exists(fullNewFileName)) { + QFile::remove(fullNewFileName); + } + QFile::copy(fullCurrentFilename, fullNewFileName); } catch (...) { - QMessageBox::critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); + QMessageBox::critical(0, "Error", "Could not copy file: " + fullCurrentFilename + " to " + fullNewFileName + "\n"); exit(-1); } ++i; @@ -560,9 +572,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) { const QString ws("\\h*"); //white-space character const QString functionPerformName(ws + "autoTester" + ws + "\\." + ws + "perform"); const QString quotedString("\\\".+\\\""); - const QString ownPath("Script" + ws + "\\." + ws + "resolvePath" + ws + "\\(" + ws + "\\\"\\.\\\"" + ws + "\\)"); - const QString functionParameter("function" + ws + "\\(testType" + ws + "\\)"); - QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString + "\\," + ws + ownPath + "\\," + ws + functionParameter + ws + "{" + ".*"); + QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString); QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle); @@ -811,19 +821,6 @@ void Test::createTestsOutline() { QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created"); } -void Test::copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename) { - QFile::remove(destinationPNGFullFilename); - - QImageReader reader; - reader.setFileName(sourceJPGFullFilename); - - QImage image = reader.read(); - - QImageWriter writer; - writer.setFileName(destinationPNGFullFilename); - writer.write(image); -} - QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 32564035e7..5c6d3e5686 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -37,7 +37,7 @@ class Test { public: Test(); - void startTestsEvaluation(const QString& testFolder = QString()); + void startTestsEvaluation(const QString& testFolder = QString(), const QString& branchFromCommandLine = QString(), const QString& userFromCommandLine = QString()); void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar); void createRecursiveScript(); @@ -69,13 +69,13 @@ public: QString getExpectedImageDestinationDirectory(const QString& filename); QString getExpectedImagePartialSourceDirectory(const QString& filename); - void copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename); - private: const QString TEST_FILENAME { "test.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; + const double THRESHOLD{ 0.96 }; + QDir imageDirectory; MismatchWindow mismatchWindow; @@ -102,9 +102,7 @@ private: QStringList resultImagesFullFilenames; // Used for accessing GitHub - const QString GIT_HUB_USER{ "highfidelity" }; const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; - const QString GIT_HUB_BRANCH{ "master" }; const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; @@ -115,6 +113,8 @@ private: // var pathSeparator = "."; const QString ADVANCE_KEY{ "n" }; const QString PATH_SEPARATOR{ "." }; + + bool exitWhenComplete{ false }; }; #endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index ffa7a0b237..03b8cf51ff 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -10,23 +10,63 @@ #include #include "ui/AutoTester.h" +#include + AutoTester* autoTester; int main(int argc, char *argv[]) { - // Only parameter is "--testFolder" + // If no parameters then run in interactive mode + // Parameter --testFolder + // Parameter --branch + // default is "master" + // Parameter --user + // default is "highfidelity" + // Parameter --repository + // default is "highfidelity" + QString testFolder; - if (argc == 3) { - if (QString(argv[1]) == "--testFolder") { - testFolder = QString(argv[2]); + + QString branch{ "master" }; + QString user{ "highfidelity" }; + + for (int i = 1; i < argc - 1; ++i) { + if (QString(argv[i]) == "--testFolder") { + ++i; + if (i < argc) { + testFolder = QString(argv[i]); + } else { + std::cout << "Missing parameter after --testFolder" << std::endl; + exit(-1); + } + } else if (QString(argv[i]) == "--branch") { + ++i; + if (i < argc) { + branch = QString(argv[i]); + } else { + std::cout << "Missing parameter after --branch" << std::endl; + exit(-1); + } + } else if (QString(argv[i]) == "--user") { + ++i; + if (i < argc) { + user = QString(argv[i]); + } else { + std::cout << "Missing parameter after --user" << std::endl; + exit(-1); + } + } else { + std::cout << "Unknown parameter" << std::endl; + exit(-1); } } QApplication application(argc, argv); autoTester = new AutoTester(); + autoTester->setup(); if (!testFolder.isNull()) { - autoTester->runFromCommandLine(testFolder); + autoTester->runFromCommandLine(testFolder, branch, user); } else { autoTester->show(); } diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 14329e36c2..079fa63a9d 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -29,13 +29,15 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { ui.hideTaskbarButton->setVisible(false); ui.showTaskbarButton->setVisible(false); #endif +} +void AutoTester::setup() { test = new Test(); } -void AutoTester::runFromCommandLine(const QString& testFolder) { +void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) { isRunningFromCommandline = true; - test->startTestsEvaluation(testFolder); + test->startTestsEvaluation(testFolder, branch, user); } void AutoTester::on_evaluateTestsButton_clicked() { @@ -116,6 +118,7 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director ui.progressBar->setValue(0); ui.progressBar->setVisible(true); + downloaders.clear(); for (int i = 0; i < _numberOfImagesToDownload; ++i) { QUrl imageURL(URLs[i]); downloadImage(imageURL); @@ -125,14 +128,12 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director } void AutoTester::saveImage(int index) { - QPixmap pixmap; - pixmap.loadFromData(downloaders[index]->downloadedData()); - - QImage image = pixmap.toImage(); - image = image.convertToFormat(QImage::Format_ARGB32); - - QString fullPathname = _directoryName + "/" + _filenames[index]; - if (!image.save(fullPathname, 0, 100)) { + try { + QFile file(_directoryName + "/" + _filenames[index]); + file.open(QIODevice::WriteOnly); + file.write(downloaders[index]->downloadedData()); + file.close(); + } catch (...) { QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]); ui.progressBar->setVisible(false); return; @@ -141,6 +142,7 @@ void AutoTester::saveImage(int index) { ++_numberOfImagesDownloaded; if (_numberOfImagesDownloaded == _numberOfImagesToDownload) { + disconnect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); } else { ui.progressBar->setValue(_numberOfImagesDownloaded); @@ -150,3 +152,20 @@ void AutoTester::saveImage(int index) { void AutoTester::about() { QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__); } + +void AutoTester::setUserText(const QString& user) { + ui.userTextEdit->setText(user); +} + +QString AutoTester::getSelectedUser() +{ + return ui.userTextEdit->toPlainText(); +} + +void AutoTester::setBranchText(const QString& branch) { + ui.branchTextEdit->setText(branch); +} + +QString AutoTester::getSelectedBranch() { + return ui.branchTextEdit->toPlainText(); +} diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 7b419a9af8..d47c4929c4 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -12,6 +12,7 @@ #include #include +#include #include "ui_AutoTester.h" #include "../Downloader.h" @@ -23,11 +24,19 @@ class AutoTester : public QMainWindow { public: AutoTester(QWidget *parent = Q_NULLPTR); - void runFromCommandLine(const QString& testFolder); + void setup(); + + void runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user); void downloadImage(const QUrl& url); void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames); + void setUserText(const QString& user); + QString getSelectedUser(); + + void setBranchText(const QString& branch); + QString getSelectedBranch(); + private slots: void on_evaluateTestsButton_clicked(); void on_createRecursiveScriptButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 236138acf4..e12fc70e3f 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 607 - 514 + 612 + 537 @@ -17,9 +17,9 @@ - 360 - 400 - 220 + 380 + 430 + 101 40 @@ -43,9 +43,9 @@ - 20 - 285 - 220 + 430 + 270 + 101 40 @@ -56,8 +56,8 @@ - 360 - 35 + 330 + 110 220 40 @@ -69,8 +69,8 @@ - 23 - 250 + 320 + 280 131 20 @@ -85,8 +85,8 @@ - 20 - 340 + 320 + 330 255 23 @@ -98,8 +98,8 @@ - 360 - 100 + 330 + 170 220 40 @@ -112,7 +112,7 @@ 20 - 80 + 110 220 40 @@ -125,7 +125,7 @@ 20 - 130 + 160 220 40 @@ -138,7 +138,7 @@ 20 - 180 + 250 220 40 @@ -150,27 +150,83 @@ - 490 - 280 - 91 + 10 + 440 + 211 40 - Show Taskbar + Show Windows Taskbar - 490 - 230 - 91 + 10 + 390 + 211 40 - Hide Taskbar + Hide Windows Taskbar + + + + + + 330 + 55 + 81 + 16 + + + + + 10 + + + + GitHub Branch + + + + + + 330 + 15 + 81 + 16 + + + + + 10 + + + + GitHub User + + + + + + 420 + 12 + 140 + 24 + + + + + + + 420 + 50 + 140 + 24 + @@ -179,7 +235,7 @@ 0 0 - 607 + 612 21