From 74aedd73fd23de8f023c6fc9c7d6ef2a2ddd9b27 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 12 Jun 2018 15:47:55 -0700 Subject: [PATCH 1/8] Exit at end of non-interactive recursive script. Similarity index set to -100.0 to indicate size difference. Simplified repo. path in testRecursive scripts. Wait 10" before starting recursive script. --- tools/auto-tester/src/Test.cpp | 70 +++++++++++++++++++--------------- tools/auto-tester/src/Test.h | 4 ++ 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 1a25cc5c0e..e34a5f787c 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -63,7 +63,6 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) // Loop over both lists and compare each pair of images // Quit loop if user has aborted due to a failed test. - const double THRESHOLD { 0.9995 }; bool success{ true }; bool keepOn{ true }; for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) { @@ -71,17 +70,19 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) QImage resultImage(resultImagesFullFilenames[i]); QImage expectedImage(expectedImagesFullFilenames[i]); + double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical + + // similarityIndex is set to -100.0 to indicate images are not the same size if (isInteractiveMode && (resultImage.width() != expectedImage.width() || resultImage.height() != expectedImage.height())) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); - exit(-1); - } - - double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical - try { - similarityIndex = imageComparer.compareImages(resultImage, expectedImage); - } catch (...) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format"); - exit(-1); + similarityIndex = -100.0; + } else { + try { + similarityIndex = imageComparer.compareImages(resultImage, expectedImage); + } catch (...) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format"); + exit(-1); + } } if (similarityIndex < THRESHOLD) { @@ -177,19 +178,24 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai } void Test::startTestsEvaluation(const QString& testFolder) { - // Get list of JPEG images in folder, sorted by name - QString previousSelection = snapshotDirectory; - QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); - if (!parent.isNull() && parent.right(1) != "/") { - parent += "/"; - } - snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, - QFileDialog::ShowDirsOnly); + if (testFolder.isNull()) { + // Get list of JPEG images in folder, sorted by name + QString previousSelection = snapshotDirectory; + QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); + if (!parent.isNull() && parent.right(1) != "/") { + parent += "/"; + } + snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + QFileDialog::ShowDirsOnly); - // If user cancelled then restore previous selection and return - if (snapshotDirectory == "") { - snapshotDirectory = previousSelection; - return; + // If user cancelled then restore previous selection and return + if (snapshotDirectory == "") { + snapshotDirectory = previousSelection; + return; + } + } else { + snapshotDirectory = testFolder; + exitWhenComplete = true; } // Quit if test results folder could not be created @@ -259,6 +265,10 @@ void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactive } zipAndDeleteTestResultsFolder(); + + if (exitWhenComplete) { + exit(0); + } } bool Test::isAValidDirectory(const QString& pathname) { @@ -299,10 +309,7 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) { void Test::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); - textStream << "Script.include(\"" - << "https://github.com/" << GIT_HUB_USER << "/hifi_tests/blob/" << GIT_HUB_BRANCH - << partialPath + "?raw=true\");" - << endl; + textStream << "Script.include(repositoryPath + \"" << partialPath + "?raw=true\");" << endl; } // Creates a single script in a user-selected folder. @@ -401,8 +408,11 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl; textStream << "branch = \"" + GIT_HUB_BRANCH + "/\";" << endl << endl; - textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/" - + GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl; + // Wait 10 seconds before starting + textStream << "Test.wait(10000);" << endl << endl; + + textStream << "var repositoryPath = \"https://github.com/\" + user + repository + \"blob/\" + branch;" << endl; + textStream << "var autoTester = Script.require(repositoryPath + \"tests/utils/autoTester.js?raw=true\");" << endl << endl; textStream << "autoTester.enableRecursive();" << endl; textStream << "autoTester.enableAuto();" << endl << endl; @@ -560,9 +570,7 @@ ExtractedText Test::getTestScriptLines(QString testFileName) { const QString ws("\\h*"); //white-space character const QString functionPerformName(ws + "autoTester" + ws + "\\." + ws + "perform"); const QString quotedString("\\\".+\\\""); - const QString ownPath("Script" + ws + "\\." + ws + "resolvePath" + ws + "\\(" + ws + "\\\"\\.\\\"" + ws + "\\)"); - const QString functionParameter("function" + ws + "\\(testType" + ws + "\\)"); - QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString + "\\," + ws + ownPath + "\\," + ws + functionParameter + ws + "{" + ".*"); + QString regexTestTitle(ws + functionPerformName + "\\(" + quotedString); QRegularExpression lineContainingTitle = QRegularExpression(regexTestTitle); diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 32564035e7..5833b990c2 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -76,6 +76,8 @@ private: const QString TEST_RESULTS_FOLDER { "TestResults" }; const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; + const double THRESHOLD{ 0.999 }; + QDir imageDirectory; MismatchWindow mismatchWindow; @@ -115,6 +117,8 @@ private: // var pathSeparator = "."; const QString ADVANCE_KEY{ "n" }; const QString PATH_SEPARATOR{ "." }; + + bool exitWhenComplete{ false }; }; #endif // hifi_test_h \ No newline at end of file From 9ca27468f6fe63da1c6bad4169ba1275b58a9982 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Wed, 13 Jun 2018 19:29:09 -0700 Subject: [PATCH 2/8] Added selection of branch from the command line. --- tools/auto-tester/src/Test.cpp | 14 +++++-- tools/auto-tester/src/Test.h | 6 ++- tools/auto-tester/src/main.cpp | 36 +++++++++++++--- tools/auto-tester/src/ui/AutoTester.cpp | 14 ++++++- tools/auto-tester/src/ui/AutoTester.h | 7 +++- tools/auto-tester/src/ui/AutoTester.ui | 56 ++++++++++++++++++------- 6 files changed, 106 insertions(+), 27 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index e34a5f787c..0f38a9c619 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -25,6 +25,10 @@ extern AutoTester* autoTester; Test::Test() { mismatchWindow.setModal(true); + + if (autoTester) { + autoTester->loadBranchCombo(GIT_HUB_BRANCHES); + } } bool Test::createTestResultsFolderPath(const QString& directory) { @@ -177,7 +181,7 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai comparisonImage.save(failureFolderPath + "/" + "Difference Image.png"); } -void Test::startTestsEvaluation(const QString& testFolder) { +void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine) { if (testFolder.isNull()) { // Get list of JPEG images in folder, sorted by name QString previousSelection = snapshotDirectory; @@ -225,6 +229,8 @@ void Test::startTestsEvaluation(const QString& testFolder) { expectedImagesFilenames.clear(); expectedImagesFullFilenames.clear(); + QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine ; + foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { @@ -237,7 +243,7 @@ void Test::startTestsEvaluation(const QString& testFolder) { QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; - QString imageURLString("https://raw.githubusercontent.com/" + GIT_HUB_USER + "/hifi_tests/" + GIT_HUB_BRANCH + "/" + + QString imageURLString("https://raw.githubusercontent.com/" + GIT_HUB_USER + "/hifi_tests/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); expectedImagesURLs << imageURLString; @@ -406,7 +412,9 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact textStream << "user = \"" + GIT_HUB_USER + "/\";" << endl; textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl; - textStream << "branch = \"" + GIT_HUB_BRANCH + "/\";" << endl << endl; + + QString branch = autoTester->getSelectedBranch(); + textStream << "branch = \"" + branch + "/\";" << endl << endl; // Wait 10 seconds before starting textStream << "Test.wait(10000);" << endl << endl; diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 5833b990c2..b91a49ed95 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -37,7 +37,7 @@ class Test { public: Test(); - void startTestsEvaluation(const QString& testFolder = QString()); + void startTestsEvaluation(const QString& testFolder = QString(), const QString& branchFromCommandLine = QString()); void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar); void createRecursiveScript(); @@ -106,7 +106,9 @@ private: // Used for accessing GitHub const QString GIT_HUB_USER{ "highfidelity" }; const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; - const QString GIT_HUB_BRANCH{ "master" }; + + // The branch is user-selected + const QStringList GIT_HUB_BRANCHES{ "master", "RC69" }; const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index ffa7a0b237..e01cd21f77 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -10,23 +10,49 @@ #include #include "ui/AutoTester.h" +#include + AutoTester* autoTester; int main(int argc, char *argv[]) { - // Only parameter is "--testFolder" + // If no parameters then run in interactive mode + // Parameter --testFolder + // Parameter --branch + // default is "master" + // QString testFolder; - if (argc == 3) { - if (QString(argv[1]) == "--testFolder") { - testFolder = QString(argv[2]); + QString branch; + if (argc > 2) { + for (int i = 1; i < argc - 1; ++i) + if (QString(argv[i]) == "--testFolder") { + ++i; + if (i < argc) { + testFolder = QString(argv[i]); + } else { + std::cout << "Missing parameter after --testFolder" << endl; + exit(-1); + } + } else if (QString(argv[i]) == "--branch") { + ++i; + if (i < argc) { + branch = QString(argv[i]); + } else { + std::cout << "Missing parameter after --branch" << endl; + exit(-1); + } + } else { + std::cout << "Unknown parameter" << endl; + exit(-1); } } QApplication application(argc, argv); autoTester = new AutoTester(); + autoTester->setup(); if (!testFolder.isNull()) { - autoTester->runFromCommandLine(testFolder); + autoTester->runFromCommandLine(testFolder, branch); } else { autoTester->show(); } diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 14329e36c2..d94916ecbc 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -29,13 +29,15 @@ AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { ui.hideTaskbarButton->setVisible(false); ui.showTaskbarButton->setVisible(false); #endif +} +void AutoTester::setup() { test = new Test(); } -void AutoTester::runFromCommandLine(const QString& testFolder) { +void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch) { isRunningFromCommandline = true; - test->startTestsEvaluation(testFolder); + test->startTestsEvaluation(testFolder, branch); } void AutoTester::on_evaluateTestsButton_clicked() { @@ -150,3 +152,11 @@ void AutoTester::saveImage(int index) { void AutoTester::about() { QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__); } + +void AutoTester::loadBranchCombo(const QStringList& items) { + ui.branchComboBox->addItems(items); +} + +QString AutoTester::getSelectedBranch() { + return ui.branchComboBox->currentText(); +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 7b419a9af8..5eff89e957 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -23,11 +23,16 @@ class AutoTester : public QMainWindow { public: AutoTester(QWidget *parent = Q_NULLPTR); - void runFromCommandLine(const QString& testFolder); + void setup(); + + void runFromCommandLine(const QString& testFolder, const QString& branch); void downloadImage(const QUrl& url); void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames); + void loadBranchCombo(const QStringList& items); + QString getSelectedBranch(); + private slots: void on_evaluateTestsButton_clicked(); void on_createRecursiveScriptButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 236138acf4..44c00509e6 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -43,8 +43,8 @@ - 20 - 285 + 340 + 275 220 40 @@ -56,8 +56,8 @@ - 360 - 35 + 340 + 95 220 40 @@ -69,8 +69,8 @@ - 23 - 250 + 343 + 240 131 20 @@ -85,8 +85,8 @@ - 20 - 340 + 340 + 330 255 23 @@ -98,8 +98,8 @@ - 360 - 100 + 340 + 160 220 40 @@ -150,8 +150,8 @@ - 490 - 280 + 10 + 410 91 40 @@ -163,8 +163,8 @@ - 490 - 230 + 10 + 360 91 40 @@ -173,6 +173,34 @@ Hide Taskbar + + + + 428 + 41 + 131 + 31 + + + + + + + 340 + 30 + 81 + 51 + + + + + 10 + + + + GitHub Branch + + From d59ddc47648e784c8ab3fabbf4700b660c399a97 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 18 Jun 2018 07:44:04 -0700 Subject: [PATCH 3/8] Added selection of branch from combo. --- tools/auto-tester/src/Test.cpp | 10 +++--- tools/auto-tester/src/ui/AutoTester.ui | 42 +++++++++++++------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 0f38a9c619..1cfc32e481 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -411,13 +411,15 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl; textStream << "user = \"" + GIT_HUB_USER + "/\";" << endl; - textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl; + textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl << endl; - QString branch = autoTester->getSelectedBranch(); - textStream << "branch = \"" + branch + "/\";" << endl << endl; + textStream << "Script.include(\"https://github.com/highfidelity/hifi_tests/blob/RC69/tests/utils/branchUtils.js?raw=true\");" << endl; + textStream << "branch = getBranch(Script.resolvePath(\".\"), repository) +\"/\";" << endl << endl; // Wait 10 seconds before starting - textStream << "Test.wait(10000);" << endl << endl; + textStream << "if (typeof Test !== 'undefined') {" << endl; + textStream << " Test.wait(10000);" << endl; + textStream << "};" << endl << endl; textStream << "var repositoryPath = \"https://github.com/\" + user + repository + \"blob/\" + branch;" << endl; textStream << "var autoTester = Script.require(repositoryPath + \"tests/utils/autoTester.js?raw=true\");" << endl << endl; diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 44c00509e6..df21dad1eb 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,7 +6,7 @@ 0 0 - 607 + 583 514 @@ -17,9 +17,9 @@ - 360 - 400 - 220 + 460 + 410 + 101 40 @@ -43,7 +43,7 @@ - 340 + 330 275 220 40 @@ -56,8 +56,8 @@ - 340 - 95 + 330 + 30 220 40 @@ -69,8 +69,8 @@ - 343 - 240 + 330 + 250 131 20 @@ -85,7 +85,7 @@ - 340 + 330 330 255 23 @@ -98,8 +98,8 @@ - 340 - 160 + 330 + 80 220 40 @@ -112,7 +112,7 @@ 20 - 80 + 100 220 40 @@ -125,7 +125,7 @@ 20 - 130 + 150 220 40 @@ -138,7 +138,7 @@ 20 - 180 + 220 220 40 @@ -176,8 +176,8 @@ - 428 - 41 + 418 + 211 131 31 @@ -186,10 +186,10 @@ - 340 - 30 + 330 + 220 81 - 51 + 16 @@ -207,7 +207,7 @@ 0 0 - 607 + 583 21 From 4897e11a125cdd2f39aa62d2c260cad852dfe07d Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 18 Jun 2018 12:43:42 -0700 Subject: [PATCH 4/8] Save downloaded byte array (from GitHub) to file with PNG extension. --- tools/auto-tester/src/ImageComparer.cpp | 3 --- tools/auto-tester/src/Test.cpp | 7 +------ tools/auto-tester/src/ui/AutoTester.cpp | 14 ++++++-------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 3afdeaec0a..56da37f225 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -17,9 +17,6 @@ double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const { // Make sure the image is 8 bits per colour QImage::Format format = expectedImage.format(); - if (format != QImage::Format::Format_ARGB32) { - throw -1; - } const int L = 255; // (2^number of bits per pixel) - 1 const double K1 { 0.01 }; diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 1cfc32e481..7a61782f9e 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -81,12 +81,7 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); similarityIndex = -100.0; } else { - try { - similarityIndex = imageComparer.compareImages(resultImage, expectedImage); - } catch (...) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Image not in expected format"); - exit(-1); - } + similarityIndex = imageComparer.compareImages(resultImage, expectedImage); } if (similarityIndex < THRESHOLD) { diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index d94916ecbc..a238e2e7e8 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -127,14 +127,12 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director } void AutoTester::saveImage(int index) { - QPixmap pixmap; - pixmap.loadFromData(downloaders[index]->downloadedData()); - - QImage image = pixmap.toImage(); - image = image.convertToFormat(QImage::Format_ARGB32); - - QString fullPathname = _directoryName + "/" + _filenames[index]; - if (!image.save(fullPathname, 0, 100)) { + try { + QFile file(_directoryName + "/" + _filenames[index]); + file.open(QIODevice::WriteOnly); + file.write(downloaders[index]->downloadedData()); + file.close(); + } catch (...) { QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]); ui.progressBar->setVisible(false); return; From 79293310e56a58efd40d0978e99635514d9d1f78 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Tue, 19 Jun 2018 11:59:31 -0700 Subject: [PATCH 5/8] No more use of JPG files. --- tools/auto-tester/src/ImageComparer.cpp | 4 +-- tools/auto-tester/src/Test.cpp | 35 +++++-------------------- tools/auto-tester/src/Test.h | 4 +-- 3 files changed, 9 insertions(+), 34 deletions(-) diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 56da37f225..fa73f97887 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -15,10 +15,8 @@ // Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity // The value is computed for the luminance component and the average value is returned double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const { - // Make sure the image is 8 bits per colour - QImage::Format format = expectedImage.format(); - const int L = 255; // (2^number of bits per pixel) - 1 + const double K1 { 0.01 }; const double K2 { 0.03 }; const double c1 = pow((K1 * L), 2); diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 7a61782f9e..88f4e58782 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -202,17 +202,6 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch return; } - // Before any processing - all images are converted to PNGs, as this is the format stored on GitHub - QStringList sortedSnapshotFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory); - foreach(QString filename, sortedSnapshotFilenames) { - QString filenameWithoutExtension = filename.left(filename.length() - 4); - copyJPGtoPNG(snapshotDirectory + "/" + filenameWithoutExtension + ".jpg", - snapshotDirectory + "/" + filenameWithoutExtension + ".png" - ); - - QFile::remove(snapshotDirectory + "/" + filenameWithoutExtension + ".jpg"); - } - // Create two lists. The first is the test results, the second is the expected images // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. @@ -513,13 +502,13 @@ void Test::createTests() { return; } - QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("jpg", snapshotDirectory); + QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; - if (isInSnapshotFilenameFormat("jpg", currentFilename)) { + if (isInSnapshotFilenameFormat("png", currentFilename)) { if (i >= maxImages) { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); @@ -543,9 +532,12 @@ void Test::createTests() { fullNewFileName += "/" + newFilename; try { - copyJPGtoPNG(fullCurrentFilename, fullNewFileName); + if (QFile::exists(fullNewFileName)) { + QFile::remove(fullNewFileName); + } + QFile::copy(fullCurrentFilename, fullNewFileName); } catch (...) { - QMessageBox::critical(0, "Error", "Could not delete existing file: " + currentFilename + "\nTest creation aborted"); + QMessageBox::critical(0, "Error", "Could not copy file: " + fullCurrentFilename + " to " + fullNewFileName + "\n"); exit(-1); } ++i; @@ -824,19 +816,6 @@ void Test::createTestsOutline() { QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created"); } -void Test::copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename) { - QFile::remove(destinationPNGFullFilename); - - QImageReader reader; - reader.setFileName(sourceJPGFullFilename); - - QImage image = reader.read(); - - QImageWriter writer; - writer.setFileName(destinationPNGFullFilename); - writer.write(image); -} - QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index b91a49ed95..94d868338d 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -69,14 +69,12 @@ public: QString getExpectedImageDestinationDirectory(const QString& filename); QString getExpectedImagePartialSourceDirectory(const QString& filename); - void copyJPGtoPNG(const QString& sourceJPGFullFilename, const QString& destinationPNGFullFilename); - private: const QString TEST_FILENAME { "test.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; - const double THRESHOLD{ 0.999 }; + const double THRESHOLD{ 0.96 }; QDir imageDirectory; From 8ad8713a139bd646ed9951ae5f4d968b48b83e44 Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Thu, 21 Jun 2018 17:02:51 -0700 Subject: [PATCH 6/8] New API. --- tools/auto-tester/src/Test.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 88f4e58782..1e76c19303 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -299,7 +299,9 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) { void Test::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); - textStream << "Script.include(repositoryPath + \"" << partialPath + "?raw=true\");" << endl; + QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); + + textStream << "Script.include(repositoryPath + \"" << partialPathWithoutTests + "\");" << endl; } // Creates a single script in a user-selected folder. @@ -394,20 +396,17 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm"); textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl; - textStream << "user = \"" + GIT_HUB_USER + "/\";" << endl; - textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\";" << endl << endl; + // Include 'autoTest.js' + textStream << "Script.include(\"https://raw.githubusercontent.com/highfidelity/hifi_tests/master/tests/utils/branchUtils.js\");" << endl; + textStream << "var autoTester = createAutoTester(Script.resolvePath(\".\"));" << endl << endl; - textStream << "Script.include(\"https://github.com/highfidelity/hifi_tests/blob/RC69/tests/utils/branchUtils.js?raw=true\");" << endl; - textStream << "branch = getBranch(Script.resolvePath(\".\"), repository) +\"/\";" << endl << endl; + textStream << "var repositoryPath = autoTester.getRepositoryPath();" << endl << endl; // Wait 10 seconds before starting textStream << "if (typeof Test !== 'undefined') {" << endl; textStream << " Test.wait(10000);" << endl; textStream << "};" << endl << endl; - textStream << "var repositoryPath = \"https://github.com/\" + user + repository + \"blob/\" + branch;" << endl; - textStream << "var autoTester = Script.require(repositoryPath + \"tests/utils/autoTester.js?raw=true\");" << endl << endl; - textStream << "autoTester.enableRecursive();" << endl; textStream << "autoTester.enableAuto();" << endl << endl; From 25834c38099374a92f5d171b62d50de71168cc3c Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 25 Jun 2018 09:13:58 -0700 Subject: [PATCH 7/8] Allow multiple evaluations. --- tools/auto-tester/src/Test.cpp | 20 ++++-- tools/auto-tester/src/Test.h | 6 +- tools/auto-tester/src/main.cpp | 24 +++++-- tools/auto-tester/src/ui/AutoTester.cpp | 23 ++++-- tools/auto-tester/src/ui/AutoTester.h | 8 ++- tools/auto-tester/src/ui/AutoTester.ui | 94 ++++++++++++++++--------- 6 files changed, 117 insertions(+), 58 deletions(-) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 1e76c19303..4f02544c12 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -27,7 +27,8 @@ Test::Test() { mismatchWindow.setModal(true); if (autoTester) { - autoTester->loadBranchCombo(GIT_HUB_BRANCHES); + autoTester->setUserText("highfidelity"); + autoTester->setBranchText("master"); } } @@ -176,7 +177,7 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai comparisonImage.save(failureFolderPath + "/" + "Difference Image.png"); } -void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine) { +void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine, const QString& userFromCommandLine) { if (testFolder.isNull()) { // Get list of JPEG images in folder, sorted by name QString previousSelection = snapshotDirectory; @@ -213,7 +214,8 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch expectedImagesFilenames.clear(); expectedImagesFullFilenames.clear(); - QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine ; + QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine; + QString user = (userFromCommandLine.isNull()) ? autoTester->getSelectedUser() : userFromCommandLine; foreach(QString currentFilename, sortedTestResultsFilenames) { QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; @@ -227,7 +229,7 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch QString expectedImageFilenameTail = currentFilename.left(currentFilename.length() - 4).right(NUM_DIGITS); QString expectedImageStoredFilename = EXPECTED_IMAGE_PREFIX + expectedImageFilenameTail + ".png"; - QString imageURLString("https://raw.githubusercontent.com/" + GIT_HUB_USER + "/hifi_tests/" + branch + "/" + + QString imageURLString("https://raw.githubusercontent.com/" + user + "/" + GIT_HUB_REPOSITORY + "/" + branch + "/" + expectedImagePartialSourceDirectory + "/" + expectedImageStoredFilename); expectedImagesURLs << imageURLString; @@ -301,7 +303,7 @@ void Test::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); QString partialPathWithoutTests = partialPath.right(partialPath.length() - 7); - textStream << "Script.include(repositoryPath + \"" << partialPathWithoutTests + "\");" << endl; + textStream << "Script.include(testsRootPath + \"" << partialPathWithoutTests + "\");" << endl; } // Creates a single script in a user-selected folder. @@ -397,10 +399,14 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl; // Include 'autoTest.js' - textStream << "Script.include(\"https://raw.githubusercontent.com/highfidelity/hifi_tests/master/tests/utils/branchUtils.js\");" << endl; + QString branch = autoTester->getSelectedBranch(); + QString user = autoTester->getSelectedUser(); + + textStream << "PATH_TO_THE_REPO_PATH_UTILS_FILE = \"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + "/tests/utils/branchUtils.js\";" << endl; + textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl; textStream << "var autoTester = createAutoTester(Script.resolvePath(\".\"));" << endl << endl; - textStream << "var repositoryPath = autoTester.getRepositoryPath();" << endl << endl; + textStream << "var testsRootPath = autoTester.getTestsRootPath();" << endl << endl; // Wait 10 seconds before starting textStream << "if (typeof Test !== 'undefined') {" << endl; diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 94d868338d..5c6d3e5686 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -37,7 +37,7 @@ class Test { public: Test(); - void startTestsEvaluation(const QString& testFolder = QString(), const QString& branchFromCommandLine = QString()); + void startTestsEvaluation(const QString& testFolder = QString(), const QString& branchFromCommandLine = QString(), const QString& userFromCommandLine = QString()); void finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar); void createRecursiveScript(); @@ -102,12 +102,8 @@ private: QStringList resultImagesFullFilenames; // Used for accessing GitHub - const QString GIT_HUB_USER{ "highfidelity" }; const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; - // The branch is user-selected - const QStringList GIT_HUB_BRANCHES{ "master", "RC69" }; - const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; ExtractedText getTestScriptLines(QString testFileName); diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index e01cd21f77..ed7f4b4e74 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -19,11 +19,17 @@ int main(int argc, char *argv[]) { // Parameter --testFolder // Parameter --branch // default is "master" - // + // Parameter --user + // default is "highfidelity" + // Parameter --repository + // default is "highfidelity" + QString testFolder; - QString branch; - if (argc > 2) { - for (int i = 1; i < argc - 1; ++i) + + QString branch{ "master" }; + QString user{ "highfidelity" }; + + for (int i = 1; i < argc - 1; ++i) { if (QString(argv[i]) == "--testFolder") { ++i; if (i < argc) { @@ -40,6 +46,14 @@ int main(int argc, char *argv[]) { std::cout << "Missing parameter after --branch" << endl; exit(-1); } + } else if (QString(argv[i]) == "--user") { + ++i; + if (i < argc) { + user = QString(argv[i]); + } else { + std::cout << "Missing parameter after --user" << endl; + exit(-1); + } } else { std::cout << "Unknown parameter" << endl; exit(-1); @@ -52,7 +66,7 @@ int main(int argc, char *argv[]) { autoTester->setup(); if (!testFolder.isNull()) { - autoTester->runFromCommandLine(testFolder, branch); + autoTester->runFromCommandLine(testFolder, branch, user); } else { autoTester->show(); } diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index a238e2e7e8..079fa63a9d 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -35,9 +35,9 @@ void AutoTester::setup() { test = new Test(); } -void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch) { +void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) { isRunningFromCommandline = true; - test->startTestsEvaluation(testFolder, branch); + test->startTestsEvaluation(testFolder, branch, user); } void AutoTester::on_evaluateTestsButton_clicked() { @@ -118,6 +118,7 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director ui.progressBar->setValue(0); ui.progressBar->setVisible(true); + downloaders.clear(); for (int i = 0; i < _numberOfImagesToDownload; ++i) { QUrl imageURL(URLs[i]); downloadImage(imageURL); @@ -141,6 +142,7 @@ void AutoTester::saveImage(int index) { ++_numberOfImagesDownloaded; if (_numberOfImagesDownloaded == _numberOfImagesToDownload) { + disconnect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); } else { ui.progressBar->setValue(_numberOfImagesDownloaded); @@ -151,10 +153,19 @@ void AutoTester::about() { QMessageBox::information(0, "About", QString("Built ") + __DATE__ + " : " + __TIME__); } -void AutoTester::loadBranchCombo(const QStringList& items) { - ui.branchComboBox->addItems(items); +void AutoTester::setUserText(const QString& user) { + ui.userTextEdit->setText(user); +} + +QString AutoTester::getSelectedUser() +{ + return ui.userTextEdit->toPlainText(); +} + +void AutoTester::setBranchText(const QString& branch) { + ui.branchTextEdit->setText(branch); } QString AutoTester::getSelectedBranch() { - return ui.branchComboBox->currentText(); -} \ No newline at end of file + return ui.branchTextEdit->toPlainText(); +} diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 5eff89e957..d47c4929c4 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -12,6 +12,7 @@ #include #include +#include #include "ui_AutoTester.h" #include "../Downloader.h" @@ -25,12 +26,15 @@ public: void setup(); - void runFromCommandLine(const QString& testFolder, const QString& branch); + void runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user); void downloadImage(const QUrl& url); void downloadImages(const QStringList& URLs, const QString& directoryName, const QStringList& filenames); - void loadBranchCombo(const QStringList& items); + void setUserText(const QString& user); + QString getSelectedUser(); + + void setBranchText(const QString& branch); QString getSelectedBranch(); private slots: diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index df21dad1eb..e12fc70e3f 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 583 - 514 + 612 + 537 @@ -17,8 +17,8 @@ - 460 - 410 + 380 + 430 101 40 @@ -43,9 +43,9 @@ - 330 - 275 - 220 + 430 + 270 + 101 40 @@ -57,7 +57,7 @@ 330 - 30 + 110 220 40 @@ -69,8 +69,8 @@ - 330 - 250 + 320 + 280 131 20 @@ -85,7 +85,7 @@ - 330 + 320 330 255 23 @@ -99,7 +99,7 @@ 330 - 80 + 170 220 40 @@ -112,7 +112,7 @@ 20 - 100 + 110 220 40 @@ -125,7 +125,7 @@ 20 - 150 + 160 220 40 @@ -138,7 +138,7 @@ 20 - 220 + 250 220 40 @@ -151,43 +151,33 @@ 10 - 410 - 91 + 440 + 211 40 - Show Taskbar + Show Windows Taskbar 10 - 360 - 91 + 390 + 211 40 - Hide Taskbar - - - - - - 418 - 211 - 131 - 31 - + Hide Windows Taskbar 330 - 220 + 55 81 16 @@ -201,13 +191,51 @@ GitHub Branch + + + + 330 + 15 + 81 + 16 + + + + + 10 + + + + GitHub User + + + + + + 420 + 12 + 140 + 24 + + + + + + + 420 + 50 + 140 + 24 + + + 0 0 - 583 + 612 21 From bd8e8aab85cd3cdd630745b1f0255671bb4d316a Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Mon, 25 Jun 2018 16:40:43 -0700 Subject: [PATCH 8/8] Fixed gcc warnings. --- tools/auto-tester/src/main.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/auto-tester/src/main.cpp b/tools/auto-tester/src/main.cpp index ed7f4b4e74..03b8cf51ff 100644 --- a/tools/auto-tester/src/main.cpp +++ b/tools/auto-tester/src/main.cpp @@ -35,7 +35,7 @@ int main(int argc, char *argv[]) { if (i < argc) { testFolder = QString(argv[i]); } else { - std::cout << "Missing parameter after --testFolder" << endl; + std::cout << "Missing parameter after --testFolder" << std::endl; exit(-1); } } else if (QString(argv[i]) == "--branch") { @@ -43,7 +43,7 @@ int main(int argc, char *argv[]) { if (i < argc) { branch = QString(argv[i]); } else { - std::cout << "Missing parameter after --branch" << endl; + std::cout << "Missing parameter after --branch" << std::endl; exit(-1); } } else if (QString(argv[i]) == "--user") { @@ -51,11 +51,11 @@ int main(int argc, char *argv[]) { if (i < argc) { user = QString(argv[i]); } else { - std::cout << "Missing parameter after --user" << endl; + std::cout << "Missing parameter after --user" << std::endl; exit(-1); } } else { - std::cout << "Unknown parameter" << endl; + std::cout << "Unknown parameter" << std::endl; exit(-1); } }