diff --git a/tools/auto-tester/src/ImageComparer.cpp b/tools/auto-tester/src/ImageComparer.cpp index 99f7d22c5f..eb892484a2 100644 --- a/tools/auto-tester/src/ImageComparer.cpp +++ b/tools/auto-tester/src/ImageComparer.cpp @@ -15,12 +15,6 @@ // Computes SSIM - see https://en.wikipedia.org/wiki/Structural_similarity // The value is computed for the luminance component and the average value is returned double ImageComparer::compareImages(QImage resultImage, QImage expectedImage) const { - // Make sure the image is 8 bits per colour - QImage::Format format = expectedImage.format(); - if (format != QImage::Format::Format_ARGB32) { - throw -1; - } - const int L = 255; // (2^number of bits per pixel) - 1 const double K1 { 0.01 }; const double K2 { 0.03 }; diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 253f30a3b1..11c5428c26 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -158,20 +158,20 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai QString destinationFile; sourceFile = testFailure._pathname + testFailure._expectedImageFilename; - destinationFile = failureFolderPath + "/" + "Expected Image.jpg"; + destinationFile = failureFolderPath + "/" + "Expected Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } sourceFile = testFailure._pathname + testFailure._actualImageFilename; - destinationFile = failureFolderPath + "/" + "Actual Image.jpg"; + destinationFile = failureFolderPath + "/" + "Actual Image.png"; if (!QFile::copy(sourceFile, destinationFile)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + sourceFile + " to " + destinationFile); exit(-1); } - comparisonImage.save(failureFolderPath + "/" + "Difference Image.jpg"); + comparisonImage.save(failureFolderPath + "/" + "Difference Image.png"); } void Test::startTestsEvaluation(const QString& testFolder) { @@ -295,7 +295,7 @@ QString Test::extractPathFromTestsDown(const QString& fullPath) { return partialPath; } -void Test::importTest(QTextStream& textStream, const QString& testPathname) { +void Test::includeTest(QTextStream& textStream, const QString& testPathname) { QString partialPath = extractPathFromTestsDown(testPathname); textStream << "Script.include(\"" << "https://github.com/" << GIT_HUB_USER << "/hifi_tests/blob/" << GIT_HUB_BRANCH @@ -329,24 +329,24 @@ void Test::createRecursiveScript() { // This method creates a `testRecursive.js` script in every sub-folder. void Test::createAllRecursiveScripts() { // Select folder to start recursing from - QString previousSelection = testDirectory; + QString previousSelection = testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", + testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (testsRootDirectory == "") { + testsRootDirectory = previousSelection; return; } - createRecursiveScript(testDirectory, false); + createRecursiveScript(testsRootDirectory, false); - QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -395,22 +395,31 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact const QString DATE_TIME_FORMAT("MMM d yyyy, h:mm"); textStream << "// This is an automatically generated file, created by auto-tester on " << QDateTime::currentDateTime().toString(DATE_TIME_FORMAT) << endl << endl; + textStream << "user = \"" + GIT_HUB_USER + "/\"" << endl; + textStream << "repository = \"" + GIT_HUB_REPOSITORY + "/\"" << endl; + textStream << "branch = \"" + GIT_HUB_BRANCH + "/\"" << endl << endl; + textStream << "var autoTester = Script.require(\"https://github.com/" + GIT_HUB_USER + "/hifi_tests/blob/" + GIT_HUB_BRANCH + "/tests/utils/autoTester.js?raw=true\");" << endl << endl; textStream << "autoTester.enableRecursive();" << endl; textStream << "autoTester.enableAuto();" << endl << endl; - QVector testPathnames; + // This is used to verify that the recursive test contains at least one test + bool testFound{ false }; + + // Directories are included in reverse order. The autoTester scripts use a stack mechanism, + // so this ensures that the tests run in alphabetical order (a convenience when debugging) + QStringList directories; // First test if top-level folder has a test.js file const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { // Current folder contains a test - importTest(textStream, testPathname); + directories.push_front(testPathname); - testPathnames << testPathname; + testFound = true; } QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); @@ -427,18 +436,23 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { // Current folder contains a test - importTest(textStream, testPathname); + directories.push_front(testPathname); - testPathnames << testPathname; + testFound = true; } } - if (interactiveMode && testPathnames.length() <= 0) { + if (interactiveMode && !testFound) { QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found"); allTestsFilename.close(); return; } + // Now include the test scripts + for (int i = 0; i < directories.length(); ++i) { + includeTest(textStream, directories.at(i)); + } + textStream << endl; textStream << "autoTester.runRecursive();" << endl; @@ -449,7 +463,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact } } -void Test::createTest() { +void Test::createTests() { // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on // Any existing expected result images will be deleted QString previousSelection = snapshotDirectory; @@ -467,18 +481,18 @@ void Test::createTest() { return; } - previousSelection = testDirectory; + previousSelection = testsRootDirectory; parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder to save the test images", parent, + testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (testsRootDirectory == "") { + testsRootDirectory = previousSelection; return; } @@ -493,8 +507,23 @@ void Test::createTest() { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); exit(-1); } - QString newFilename = "ExpectedImage_" + QString::number(i - 1).rightJustified(5, '0') + ".png"; - QString fullNewFileName = testDirectory + "/" + newFilename; + + // Path to test is extracted from the file name + // Example: + // filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg + // path is /engine/interaction/pointer/laser/distanceScaleEnd + // + // Note: we don't use the first part and the last 2 parts of the filename at this stage + // + QStringList pathParts = currentFilename.split("."); + QString fullNewFileName = testsRootDirectory; + for (int j = 1; j < pathParts.size() - 2; ++j) { + fullNewFileName += "/" + pathParts[j]; + } + + // The image index is the penultimate component of the path parts (the last being the file extension) + QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png"; + fullNewFileName += "/" + newFilename; try { copyJPGtoPNG(fullCurrentFilename, fullNewFileName); @@ -602,29 +631,29 @@ void Test::createMDFile() { void Test::createAllMDFiles() { // Select folder to start recursing from - QString previousSelection = testDirectory; + QString previousSelection = testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent, + testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (testsRootDirectory == "") { + testsRootDirectory = previousSelection; return; } // First test if top-level folder has a test.js file - const QString testPathname { testDirectory + "/" + TEST_FILENAME }; + const QString testPathname{ testsRootDirectory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { - createMDFile(testDirectory); + createMDFile(testsRootDirectory); } - QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 2fedc66959..78c5dfb14c 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -44,7 +44,7 @@ public: void createAllRecursiveScripts(); void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); - void createTest(); + void createTests(); void createMDFile(); void createAllMDFiles(); void createMDFile(const QString& topLevelDirectory); @@ -57,7 +57,7 @@ public: bool isInSnapshotFilenameFormat(const QString& imageFormat, const QString& filename); - void importTest(QTextStream& textStream, const QString& testPathname); + void includeTest(QTextStream& textStream, const QString& testPathname); void appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage); @@ -91,8 +91,10 @@ private: // We have two directories to work with. // The first is the directory containing the test we are working with - // The second contains the snapshots taken for test runs that need to be evaluated + // The second is the root directory of all tests + // The third contains the snapshots taken for test runs that need to be evaluated QString testDirectory; + QString testsRootDirectory; QString snapshotDirectory; QStringList expectedImagesFilenames; @@ -100,9 +102,11 @@ private: QStringList resultImagesFullFilenames; // Used for accessing GitHub - const QString GIT_HUB_USER{ "highfidelity" }; - const QString GIT_HUB_BRANCH { "master" }; - const QString DATETIME_FORMAT { "yyyy-MM-dd_hh-mm-ss" }; + const QString GIT_HUB_USER{ "NissimHadar" }; + const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; + const QString GIT_HUB_BRANCH{ "newAvatar" }; + + const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; ExtractedText getTestScriptLines(QString testFileName); diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index db9974a250..ea2633edf4 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -40,8 +40,8 @@ void AutoTester::on_createAllRecursiveScriptsButton_clicked() { test->createAllRecursiveScripts(); } -void AutoTester::on_createTestButton_clicked() { - test->createTest(); +void AutoTester::on_createTestsButton_clicked() { + test->createTests(); } void AutoTester::on_createMDFileButton_clicked() { @@ -95,7 +95,7 @@ void AutoTester::saveImage(int index) { pixmap.loadFromData(downloaders[index]->downloadedData()); QImage image = pixmap.toImage(); - image = image.convertToFormat(QImage::Format_ARGB32); + image = image.convertToFormat(QImage::Format_RGB32); QString fullPathname = _directoryName + "/" + _filenames[index]; if (!image.save(fullPathname, 0, 100)) { diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index fe37f2298d..5991afed1b 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -32,7 +32,7 @@ private slots: void on_evaluateTestsButton_clicked(); void on_createRecursiveScriptButton_clicked(); void on_createAllRecursiveScriptsButton_clicked(); - void on_createTestButton_clicked(); + void on_createTestsButton_clicked(); void on_createMDFileButton_clicked(); void on_createAllMDFilesButton_clicked(); void on_createTestsOutlineButton_clicked(); diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index 8c534eb7c7..c9ea229012 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -27,7 +27,7 @@ Close - + 20 @@ -37,7 +37,7 @@ - Create Test + Create Tests