From 816b3a4ef6f3e64ea7675edc4df6b901b8cda34f Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Wed, 25 Jul 2018 15:49:41 -0700 Subject: [PATCH] WIP. --- tools/auto-tester/src/Test.cpp | 211 +++++++++--------- tools/auto-tester/src/Test.h | 28 +-- ...SuiteCreator.cpp => TestRailInterface.cpp} | 15 +- ...TestSuiteCreator.h => TestRailInterface.h} | 15 +- tools/auto-tester/src/ui/AutoTester.cpp | 74 +++--- tools/auto-tester/src/ui/AutoTester.h | 10 +- tools/auto-tester/src/ui/MismatchWindow.cpp | 6 +- tools/auto-tester/src/ui/MismatchWindow.h | 2 +- 8 files changed, 188 insertions(+), 173 deletions(-) rename tools/auto-tester/src/{TestSuiteCreator.cpp => TestRailInterface.cpp} (92%) rename tools/auto-tester/src/{TestSuiteCreator.h => TestRailInterface.h} (65%) diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 11e1696bb1..22ec78e284 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -18,14 +18,14 @@ #include #include -#include "TestSuiteCreator.h" +#include "TestRailInterface.h" #include "ui/AutoTester.h" extern AutoTester* autoTester; #include Test::Test() { - mismatchWindow.setModal(true); + _mismatchWindow.setModal(true); if (autoTester) { autoTester->setUserText("highfidelity"); @@ -35,35 +35,35 @@ Test::Test() { bool Test::createTestResultsFolderPath(const QString& directory) { QDateTime now = QDateTime::currentDateTime(); - testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT); - QDir testResultsFolder(testResultsFolderPath); + _testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT); + QDir testResultsFolder(_testResultsFolderPath); // Create a new test results folder - return QDir().mkdir(testResultsFolderPath); + return QDir().mkdir(_testResultsFolderPath); } void Test::zipAndDeleteTestResultsFolder() { - QString zippedResultsFileName { testResultsFolderPath + ".zip" }; + QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; QFileInfo fileInfo(zippedResultsFileName); if (!fileInfo.exists()) { QFile::remove(zippedResultsFileName); } - QDir testResultsFolder(testResultsFolderPath); + QDir testResultsFolder(_testResultsFolderPath); if (!testResultsFolder.isEmpty()) { - JlCompress::compressDir(testResultsFolderPath + ".zip", testResultsFolderPath); + JlCompress::compressDir(_testResultsFolderPath + ".zip", _testResultsFolderPath); } testResultsFolder.removeRecursively(); //In all cases, for the next evaluation - testResultsFolderPath = ""; - index = 1; + _testResultsFolderPath = ""; + _index = 1; } bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) { progressBar->setMinimum(0); - progressBar->setMaximum(expectedImagesFullFilenames.length() - 1); + progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); progressBar->setValue(0); progressBar->setVisible(true); @@ -71,10 +71,10 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) // Quit loop if user has aborted due to a failed test. bool success{ true }; bool keepOn{ true }; - for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) { + for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size - QImage resultImage(resultImagesFullFilenames[i]); - QImage expectedImage(expectedImagesFullFilenames[i]); + QImage resultImage(_resultImagesFullFilenames[i]); + QImage expectedImage(_expectedImagesFullFilenames[i]); double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical @@ -83,30 +83,30 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); similarityIndex = -100.0; } else { - similarityIndex = imageComparer.compareImages(resultImage, expectedImage); + similarityIndex = _imageComparer.compareImages(resultImage, expectedImage); } if (similarityIndex < THRESHOLD) { TestFailure testFailure = TestFailure{ (float)similarityIndex, - expectedImagesFullFilenames[i].left(expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) - QFileInfo(expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image - QFileInfo(resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image + _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) + QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image }; - mismatchWindow.setTestFailure(testFailure); + _mismatchWindow.setTestFailure(testFailure); if (!isInteractiveMode) { - appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); + appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); success = false; } else { - mismatchWindow.exec(); + _mismatchWindow.exec(); - switch (mismatchWindow.getUserResponse()) { + switch (_mismatchWindow.getUserResponse()) { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: - appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); + appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); success = false; break; case USER_RESPONSE_ABORT: @@ -127,20 +127,20 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) return success; } -void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { - if (!QDir().exists(testResultsFolderPath)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found"); +void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { + if (!QDir().exists(_testResultsFolderPath)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } QString err = QString::number(testFailure._error).left(6); - QString failureFolderPath { testResultsFolderPath + "/" + err + "-Failure_" + QString::number(index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) }; + QString failureFolderPath { _testResultsFolderPath + "/" + err + "-Failure_" + QString::number(_index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) }; if (!QDir().mkdir(failureFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath); exit(-1); } - ++index; + ++_index; QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME); if (!descriptionFile.open(QIODevice::ReadWrite)) { @@ -153,7 +153,7 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/' stream << "Expected image was " << testFailure._expectedImageFilename << endl; stream << "Actual image was " << testFailure._actualImageFilename << endl; - stream << "Similarity index was " << testFailure._error << endl; + stream << "Similarity _index was " << testFailure._error << endl; descriptionFile.close(); @@ -181,26 +181,26 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine, const QString& userFromCommandLine) { if (testFolder.isNull()) { // Get list of JPEG images in folder, sorted by name - QString previousSelection = snapshotDirectory; + QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (snapshotDirectory == "") { - snapshotDirectory = previousSelection; + if (_snapshotDirectory == "") { + _snapshotDirectory = previousSelection; return; } } else { - snapshotDirectory = testFolder; - exitWhenComplete = true; + _snapshotDirectory = testFolder; + _exitWhenComplete = true; } // Quit if test results folder could not be created - if (!createTestResultsFolderPath(snapshotDirectory)) { + if (!createTestResultsFolderPath(_snapshotDirectory)) { return; } @@ -208,20 +208,20 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. - QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory); + QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); QStringList expectedImagesURLs; - resultImagesFullFilenames.clear(); - expectedImagesFilenames.clear(); - expectedImagesFullFilenames.clear(); + _resultImagesFullFilenames.clear(); + _expectedImagesFilenames.clear(); + _expectedImagesFullFilenames.clear(); QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine; QString user = (userFromCommandLine.isNull()) ? autoTester->getSelectedUser() : userFromCommandLine; foreach(QString currentFilename, sortedTestResultsFilenames) { - QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; + QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { - resultImagesFullFilenames << fullCurrentFilename; + _resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); @@ -238,12 +238,12 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch // The image retrieved from GitHub needs a unique name QString expectedImageFilename = currentFilename.replace("/", "_").replace(".png", "_EI.png"); - expectedImagesFilenames << expectedImageFilename; - expectedImagesFullFilenames << snapshotDirectory + "/" + expectedImageFilename; + _expectedImagesFilenames << expectedImageFilename; + _expectedImagesFullFilenames << _snapshotDirectory + "/" + expectedImageFilename; } } - autoTester->downloadImages(expectedImagesURLs, snapshotDirectory, expectedImagesFilenames); + autoTester->downloadImages(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames); } void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) { @@ -259,7 +259,7 @@ void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactive zipAndDeleteTestResultsFolder(); - if (exitWhenComplete) { + if (_exitWhenComplete) { exit(0); } } @@ -311,46 +311,46 @@ void Test::includeTest(QTextStream& textStream, const QString& testPathname) { // This script will run all text.js scripts in every applicable sub-folder void Test::createRecursiveScript() { // Select folder to start recursing from - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } - createRecursiveScript(testDirectory, true); + createRecursiveScript(_testDirectory, true); } // This method creates a `testRecursive.js` script in every sub-folder. void Test::createAllRecursiveScripts() { // Select folder to start recursing from - QString previousSelection = testsRootDirectory; + QString previousSelection = _testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", + _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testsRootDirectory == "") { - testsRootDirectory = previousSelection; + if (_testsRootDirectory == "") { + _testsRootDirectory = previousSelection; return; } - createRecursiveScript(testsRootDirectory, false); + createRecursiveScript(_testsRootDirectory, false); - QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -478,42 +478,42 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact void Test::createTests() { // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on // Any existing expected result images will be deleted - QString previousSelection = snapshotDirectory; + QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (snapshotDirectory == "") { - snapshotDirectory = previousSelection; + if (_snapshotDirectory == "") { + _snapshotDirectory = previousSelection; return; } - previousSelection = testsRootDirectory; + previousSelection = _testsRootDirectory; parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent, + _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testsRootDirectory == "") { - testsRootDirectory = previousSelection; + if (_testsRootDirectory == "") { + _testsRootDirectory = previousSelection; return; } - QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory); + QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; + QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { if (i >= maxImages) { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); @@ -523,17 +523,17 @@ void Test::createTests() { // Path to test is extracted from the file name // Example: // filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg - // path is /engine/interaction/pointer/laser/distanceScaleEnd + // path is <_testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd // // Note: we don't use the first part and the last 2 parts of the filename at this stage // QStringList pathParts = currentFilename.split("."); - QString fullNewFileName = testsRootDirectory; + QString fullNewFileName = _testsRootDirectory; for (int j = 1; j < pathParts.size() - 2; ++j) { fullNewFileName += "/" + pathParts[j]; } - // The image index is the penultimate component of the path parts (the last being the file extension) + // The image _index is the penultimate component of the path parts (the last being the file extension) QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png"; fullNewFileName += "/" + newFilename; @@ -622,51 +622,51 @@ ExtractedText Test::getTestScriptLines(QString testFileName) { // The folder selected must contain a script named "test.js", the file produced is named "test.md" void Test::createMDFile() { // Folder selection - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent, + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } - createMDFile(testDirectory); + createMDFile(_testDirectory); QMessageBox::information(0, "Success", "MD file has been created"); } void Test::createAllMDFiles() { // Select folder to start recursing from - QString previousSelection = testsRootDirectory; + QString previousSelection = _testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent, + _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testsRootDirectory == "") { - testsRootDirectory = previousSelection; + if (_testsRootDirectory == "") { + _testsRootDirectory = previousSelection; return; } // First test if top-level folder has a test.js file - const QString testPathname{ testsRootDirectory + "/" + TEST_FILENAME }; + const QString testPathname{ _testsRootDirectory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { - createMDFile(testsRootDirectory); + createMDFile(_testsRootDirectory); } - QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -686,9 +686,9 @@ void Test::createAllMDFiles() { QMessageBox::information(0, "Success", "MD files have been created"); } -void Test::createMDFile(const QString& testDirectory) { +void Test::createMDFile(const QString& _testDirectory) { // Verify folder contains test.js file - QString testFileName(testDirectory + "/" + TEST_FILENAME); + QString testFileName(_testDirectory + "/" + TEST_FILENAME); QFileInfo testFileInfo(testFileName); if (!testFileInfo.exists()) { QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME); @@ -697,7 +697,7 @@ void Test::createMDFile(const QString& testDirectory) { ExtractedText testScriptLines = getTestScriptLines(testFileName); - QString mdFilename(testDirectory + "/" + "test.md"); + QString mdFilename(_testDirectory + "/" + "test.md"); QFile mdFile(mdFilename); if (!mdFile.open(QIODevice::WriteOnly)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); @@ -711,7 +711,7 @@ void Test::createMDFile(const QString& testDirectory) { stream << "# " << testName << "\n"; // Find the relevant part of the path to the test (i.e. from "tests" down - QString partialPath = extractPathFromTestsDown(testDirectory); + QString partialPath = extractPathFromTestsDown(_testDirectory); stream << "## Run this script URL: [Manual](./test.js?raw=true) [Auto](./testAuto.js?raw=true)(from menu/Edit/Open and Run scripts from URL...)." << "\n\n"; @@ -735,23 +735,23 @@ void Test::createMDFile(const QString& testDirectory) { } void Test::createTestsOutline() { - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } const QString testsOutlineFilename { "testsOutline.md" }; - QString mdFilename(testDirectory + "/" + testsOutlineFilename); + QString mdFilename(_testDirectory + "/" + testsOutlineFilename); QFile mdFile(mdFilename); if (!mdFile.open(QIODevice::WriteOnly)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); @@ -765,10 +765,10 @@ void Test::createTestsOutline() { stream << "Directories with an appended (*) have an automatic test\n\n"; // We need to know our current depth, as this isn't given by QDirIterator - int rootDepth { testDirectory.count('/') }; + int rootDepth { _testDirectory.count('/') }; // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file - QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -823,31 +823,40 @@ void Test::createTestsOutline() { } void Test::createTestRailTestSuite() { - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } - TestSuiteCreator testSuiteCreator; - testSuiteCreator.createTestSuite(testDirectory, autoTester->getSelectedUser(), autoTester->getSelectedBranch()); + QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in", + parent, QFileDialog::ShowDirsOnly); + + // If user cancelled then restore previous selection and return + if (outputDirectory == "") { + return; + } + + TestRailInterface testRailInterface; + testRailInterface.createTestSuite(_testDirectory, outputDirectory, autoTester->getSelectedUser(), + autoTester->getSelectedBranch()); } QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { - imageDirectory = QDir(pathToImageDirectory); + _imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; nameFilters << "*." + imageFormat; - return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); + return _imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } // Snapshots are files in the following format: diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 47786664fd..3648906086 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -70,6 +70,8 @@ public: QString getExpectedImageDestinationDirectory(const QString& filename); QString getExpectedImagePartialSourceDirectory(const QString& filename); + ExtractedText getTestScriptLines(QString testFileName); + private: const QString TEST_FILENAME { "test.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; @@ -77,14 +79,14 @@ private: const double THRESHOLD{ 0.96 }; - QDir imageDirectory; + QDir _imageDirectory; - MismatchWindow mismatchWindow; + MismatchWindow _mismatchWindow; - ImageComparer imageComparer; + ImageComparer _imageComparer; - QString testResultsFolderPath; - int index { 1 }; + QString _testResultsFolderPath; + int _index { 1 }; // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) const int NUM_DIGITS { 5 }; @@ -94,28 +96,26 @@ private: // The first is the directory containing the test we are working with // The second is the root directory of all tests // The third contains the snapshots taken for test runs that need to be evaluated - QString testDirectory; - QString testsRootDirectory; - QString snapshotDirectory; + QString _testDirectory; + QString _testsRootDirectory; + QString _snapshotDirectory; - QStringList expectedImagesFilenames; - QStringList expectedImagesFullFilenames; - QStringList resultImagesFullFilenames; + QStringList _expectedImagesFilenames; + QStringList _expectedImagesFullFilenames; + QStringList _resultImagesFullFilenames; // Used for accessing GitHub const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; - ExtractedText getTestScriptLines(QString testFileName); - // NOTE: these need to match the appropriate var's in autoTester.js // var advanceKey = "n"; // var pathSeparator = "."; const QString ADVANCE_KEY{ "n" }; const QString PATH_SEPARATOR{ "." }; - bool exitWhenComplete{ false }; + bool _exitWhenComplete{ false }; }; #endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/TestSuiteCreator.cpp b/tools/auto-tester/src/TestRailInterface.cpp similarity index 92% rename from tools/auto-tester/src/TestSuiteCreator.cpp rename to tools/auto-tester/src/TestRailInterface.cpp index 627012deec..189dc9a551 100644 --- a/tools/auto-tester/src/TestSuiteCreator.cpp +++ b/tools/auto-tester/src/TestRailInterface.cpp @@ -1,5 +1,5 @@ // -// TestSuiteCreator.cpp +// TestRailInterface.cpp // // Created by Nissim Hadar on 6 Jul 2018. // Copyright 2013 High Fidelity, Inc. @@ -8,7 +8,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "TestSuiteCreator.h" +#include "TestRailInterface.h" #include "Test.h" #include @@ -16,7 +16,10 @@ #include #include -void TestSuiteCreator::createTestSuite(const QString& testDirectory, const QString& user, const QString& branch) { +void TestRailInterface::createTestSuite(const QString& testDirectory, + const QString& outputDirectory, + const QString& user, + const QString& branch) { QDomProcessingInstruction instruction = document.createProcessingInstruction("xml", "version='1.0' encoding='UTF-8'"); document.appendChild(instruction); @@ -37,7 +40,7 @@ void TestSuiteCreator::createTestSuite(const QString& testDirectory, const QStri root.appendChild(topLevelSection); // Write to file - const QString testRailsFilename{ "D:/t/TestRailSuite.xml" }; + const QString testRailsFilename{ outputDirectory + "/TestRailSuite.xml" }; QFile file(testRailsFilename); if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Could not create XML file"); @@ -52,7 +55,7 @@ void TestSuiteCreator::createTestSuite(const QString& testDirectory, const QStri QMessageBox::information(0, "Success", "TestRail XML file has been created"); } -QDomElement TestSuiteCreator::processDirectory(const QString& directory, const QString& user, const QString& branch, const QDomElement& element) { +QDomElement TestRailInterface::processDirectory(const QString& directory, const QString& user, const QString& branch, const QDomElement& element) { QDomElement result = element; // Loop over all entries in directory @@ -95,7 +98,7 @@ QDomElement TestSuiteCreator::processDirectory(const QString& directory, const Q return result; } -QDomElement TestSuiteCreator::processTest(const QString& fullDirectory, const QString& test, const QString& user, const QString& branch, const QDomElement& element) { +QDomElement TestRailInterface::processTest(const QString& fullDirectory, const QString& test, const QString& user, const QString& branch, const QDomElement& element) { QDomElement result = element; QDomElement caseElement = document.createElement("case"); diff --git a/tools/auto-tester/src/TestSuiteCreator.h b/tools/auto-tester/src/TestRailInterface.h similarity index 65% rename from tools/auto-tester/src/TestSuiteCreator.h rename to tools/auto-tester/src/TestRailInterface.h index a6ffa762e9..12e2f058c5 100644 --- a/tools/auto-tester/src/TestSuiteCreator.h +++ b/tools/auto-tester/src/TestRailInterface.h @@ -1,5 +1,5 @@ // -// TestSuiteCreator.h +// TestRailInterface.h // // Created by Nissim Hadar on 6 Jul 2018. // Copyright 2013 High Fidelity, Inc. @@ -8,16 +8,19 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef hifi_test_suite_creator_h -#define hifi_test_suite_creator_h +#ifndef hifi_test_testrail_interface_h +#define hifi_test_testrail_interface_h #include #include #include -class TestSuiteCreator { +class TestRailInterface { public: - void createTestSuite(const QString& testDirectory, const QString& user, const QString& branch); + void createTestSuite(const QString& testDirectory, + const QString& outputDirectory, + const QString& user, + const QString& branch); QDomElement processDirectory(const QString& directory, const QString& user, const QString& branch, const QDomElement& element); QDomElement processTest(const QString& fullDirectory, const QString& test, const QString& user, const QString& branch, const QDomElement& element); @@ -26,4 +29,4 @@ private: QDomDocument document; }; -#endif // hifi_test_suite_creator_h \ No newline at end of file +#endif \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 76ae6c5385..b4dbb5a7da 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -16,60 +16,60 @@ #endif AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { - ui.setupUi(this); - ui.checkBoxInteractiveMode->setChecked(true); - ui.progressBar->setVisible(false); + _ui.setupUi(this); + _ui.checkBoxInteractiveMode->setChecked(true); + _ui.progressBar->setVisible(false); - signalMapper = new QSignalMapper(); + _signalMapper = new QSignalMapper(); - connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked); - connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about); + connect(_ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked); + connect(_ui.actionAbout, &QAction::triggered, this, &AutoTester::about); #ifndef Q_OS_WIN - ui.hideTaskbarButton->setVisible(false); - ui.showTaskbarButton->setVisible(false); + _ui.hideTaskbarButton->setVisible(false); + _ui.showTaskbarButton->setVisible(false); #endif } void AutoTester::setup() { - test = new Test(); + _test = new Test(); } void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) { - isRunningFromCommandline = true; - test->startTestsEvaluation(testFolder, branch, user); + _isRunningFromCommandline = true; + _test->startTestsEvaluation(testFolder, branch, user); } void AutoTester::on_evaluateTestsButton_clicked() { - test->startTestsEvaluation(); + _test->startTestsEvaluation(); } void AutoTester::on_createRecursiveScriptButton_clicked() { - test->createRecursiveScript(); + _test->createRecursiveScript(); } void AutoTester::on_createAllRecursiveScriptsButton_clicked() { - test->createAllRecursiveScripts(); + _test->createAllRecursiveScripts(); } void AutoTester::on_createTestsButton_clicked() { - test->createTests(); + _test->createTests(); } void AutoTester::on_createMDFileButton_clicked() { - test->createMDFile(); + _test->createMDFile(); } void AutoTester::on_createAllMDFilesButton_clicked() { - test->createAllMDFiles(); + _test->createAllMDFiles(); } void AutoTester::on_createTestsOutlineButton_clicked() { - test->createTestsOutline(); + _test->createTestsOutline(); } void AutoTester::on_createTestRailTestSuiteButton_clicked() { - test->createTestRailTestSuite(); + _test->createTestRailTestSuite(); } // To toggle between show and hide @@ -101,10 +101,10 @@ void AutoTester::on_closeButton_clicked() { } void AutoTester::downloadImage(const QUrl& url) { - downloaders.emplace_back(new Downloader(url, this)); - connect(downloaders[_index], SIGNAL (downloaded()), signalMapper, SLOT (map())); + _downloaders.emplace_back(new Downloader(url, this)); + connect(_downloaders[_index], SIGNAL (downloaded()), _signalMapper, SLOT (map())); - signalMapper->setMapping(downloaders[_index], _index); + _signalMapper->setMapping(_downloaders[_index], _index); ++_index; } @@ -117,39 +117,39 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director _numberOfImagesDownloaded = 0; _index = 0; - ui.progressBar->setMinimum(0); - ui.progressBar->setMaximum(_numberOfImagesToDownload - 1); - ui.progressBar->setValue(0); - ui.progressBar->setVisible(true); + _ui.progressBar->setMinimum(0); + _ui.progressBar->setMaximum(_numberOfImagesToDownload - 1); + _ui.progressBar->setValue(0); + _ui.progressBar->setVisible(true); - downloaders.clear(); + _downloaders.clear(); for (int i = 0; i < _numberOfImagesToDownload; ++i) { QUrl imageURL(URLs[i]); downloadImage(imageURL); } - connect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); + connect(_signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); } void AutoTester::saveImage(int index) { try { QFile file(_directoryName + "/" + _filenames[index]); file.open(QIODevice::WriteOnly); - file.write(downloaders[index]->downloadedData()); + file.write(_downloaders[index]->downloadedData()); file.close(); } catch (...) { QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]); - ui.progressBar->setVisible(false); + _ui.progressBar->setVisible(false); return; } ++_numberOfImagesDownloaded; if (_numberOfImagesDownloaded == _numberOfImagesToDownload) { - disconnect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); - test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); + disconnect(_signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); + _test->finishTestsEvaluation(_isRunningFromCommandline, _ui.checkBoxInteractiveMode->isChecked(), _ui.progressBar); } else { - ui.progressBar->setValue(_numberOfImagesDownloaded); + _ui.progressBar->setValue(_numberOfImagesDownloaded); } } @@ -158,18 +158,18 @@ void AutoTester::about() { } void AutoTester::setUserText(const QString& user) { - ui.userTextEdit->setText(user); + _ui.userTextEdit->setText(user); } QString AutoTester::getSelectedUser() { - return ui.userTextEdit->toPlainText(); + return _ui.userTextEdit->toPlainText(); } void AutoTester::setBranchText(const QString& branch) { - ui.branchTextEdit->setText(branch); + _ui.branchTextEdit->setText(branch); } QString AutoTester::getSelectedBranch() { - return ui.branchTextEdit->toPlainText(); + return _ui.branchTextEdit->toPlainText(); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index 76b4741d9a..cb99b0a79c 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -56,23 +56,23 @@ private slots: void about(); private: - Ui::AutoTesterClass ui; - Test* test; + Ui::AutoTesterClass _ui; + Test* _test; - std::vector downloaders; + std::vector _downloaders; // local storage for parameters - folder to store downloaded files in, and a list of their names QString _directoryName; QStringList _filenames; // Used to enable passing a parameter to slots - QSignalMapper* signalMapper; + QSignalMapper* _signalMapper; int _numberOfImagesToDownload { 0 }; int _numberOfImagesDownloaded { 0 }; int _index { 0 }; - bool isRunningFromCommandline { false }; + bool _isRunningFromCommandline { false }; }; #endif // hifi_AutoTester_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp index d880a1abdc..79d2ce9f61 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.cpp +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -66,14 +66,14 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) { QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename); QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename); - diffPixmap = computeDiffPixmap( + _diffPixmap = computeDiffPixmap( QImage(testFailure._pathname + testFailure._expectedImageFilename), QImage(testFailure._pathname + testFailure._actualImageFilename) ); expectedImage->setPixmap(expectedPixmap); resultImage->setPixmap(actualPixmap); - diffImage->setPixmap(diffPixmap); + diffImage->setPixmap(_diffPixmap); } void MismatchWindow::on_passTestButton_clicked() { @@ -92,5 +92,5 @@ void MismatchWindow::on_abortTestsButton_clicked() { } QPixmap MismatchWindow::getComparisonImage() { - return diffPixmap; + return _diffPixmap; } diff --git a/tools/auto-tester/src/ui/MismatchWindow.h b/tools/auto-tester/src/ui/MismatchWindow.h index cdbdcb4098..30fca94469 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.h +++ b/tools/auto-tester/src/ui/MismatchWindow.h @@ -36,7 +36,7 @@ private slots: private: UserResponse _userResponse{ USER_RESPONSE_INVALID }; - QPixmap diffPixmap; + QPixmap _diffPixmap; };