diff --git a/tools/auto-tester/CMakeLists.txt b/tools/auto-tester/CMakeLists.txt index a2589bb760..1546a35f4c 100644 --- a/tools/auto-tester/CMakeLists.txt +++ b/tools/auto-tester/CMakeLists.txt @@ -5,7 +5,7 @@ project(${TARGET_NAME}) SET (CMAKE_AUTOUIC ON) SET (CMAKE_AUTOMOC ON) -setup_hifi_project (Core Widgets Network) +setup_hifi_project (Core Widgets Network Xml) link_hifi_libraries () # FIX: Qt was built with -reduce-relocations @@ -18,7 +18,7 @@ include_directories (${CMAKE_CURRENT_SOURCE_DIR}) include_directories (${Qt5Core_INCLUDE_DIRS}) include_directories (${Qt5Widgets_INCLUDE_DIRS}) -set (QT_LIBRARIES Qt5::Core Qt5::Widgets) +set (QT_LIBRARIES Qt5::Core Qt5::Widgets QT::Gui Qt5::Xml) if (WIN32) # Do not show Console diff --git a/tools/auto-tester/src/Test.cpp b/tools/auto-tester/src/Test.cpp index 4f02544c12..eba1cf231b 100644 --- a/tools/auto-tester/src/Test.cpp +++ b/tools/auto-tester/src/Test.cpp @@ -24,7 +24,7 @@ extern AutoTester* autoTester; #include Test::Test() { - mismatchWindow.setModal(true); + _mismatchWindow.setModal(true); if (autoTester) { autoTester->setUserText("highfidelity"); @@ -34,35 +34,35 @@ Test::Test() { bool Test::createTestResultsFolderPath(const QString& directory) { QDateTime now = QDateTime::currentDateTime(); - testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT); - QDir testResultsFolder(testResultsFolderPath); + _testResultsFolderPath = directory + "/" + TEST_RESULTS_FOLDER + "--" + now.toString(DATETIME_FORMAT); + QDir testResultsFolder(_testResultsFolderPath); // Create a new test results folder - return QDir().mkdir(testResultsFolderPath); + return QDir().mkdir(_testResultsFolderPath); } void Test::zipAndDeleteTestResultsFolder() { - QString zippedResultsFileName { testResultsFolderPath + ".zip" }; + QString zippedResultsFileName { _testResultsFolderPath + ".zip" }; QFileInfo fileInfo(zippedResultsFileName); if (!fileInfo.exists()) { QFile::remove(zippedResultsFileName); } - QDir testResultsFolder(testResultsFolderPath); + QDir testResultsFolder(_testResultsFolderPath); if (!testResultsFolder.isEmpty()) { - JlCompress::compressDir(testResultsFolderPath + ".zip", testResultsFolderPath); + JlCompress::compressDir(_testResultsFolderPath + ".zip", _testResultsFolderPath); } testResultsFolder.removeRecursively(); //In all cases, for the next evaluation - testResultsFolderPath = ""; - index = 1; + _testResultsFolderPath = ""; + _index = 1; } bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) { progressBar->setMinimum(0); - progressBar->setMaximum(expectedImagesFullFilenames.length() - 1); + progressBar->setMaximum(_expectedImagesFullFilenames.length() - 1); progressBar->setValue(0); progressBar->setVisible(true); @@ -70,10 +70,10 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) // Quit loop if user has aborted due to a failed test. bool success{ true }; bool keepOn{ true }; - for (int i = 0; keepOn && i < expectedImagesFullFilenames.length(); ++i) { + for (int i = 0; keepOn && i < _expectedImagesFullFilenames.length(); ++i) { // First check that images are the same size - QImage resultImage(resultImagesFullFilenames[i]); - QImage expectedImage(expectedImagesFullFilenames[i]); + QImage resultImage(_resultImagesFullFilenames[i]); + QImage expectedImage(_expectedImagesFullFilenames[i]); double similarityIndex; // in [-1.0 .. 1.0], where 1.0 means images are identical @@ -82,30 +82,30 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Images are not the same size"); similarityIndex = -100.0; } else { - similarityIndex = imageComparer.compareImages(resultImage, expectedImage); + similarityIndex = _imageComparer.compareImages(resultImage, expectedImage); } if (similarityIndex < THRESHOLD) { TestFailure testFailure = TestFailure{ (float)similarityIndex, - expectedImagesFullFilenames[i].left(expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) - QFileInfo(expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image - QFileInfo(resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image + _expectedImagesFullFilenames[i].left(_expectedImagesFullFilenames[i].lastIndexOf("/") + 1), // path to the test (including trailing /) + QFileInfo(_expectedImagesFullFilenames[i].toStdString().c_str()).fileName(), // filename of expected image + QFileInfo(_resultImagesFullFilenames[i].toStdString().c_str()).fileName() // filename of result image }; - mismatchWindow.setTestFailure(testFailure); + _mismatchWindow.setTestFailure(testFailure); if (!isInteractiveMode) { - appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); + appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); success = false; } else { - mismatchWindow.exec(); + _mismatchWindow.exec(); - switch (mismatchWindow.getUserResponse()) { + switch (_mismatchWindow.getUserResponse()) { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: - appendTestResultsToFile(testResultsFolderPath, testFailure, mismatchWindow.getComparisonImage()); + appendTestResultsToFile(_testResultsFolderPath, testFailure, _mismatchWindow.getComparisonImage()); success = false; break; case USER_RESPONSE_ABORT: @@ -126,20 +126,20 @@ bool Test::compareImageLists(bool isInteractiveMode, QProgressBar* progressBar) return success; } -void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { - if (!QDir().exists(testResultsFolderPath)) { - QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + testResultsFolderPath + " not found"); +void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestFailure testFailure, QPixmap comparisonImage) { + if (!QDir().exists(_testResultsFolderPath)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } QString err = QString::number(testFailure._error).left(6); - QString failureFolderPath { testResultsFolderPath + "/" + err + "-Failure_" + QString::number(index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) }; + QString failureFolderPath { _testResultsFolderPath + "/" + err + "-Failure_" + QString::number(_index) + "--" + testFailure._actualImageFilename.left(testFailure._actualImageFilename.length() - 4) }; if (!QDir().mkdir(failureFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create folder " + failureFolderPath); exit(-1); } - ++index; + ++_index; QFile descriptionFile(failureFolderPath + "/" + TEST_RESULTS_FILENAME); if (!descriptionFile.open(QIODevice::ReadWrite)) { @@ -152,7 +152,7 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai stream << "Test failed in folder " << testFailure._pathname.left(testFailure._pathname.length() - 1) << endl; // remove trailing '/' stream << "Expected image was " << testFailure._expectedImageFilename << endl; stream << "Actual image was " << testFailure._actualImageFilename << endl; - stream << "Similarity index was " << testFailure._error << endl; + stream << "Similarity _index was " << testFailure._error << endl; descriptionFile.close(); @@ -180,26 +180,26 @@ void Test::appendTestResultsToFile(const QString& testResultsFolderPath, TestFai void Test::startTestsEvaluation(const QString& testFolder, const QString& branchFromCommandLine, const QString& userFromCommandLine) { if (testFolder.isNull()) { // Get list of JPEG images in folder, sorted by name - QString previousSelection = snapshotDirectory; + QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (snapshotDirectory == "") { - snapshotDirectory = previousSelection; + if (_snapshotDirectory == "") { + _snapshotDirectory = previousSelection; return; } } else { - snapshotDirectory = testFolder; - exitWhenComplete = true; + _snapshotDirectory = testFolder; + _exitWhenComplete = true; } // Quit if test results folder could not be created - if (!createTestResultsFolderPath(snapshotDirectory)) { + if (!createTestResultsFolderPath(_snapshotDirectory)) { return; } @@ -207,20 +207,20 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch // The expected images are represented as a URL to enable download from GitHub // Images that are in the wrong format are ignored. - QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory); + QStringList sortedTestResultsFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); QStringList expectedImagesURLs; - resultImagesFullFilenames.clear(); - expectedImagesFilenames.clear(); - expectedImagesFullFilenames.clear(); + _resultImagesFullFilenames.clear(); + _expectedImagesFilenames.clear(); + _expectedImagesFullFilenames.clear(); QString branch = (branchFromCommandLine.isNull()) ? autoTester->getSelectedBranch() : branchFromCommandLine; QString user = (userFromCommandLine.isNull()) ? autoTester->getSelectedUser() : userFromCommandLine; foreach(QString currentFilename, sortedTestResultsFilenames) { - QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; + QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { - resultImagesFullFilenames << fullCurrentFilename; + _resultImagesFullFilenames << fullCurrentFilename; QString expectedImagePartialSourceDirectory = getExpectedImagePartialSourceDirectory(currentFilename); @@ -237,12 +237,12 @@ void Test::startTestsEvaluation(const QString& testFolder, const QString& branch // The image retrieved from GitHub needs a unique name QString expectedImageFilename = currentFilename.replace("/", "_").replace(".png", "_EI.png"); - expectedImagesFilenames << expectedImageFilename; - expectedImagesFullFilenames << snapshotDirectory + "/" + expectedImageFilename; + _expectedImagesFilenames << expectedImageFilename; + _expectedImagesFullFilenames << _snapshotDirectory + "/" + expectedImageFilename; } } - autoTester->downloadImages(expectedImagesURLs, snapshotDirectory, expectedImagesFilenames); + autoTester->downloadImages(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames); } void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactiveMode, QProgressBar* progressBar) { @@ -258,7 +258,7 @@ void Test::finishTestsEvaluation(bool isRunningFromCommandline, bool interactive zipAndDeleteTestResultsFolder(); - if (exitWhenComplete) { + if (_exitWhenComplete) { exit(0); } } @@ -310,46 +310,46 @@ void Test::includeTest(QTextStream& textStream, const QString& testPathname) { // This script will run all text.js scripts in every applicable sub-folder void Test::createRecursiveScript() { // Select folder to start recursing from - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder that will contain the top level test script", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } - createRecursiveScript(testDirectory, true); + createRecursiveScript(_testDirectory, true); } // This method creates a `testRecursive.js` script in every sub-folder. void Test::createAllRecursiveScripts() { // Select folder to start recursing from - QString previousSelection = testsRootDirectory; + QString previousSelection = _testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", + _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the recursive scripts", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testsRootDirectory == "") { - testsRootDirectory = previousSelection; + if (_testsRootDirectory == "") { + _testsRootDirectory = previousSelection; return; } - createRecursiveScript(testsRootDirectory, false); + createRecursiveScript(_testsRootDirectory, false); - QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -477,42 +477,42 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact void Test::createTests() { // Rename files sequentially, as ExpectedResult_00000.jpeg, ExpectedResult_00001.jpg and so on // Any existing expected result images will be deleted - QString previousSelection = snapshotDirectory; + QString previousSelection = _snapshotDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (snapshotDirectory == "") { - snapshotDirectory = previousSelection; + if (_snapshotDirectory == "") { + _snapshotDirectory = previousSelection; return; } - previousSelection = testsRootDirectory; + previousSelection = _testsRootDirectory; parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent, + _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select test root folder", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testsRootDirectory == "") { - testsRootDirectory = previousSelection; + if (_testsRootDirectory == "") { + _testsRootDirectory = previousSelection; return; } - QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", snapshotDirectory); + QStringList sortedImageFilenames = createListOfAll_imagesInDirectory("png", _snapshotDirectory); int i = 1; const int maxImages = pow(10, NUM_DIGITS); foreach (QString currentFilename, sortedImageFilenames) { - QString fullCurrentFilename = snapshotDirectory + "/" + currentFilename; + QString fullCurrentFilename = _snapshotDirectory + "/" + currentFilename; if (isInSnapshotFilenameFormat("png", currentFilename)) { if (i >= maxImages) { QMessageBox::critical(0, "Error", "More than " + QString::number(maxImages) + " images not supported"); @@ -522,17 +522,17 @@ void Test::createTests() { // Path to test is extracted from the file name // Example: // filename is tests.engine.interaction.pointer.laser.distanceScaleEnd.00000.jpg - // path is /engine/interaction/pointer/laser/distanceScaleEnd + // path is <_testDirectory>/engine/interaction/pointer/laser/distanceScaleEnd // // Note: we don't use the first part and the last 2 parts of the filename at this stage // QStringList pathParts = currentFilename.split("."); - QString fullNewFileName = testsRootDirectory; + QString fullNewFileName = _testsRootDirectory; for (int j = 1; j < pathParts.size() - 2; ++j) { fullNewFileName += "/" + pathParts[j]; } - // The image index is the penultimate component of the path parts (the last being the file extension) + // The image _index is the penultimate component of the path parts (the last being the file extension) QString newFilename = "ExpectedImage_" + pathParts[pathParts.size() - 2].rightJustified(5, '0') + ".png"; fullNewFileName += "/" + newFilename; @@ -621,51 +621,51 @@ ExtractedText Test::getTestScriptLines(QString testFileName) { // The folder selected must contain a script named "test.js", the file produced is named "test.md" void Test::createMDFile() { // Folder selection - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent, + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } - createMDFile(testDirectory); + createMDFile(_testDirectory); QMessageBox::information(0, "Success", "MD file has been created"); } void Test::createAllMDFiles() { // Select folder to start recursing from - QString previousSelection = testsRootDirectory; + QString previousSelection = _testsRootDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent, + _testsRootDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the root folder for the MD files", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testsRootDirectory == "") { - testsRootDirectory = previousSelection; + if (_testsRootDirectory == "") { + _testsRootDirectory = previousSelection; return; } // First test if top-level folder has a test.js file - const QString testPathname{ testsRootDirectory + "/" + TEST_FILENAME }; + const QString testPathname{ _testsRootDirectory + "/" + TEST_FILENAME }; QFileInfo fileInfo(testPathname); if (fileInfo.exists()) { - createMDFile(testsRootDirectory); + createMDFile(_testsRootDirectory); } - QDirIterator it(testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -685,9 +685,9 @@ void Test::createAllMDFiles() { QMessageBox::information(0, "Success", "MD files have been created"); } -void Test::createMDFile(const QString& testDirectory) { +void Test::createMDFile(const QString& _testDirectory) { // Verify folder contains test.js file - QString testFileName(testDirectory + "/" + TEST_FILENAME); + QString testFileName(_testDirectory + "/" + TEST_FILENAME); QFileInfo testFileInfo(testFileName); if (!testFileInfo.exists()) { QMessageBox::critical(0, "Error", "Could not find file: " + TEST_FILENAME); @@ -696,7 +696,7 @@ void Test::createMDFile(const QString& testDirectory) { ExtractedText testScriptLines = getTestScriptLines(testFileName); - QString mdFilename(testDirectory + "/" + "test.md"); + QString mdFilename(_testDirectory + "/" + "test.md"); QFile mdFile(mdFilename); if (!mdFile.open(QIODevice::WriteOnly)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); @@ -710,7 +710,7 @@ void Test::createMDFile(const QString& testDirectory) { stream << "# " << testName << "\n"; // Find the relevant part of the path to the test (i.e. from "tests" down - QString partialPath = extractPathFromTestsDown(testDirectory); + QString partialPath = extractPathFromTestsDown(_testDirectory); stream << "## Run this script URL: [Manual](./test.js?raw=true) [Auto](./testAuto.js?raw=true)(from menu/Edit/Open and Run scripts from URL...)." << "\n\n"; @@ -734,23 +734,23 @@ void Test::createMDFile(const QString& testDirectory) { } void Test::createTestsOutline() { - QString previousSelection = testDirectory; + QString previousSelection = _testDirectory; QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - testDirectory = + _testDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly); // If user cancelled then restore previous selection and return - if (testDirectory == "") { - testDirectory = previousSelection; + if (_testDirectory == "") { + _testDirectory = previousSelection; return; } const QString testsOutlineFilename { "testsOutline.md" }; - QString mdFilename(testDirectory + "/" + testsOutlineFilename); + QString mdFilename(_testDirectory + "/" + testsOutlineFilename); QFile mdFile(mdFilename); if (!mdFile.open(QIODevice::WriteOnly)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to create file " + mdFilename); @@ -764,10 +764,10 @@ void Test::createTestsOutline() { stream << "Directories with an appended (*) have an automatic test\n\n"; // We need to know our current depth, as this isn't given by QDirIterator - int rootDepth { testDirectory.count('/') }; + int rootDepth { _testDirectory.count('/') }; // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file - QDirIterator it(testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -821,12 +821,51 @@ void Test::createTestsOutline() { QMessageBox::information(0, "Success", "Test outline file " + testsOutlineFilename + " has been created"); } +void Test::createTestRailTestCases() { + QString previousSelection = _testDirectory; + QString parent = previousSelection.left(previousSelection.lastIndexOf('/')); + if (!parent.isNull() && parent.right(1) != "/") { + parent += "/"; + } + + _testDirectory = + QFileDialog::getExistingDirectory(nullptr, "Please select the tests root folder", parent, QFileDialog::ShowDirsOnly); + + // If user cancelled then restore previous selection and return + if (_testDirectory == "") { + _testDirectory = previousSelection; + return; + } + + QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in", + nullptr, QFileDialog::ShowDirsOnly); + + // If user cancelled then return + if (outputDirectory == "") { + return; + } + + if (_testRailCreateMode == PYTHON) { + _testRailInterface.createTestSuitePython(_testDirectory, outputDirectory, autoTester->getSelectedUser(), + autoTester->getSelectedBranch()); + } else { + _testRailInterface.createTestSuiteXML(_testDirectory, outputDirectory, autoTester->getSelectedUser(), + autoTester->getSelectedBranch()); + } +} + +void Test::createTestRailRun() { + QString outputDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store generated files in", + nullptr, QFileDialog::ShowDirsOnly); + _testRailInterface.createTestRailRun(outputDirectory); +} + QStringList Test::createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory) { - imageDirectory = QDir(pathToImageDirectory); + _imageDirectory = QDir(pathToImageDirectory); QStringList nameFilters; nameFilters << "*." + imageFormat; - return imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); + return _imageDirectory.entryList(nameFilters, QDir::Files, QDir::Name); } // Snapshots are files in the following format: @@ -889,3 +928,7 @@ QString Test::getExpectedImagePartialSourceDirectory(const QString& filename) { return result; } + +void Test::setTestRailCreateMode(TestRailCreateMode testRailCreateMode) { + _testRailCreateMode = testRailCreateMode; +} diff --git a/tools/auto-tester/src/Test.h b/tools/auto-tester/src/Test.h index 5c6d3e5686..6d7506b738 100644 --- a/tools/auto-tester/src/Test.h +++ b/tools/auto-tester/src/Test.h @@ -18,6 +18,7 @@ #include "ImageComparer.h" #include "ui/MismatchWindow.h" +#include "TestRailInterface.h" class Step { public: @@ -33,6 +34,11 @@ public: StepList stepList; }; +enum TestRailCreateMode { + PYTHON, + XML +}; + class Test { public: Test(); @@ -51,6 +57,9 @@ public: void createTestsOutline(); + void createTestRailTestCases(); + void createTestRailRun(); + bool compareImageLists(bool isInteractiveMode, QProgressBar* progressBar); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); @@ -64,11 +73,15 @@ public: bool createTestResultsFolderPath(const QString& directory); void zipAndDeleteTestResultsFolder(); - bool isAValidDirectory(const QString& pathname); + static bool isAValidDirectory(const QString& pathname); QString extractPathFromTestsDown(const QString& fullPath); QString getExpectedImageDestinationDirectory(const QString& filename); QString getExpectedImagePartialSourceDirectory(const QString& filename); + ExtractedText getTestScriptLines(QString testFileName); + + void setTestRailCreateMode(TestRailCreateMode testRailCreateMode); + private: const QString TEST_FILENAME { "test.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; @@ -76,14 +89,14 @@ private: const double THRESHOLD{ 0.96 }; - QDir imageDirectory; + QDir _imageDirectory; - MismatchWindow mismatchWindow; + MismatchWindow _mismatchWindow; - ImageComparer imageComparer; + ImageComparer _imageComparer; - QString testResultsFolderPath; - int index { 1 }; + QString _testResultsFolderPath; + int _index { 1 }; // Expected images are in the format ExpectedImage_dddd.jpg (d == decimal digit) const int NUM_DIGITS { 5 }; @@ -93,28 +106,30 @@ private: // The first is the directory containing the test we are working with // The second is the root directory of all tests // The third contains the snapshots taken for test runs that need to be evaluated - QString testDirectory; - QString testsRootDirectory; - QString snapshotDirectory; + QString _testDirectory; + QString _testsRootDirectory; + QString _snapshotDirectory; - QStringList expectedImagesFilenames; - QStringList expectedImagesFullFilenames; - QStringList resultImagesFullFilenames; + QStringList _expectedImagesFilenames; + QStringList _expectedImagesFullFilenames; + QStringList _resultImagesFullFilenames; // Used for accessing GitHub const QString GIT_HUB_REPOSITORY{ "hifi_tests" }; const QString DATETIME_FORMAT{ "yyyy-MM-dd_hh-mm-ss" }; - ExtractedText getTestScriptLines(QString testFileName); - // NOTE: these need to match the appropriate var's in autoTester.js // var advanceKey = "n"; // var pathSeparator = "."; const QString ADVANCE_KEY{ "n" }; const QString PATH_SEPARATOR{ "." }; - bool exitWhenComplete{ false }; + bool _exitWhenComplete{ false }; + + TestRailInterface _testRailInterface; + + TestRailCreateMode _testRailCreateMode { PYTHON }; }; #endif // hifi_test_h \ No newline at end of file diff --git a/tools/auto-tester/src/TestRailInterface.cpp b/tools/auto-tester/src/TestRailInterface.cpp new file mode 100644 index 0000000000..f5fe8cefdb --- /dev/null +++ b/tools/auto-tester/src/TestRailInterface.cpp @@ -0,0 +1,837 @@ +// +// TestRailInterface.cpp +// +// Created by Nissim Hadar on 6 Jul 2018. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "TestRailInterface.h" +#include "Test.h" + +#include +#include +#include +#include + +TestRailInterface::TestRailInterface() { + _testRailTestCasesSelectorWindow.setURL("https://highfidelity.testrail.net"); + ////_testRailTestCasesSelectorWindow.setURL("https://nissimhadar.testrail.io"); + _testRailTestCasesSelectorWindow.setUser("@highfidelity.io"); + ////_testRailSelectorWindow.setUser("nissim.hadar@gmail.com"); + + _testRailTestCasesSelectorWindow.setProjectID(INTERFACE_PROJECT_ID); + ////_testRailSelectorWindow.setProject(1); + + _testRailTestCasesSelectorWindow.setSuiteID(INTERFACE_SUITE_ID); + + _testRailRunSelectorWindow.setURL("https://highfidelity.testrail.net"); + ////_testRailRunSelectorWindow.setURL("https://nissimhadar.testrail.io"); + _testRailRunSelectorWindow.setUser("@highfidelity.io"); + ////_testRailSelectorWindow.setUser("nissim.hadar@gmail.com"); + + _testRailRunSelectorWindow.setProjectID(INTERFACE_PROJECT_ID); + ////_testRailSelectorWindow.setProject(1); + + _testRailRunSelectorWindow.setSuiteID(INTERFACE_SUITE_ID); +} + +QString TestRailInterface::getObject(const QString& path) { + return path.right(path.length() - path.lastIndexOf("/") - 1); +} + + +bool TestRailInterface::setPythonCommand() { + if (QProcessEnvironment::systemEnvironment().contains("PYTHON_PATH")) { + QString _pythonPath = QProcessEnvironment::systemEnvironment().value("PYTHON_PATH"); + if (!QFile::exists(_pythonPath + "/" + pythonExe)) { + QMessageBox::critical(0, pythonExe, QString("Python executable not found in ") + _pythonPath); + } + _pythonCommand = _pythonPath + "/" + pythonExe; + return true; + } else { + QMessageBox::critical(0, "PYTHON_PATH not defined", + "Please set PYTHON_PATH to directory containing the Python executable"); + return false; + } + + return false; +} + +// Creates the testrail.py script +// This is the file linked to from http://docs.gurock.com/testrail-api2/bindings-python +void TestRailInterface::createTestRailDotPyScript() { + QFile file(_outputDirectory + "/testrail.py"); + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'testrail.py'"); + exit(-1); + } + + QTextStream stream(&file); + + stream << "#\n"; + stream << "# TestRail API binding for Python 3.x (API v2, available since \n"; + stream << "# TestRail 3.0)\n"; + stream << "#\n"; + stream << "# Learn more:\n"; + stream << "#\n"; + stream << "# http://docs.gurock.com/testrail-api2/start\n"; + stream << "# http://docs.gurock.com/testrail-api2/accessing\n"; + stream << "#\n"; + stream << "# Copyright Gurock Software GmbH. See license.md for details.\n"; + stream << "#\n"; + stream << "\n"; + stream << "import urllib.request, urllib.error\n"; + stream << "import json, base64\n"; + stream << "\n"; + stream << "class APIClient:\n"; + stream << "\tdef __init__(self, base_url):\n"; + stream << "\t\tself.user = ''\n"; + stream << "\t\tself.password = ''\n"; + stream << "\t\tif not base_url.endswith('/'):\n"; + stream << "\t\t\tbase_url += '/'\n"; + stream << "\t\tself.__url = base_url + 'index.php?/api/v2/'\n"; + stream << "\n"; + stream << "\t#\n"; + stream << "\t# Send Get\n"; + stream << "\t#\n"; + stream << "\t# Issues a GET request (read) against the API and returns the result\n"; + stream << "\t# (as Python dict).\n"; + stream << "\t#\n"; + stream << "\t# Arguments:\n"; + stream << "\t#\n"; + stream << "\t# uri The API method to call including parameters\n"; + stream << "\t# (e.g. get_case/1)\n"; + stream << "\t#\n"; + stream << "\tdef send_get(self, uri):\n"; + stream << "\t\treturn self.__send_request('GET', uri, None)\n"; + stream << "\n"; + stream << "\t#\n"; + stream << "\t# Send POST\n"; + stream << "\t#\n"; + stream << "\t# Issues a POST request (write) against the API and returns the result\n"; + stream << "\t# (as Python dict).\n"; + stream << "\t#\n"; + stream << "\t# Arguments:\n"; + stream << "\t#\n"; + stream << "\t# uri The API method to call including parameters\n"; + stream << "\t# (e.g. add_case/1)\n"; + stream << "\t# data The data to submit as part of the request (as\n"; + stream << "\t# Python dict, strings must be UTF-8 encoded)\n"; + stream << "\t#\n"; + stream << "\tdef send_post(self, uri, data):\n"; + stream << "\t\treturn self.__send_request('POST', uri, data)\n"; + stream << "\n"; + stream << "\tdef __send_request(self, method, uri, data):\n"; + stream << "\t\turl = self.__url + uri\n"; + stream << "\t\trequest = urllib.request.Request(url)\n"; + stream << "\t\tif (method == 'POST'):\n"; + stream << "\t\t\trequest.data = bytes(json.dumps(data), 'utf-8')\n"; + stream << "\t\tauth = str(\n"; + stream << "\t\t\tbase64.b64encode(\n"; + stream << "\t\t\t\tbytes('%s:%s' % (self.user, self.password), 'utf-8')\n"; + stream << "\t\t\t),\n"; + stream << "\t\t\t'ascii'\n"; + stream << "\t\t).strip()\n"; + stream << "\t\trequest.add_header('Authorization', 'Basic %s' % auth)\n"; + stream << "\t\trequest.add_header('Content-Type', 'application/json')\n"; + stream << "\n"; + stream << "\t\te = None\n"; + stream << "\t\ttry:\n"; + stream << "\t\t\tresponse = urllib.request.urlopen(request).read()\n"; + stream << "\t\texcept urllib.error.HTTPError as ex:\n"; + stream << "\t\t\tresponse = ex.read()\n"; + stream << "\t\t\te = ex\n"; + stream << "\n"; + stream << "\t\tif response:\n"; + stream << "\t\t\tresult = json.loads(response.decode())\n"; + stream << "\t\telse:\n"; + stream << "\t\t\tresult = {}\n"; + stream << "\n"; + stream << "\t\tif e != None:\n"; + stream << "\t\t\tif result and 'error' in result:\n"; + stream << "\t\t\t\terror = '\"' + result['error'] + '\"'\n"; + stream << "\t\t\telse:\n"; + stream << "\t\t\t\terror = 'No additional error message received'\n"; + stream << "\t\t\traise APIError('TestRail API returned HTTP %s (%s)' % \n"; + stream << "\t\t\t\t(e.code, error))\n"; + stream << "\n"; + stream << "\t\treturn result\n"; + stream << "\n"; + stream << "class APIError(Exception):\n"; + stream << "\tpass\n"; + + file.close(); +} + +// Creates a Stack class +void TestRailInterface::createStackDotPyScript() { + QString filename = _outputDirectory + "/stack.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); + + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'stack.py'"); + exit(-1); + } + + QTextStream stream(&file); + + stream << "class Stack:\n"; + + stream << "\tdef __init__(self):\n"; + stream << "\t\tself.items = []\n"; + stream << "\n"; + + stream << "\tdef is_empty(self):\n"; + stream << "\t\treturn self.items == []\n"; + stream << "\n"; + + stream << "\tdef push(self, item):\n"; + stream << "\t\tself.items.append(item)\n"; + stream << "\n"; + + stream << "\tdef pop(self):\n"; + stream << "\t\treturn self.items.pop()\n"; + stream << "\n"; + + stream << "\tdef peek(self):\n"; + stream << "\t\treturn self.items[len(self.items)-1]\n"; + stream << "\n"; + + stream << "\tdef size(self):\n"; + stream << "\t\treturn len(self.items)\n"; + stream << "\n"; + + file.close(); +} + +void TestRailInterface::requestTestRailTestCasesDataFromUser() { + // Make sure correct fields are enabled before calling + _testRailTestCasesSelectorWindow.reset(); + _testRailTestCasesSelectorWindow.exec(); + + if (_testRailTestCasesSelectorWindow.getUserCancelled()) { + return; + } + + _url = _testRailTestCasesSelectorWindow.getURL() + "/"; + _user = _testRailTestCasesSelectorWindow.getUser(); + _password = _testRailTestCasesSelectorWindow.getPassword(); + ////_password = "tutKA76"; + _projectID = QString::number(_testRailTestCasesSelectorWindow.getProjectID()); + _suiteID = QString::number(_testRailTestCasesSelectorWindow.getSuiteID()); +} + +bool TestRailInterface::isAValidTestDirectory(const QString& directory) { + if (Test::isAValidDirectory(directory)) { + // Ignore the utils and preformance directories + if (directory.right(QString("utils").length()) == "utils" || + directory.right(QString("performance").length()) == "performance") { + return false; + } + return true; + } + + return false; +} + +void TestRailInterface::processDirectoryPython(const QString& directory, + QTextStream& stream, + const QString& userGitHub, + const QString& branchGitHub) { + // Loop over all entries in directory + QDirIterator it(directory.toStdString().c_str()); + while (it.hasNext()) { + QString nextDirectory = it.next(); + + QString objectName = getObject(nextDirectory); + + if (isAValidTestDirectory(nextDirectory)) { + // The name of the section is the directory at the end of the path + stream << "parent_id = parent_ids.peek()\n"; + stream << "data = { 'name': '" << objectName << "', 'suite_id': " + _suiteID + ", 'parent_id': parent_id }\n"; + + stream << "section = client.send_post('add_section/' + str(" << _projectID << "), data)\n"; + + // Now we push the parent_id, and recursively process each directory + stream << "parent_ids.push(section['id'])\n\n"; + processDirectoryPython(nextDirectory, stream, userGitHub, branchGitHub); + } else if (objectName == "test.js") { + processTestPython(nextDirectory, stream, userGitHub, branchGitHub); + } + } + + // pop the parent directory before leaving + stream << "parent_ids.pop()\n\n"; +} + +// A suite of TestRail test cases contains trees. +// The nodes of the trees are sections +// The leaves are the test cases +// +// Each node and leaf have an ID and a parent ID. +// Therefore, the tree is built top-down, using a stack to store the IDs of each node +// +void TestRailInterface::createAddTestCasesPythonScript(const QString& testDirectory, + const QString& userGitHub, + const QString& branchGitHub) { + QString filename = _outputDirectory + "/addTestCases.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); + + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'addTestCases.py'"); + exit(-1); + } + + QTextStream stream(&file); + + // Code to access TestRail + stream << "from testrail import *\n"; + stream << "client = APIClient('" << _url.toStdString().c_str() << "')\n"; + stream << "client.user = '" << _user << "'\n"; + stream << "client.password = '" << _password << "'\n\n"; + + stream << "from stack import *\n"; + stream << "parent_ids = Stack()\n\n"; + + // top-level section + stream << "data = { 'name': '" + << "Test Suite - " << QDateTime::currentDateTime().toString("yyyy-MM-ddTHH:mm") + "', " + << "'suite_id': " + _suiteID + "}\n"; + + stream << "section = client.send_post('add_section/' + str(" << _projectID << "), data)\n"; + + // Now we push the parent_id, and recursively process each directory + stream << "parent_ids.push(section['id'])\n\n"; + processDirectoryPython(testDirectory, stream, userGitHub, branchGitHub); + + file.close(); + + if (QMessageBox::Yes == QMessageBox(QMessageBox::Information, "Python script has been created", + "Do you want to run the script and update TestRail?", + QMessageBox::Yes | QMessageBox::No) + .exec()) { + QProcess* process = new QProcess(); + connect(process, &QProcess::started, this, [=]() { _busyWindow.exec(); }); + + connect(process, static_cast(&QProcess::finished), this, + [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); + + QStringList parameters = QStringList() << _outputDirectory + "/addTestCases.py"; + process->start(_pythonCommand, parameters); + } +} + +void TestRailInterface::updateMilestonesComboData(int exitCode, QProcess::ExitStatus exitStatus) { + // Quit if user has previously cancelled + if (_testRailTestCasesSelectorWindow.getUserCancelled()) { + return; + } + + // Check if process completed successfully + if (exitStatus != QProcess::NormalExit) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not get milestones from TestRail"); + exit(-1); + } + + // Create map of milestones from the file created by the process + _milestoneNames.clear(); + + QString filename = _outputDirectory + "/milestones.txt"; + if (!QFile::exists(filename)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not find milestones.txt in " + _outputDirectory); + exit(-1); + } + + QFile file(filename); + if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not open " + _outputDirectory + "/milestones.txt"); + exit(-1); + } + + QTextStream in(&file); + QString line = in.readLine(); + while (!line.isNull()) { + QStringList words = line.split(' '); + _milestones[words[0]] = words[1].toInt(); + _milestoneNames << words[0]; + + line = in.readLine(); + } + + file.close(); + + // Update the combo + _testRailTestCasesSelectorWindow.updateMilestonesComboBoxData(_milestoneNames); + + _testRailTestCasesSelectorWindow.exec(); + + if (_testRailTestCasesSelectorWindow.getUserCancelled()) { + return; + } + + createAddTestCasesPythonScript(_testDirectory, _userGitHub, _branchGitHub); +} + +void TestRailInterface::updateSectionsComboData(int exitCode, QProcess::ExitStatus exitStatus) { + // Quit if user has previously cancelled + if (_testRailRunSelectorWindow.getUserCancelled()) { + return; + } + + // Check if process completed successfully + if (exitStatus != QProcess::NormalExit) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not get sections from TestRail"); + exit(-1); + } + + // Create map of sections from the file created by the process + _sectionNames.clear(); + + QString filename = _outputDirectory + "/sections.txt"; + if (!QFile::exists(filename)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not find sections.txt in " + _outputDirectory); + exit(-1); + } + + QFile file(filename); + if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not open " + _outputDirectory + "/sections.txt"); + exit(-1); + } + + QTextStream in(&file); + QString line = in.readLine(); + while (!line.isNull()) { + // The section name is all the words except for the last + // The id is the last word + QString section = line.left(line.lastIndexOf(" ")); + QString id = line.right(line.length() - line.lastIndexOf(" ") - 1); + + _sections[section] = id.toInt(); + _sectionNames << section; + + line = in.readLine(); + } + + file.close(); + + // Update the combo + _testRailRunSelectorWindow.updateSectionsComboBoxData(_sectionNames); + + _testRailRunSelectorWindow.exec(); + + if (_testRailRunSelectorWindow.getUserCancelled()) { + return; + } + + ////createAddTestCasesPythonScript(_testDirectory, _userGitHub, _branchGitHub); +} + +void TestRailInterface::getMilestonesFromTestRail() { + QString filename = _outputDirectory + "/getMilestones.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); + + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'getMilestones.py'"); + exit(-1); + } + + QTextStream stream(&file); + + // Code to access TestRail + stream << "from testrail import *\n"; + stream << "client = APIClient('" << _url.toStdString().c_str() << "')\n"; + stream << "client.user = '" << _user << "'\n"; + stream << "client.password = '" << _password << "'\n\n"; + + // Print the list of uncompleted milestones + stream << "file = open('" + _outputDirectory + "/milestones.txt', 'w')\n\n"; + stream << "milestones = client.send_get('get_milestones/" + _projectID + "')\n"; + stream << "for milestone in milestones:\n"; + stream << "\tif milestone['is_completed'] == False:\n"; + stream << "\t\tfile.write(milestone['name'] + ' ' + str(milestone['id']) + '\\n')\n\n"; + stream << "file.close()\n"; + + file.close(); + + QProcess* process = new QProcess(); + connect(process, static_cast(&QProcess::finished), this, + [=](int exitCode, QProcess::ExitStatus exitStatus) { updateMilestonesComboData(exitCode, exitStatus); }); + + QStringList parameters = QStringList() << _outputDirectory + "/getMilestones.py "; + process->start(_pythonCommand, parameters); +} + +void TestRailInterface::createTestSuitePython(const QString& testDirectory, + const QString& outputDirectory, + const QString& userGitHub, + const QString& branchGitHub) { + _testDirectory = testDirectory; + _outputDirectory = outputDirectory; + _userGitHub = userGitHub; + _branchGitHub = branchGitHub; + + if (!setPythonCommand()) { + return; + } + + requestTestRailTestCasesDataFromUser(); + createTestRailDotPyScript(); + createStackDotPyScript(); + + // TestRail will be updated after the process initiated by getMilestonesFromTestRail has completed + getMilestonesFromTestRail(); +} + +void TestRailInterface::createTestSuiteXML(const QString& testDirectory, + const QString& outputDirectory, + const QString& userGitHub, + const QString& branchGitHub) { + _outputDirectory = outputDirectory; + + QDomProcessingInstruction instruction = _document.createProcessingInstruction("xml", "version='1.0' encoding='UTF-8'"); + _document.appendChild(instruction); + + // We create a single section, within sections + QDomElement root = _document.createElement("sections"); + _document.appendChild(root); + + QDomElement topLevelSection = _document.createElement("section"); + + QDomElement suiteName = _document.createElement("name"); + suiteName.appendChild( + _document.createTextNode("Test Suite - " + QDateTime::currentDateTime().toString("yyyy-MM-ddTHH:mm"))); + topLevelSection.appendChild(suiteName); + + // This is the first call to 'process'. This is then called recursively to build the full XML tree + QDomElement secondLevelSections = _document.createElement("sections"); + topLevelSection.appendChild(processDirectoryXML(testDirectory, userGitHub, branchGitHub, secondLevelSections)); + + topLevelSection.appendChild(secondLevelSections); + root.appendChild(topLevelSection); + + // Write to file + const QString testRailsFilename{ _outputDirectory + "/TestRailSuite.xml" }; + QFile file(testRailsFilename); + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create XML file"); + exit(-1); + } + + QTextStream stream(&file); + stream << _document.toString(); + + file.close(); + + QMessageBox::information(0, "Success", "TestRail XML file has been created"); +} + +QDomElement TestRailInterface::processDirectoryXML(const QString& directory, + const QString& userGitHub, + const QString& branchGitHub, + const QDomElement& element) { + QDomElement result = element; + + // Loop over all entries in directory + QDirIterator it(directory.toStdString().c_str()); + while (it.hasNext()) { + QString nextDirectory = it.next(); + + // The object name appears after the last slash (we are assured there is at least 1). + QString objectName = getObject(nextDirectory); + + // Only process directories + if (isAValidTestDirectory(nextDirectory)) { + // Create a section and process it + QDomElement sectionElement = _document.createElement("section"); + + QDomElement sectionElementName = _document.createElement("name"); + sectionElementName.appendChild(_document.createTextNode(objectName)); + sectionElement.appendChild(sectionElementName); + + QDomElement testsElement = _document.createElement("sections"); + sectionElement.appendChild(processDirectoryXML(nextDirectory, userGitHub, branchGitHub, testsElement)); + + result.appendChild(sectionElement); + } else if (objectName == "test.js" || objectName == "testStory.js") { + QDomElement sectionElement = _document.createElement("section"); + QDomElement sectionElementName = _document.createElement("name"); + sectionElementName.appendChild(_document.createTextNode("all")); + sectionElement.appendChild(sectionElementName); + sectionElement.appendChild( + processTestXML(nextDirectory, objectName, userGitHub, branchGitHub, _document.createElement("cases"))); + + result.appendChild(sectionElement); + } + } + + return result; +} + +QDomElement TestRailInterface::processTestXML(const QString& fullDirectory, + const QString& test, + const QString& userGitHub, + const QString& branchGitHub, + const QDomElement& element) { + QDomElement result = element; + + QDomElement caseElement = _document.createElement("case"); + + caseElement.appendChild(_document.createElement("id")); + + // The name of the test is derived from the full path. + // The first term is the first word after "tests" + // The last word is the penultimate word + QStringList words = fullDirectory.split('/'); + int i = 0; + while (words[i] != "tests") { + ++i; + if (i >= words.length() - 1) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Folder \"tests\" not found in " + fullDirectory); + exit(-1); + } + } + + ++i; + QString title{ words[i] }; + for (++i; i < words.length() - 1; ++i) { + title += " / " + words[i]; + } + + QDomElement titleElement = _document.createElement("title"); + titleElement.appendChild(_document.createTextNode(title)); + caseElement.appendChild(titleElement); + + QDomElement templateElement = _document.createElement("template"); + templateElement.appendChild(_document.createTextNode("Test Case (Steps)")); + caseElement.appendChild(templateElement); + + QDomElement typeElement = _document.createElement("type"); + typeElement.appendChild(_document.createTextNode("3 - Regression")); + caseElement.appendChild(typeElement); + + QDomElement priorityElement = _document.createElement("priority"); + priorityElement.appendChild(_document.createTextNode("Medium")); + caseElement.appendChild(priorityElement); + + QDomElement estimateElementName = _document.createElement("estimate"); + estimateElementName.appendChild(_document.createTextNode("60")); + caseElement.appendChild(estimateElementName); + + caseElement.appendChild(_document.createElement("references")); + + QDomElement customElement = _document.createElement("custom"); + + QDomElement tester_countElement = _document.createElement("tester_count"); + tester_countElement.appendChild(_document.createTextNode("1")); + customElement.appendChild(tester_countElement); + + QDomElement domain_bot_loadElement = _document.createElement("domain_bot_load"); + QDomElement domain_bot_loadElementId = _document.createElement("id"); + domain_bot_loadElementId.appendChild(_document.createTextNode("1")); + domain_bot_loadElement.appendChild(domain_bot_loadElementId); + QDomElement domain_bot_loadElementValue = _document.createElement("value"); + domain_bot_loadElementValue.appendChild( + _document.createTextNode(" Without Bots (hifiqa-rc / hifi-qa-stable / hifiqa-master)")); + domain_bot_loadElement.appendChild(domain_bot_loadElementValue); + customElement.appendChild(domain_bot_loadElement); + + QDomElement automation_typeElement = _document.createElement("automation_type"); + QDomElement automation_typeElementId = _document.createElement("id"); + automation_typeElementId.appendChild(_document.createTextNode("0")); + automation_typeElement.appendChild(automation_typeElementId); + QDomElement automation_typeElementValue = _document.createElement("value"); + automation_typeElementValue.appendChild(_document.createTextNode("None")); + automation_typeElement.appendChild(automation_typeElementValue); + customElement.appendChild(automation_typeElement); + + QDomElement added_to_releaseElement = _document.createElement("added_to_release"); + QDomElement added_to_releaseElementId = _document.createElement("id"); + added_to_releaseElementId.appendChild(_document.createTextNode("4")); + added_to_releaseElement.appendChild(added_to_releaseElementId); + QDomElement added_to_releaseElementValue = _document.createElement("value"); + added_to_releaseElementValue.appendChild(_document.createTextNode(branchGitHub)); + added_to_releaseElement.appendChild(added_to_releaseElementValue); + customElement.appendChild(added_to_releaseElement); + + QDomElement precondsElement = _document.createElement("preconds"); + precondsElement.appendChild(_document.createTextNode( + "Tester is in an empty region of a domain in which they have edit rights\n\n*Note: Press 'n' to advance test script")); + customElement.appendChild(precondsElement); + + QString testMDName = QString("https://github.com/") + userGitHub + "/hifi_tests/blob/" + branchGitHub + + "/tests/content/entity/light/point/create/test.md"; + + QDomElement steps_seperatedElement = _document.createElement("steps_separated"); + QDomElement stepElement = _document.createElement("step"); + QDomElement stepIndexElement = _document.createElement("index"); + stepIndexElement.appendChild(_document.createTextNode("1")); + stepElement.appendChild(stepIndexElement); + QDomElement stepContentElement = _document.createElement("content"); + stepContentElement.appendChild( + _document.createTextNode(QString("Execute instructions in [THIS TEST](") + testMDName + ")")); + stepElement.appendChild(stepContentElement); + QDomElement stepExpectedElement = _document.createElement("expected"); + stepExpectedElement.appendChild(_document.createTextNode("Refer to the expected result in the linked description.")); + stepElement.appendChild(stepExpectedElement); + steps_seperatedElement.appendChild(stepElement); + customElement.appendChild(steps_seperatedElement); + + QDomElement notesElement = _document.createElement("notes"); + notesElement.appendChild(_document.createTextNode(testMDName)); + customElement.appendChild(notesElement); + + caseElement.appendChild(customElement); + + result.appendChild(caseElement); + + return result; +} + +void TestRailInterface::processTestPython(const QString& fullDirectory, + QTextStream& stream, + const QString& userGitHub, + const QString& branchGitHub) { + // The name of the test is derived from the full path. + // The first term is the first word after "tests" + // The last word is the penultimate word + QStringList words = fullDirectory.split('/'); + int i = 0; + while (words[i] != "tests") { + ++i; + if (i >= words.length() - 1) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Folder \"tests\" not found in " + fullDirectory); + exit(-1); + } + } + + ++i; + QString title{ words[i] }; + for (++i; i < words.length() - 1; ++i) { + title += " / " + words[i]; + } + + // To create the path to test.md, prefix by tests, and remove blanks + QString pathToTestMD = QString("/tests/") + title.remove(" "); + + stream << "section_id = parent_ids.peek()\n"; + + QString testMDName = + QString("https://github.com/") + userGitHub + "/hifi_tests/blob/" + branchGitHub + pathToTestMD + "/test.md "; + + QString testContent = QString("Execute instructions in [THIS TEST](") + testMDName + ")"; + QString testExpected = QString("Refer to the expected result in the linked description."); + + int milestone_id = _milestones[_milestoneNames[_testRailTestCasesSelectorWindow.getMilestoneID()]]; + + stream << "data = {\n" + << "\t'title': '" << title << "',\n" + << "\t'template_id': 2,\n" + << "\t'milestone_id': " << milestone_id << ",\n" + << "\t'custom_tester_count': 1,\n" + << "\t'custom_domain_bot_load': 1,\n" + << "\t'custom_added_to_release': 4,\n" + << "\t'custom_preconds': " + << "'Tester is in an empty region of a domain in which they have edit rights\\n\\n*Note: Press \\'n\\' to advance " + "test script',\n" + << "\t'custom_steps_separated': " + << "[\n\t\t{\n\t\t\t'content': '" << testContent << "',\n\t\t\t'expected': '" << testExpected << "'\n\t\t}\n\t]\n" + << "}\n"; + + stream << "case = client.send_post('add_case/' + str(section_id), data)\n"; +} + +void TestRailInterface::requestTestRailRunDataFromUser() { + _testRailRunSelectorWindow.reset(); + _testRailRunSelectorWindow.exec(); + + if (_testRailRunSelectorWindow.getUserCancelled()) { + return; + } + + _url = _testRailRunSelectorWindow.getURL() + "/"; + _user = _testRailRunSelectorWindow.getUser(); + _password = _testRailRunSelectorWindow.getPassword(); + ////_password = "tutKA76"; + _projectID = QString::number(_testRailRunSelectorWindow.getProjectID()); + _suiteID = QString::number(_testRailRunSelectorWindow.getSuiteID()); +} + +void TestRailInterface::getTestSectionsFromTestRail() { + QString filename = _outputDirectory + "/getSections.py"; + if (QFile::exists(filename)) { + QFile::remove(filename); + } + QFile file(filename); + + if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Could not create 'getSections.py'"); + exit(-1); + } + + QTextStream stream(&file); + + // Code to access TestRail + stream << "from testrail import *\n"; + stream << "client = APIClient('" << _url.toStdString().c_str() << "')\n"; + stream << "client.user = '" << _user << "'\n"; + stream << "client.password = '" << _password << "'\n\n"; + + // Print the list of sections without parents + stream << "sections = client.send_get('get_sections/" + _projectID + "&suite_id=" + _suiteID + "')\n\n"; + stream << "file = open('" + _outputDirectory + "/sections.txt', 'w')\n\n"; + stream << "for section in sections:\n"; + stream << "\tif section['parent_id'] == None:\n"; + stream << "\t\tfile.write(section['name'] + ' ' + str(section['id']) + '\\n')\n\n"; + stream << "file.close()\n"; + + file.close(); + + QProcess* process = new QProcess(); + connect(process, static_cast(&QProcess::finished), this, + [=](int exitCode, QProcess::ExitStatus exitStatus) { updateSectionsComboData(exitCode, exitStatus); }); + + QStringList parameters = QStringList() << _outputDirectory + "/getSections.py "; + process->start(_pythonCommand, parameters); +} + +void TestRailInterface::createTestRailRun(const QString& outputDirectory) { + _outputDirectory = outputDirectory; + + if (!setPythonCommand()) { + return; + } + + requestTestRailRunDataFromUser(); + createTestRailDotPyScript(); + createStackDotPyScript(); + + // TestRail will be updated after the process initiated by getTestCasesFromTestRail has completed + getTestSectionsFromTestRail(); +} \ No newline at end of file diff --git a/tools/auto-tester/src/TestRailInterface.h b/tools/auto-tester/src/TestRailInterface.h new file mode 100644 index 0000000000..4c2bc7b937 --- /dev/null +++ b/tools/auto-tester/src/TestRailInterface.h @@ -0,0 +1,119 @@ +// +// TestRailInterface.h +// +// Created by Nissim Hadar on 6 Jul 2018. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_test_testrail_interface_h +#define hifi_test_testrail_interface_h + +#include "ui/BusyWindow.h" + +#include "ui/TestRailTestCasesSelectorWindow.h" +#include "ui/TestRailRunSelectorWindow.h" + +#include +#include +#include +#include + +class TestRailInterface : public QObject{ + Q_OBJECT + +public: + TestRailInterface(); + + void createTestSuiteXML(const QString& testDirectory, + const QString& outputDirectory, + const QString& userGitHub, + const QString& branchGitHub); + + void createTestSuitePython(const QString& testDirectory, + const QString& outputDirectory, + const QString& userGitHub, + const QString& branchGitHub); + + QDomElement processDirectoryXML(const QString& directory, + const QString& useGitHubr, + const QString& branchGitHub, + const QDomElement& element); + + QDomElement processTestXML(const QString& fullDirectory, + const QString& test, + const QString& userGitHub, + const QString& branchGitHub, + const QDomElement& element); + + void processTestPython(const QString& fullDirectory, + QTextStream& stream, + const QString& userGitHub, + const QString& branchGitHub); + + void getMilestonesFromTestRail(); + void getTestSectionsFromTestRail(); + + void createTestRailDotPyScript(); + void createStackDotPyScript(); + + void requestTestRailTestCasesDataFromUser(); + void requestTestRailRunDataFromUser(); + + void createAddTestCasesPythonScript(const QString& testDirectory, + const QString& userGitHub, + const QString& branchGitHub); + + void processDirectoryPython(const QString& directory, + QTextStream& stream, + const QString& userGitHub, + const QString& branchGitHub); + + bool isAValidTestDirectory(const QString& directory); + + QString getObject(const QString& path); + + void updateMilestonesComboData(int exitCode, QProcess::ExitStatus exitStatus); + void updateSectionsComboData(int exitCode, QProcess::ExitStatus exitStatus); + + void createTestRailRun(const QString& outputDirectory); + + bool setPythonCommand(); + +private: + // HighFidelity Interface project ID in TestRail + const int INTERFACE_PROJECT_ID{ 24 }; + + // Rendering suite ID + const int INTERFACE_SUITE_ID{ 1147 }; + + QDomDocument _document; + + BusyWindow _busyWindow; + TestRailTestCasesSelectorWindow _testRailTestCasesSelectorWindow; + TestRailRunSelectorWindow _testRailRunSelectorWindow; + + QString _url; + QString _user; + QString _password; + QString _projectID; + QString _suiteID; + + QString _testDirectory; + QString _outputDirectory; + QString _userGitHub; + QString _branchGitHub; + + const QString pythonExe{ "python.exe" }; + QString _pythonCommand; + + std::map _milestones; + QStringList _milestoneNames; + + std::map _sections; + QStringList _sectionNames; +}; + +#endif \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.cpp b/tools/auto-tester/src/ui/AutoTester.cpp index 079fa63a9d..697711c4eb 100644 --- a/tools/auto-tester/src/ui/AutoTester.cpp +++ b/tools/auto-tester/src/ui/AutoTester.cpp @@ -16,56 +16,64 @@ #endif AutoTester::AutoTester(QWidget *parent) : QMainWindow(parent) { - ui.setupUi(this); - ui.checkBoxInteractiveMode->setChecked(true); - ui.progressBar->setVisible(false); + _ui.setupUi(this); + _ui.checkBoxInteractiveMode->setChecked(true); + _ui.progressBar->setVisible(false); - signalMapper = new QSignalMapper(); + _signalMapper = new QSignalMapper(); - connect(ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked); - connect(ui.actionAbout, &QAction::triggered, this, &AutoTester::about); + connect(_ui.actionClose, &QAction::triggered, this, &AutoTester::on_closeButton_clicked); + connect(_ui.actionAbout, &QAction::triggered, this, &AutoTester::about); #ifndef Q_OS_WIN - ui.hideTaskbarButton->setVisible(false); - ui.showTaskbarButton->setVisible(false); + _ui.hideTaskbarButton->setVisible(false); + _ui.showTaskbarButton->setVisible(false); #endif } void AutoTester::setup() { - test = new Test(); + _test = new Test(); } void AutoTester::runFromCommandLine(const QString& testFolder, const QString& branch, const QString& user) { - isRunningFromCommandline = true; - test->startTestsEvaluation(testFolder, branch, user); + _isRunningFromCommandline = true; + _test->startTestsEvaluation(testFolder, branch, user); } void AutoTester::on_evaluateTestsButton_clicked() { - test->startTestsEvaluation(); + _test->startTestsEvaluation(); } void AutoTester::on_createRecursiveScriptButton_clicked() { - test->createRecursiveScript(); + _test->createRecursiveScript(); } void AutoTester::on_createAllRecursiveScriptsButton_clicked() { - test->createAllRecursiveScripts(); + _test->createAllRecursiveScripts(); } void AutoTester::on_createTestsButton_clicked() { - test->createTests(); + _test->createTests(); } void AutoTester::on_createMDFileButton_clicked() { - test->createMDFile(); + _test->createMDFile(); } void AutoTester::on_createAllMDFilesButton_clicked() { - test->createAllMDFiles(); + _test->createAllMDFiles(); } void AutoTester::on_createTestsOutlineButton_clicked() { - test->createTestsOutline(); + _test->createTestsOutline(); +} + +void AutoTester::on_createTestRailTestCasesButton_clicked() { + _test->createTestRailTestCases(); +} + +void AutoTester::on_createTestRailRunButton_clicked() { + _test->createTestRailRun(); } // To toggle between show and hide @@ -96,11 +104,19 @@ void AutoTester::on_closeButton_clicked() { exit(0); } -void AutoTester::downloadImage(const QUrl& url) { - downloaders.emplace_back(new Downloader(url, this)); - connect(downloaders[_index], SIGNAL (downloaded()), signalMapper, SLOT (map())); +void AutoTester::on_createPythonScriptRadioButton_clicked() { + _test->setTestRailCreateMode(PYTHON); +} - signalMapper->setMapping(downloaders[_index], _index); +void AutoTester::on_createXMLScriptRadioButton_clicked() { + _test->setTestRailCreateMode(XML); +} + +void AutoTester::downloadImage(const QUrl& url) { + _downloaders.emplace_back(new Downloader(url, this)); + connect(_downloaders[_index], SIGNAL (downloaded()), _signalMapper, SLOT (map())); + + _signalMapper->setMapping(_downloaders[_index], _index); ++_index; } @@ -113,39 +129,39 @@ void AutoTester::downloadImages(const QStringList& URLs, const QString& director _numberOfImagesDownloaded = 0; _index = 0; - ui.progressBar->setMinimum(0); - ui.progressBar->setMaximum(_numberOfImagesToDownload - 1); - ui.progressBar->setValue(0); - ui.progressBar->setVisible(true); + _ui.progressBar->setMinimum(0); + _ui.progressBar->setMaximum(_numberOfImagesToDownload - 1); + _ui.progressBar->setValue(0); + _ui.progressBar->setVisible(true); - downloaders.clear(); + _downloaders.clear(); for (int i = 0; i < _numberOfImagesToDownload; ++i) { QUrl imageURL(URLs[i]); downloadImage(imageURL); } - connect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); + connect(_signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); } void AutoTester::saveImage(int index) { try { QFile file(_directoryName + "/" + _filenames[index]); file.open(QIODevice::WriteOnly); - file.write(downloaders[index]->downloadedData()); + file.write(_downloaders[index]->downloadedData()); file.close(); } catch (...) { QMessageBox::information(0, "Test Aborted", "Failed to save image: " + _filenames[index]); - ui.progressBar->setVisible(false); + _ui.progressBar->setVisible(false); return; } ++_numberOfImagesDownloaded; if (_numberOfImagesDownloaded == _numberOfImagesToDownload) { - disconnect(signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); - test->finishTestsEvaluation(isRunningFromCommandline, ui.checkBoxInteractiveMode->isChecked(), ui.progressBar); + disconnect(_signalMapper, SIGNAL (mapped(int)), this, SLOT (saveImage(int))); + _test->finishTestsEvaluation(_isRunningFromCommandline, _ui.checkBoxInteractiveMode->isChecked(), _ui.progressBar); } else { - ui.progressBar->setValue(_numberOfImagesDownloaded); + _ui.progressBar->setValue(_numberOfImagesDownloaded); } } @@ -154,18 +170,18 @@ void AutoTester::about() { } void AutoTester::setUserText(const QString& user) { - ui.userTextEdit->setText(user); + _ui.userTextEdit->setText(user); } QString AutoTester::getSelectedUser() { - return ui.userTextEdit->toPlainText(); + return _ui.userTextEdit->toPlainText(); } void AutoTester::setBranchText(const QString& branch) { - ui.branchTextEdit->setText(branch); + _ui.branchTextEdit->setText(branch); } QString AutoTester::getSelectedBranch() { - return ui.branchTextEdit->toPlainText(); + return _ui.branchTextEdit->toPlainText(); } diff --git a/tools/auto-tester/src/ui/AutoTester.h b/tools/auto-tester/src/ui/AutoTester.h index d47c4929c4..236a1ed733 100644 --- a/tools/auto-tester/src/ui/AutoTester.h +++ b/tools/auto-tester/src/ui/AutoTester.h @@ -45,10 +45,15 @@ private slots: void on_createMDFileButton_clicked(); void on_createAllMDFilesButton_clicked(); void on_createTestsOutlineButton_clicked(); + void on_createTestRailTestCasesButton_clicked(); + void on_createTestRailRunButton_clicked(); void on_hideTaskbarButton_clicked(); void on_showTaskbarButton_clicked(); + void on_createPythonScriptRadioButton_clicked(); + void on_createXMLScriptRadioButton_clicked(); + void on_closeButton_clicked(); void saveImage(int index); @@ -56,23 +61,23 @@ private slots: void about(); private: - Ui::AutoTesterClass ui; - Test* test; + Ui::AutoTesterClass _ui; + Test* _test; - std::vector downloaders; + std::vector _downloaders; // local storage for parameters - folder to store downloaded files in, and a list of their names QString _directoryName; QStringList _filenames; // Used to enable passing a parameter to slots - QSignalMapper* signalMapper; + QSignalMapper* _signalMapper; int _numberOfImagesToDownload { 0 }; int _numberOfImagesDownloaded { 0 }; int _index { 0 }; - bool isRunningFromCommandline { false }; + bool _isRunningFromCommandline { false }; }; #endif // hifi_AutoTester_h \ No newline at end of file diff --git a/tools/auto-tester/src/ui/AutoTester.ui b/tools/auto-tester/src/ui/AutoTester.ui index e12fc70e3f..680fe17785 100644 --- a/tools/auto-tester/src/ui/AutoTester.ui +++ b/tools/auto-tester/src/ui/AutoTester.ui @@ -6,8 +6,8 @@ 0 0 - 612 - 537 + 645 + 814 @@ -18,7 +18,7 @@ 380 - 430 + 620 101 40 @@ -44,7 +44,7 @@ 430 - 270 + 490 101 40 @@ -57,7 +57,7 @@ 330 - 110 + 340 220 40 @@ -70,7 +70,7 @@ 320 - 280 + 500 131 20 @@ -86,7 +86,7 @@ 320 - 330 + 550 255 23 @@ -99,7 +99,7 @@ 330 - 170 + 400 220 40 @@ -229,13 +229,68 @@ + + + + 410 + 100 + 140 + 40 + + + + Create TestRail Test Cases + + + + + + 310 + 100 + 95 + 20 + + + + Python + + + true + + + + + + 310 + 120 + 95 + 20 + + + + XML + + + + + + 410 + 180 + 140 + 40 + + + + Create TestRail Run + + 0 0 - 612 + 645 21 diff --git a/tools/auto-tester/src/ui/BusyWindow.cpp b/tools/auto-tester/src/ui/BusyWindow.cpp new file mode 100644 index 0000000000..904dc47d75 --- /dev/null +++ b/tools/auto-tester/src/ui/BusyWindow.cpp @@ -0,0 +1,18 @@ +// +// BusyWindow.cpp +// +// Created by Nissim Hadar on 26 Jul 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "BusyWindow.h" + +#include + +#include + +BusyWindow::BusyWindow(QWidget *parent) { + setupUi(this); +} diff --git a/tools/auto-tester/src/ui/BusyWindow.h b/tools/auto-tester/src/ui/BusyWindow.h new file mode 100644 index 0000000000..62f2df7e04 --- /dev/null +++ b/tools/auto-tester/src/ui/BusyWindow.h @@ -0,0 +1,22 @@ +// +// BusyWindow.h +// +// Created by Nissim Hadar on 29 Jul 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_BusyWindow_h +#define hifi_BusyWindow_h + +#include "ui_BusyWindow.h" + +class BusyWindow : public QDialog, public Ui::BusyWindow { + Q_OBJECT + +public: + BusyWindow(QWidget* parent = Q_NULLPTR); +}; + +#endif \ No newline at end of file diff --git a/tools/auto-tester/src/ui/BusyWindow.ui b/tools/auto-tester/src/ui/BusyWindow.ui new file mode 100644 index 0000000000..c237566a5e --- /dev/null +++ b/tools/auto-tester/src/ui/BusyWindow.ui @@ -0,0 +1,75 @@ + + + BusyWindow + + + Qt::ApplicationModal + + + + 0 + 0 + 542 + 189 + + + + Updating TestRail - please wait + + + + + 30 + 850 + 500 + 28 + + + + + 12 + + + + similarity + + + + + + 40 + 40 + 481 + 101 + + + + 0 + + + 0 + + + + + + 50 + 60 + 431 + 61 + + + + + 20 + + + + Please wait for this window to close + + + + + + + diff --git a/tools/auto-tester/src/ui/MismatchWindow.cpp b/tools/auto-tester/src/ui/MismatchWindow.cpp index d880a1abdc..79d2ce9f61 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.cpp +++ b/tools/auto-tester/src/ui/MismatchWindow.cpp @@ -66,14 +66,14 @@ void MismatchWindow::setTestFailure(TestFailure testFailure) { QPixmap expectedPixmap = QPixmap(testFailure._pathname + testFailure._expectedImageFilename); QPixmap actualPixmap = QPixmap(testFailure._pathname + testFailure._actualImageFilename); - diffPixmap = computeDiffPixmap( + _diffPixmap = computeDiffPixmap( QImage(testFailure._pathname + testFailure._expectedImageFilename), QImage(testFailure._pathname + testFailure._actualImageFilename) ); expectedImage->setPixmap(expectedPixmap); resultImage->setPixmap(actualPixmap); - diffImage->setPixmap(diffPixmap); + diffImage->setPixmap(_diffPixmap); } void MismatchWindow::on_passTestButton_clicked() { @@ -92,5 +92,5 @@ void MismatchWindow::on_abortTestsButton_clicked() { } QPixmap MismatchWindow::getComparisonImage() { - return diffPixmap; + return _diffPixmap; } diff --git a/tools/auto-tester/src/ui/MismatchWindow.h b/tools/auto-tester/src/ui/MismatchWindow.h index cdbdcb4098..f203a2be6a 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.h +++ b/tools/auto-tester/src/ui/MismatchWindow.h @@ -14,8 +14,7 @@ #include "../common.h" -class MismatchWindow : public QDialog, public Ui::MismatchWindow -{ +class MismatchWindow : public QDialog, public Ui::MismatchWindow { Q_OBJECT public: @@ -36,7 +35,7 @@ private slots: private: UserResponse _userResponse{ USER_RESPONSE_INVALID }; - QPixmap diffPixmap; + QPixmap _diffPixmap; }; diff --git a/tools/auto-tester/src/ui/MismatchWindow.ui b/tools/auto-tester/src/ui/MismatchWindow.ui index 72f86261ab..8a174989d4 100644 --- a/tools/auto-tester/src/ui/MismatchWindow.ui +++ b/tools/auto-tester/src/ui/MismatchWindow.ui @@ -2,6 +2,9 @@ MismatchWindow + + Qt::ApplicationModal + 0 @@ -193,4 +196,4 @@ - \ No newline at end of file + diff --git a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp new file mode 100644 index 0000000000..9d960b16c1 --- /dev/null +++ b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.cpp @@ -0,0 +1,101 @@ +// +// TestRailRunSelectorWindow.cpp +// +// Created by Nissim Hadar on 31 Jul 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "TestRailRunSelectorWindow.h" + +#include + +#include + +TestRailRunSelectorWindow::TestRailRunSelectorWindow(QWidget *parent) { + setupUi(this); + + projectIDLineEdit->setValidator(new QIntValidator(1, 999, this)); +} + + +void TestRailRunSelectorWindow::reset() { + urlLineEdit->setDisabled(false); + userLineEdit->setDisabled(false); + passwordLineEdit->setDisabled(false); + projectIDLineEdit->setDisabled(false); + + OKButton->setDisabled(true); + sectionsComboBox->setDisabled(true); +} + +void TestRailRunSelectorWindow::on_acceptButton_clicked() { + urlLineEdit->setDisabled(true); + userLineEdit->setDisabled(true); + passwordLineEdit->setDisabled(true); + projectIDLineEdit->setDisabled(true); + + OKButton->setDisabled(false); + sectionsComboBox->setDisabled(false); + close(); +} + +void TestRailRunSelectorWindow::on_OKButton_clicked() { + userCancelled = false; + close(); +} + +void TestRailRunSelectorWindow::on_cancelButton_clicked() { + userCancelled = true; + close(); +} + +bool TestRailRunSelectorWindow::getUserCancelled() { + return userCancelled; +} + +void TestRailRunSelectorWindow::setURL(const QString& user) { + urlLineEdit->setText(user); +} + +QString TestRailRunSelectorWindow::getURL() { + return urlLineEdit->text(); +} + +void TestRailRunSelectorWindow::setUser(const QString& user) { + userLineEdit->setText(user); +} + +QString TestRailRunSelectorWindow::getUser() { + return userLineEdit->text(); +} + +QString TestRailRunSelectorWindow::getPassword() { + return passwordLineEdit->text(); +} + +void TestRailRunSelectorWindow::setProjectID(const int project) { + projectIDLineEdit->setText(QString::number(project)); +} + +int TestRailRunSelectorWindow::getProjectID() { + return projectIDLineEdit->text().toInt(); +} + +void TestRailRunSelectorWindow::setSuiteID(const int project) { + suiteIDLineEdit->setText(QString::number(project)); +} + +int TestRailRunSelectorWindow::getSuiteID() { + return suiteIDLineEdit->text().toInt(); +} + +void TestRailRunSelectorWindow::updateSectionsComboBoxData(QStringList data) { + sectionsComboBox->insertItems(0, data); +} + +int TestRailRunSelectorWindow::getSectionID() { + return 0; + sectionsComboBox->currentIndex(); +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.h b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.h new file mode 100644 index 0000000000..d6428bb476 --- /dev/null +++ b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.h @@ -0,0 +1,50 @@ +// +// TestRailRunSelectorWindow.h +// +// Created by Nissim Hadar on 31 Jul 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_TestRailRunSelectorWindow_h +#define hifi_TestRailRunSelectorWindow_h + +#include "ui_TestRailRunSelectorWindow.h" + +class TestRailRunSelectorWindow : public QDialog, public Ui::TestRailRunSelectorWindow { + Q_OBJECT + +public: + TestRailRunSelectorWindow(QWidget* parent = Q_NULLPTR); + + void reset(); + + bool getUserCancelled(); + + void setURL(const QString& user); + QString getURL(); + + void setUser(const QString& user); + QString getUser(); + + QString getPassword(); + + void setProjectID(const int project); + int getProjectID(); + + void setSuiteID(const int project); + int getSuiteID(); + + bool userCancelled{ false }; + + void updateSectionsComboBoxData(QStringList data); + int getSectionID(); + +private slots: + void on_acceptButton_clicked(); + void on_OKButton_clicked(); + void on_cancelButton_clicked(); +}; + +#endif \ No newline at end of file diff --git a/tools/auto-tester/src/ui/TestRailRunSelectorWindow.ui b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.ui new file mode 100644 index 0000000000..ad39b5cc64 --- /dev/null +++ b/tools/auto-tester/src/ui/TestRailRunSelectorWindow.ui @@ -0,0 +1,283 @@ + + + TestRailRunSelectorWindow + + + Qt::ApplicationModal + + + + 0 + 0 + 489 + 474 + + + + TestRail Run Selector Window + + + + + 30 + 850 + 500 + 28 + + + + + 12 + + + + similarity + + + + + + 70 + 125 + 121 + 20 + + + + + 10 + + + + TestRail Password + + + + + + 70 + 25 + 121 + 20 + + + + + 10 + + + + TestRail URL + + + + + false + + + + 120 + 420 + 93 + 28 + + + + OK + + + + + + 280 + 420 + 93 + 28 + + + + Cancel + + + + + + 200 + 120 + 231 + 24 + + + + QLineEdit::Password + + + + + + 70 + 75 + 121 + 20 + + + + + 10 + + + + TestRail User + + + + + + 200 + 170 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 70 + 175 + 121 + 20 + + + + + 10 + + + + TestRail Project ID + + + + + + 200 + 270 + 231 + 28 + + + + Accept + + + + + false + + + + 140 + 350 + 311 + 22 + + + + + + true + + + + 20 + 350 + 121 + 20 + + + + + 10 + + + + TestRail Sections + + + + + + 200 + 20 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 200 + 70 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 200 + 215 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 70 + 220 + 121 + 20 + + + + + 10 + + + + TestRail Suite ID + + + + + + urlLineEdit + userLineEdit + passwordLineEdit + projectIDLineEdit + suiteIDLineEdit + acceptButton + sectionsComboBox + OKButton + cancelButton + + + + diff --git a/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.cpp b/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.cpp new file mode 100644 index 0000000000..27d91df1ac --- /dev/null +++ b/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.cpp @@ -0,0 +1,100 @@ +// +// TestRailTestCasesSelectorWindow.cpp +// +// Created by Nissim Hadar on 26 Jul 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#include "TestRailTestCasesSelectorWindow.h" + +#include + +#include + +TestRailTestCasesSelectorWindow::TestRailTestCasesSelectorWindow(QWidget *parent) { + setupUi(this); + + projectIDLineEdit->setValidator(new QIntValidator(1, 999, this)); +} + + +void TestRailTestCasesSelectorWindow::reset() { + urlLineEdit->setDisabled(false); + userLineEdit->setDisabled(false); + passwordLineEdit->setDisabled(false); + projectIDLineEdit->setDisabled(false); + + OKButton->setDisabled(true); + milestonesComboBox->setDisabled(true); +} + +void TestRailTestCasesSelectorWindow::on_acceptButton_clicked() { + urlLineEdit->setDisabled(true); + userLineEdit->setDisabled(true); + passwordLineEdit->setDisabled(true); + projectIDLineEdit->setDisabled(true); + + OKButton->setDisabled(false); + milestonesComboBox->setDisabled(false); + close(); +} + +void TestRailTestCasesSelectorWindow::on_OKButton_clicked() { + userCancelled = false; + close(); +} + +void TestRailTestCasesSelectorWindow::on_cancelButton_clicked() { + userCancelled = true; + close(); +} + +bool TestRailTestCasesSelectorWindow::getUserCancelled() { + return userCancelled; +} + +void TestRailTestCasesSelectorWindow::setURL(const QString& user) { + urlLineEdit->setText(user); +} + +QString TestRailTestCasesSelectorWindow::getURL() { + return urlLineEdit->text(); +} + +void TestRailTestCasesSelectorWindow::setUser(const QString& user) { + userLineEdit->setText(user); +} + +QString TestRailTestCasesSelectorWindow::getUser() { + return userLineEdit->text(); +} + +QString TestRailTestCasesSelectorWindow::getPassword() { + return passwordLineEdit->text(); +} + +void TestRailTestCasesSelectorWindow::setProjectID(const int project) { + projectIDLineEdit->setText(QString::number(project)); +} + +int TestRailTestCasesSelectorWindow::getProjectID() { + return projectIDLineEdit->text().toInt(); +} + +void TestRailTestCasesSelectorWindow::setSuiteID(const int project) { + suiteIDLineEdit->setText(QString::number(project)); +} + +int TestRailTestCasesSelectorWindow::getSuiteID() { + return suiteIDLineEdit->text().toInt(); +} + +void TestRailTestCasesSelectorWindow::updateMilestonesComboBoxData(QStringList data) { + milestonesComboBox->insertItems(0, data); +} + +int TestRailTestCasesSelectorWindow::getMilestoneID() { + return milestonesComboBox->currentIndex(); +} \ No newline at end of file diff --git a/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.h b/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.h new file mode 100644 index 0000000000..51e3a562ae --- /dev/null +++ b/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.h @@ -0,0 +1,50 @@ +// +// TestRailTestCasesSelectorWindow.h +// +// Created by Nissim Hadar on 26 Jul 2017. +// Copyright 2013 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#ifndef hifi_TestRailTestCasesSelectorWindow_h +#define hifi_TestRailTestCasesSelectorWindow_h + +#include "ui_TestRailTestCasesSelectorWindow.h" + +class TestRailTestCasesSelectorWindow : public QDialog, public Ui::TestRailTestCasesSelectorWindow { + Q_OBJECT + +public: + TestRailTestCasesSelectorWindow(QWidget* parent = Q_NULLPTR); + + void reset(); + + bool getUserCancelled(); + + void setURL(const QString& user); + QString getURL(); + + void setUser(const QString& user); + QString getUser(); + + QString getPassword(); + + void setProjectID(const int project); + int getProjectID(); + + void setSuiteID(const int project); + int getSuiteID(); + + bool userCancelled{ false }; + + void updateMilestonesComboBoxData(QStringList data); + int getMilestoneID(); + +private slots: + void on_acceptButton_clicked(); + void on_OKButton_clicked(); + void on_cancelButton_clicked(); +}; + +#endif \ No newline at end of file diff --git a/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.ui b/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.ui new file mode 100644 index 0000000000..2dc43d08f1 --- /dev/null +++ b/tools/auto-tester/src/ui/TestRailTestCasesSelectorWindow.ui @@ -0,0 +1,280 @@ + + + TestRailTestCasesSelectorWindow + + + + 0 + 0 + 489 + 474 + + + + TestRail Test Case Selector Window + + + + + 30 + 850 + 500 + 28 + + + + + 12 + + + + similarity + + + + + + 70 + 125 + 121 + 20 + + + + + 10 + + + + TestRail Password + + + + + + 70 + 25 + 121 + 20 + + + + + 10 + + + + TestRail URL + + + + + false + + + + 120 + 420 + 93 + 28 + + + + OK + + + + + + 280 + 420 + 93 + 28 + + + + Cancel + + + + + + 200 + 120 + 231 + 24 + + + + QLineEdit::Password + + + + + + 70 + 75 + 121 + 20 + + + + + 10 + + + + TestRail User + + + + + + 200 + 170 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 70 + 175 + 121 + 20 + + + + + 10 + + + + TestRail Project ID + + + + + + 200 + 270 + 231 + 28 + + + + Accept + + + + + false + + + + 270 + 350 + 161 + 22 + + + + + + true + + + + 140 + 350 + 121 + 20 + + + + + 10 + + + + TestRail Milestone + + + + + + 200 + 20 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 200 + 70 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 200 + 215 + 231 + 24 + + + + QLineEdit::Normal + + + + + + 70 + 220 + 121 + 20 + + + + + 10 + + + + TestRail Suite ID + + + + + + urlLineEdit + userLineEdit + passwordLineEdit + projectIDLineEdit + suiteIDLineEdit + acceptButton + milestonesComboBox + OKButton + cancelButton + + + +